Compare commits

..
3 Commits
Author SHA1 Message Date
Ryan Houdek 9fbdc00bd6 Docs: Update for release FEX-2609.1 2026-09-21 11:44:20 -07:00
Ryan Houdek 116053cd29 InstcountCI: Update 2026-09-21 11:43:34 -07:00
Ryan Houdek 536722f59d FEXCore/OpcodeDispatcher: Fixes syscall instruction on Linux
I forgot on Linux by default we didn't have the syscall instructions
count as block end. Change this so that it counts as block end now.

This has the additional benefit that now the frontend needs to modify
the RIP manually as well which is fine as it's what arm64ec and wow64
does.

Also add back the UnimplementedOp in RDPID that accidentally got caught
up. Also increment DiskCache version as both changes will change
codegen.

Fixes #5942
2026-09-21 11:43:28 -07:00
366 changed files with 17614 additions and 74990 deletions

No files matched your search

+2 -2
View File
@@ -24,7 +24,7 @@ jobs:
fail-fast: false
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
fetch-depth: '0'
fetch-tags: 'true'
@@ -154,7 +154,7 @@ jobs:
- name: Upload results
if: ${{ always() }}
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
timeout-minutes: 1
with:
name: Results-${{ env.runner_name }}-${{ env.runner_label }}
+2 -2
View File
@@ -31,7 +31,7 @@ jobs:
fail-fast: false
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
fetch-depth: '0'
fetch-tags: 'true'
@@ -115,7 +115,7 @@ jobs:
- name: Upload results
if: ${{ always() }}
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
timeout-minutes: 1
with:
name: Results-${{ env.runner_name }}-${{ env.runner_label }}
+3 -10
View File
@@ -24,7 +24,7 @@ jobs:
fail-fast: false
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
fetch-depth: '0'
fetch-tags: 'true'
@@ -40,7 +40,7 @@ jobs:
- name: Configure CMake
run: |
cmake -S . -B build -DCMAKE_BUILD_TYPE=$BUILD_TYPE -G Ninja -DENABLE_LTO=False \
-DENABLE_ASSERTIONS=True -DENABLE_X86_HOST_DEBUG=True -DBUILD_FEX_LINUX_TESTS=True
-DENABLE_ASSERTIONS=True -DENABLE_X86_HOST_DEBUG=True
# These steps make a lot of noise but rarely fail.
# Put them in a separate step to make normal build logs easier to parse
@@ -57,16 +57,9 @@ jobs:
with:
target: asm_tests
# host fex tests tests
- name: Host Tests
if: ${{ always() }}
uses: ./.github/workflows/test
with:
target: fex_linux_tests_host
- name: Upload results
if: ${{ always() }}
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
timeout-minutes: 1
with:
name: Results-${{ env.runner_name }}-${{ env.runner_label }}
+2 -2
View File
@@ -23,7 +23,7 @@ jobs:
fail-fast: false
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
fetch-depth: '0'
fetch-tags: 'true'
@@ -70,7 +70,7 @@ jobs:
- name: Upload results
if: ${{ always() }}
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
timeout-minutes: 1
with:
name: Results-${{ env.runner_name }}-${{ env.runner_label }}
+2 -3
View File
@@ -20,7 +20,7 @@ jobs:
fail-fast: false
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
fetch-depth: '0'
fetch-tags: 'true'
@@ -47,8 +47,7 @@ jobs:
run: |
cmake -S . -B build -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_TOOLCHAIN_FILE=$GITHUB_WORKSPACE/Data/CMake/toolchain_mingw.cmake \
-DMINGW_TRIPLE=$MINGW_TRIPLE -G Ninja -DENABLE_LTO=False -DENABLE_ASSERTIONS=True -DENABLE_X86_HOST_DEBUG=True -DBUILD_TESTING=False \
-DCMAKE_INSTALL_PREFIX="$PWD"/build/install \
-DBUILD_FEX_MINGW_TESTS=True -DLLVM_MINGW="$HOME/llvm-mingw/build/"
-DCMAKE_INSTALL_PREFIX="$PWD"/build/install
- name: Build
run: cmake --build build
+2 -18
View File
@@ -14,7 +14,7 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v7
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.sha }}
@@ -39,7 +39,7 @@ jobs:
run: which git-clang-format-19
- name: Setup Python env
uses: actions/setup-python@v7
uses: actions/setup-python@v4
with:
python-version: 3.11
cache: pip
@@ -61,19 +61,3 @@ jobs:
--start-rev "$START_REV" \
--end-rev "$END_REV" \
--changed-files "$CHANGED_FILES"
- name: Check long CMake lines
if: always()
run: |
# check lines > 140 chars
# unit tests are left alone for now
# FEXInterpreter is also left alone until its long echo lines can be removed
echo "$CHANGED_FILES" | tr ',' '\n' | grep -ve "unittests\|External\|FEXInterpreter" \
| grep -e "CMakeLists.txt\|\.cmake" | xargs -L1 wc -L | while read -r line; do
if [ "$(echo "$line" | cut -d' ' -f1)" -gt 140 ]; then
echo "$line"
fi
done | tee lines.txt
# if any lines were emitted, fail
[ ! -s lines.txt ]
+2 -2
View File
@@ -23,7 +23,7 @@ jobs:
fail-fast: false
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
fetch-depth: '0'
fetch-tags: 'true'
@@ -62,7 +62,7 @@ jobs:
run: DESTDIR="$PWD"/install distrobox enter --name steamrt4 -- cmake --build build -t install
- name: Upload libraries
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
timeout-minutes: 1
with:
overwrite: true
+2 -2
View File
@@ -25,7 +25,7 @@ jobs:
fail-fast: false
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
fetch-depth: '0'
fetch-tags: 'true'
@@ -75,7 +75,7 @@ jobs:
- name: Upload results
if: ${{ always() }}
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
timeout-minutes: 1
with:
name: Results-${{ env.runner_name }}-${{ env.runner_label }}
+2 -2
View File
@@ -17,7 +17,7 @@ jobs:
fail-fast: false
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
fetch-depth: '0'
fetch-tags: 'true'
@@ -45,7 +45,7 @@ jobs:
target: arm64ec
- name: Upload libraries
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
timeout-minutes: 1
with:
overwrite: true
+1 -1
View File
@@ -57,7 +57,7 @@ promote:
- arm64
- aarch64
rules:
- if: $PROMOTE_BRANCH && $CI_COMMIT_BRANCH == 'main'
- if: '$PROMOTE_BRANCH'
before_script:
- apt-get -y update
- apt-get install -y tmux curl
-1
View File
@@ -1 +0,0 @@
line_max_words = 140
+49 -130
View File
@@ -5,69 +5,40 @@ include(CheckIncludeFiles)
check_include_files("gdb/jit-reader.h" HAVE_GDB_JIT_READER_H)
option(BUILD_FEX_LINUX_TESTS "Build FEXLinuxTests (requires x86 compiler)" FALSE)
option(BUILD_FEX_MINGW_TESTS "Build FEXMingwTests (requires llvm-mingw compiler)" FALSE)
option(BUILD_THUNKS "Build thunks" FALSE)
option(ENABLE_CLANG_THUNKS "Build thunks with clang" TRUE)
option(BUILD_FEXCONFIG "Build FEXConfig" TRUE)
option(ENABLE_CLANG_THUNKS "Build thunks with clang" TRUE)
option(ENABLE_IWYU "Enable the Include What You Use sanitizer" FALSE)
option(ENABLE_LTO "Enable LTO with compilation" TRUE)
option(ENABLE_COVERAGE "Enable Code Coverage" FALSE)
set(USE_LINKER "" CACHE STRING "Path to a custom linker program")
option(ENABLE_XRAY "Enable building with LLVM X-Ray" FALSE)
set(USE_LINKER "" CACHE STRING "Path to a custom linker program")
option(ENABLE_UBSAN "Enable the Clang Undefined Behavior Sanitizer" FALSE)
option(ENABLE_ASAN "Enable the Clang Address Sanitizer" FALSE)
option(ENABLE_TSAN "Enable the Clang Thread Sanitizer" FALSE)
option(ENABLE_COVERAGE "Enable Code Coverage" FALSE)
option(ENABLE_ASSERTIONS "Enable debug assertions" FALSE)
option(ENABLE_GDB_SYMBOLS "Enable GDBSymbols integration support" ${HAVE_GDB_JIT_READER_H})
option(ENABLE_STRICT_WERROR "Enable stricter -Werror" FALSE)
option(ENABLE_WERROR "Enable -Werror" FALSE)
option(ENABLE_FEX_ALLOCATOR "Enable allocator for FEX" TRUE)
option(ENABLE_JEMALLOC_GLIBC_ALLOC "Enable jemalloc glibc allocator" TRUE)
option(ENABLE_OFFLINE_TELEMETRY "Enable FEX offline telemetry" TRUE)
option(ENABLE_COMPILE_TIME_TRACE "Enable time trace compile option" FALSE)
option(ENABLE_LIBCXX "Use LLVM's libc++ instead of the GNU libstdc++" FALSE)
option(ENABLE_CCACHE "Enable ccache for build caching" TRUE)
option(ENABLE_VIXL_SIMULATOR "Use the VIXL simulator for emulation (only useful for CI testing)" FALSE)
option(ENABLE_VIXL_DISASSEMBLER "Enable debug disassembler output with VIXL" FALSE)
option(ENABLE_ZYDIS "Enable x86/x86-64 guest disassembler output with Zydis" FALSE)
option(USE_LEGACY_BINFMTMISC "Use legacy method of setting up binfmt_misc" FALSE)
option(ENABLE_FEXCORE_PROFILER "Enable FEXCore's timeline profiling capabilities" FALSE)
set(FEXCORE_PROFILER_BACKEND "gpuvis" CACHE STRING "Set which backend to use for FEXCore's profiler")
set_property(CACHE FEXCORE_PROFILER_BACKEND PROPERTY STRINGS gpuvis tracy)
option(ENABLE_GLIBC_ALLOCATOR_HOOK_FAULT "Enables glibc memory allocation hooking with fault for CI testing")
option(USE_PDB_DEBUGINFO "Build debug info in PDB format" FALSE)
option(BUILD_STEAM_SUPPORT "Enable Steam integration" FALSE)
set(X86_32_TOOLCHAIN_FILE
"${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_x86_32.cmake"
CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting i686")
set(X86_64_TOOLCHAIN_FILE
"${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_x86_64.cmake"
CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting x86_64")
if (BUILD_FEX_MINGW_TESTS)
set(LLVM_MINGW "" CACHE FILEPATH "Path to llvm_mingw")
set(X86_32_MINGW_TOOLCHAIN_FILE
"${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_mingw_x86_32.cmake"
CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting mingw_i686")
set(X86_64_MINGW_TOOLCHAIN_FILE
"${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_mingw_x86_64.cmake"
CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting mingw_x86_64")
endif()
set(X86_32_TOOLCHAIN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_x86_32.cmake" CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting i686")
set(X86_64_TOOLCHAIN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_x86_64.cmake" CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting x86_64")
set(X86_DEV_ROOTFS "/" CACHE FILEPATH "Path to the sysroot used for cross-compiling for i686 and x86_64")
set(DATA_DIRECTORY "" CACHE PATH "Global data directory (override)")
set(HOSTLIBS_DATA_DIRECTORY "" CACHE PATH "Global data directory (override)")
@@ -110,8 +81,7 @@ else ()
message(STATUS "Clang version ${CMAKE_CXX_COMPILER_VERSION}")
set(CLANG_MINIMUM_VERSION 13.0)
if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION})
message(FATAL_ERROR "Clang version too old for FEX. Need at least "
"${CLANG_MINIMUM_VERSION} but has ${CMAKE_CXX_COMPILER_VERSION}")
message(FATAL_ERROR "Clang version too old for FEX. Need at least ${CLANG_MINIMUM_VERSION} but has ${CMAKE_CXX_COMPILER_VERSION}")
endif()
endif()
@@ -167,8 +137,7 @@ if (ENABLE_FEXCORE_PROFILER)
# Tracy can gather call stack samples in regular intervals, but this
# isn't useful for us since it would usually sample opaque JIT code
add_compile_definitions(TRACY_NO_SAMPLING=1)
# This pulls in libbacktrace which allocators in global constructors
# (before FEX can set up its allocator hooks)
# This pulls in libbacktrace which allocators in global constructors (before FEX can set up its allocator hooks)
add_compile_definitions(TRACY_NO_CALLSTACK=1)
if (MINGW)
message(FATAL_ERROR "Tracy profiler not supported on MinGW")
@@ -205,7 +174,8 @@ set(OVERRIDE_HASH "detect" CACHE STRING "Override the FEX git hash")
get_property(IS_MULTI_CONFIG GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
if (NOT IS_MULTI_CONFIG AND NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE)
set(CMAKE_BUILD_TYPE Release
CACHE STRING "Choose the type of build." FORCE)
message(STATUS "No build type set, defaulting to a Release build")
endif()
@@ -254,17 +224,11 @@ check_cxx_source_compiles(
HAS_CLANG_PRESERVE_ALL)
unset(CMAKE_REQUIRED_FLAGS)
if (HAS_CLANG_PRESERVE_ALL)
message(STATUS "Clang version ${CMAKE_CXX_COMPILER_VERSION}")
set(CLANG_MINIMUM_VERSION 22.1)
if (MINGW AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION})
if (MINGW)
message(STATUS "Ignoring broken clang::preserve_all support")
set(HAS_CLANG_PRESERVE_ALL FALSE)
else()
message(STATUS "Has clang::preserve_all")
if (MINGW)
message(STATUS "Disabling asynchronous-unwind-tables due to bug with preserve_all and mingw")
add_compile_options(-fno-asynchronous-unwind-tables)
endif()
endif()
endif()
@@ -282,7 +246,7 @@ check_cxx_source_compiles(
return program_invocation_name == nullptr;
}"
HAS_PROGRAM_INVOCATION_NAME)
add_compile_definitions( "HAS_PROGRAM_INVOCATION_NAME=${HAS_PROGRAM_INVOCATION_NAME}")
add_compile_definitions("HAS_PROGRAM_INVOCATION_NAME=${HAS_PROGRAM_INVOCATION_NAME}")
if (ENABLE_VIXL_SIMULATOR)
# We can run the simulator on both x86-64 or AArch64 hosts
@@ -314,8 +278,7 @@ if (ENABLE_COMPILE_TIME_TRACE)
link_libraries(-ftime-trace)
endif()
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
set(PTHREAD_LIB pthread)
if (USE_LINKER)
message(STATUS "Overriding linker to: ${USE_LINKER}")
@@ -324,8 +287,8 @@ endif()
if (ENABLE_LIBCXX)
message(WARNING "This is an unsupported configuration and should only be used for testing")
string(APPEND CMAKE_CXX_FLAGS " -std=c++11 -stdlib=libc++")
string(APPEND CMAKE_EXE_LINKER_FLAGS " -stdlib=libc++ -lc++abi")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -stdlib=libc++")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -stdlib=libc++ -lc++abi")
endif()
if (NOT ENABLE_OFFLINE_TELEMETRY)
@@ -335,16 +298,15 @@ endif()
if (ENABLE_UBSAN)
# See https://github.com/FEX-Emu/FEX/pull/4494#issuecomment-2800608944
# and related discussion for the use of -fno-sanitize=alignment and -fno-sanitize=function with UBSAN.
# alignment: we don't follow a strict alignment policy, for example IR uses packed structs that are regularly access unaligned.
# function: syscalls cast function pointers to void (*)(unsigned long...), causing warnings related to this access.
# and related discussion for the use of -fno-sanitize=alignment -fno-sanitize=function
# with UBSAN.
# alignment: we don't follow a strict alignment policy, for example IR uses packed structs
# that are regularly access unaligned.
# function: syscalls cast function pointers to void (*)(unsigned long...), causing warnings
# related to this access.
add_compile_definitions(ENABLE_UBSAN=1)
add_compile_options(-fno-omit-frame-pointer -fsanitize=undefined
-fno-sanitize=alignment -fno-sanitize=function
-fno-sanitize-recover=undefined)
link_libraries(-fno-omit-frame-pointer -fsanitize=undefined
-fno-sanitize=alignment -fno-sanitize=function
-fno-sanitize-recover=undefined)
add_compile_options(-fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize=alignment -fno-sanitize=function -fno-sanitize-recover=undefined)
link_libraries(-fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize=alignment -fno-sanitize=function -fno-sanitize-recover=undefined)
endif()
if (ENABLE_ASAN)
@@ -366,8 +328,7 @@ endif()
if (ENABLE_JEMALLOC_GLIBC_ALLOC)
# The glibc jemalloc subproject which hooks the glibc allocator.
# Required for thunks to work.
# All host native libraries will use this allocator, while *most*
# other FEX internal allocations will use the other jemalloc allocator.
# All host native libraries will use this allocator, while *most* other FEX internal allocations will use the other jemalloc allocator.
add_subdirectory(External/jemalloc_glibc/)
elseif (NOT MINGW)
message(STATUS
@@ -393,11 +354,11 @@ if (USE_PDB_DEBUGINFO)
add_link_options(-g -Wl,--pdb=)
endif()
string(APPEND CMAKE_CXX_FLAGS_RELWITHDEBINFO " -fno-omit-frame-pointer")
string(APPEND CMAKE_LINKER_FLAGS_RELWITHDEBINFO " -fno-omit-frame-pointer")
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-omit-frame-pointer")
set(CMAKE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_LINKER_FLAGS_RELWITHDEBINFO} -fno-omit-frame-pointer")
string(APPEND CMAKE_CXX_FLAGS_RELEASE " -fomit-frame-pointer")
string(APPEND CMAKE_LINKER_FLAGS_RELEASE " -fomit-frame-pointer")
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fomit-frame-pointer")
set(CMAKE_LINKER_FLAGS_RELEASE "${CMAKE_LINKER_FLAGS_RELEASE} -fomit-frame-pointer")
## Modules ##
list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/Data/CMake/)
@@ -406,10 +367,7 @@ include(LinkerGC)
## Externals ##
if (NOT BUILD_STEAM_SUPPORT)
find_package(unordered_dense QUIET CONFIG)
endif()
find_package(unordered_dense QUIET CONFIG)
if (NOT unordered_dense_FOUND)
add_subdirectory(External/unordered_dense)
endif()
@@ -420,10 +378,8 @@ if (BUILD_TESTING OR ENABLE_VIXL_DISASSEMBLER OR ENABLE_VIXL_SIMULATOR)
endif()
if (ENABLE_ZYDIS)
if (NOT BUILD_STEAM_SUPPORT)
find_package(Zycore 1.5 MODULE QUIET)
find_package(Zydis 4.0 MODULE QUIET)
endif()
find_package(Zycore 1.5 MODULE QUIET)
find_package(Zydis 4.0 MODULE QUIET)
if (TARGET Zydis::Zydis AND TARGET Zycore::Zycore)
message(STATUS "Using system Zydis")
@@ -444,7 +400,7 @@ find_package(Python 3.9 REQUIRED COMPONENTS Interpreter)
set(BUILD_SHARED_LIBS OFF)
if (NOT CMAKE_CROSSCOMPILING AND NOT BUILD_STEAM_SUPPORT)
if (NOT CMAKE_CROSSCOMPILING)
find_package(xxhash MODULE QUIET)
endif()
@@ -463,8 +419,7 @@ if (BUILD_TESTING)
add_subdirectory(External/Catch2/)
# Pull in catch_discover_tests definition
list(APPEND CMAKE_MODULE_PATH
"${CMAKE_CURRENT_SOURCE_DIR}/External/Catch2/contrib/")
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/External/Catch2/contrib/")
endif()
include(Catch)
@@ -473,7 +428,7 @@ else ()
file(GENERATE OUTPUT CTestTestfile.cmake CONTENT "# No tests since BUILD_TESTING is disabled")
endif()
if (MINGW OR BUILD_STEAM_SUPPORT)
if (MINGW)
set(FMT_INSTALL OFF)
add_subdirectory(External/fmt/)
else()
@@ -485,14 +440,10 @@ else()
endif()
endif()
if (NOT BUILD_STEAM_SUPPORT)
find_package(range-v3 QUIET)
endif()
find_package(range-v3 QUIET)
if (NOT range-v3_FOUND)
add_subdirectory(External/range-v3/)
target_compile_definitions(range-v3 INTERFACE
RANGES_DISABLE_DEPRECATED_WARNINGS)
target_compile_definitions(range-v3 INTERFACE RANGES_DISABLE_DEPRECATED_WARNINGS)
endif()
add_subdirectory(External/tiny-json/)
@@ -533,13 +484,11 @@ if(ENABLE_WERROR OR ENABLE_STRICT_WERROR)
endif()
set(FEX_TUNE_COMPILE_FLAGS)
if (NOT TUNE_ARCH STREQUAL "generic")
set(TUNE_ARCH_STRING "${TUNE_ARCH}")
if(ARCHITECTURE_arm64)
set(TUNE_ARCH_STRING "${TUNE_ARCH}+crc")
endif()
check_cxx_compiler_flag("-march=${TUNE_ARCH_STRING}" COMPILER_SUPPORTS_ARCH_TYPE)
if(COMPILER_SUPPORTS_ARCH_TYPE)
list(APPEND FEX_TUNE_COMPILE_FLAGS "-march=${TUNE_ARCH_STRING}")
@@ -556,23 +505,19 @@ if (TUNE_CPU STREQUAL "native")
if(ARCHITECTURE_arm64)
if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 999999.0)
# Clang 12.0 fixed the -mcpu=native bug with mixed big.little implementers
# Clang can not currently check for native Apple M1 type in hypervisor.
# Currently disabled
# Clang can not currently check for native Apple M1 type in hypervisor. Currently disabled
check_cxx_compiler_flag("-mcpu=native" COMPILER_SUPPORTS_CPU_TYPE)
if(COMPILER_SUPPORTS_CPU_TYPE)
list(APPEND FEX_TUNE_COMPILE_FLAGS "-mcpu=native")
endif()
else()
execute_process(COMMAND python3
"${PROJECT_SOURCE_DIR}/Scripts/aarch64_fit_native.py"
"/proc/cpuinfo" "${CMAKE_CXX_COMPILER_VERSION}"
execute_process(COMMAND python3 "${PROJECT_SOURCE_DIR}/Scripts/aarch64_fit_native.py" "/proc/cpuinfo" "${CMAKE_CXX_COMPILER_VERSION}"
OUTPUT_VARIABLE AARCH64_CPU)
string(STRIP ${AARCH64_CPU} AARCH64_CPU)
execute_process(COMMAND python3 "${PROJECT_SOURCE_DIR}/Scripts/NeedDisabledSVE.py"
RESULT_VARIABLE NEEDS_SVE_DISABLED)
if (NEEDS_SVE_DISABLED)
message(STATUS "Platform has bugged SVE. Disabling")
set(AARCH64_CPU "cortex-a78")
@@ -594,8 +539,7 @@ elseif (NOT TUNE_CPU STREQUAL "none")
if(COMPILER_SUPPORTS_CPU_TYPE)
list(APPEND FEX_TUNE_COMPILE_FLAGS "-mcpu=${TUNE_CPU}")
else()
message(FATAL_ERROR "Trying to compile cpu type '${TUNE_CPU}' "
"but the compiler doesn't support this")
message(FATAL_ERROR "Trying to compile cpu type '${TUNE_CPU}' but the compiler doesn't support this")
endif()
endif()
@@ -658,38 +602,13 @@ add_compile_options(-Wall)
if (BUILD_TESTING)
message(STATUS "Unit tests are enabled")
set(TEST_JOB_COUNT "" CACHE STRING
"Override number of parallel jobs to use while running tests")
set(TEST_JOB_COUNT "" CACHE STRING "Override number of parallel jobs to use while running tests")
if (TEST_JOB_COUNT)
message(STATUS "Running tests with ${TEST_JOB_COUNT} jobs")
elseif(CMAKE_VERSION VERSION_LESS "3.29")
execute_process(COMMAND "nproc"
OUTPUT_STRIP_TRAILING_WHITESPACE
OUTPUT_VARIABLE TEST_JOB_COUNT)
execute_process(COMMAND "nproc" OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE TEST_JOB_COUNT)
endif()
set(TEST_JOB_FLAG "-j${TEST_JOB_COUNT}")
# When running "ninja tests", defined below, it is useful to intentionally
# oversubscribe the CPU cores since many of the tests spend their time idle.
if ("$CACHE{TEST_JOB_COUNT}" STREQUAL "")
execute_process(COMMAND "nproc"
OUTPUT_STRIP_TRAILING_WHITESPACE
OUTPUT_VARIABLE TESTS_TARGET_JOB_COUNT)
math(EXPR TESTS_TARGET_JOB_COUNT "${TESTS_TARGET_JOB_COUNT} * 3 / 2")
set(TESTS_TARGET_JOB_FLAG "-j${TESTS_TARGET_JOB_COUNT}")
else()
set(TESTS_TARGET_JOB_FLAG ${TEST_JOB_FLAG})
endif()
# Runs the whole test suite, with large pages enabled to
# reduce the cost of fork() in ctest.
add_custom_target(tests
WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
USES_TERMINAL
COMMAND ${CMAKE_COMMAND} -E env GLIBC_TUNABLES=glibc.malloc.hugetlb=1
ctest "--progress" "--timeout" "302" ${TESTS_TARGET_JOB_FLAG})
endif()
add_subdirectory(External/SoftFloat-3e/)
@@ -709,15 +628,15 @@ if (NOT BUILD_STEAM_SUPPORT)
add_subdirectory(Data/AppConfig/)
endif()
if (NOT MINGW)
# Install the ThunksDB file
file(GLOB CONFIG_SOURCES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/Data/*.json)
foreach(CONFIG_SRC ${CONFIG_SOURCES})
install(FILES ${CONFIG_SRC}
DESTINATION ${DATA_DIRECTORY}/
COMPONENT Runtime)
endforeach()
endif()
# Install the ThunksDB file
file(GLOB CONFIG_SOURCES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/Data/*.json)
# Any application configuration json file gets installed
foreach(CONFIG_SRC ${CONFIG_SOURCES})
install(FILES ${CONFIG_SRC}
DESTINATION ${DATA_DIRECTORY}/
COMPONENT Runtime)
endforeach()
if (BUILD_TESTING)
add_subdirectory(unittests/)
-10
View File
@@ -768,16 +768,6 @@ public:
ASIMD2RegMisc(0, size, 0b11101, rd, rn);
}
}
///< size is the destination size.
///< source size is the next size up.
void bfcvtn(QRegister rd, QRegister rn) {
ASIMD2RegMisc(0, SubRegSize::i32Bit, 0b10110, rd.D(), rn.D());
}
///< size is the destination size.
///< source size is the next size up.
void bfcvtn2(QRegister rd, QRegister rn) {
ASIMD2RegMisc(0, SubRegSize::i32Bit, 0b10110, rd, rn);
}
template<IsQOrDRegister T>
void rev32(SubRegSize size, T rd, T rn) {
+1 -1
View File
@@ -3,7 +3,7 @@
# This applies some common linker options that reduce code size and linking time in Release mode. Namely:
# --gc-sections: Linktime garbage collection, discards unused sections from the final output
# --strip-all : Similar to running `strip`, discards the symbol table from the final output
# --as-needed : Only includes libraries that are actually needed in the final output
# --as-needed : Only includes libraries that are actually needed in the final output.
macro(LinkerGC target)
if (CMAKE_BUILD_TYPE MATCHES "RELEASE")
+2 -5
View File
@@ -6,17 +6,14 @@ set(CMAKE_CXX_COMPILER ${MINGW_TRIPLE}-clang++)
set(CMAKE_DLLTOOL ${MINGW_TRIPLE}-dlltool)
set(CMAKE_AR ${MINGW_TRIPLE}-ar)
# Compile everything as static to avoid requiring the MinGW runtime libraries,
# force page aligned sections so that debug symbols work correctly,
# and disable loop alignment to workaround an LLVM bug
# Compile everything as static to avoid requiring the MinGW runtime libraries, force page aligned sections so that
# debug symbols work correctly, and disable loop alignment to workaround an LLVM bug
# (https://github.com/llvm/llvm-project/issues/47432)
set(CMAKE_SHARED_LINKER_FLAGS_INIT "-static -Wl,--file-alignment=4096,/mllvm:-align-loops=1")
set(CMAKE_EXE_LINKER_FLAGS_INIT "-static -Wl,--file-alignment=4096,/mllvm:-align-loops=1")
set(CMAKE_C_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
set(CMAKE_CXX_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
set(CMAKE_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
set(CMAKE_SYSTEM_NAME Windows)
set(CMAKE_SYSTEM_PROCESSOR ${MINGW_TRIPLE})
-30
View File
@@ -1,30 +0,0 @@
set(CMAKE_SYSTEM_PROCESSOR i686)
set(LLVM_MINGW "" CACHE FILEPATH "Path to LLVM_MINGW")
if (LLVM_MINGW)
set(LLVM_MINGW_BIN "${LLVM_MINGW}/bin/")
else()
set(LLVM_MINGW_BIN "")
endif()
set(MINGW_TRIPLE "i686-w64-mingw32")
set(CMAKE_RC_COMPILER ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-windres)
set(CMAKE_C_COMPILER ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-clang)
set(CMAKE_CXX_COMPILER ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-clang++)
set(CMAKE_DLLTOOL ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-dlltool)
set(CMAKE_AR ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-ar)
# Compile everything as static to avoid requiring the MinGW runtime libraries, force page aligned sections so that
# debug symbols work correctly, and disable loop alignment to workaround an LLVM bug
# (https://github.com/llvm/llvm-project/issues/47432)
set(CMAKE_SHARED_LINKER_FLAGS_INIT "-static -Wl,--file-alignment=4096,/mllvm:-align-loops=1")
set(CMAKE_EXE_LINKER_FLAGS_INIT "-static -Wl,--file-alignment=4096,/mllvm:-align-loops=1")
set(CMAKE_C_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
set(CMAKE_CXX_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
set(CMAKE_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
set(CMAKE_SYSTEM_NAME Windows)
set(CMAKE_SYSTEM_PROCESSOR ${MINGW_TRIPLE})
set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)
-29
View File
@@ -1,29 +0,0 @@
set(CMAKE_SYSTEM_PROCESSOR x86_64)
set(LLVM_MINGW "" CACHE FILEPATH "Path to LLVM_MINGW")
if (LLVM_MINGW)
set(LLVM_MINGW_BIN "${LLVM_MINGW}/bin/")
else()
set(LLVM_MINGW_BIN "")
endif()
set(MINGW_TRIPLE "x86_64-w64-mingw32")
set(CMAKE_RC_COMPILER ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-windres)
set(CMAKE_C_COMPILER ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-clang)
set(CMAKE_CXX_COMPILER ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-clang++)
set(CMAKE_DLLTOOL ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-dlltool)
set(CMAKE_AR ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-ar)
# Compile everything as static to avoid requiring the MinGW runtime libraries, force page aligned sections so that
# debug symbols work correctly, and disable loop alignment to workaround an LLVM bug
# (https://github.com/llvm/llvm-project/issues/47432)
set(CMAKE_SHARED_LINKER_FLAGS_INIT "-static -Wl,--file-alignment=4096,/mllvm:-align-loops=1")
set(CMAKE_EXE_LINKER_FLAGS_INIT "-static -Wl,--file-alignment=4096,/mllvm:-align-loops=1")
set(CMAKE_C_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
set(CMAKE_CXX_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
set(CMAKE_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
set(CMAKE_SYSTEM_NAME Windows)
set(CMAKE_SYSTEM_PROCESSOR ${MINGW_TRIPLE})
set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)
+1 -2
View File
@@ -15,8 +15,7 @@ if (NOT USE_LEGACY_BINFMTMISC)
configure_file(FEX-x86.conf.in ${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86.conf)
configure_file(FEX-x86_64.conf.in ${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86_64.conf)
install(FILES ${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86.conf
${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86_64.conf
install(FILES ${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86.conf ${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86_64.conf
DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/binfmt.d/
COMPONENT Runtime)
else()
+37 -37
View File
@@ -4,40 +4,40 @@
#
# pip-compile --generate-hashes --output-file=requirements_formatting.txt --strip-extras requirements_formatting.txt.in
#
black==26.10.1 \
--hash=sha256:03c0ddd93bb392e71209903a691767eb366fe1a76deb9509ccbaae9e1f14bb52 \
--hash=sha256:0ce08b367307b0fd91c9dd1d4084e62b05b3055475f951f0f34a46b6e2393b64 \
--hash=sha256:182f6c32be38074b16d378498c498b32cb51928178ee611485344972c35ec9c6 \
--hash=sha256:1935b32f5326028019856e18cb42b4da63db23765dc84464cec723e0de478a9b \
--hash=sha256:19fa8f5beb5e77c54c9c7e21d00cc93ed6c8b6228ee385616906d6befe081143 \
--hash=sha256:2520037aa62f8a1454d0811b8f5c88b444445b03a4bfba480d8d220893b64c34 \
--hash=sha256:28842f9a8207cc1df6eb983a35a14c5a0dfcd603d214fe82d84bef552afd2e3a \
--hash=sha256:289282aa2e09d3162312a3be1788ff21b08e9ea9cc4a81e656024728b32428fb \
--hash=sha256:2ffbc023a12d0c729408823b8f10514490bd0baa301d0d4e21a7240249f9507f \
--hash=sha256:3414a0c52901964dceabd98c7c56beac0f964115a116ecedcce7247359b14017 \
--hash=sha256:4d9a90516db1d99c25dbb20cc0998e0e01531dd903466c7744e56d66f864220a \
--hash=sha256:51d5e417e700fe6ec0b0ecdc408c6f6cb5def80328f31f724993d82c6486b746 \
--hash=sha256:5cd88fd7b444ca51f3fc883b6f6657ea53a258b0b2eef6d9f2dfcfa17ce0e27b \
--hash=sha256:5f9f83beae62437e060dafd53d7f1fc327e3d3494f74d72ee5c2b73eb90fc4e7 \
--hash=sha256:70ccbd175b7f6be29d2b727ee7ca6b4c54053df59da653a6df80b175d20a94fa \
--hash=sha256:7bdade400bfe24d78a7762896acc2f9a8e1a17fb0fd0536bf6b7c7097cf3eec7 \
--hash=sha256:8375962579d537364cc0efa19b1474481915d3a793f9fc0774901814c5e5b5f4 \
--hash=sha256:978113a40223a6aaefc17364176a809a320e6b288683841427fff04c6d7b4130 \
--hash=sha256:9a0219b29cd70e49f920acb7081e6ce5025c719008447c521d0200dcad93206a \
--hash=sha256:b5347d760f0c02bb00dd249384cab71c3bf828b4f68d5b401eb116e0390f147d \
--hash=sha256:b6272cfd7e1e8e271f5b0e0207259fe2834687e5cb9b5f620b34a44db9754993 \
--hash=sha256:d42dd2fac7c342ae67e64ee99c9532e20b2a84e92c79ed3317fa2ef54c801d93 \
--hash=sha256:d5bd3518d8e97138fef295230b1e9804076d69fa4e3594071494a8c68abe6266 \
--hash=sha256:d8b3a9074a680b3c5749633714e9ae3992a1e5a23343a97ad61cd9b119b444d2 \
--hash=sha256:f6dba8138cdc99061ef07b958ac082d2aa057b6961d1936f9717c350f02bab5f \
--hash=sha256:fe85fc4019bee59bc495c0f2a8ee76c5cd02c7015508d94a967ba2376f39a52c \
--hash=sha256:ff57f63029aa1353fa8b1b0c8971fd88a6c92dc766608d2eee33ad2deb23270e
black==26.3.1 \
--hash=sha256:0126ae5b7c09957da2bdbd91a9ba1207453feada9e9fe51992848658c6c8e01c \
--hash=sha256:0f76ff19ec5297dd8e66eb64deda23631e642c9393ab592826fd4bdc97a4bce7 \
--hash=sha256:28ef38aee69e4b12fda8dba75e21f9b4f979b490c8ac0baa7cb505369ac9e1ff \
--hash=sha256:2bd5aa94fc267d38bb21a70d7410a89f1a1d318841855f698746f8e7f51acd1b \
--hash=sha256:2c50f5063a9641c7eed7795014ba37b0f5fa227f3d408b968936e24bc0566b07 \
--hash=sha256:2d6bfaf7fd0993b420bed691f20f9492d53ce9a2bcccea4b797d34e947318a78 \
--hash=sha256:41cd2012d35b47d589cb8a16faf8a32ef7a336f56356babd9fcf70939ad1897f \
--hash=sha256:474c27574d6d7037c1bc875a81d9be0a9a4f9ee95e62800dab3cfaadbf75acd5 \
--hash=sha256:5602bdb96d52d2d0672f24f6ffe5218795736dd34807fd0fd55ccd6bf206168b \
--hash=sha256:5e9d0d86df21f2e1677cc4bd090cd0e446278bcbbe49bf3659c308c3e402843e \
--hash=sha256:5ed0ca58586c8d9a487352a96b15272b7fa55d139fc8496b519e78023a8dab0a \
--hash=sha256:6c54a4a82e291a1fee5137371ab488866b7c86a3305af4026bdd4dc78642e1ac \
--hash=sha256:6e131579c243c98f35bce64a7e08e87fb2d610544754675d4a0e73a070a5aa3a \
--hash=sha256:855822d90f884905362f602880ed8b5df1b7e3ee7d0db2502d4388a954cc8c54 \
--hash=sha256:86a8b5035fce64f5dcd1b794cf8ec4d31fe458cf6ce3986a30deb434df82a1d2 \
--hash=sha256:8a33d657f3276328ce00e4d37fe70361e1ec7614da5d7b6e78de5426cb56332f \
--hash=sha256:92c0ec1f2cc149551a2b7b47efc32c866406b6891b0ee4625e95967c8f4acfb1 \
--hash=sha256:9a5e9f45e5d5e1c5b5c29b3bd4265dcc90e8b92cf4534520896ed77f791f4da5 \
--hash=sha256:afc622538b430aa4c8c853f7f63bc582b3b8030fd8c80b70fb5fa5b834e575c2 \
--hash=sha256:b07fc0dab849d24a80a29cfab8d8a19187d1c4685d8a5e6385a5ce323c1f015f \
--hash=sha256:b5e6f89631eb88a7302d416594a32faeee9fb8fb848290da9d0a5f2903519fc1 \
--hash=sha256:bf9bf162ed91a26f1adba8efda0b573bc6924ec1408a52cc6f82cb73ec2b142c \
--hash=sha256:c7e72339f841b5a237ff14f7d3880ddd0fc7f98a1199e8c4327f9a4f478c1839 \
--hash=sha256:ddb113db38838eb9f043623ba274cfaf7d51d5b0c22ecb30afe58b1bb8322983 \
--hash=sha256:dfdd51fc3e64ea4f35873d1b3fb25326773d55d2329ff8449139ebaad7357efb \
--hash=sha256:f1cd08e99d2f9317292a311dfe578fd2a24b15dbce97792f9c4d752275c1fa56 \
--hash=sha256:f89f2ab047c76a9c03f78d0d66ca519e389519902fa27e7a91117ef7611c0568
# via
# -r requirements_formatting.txt.in
# darker
certifi==2026.10.19 \
--hash=sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae \
--hash=sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b
certifi==2025.7.14 \
--hash=sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2 \
--hash=sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995
# via
# -r requirements_formatting.txt.in
# requests
@@ -308,9 +308,9 @@ pygithub==2.6.1 \
--hash=sha256:6f2fa6d076ccae475f9fc392cc6cdbd54db985d4f69b8833a28397de75ed6ca3 \
--hash=sha256:b5c035392991cca63959e9453286b41b54d83bf2de2daa7d7ff7e4312cebf3bf
# via -r requirements_formatting.txt.in
pyjwt==2.15.1 \
--hash=sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193 \
--hash=sha256:4f259e80cdfb6b3fc18a7de51fd1ef9ec79652f25019bae68975ca2468a34df8
pyjwt==2.13.0 \
--hash=sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423 \
--hash=sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728
# via
# -r requirements_formatting.txt.in
# pygithub
@@ -403,9 +403,9 @@ typing-extensions==4.14.1 \
--hash=sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36 \
--hash=sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76
# via pygithub
urllib3==2.8.0 \
--hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \
--hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63
urllib3==2.7.0 \
--hash=sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c \
--hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897
# via
# -r requirements_formatting.txt.in
# pygithub
+4 -4
View File
@@ -1,10 +1,10 @@
black>=26.10.1
black>=26.3.1
darker==2.1.1
PyGithub==2.6.1
cryptography>=50.0.0
urllib3>=2.8.0
urllib3>=2.7.0
requests>=2.33.0
idna>=3.15
certifi>=2026.10.19
certifi>=2024.7.4
PyNaCl>=1.6.2
PyJWT>=2.15.1
PyJWT>=2.13.0
+1 -1
+2 -4
View File
@@ -33,11 +33,9 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/include/git_version.h.in
include_directories(${CMAKE_BINARY_DIR}/generated)
# Disable strict aliasing for all build modes
# See discussion in
# https://github.com/FEX-Emu/FEX/pull/4494#issuecomment-2800608944
# See discussion in https://github.com/FEX-Emu/FEX/pull/4494#issuecomment-2800608944
# for background context.
add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-fno-strict-aliasing>
$<$<COMPILE_LANGUAGE:CXX>:-fno-exceptions>)
add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-fno-strict-aliasing> $<$<COMPILE_LANGUAGE:CXX>:-fno-exceptions>)
add_subdirectory(Source/)
+2 -2
View File
@@ -8,8 +8,8 @@ This project aims to provide a fast and functional x86-64 emulation library that
* Support a tiered recompiler to allow for fast runtime performance
* Support offline compilation and offline tooling for inspection and performance analysis
* Support threaded emulation. Including emulating x86-64's strong memory model on weak memory model architectures
* Support a majority of the x86-64 instruction space.
* Including MMX, SSE, SSE2, SSE3, SSSE3, SSE4*, AVX, AVX2, F16C, and AVX-VNNI
* Support a significant portion of the x86-64 instruction space.
* Including MMX, SSE, SSE2, SSE3, SSSE3, and SSE4*
* Support fallback routines for uncommonly used x86-64 instructions
* Including x87 and 3DNow!
* Only support userspace emulation.
+7 -17
View File
@@ -4,7 +4,6 @@ set(FEXCORE_BASE_SRCS
Interface/Config/Config.cpp
Utils/Allocator.cpp
Utils/FileLoading.cpp
Utils/FileUtils.cpp
Utils/ForcedAssert.cpp
Utils/LogManager.cpp
Utils/SpinWaitLock.cpp
@@ -104,13 +103,9 @@ if (ENABLE_ZYDIS)
endif()
if (ARCHITECTURE_arm64 AND HAS_CLANG_PRESERVE_ALL)
list(APPEND DEFINES
"-DFEXCORE_PRESERVE_ALL_ATTR=__attribute__((preserve_all))"
"-DFEXCORE_HAS_PRESERVE_ALL_ATTR=1")
list(APPEND DEFINES "-DFEXCORE_PRESERVE_ALL_ATTR=__attribute__((preserve_all));-DFEXCORE_HAS_PRESERVE_ALL_ATTR=1")
else()
list(APPEND DEFINES
"-DFEXCORE_PRESERVE_ALL_ATTR="
"-DFEXCORE_HAS_PRESERVE_ALL_ATTR=0")
list(APPEND DEFINES "-DFEXCORE_PRESERVE_ALL_ATTR=;-DFEXCORE_HAS_PRESERVE_ALL_ATTR=0")
endif()
set(LIBS fmt::fmt xxHash::xxhash FEXHeaderUtils CodeEmitter cephes_128bit)
@@ -191,8 +186,8 @@ add_custom_command(
OUTPUT "${OUTPUT_MAN_NAME}"
DEPENDS "${INPUT_CONFIG_NAME}"
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/config_generator.py"
COMMAND "python3" "${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/config_generator.py"
"${INPUT_CONFIG_NAME}" "${OUTPUT_CONFIG_NAME}" "${OUTPUT_MAN_NAME}" "${OUTPUT_CONFIG_OPTION_NAME}")
COMMAND "python3" "${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/config_generator.py" "${INPUT_CONFIG_NAME}" "${OUTPUT_CONFIG_NAME}" "${OUTPUT_MAN_NAME}"
"${OUTPUT_CONFIG_OPTION_NAME}")
add_custom_command(
OUTPUT "${OUTPUT_MAN_NAME_COMPRESS}"
@@ -266,10 +261,6 @@ endfunction()
# Build FEXCore_Base static library
add_library(FEXCore_Base STATIC ${FEXCORE_BASE_SRCS})
target_link_libraries(FEXCore_Base PUBLIC ${LIBS})
if (MINGW)
target_link_libraries(FEXCore_Base PUBLIC ntdll)
endif()
AddDefaultOptionsToTarget(FEXCore_Base)
if (ENABLE_FEXCORE_PROFILER AND FEXCORE_PROFILER_BACKEND STREQUAL "TRACY")
@@ -288,8 +279,8 @@ function(AddLibrary Name Type)
add_library(${Name} ${Type} $<TARGET_OBJECTS:${PROJECT_NAME}_object>)
set_target_properties(${Name} PROPERTIES OUTPUT_NAME FEXCore)
# During generation of the import library (dll.a), MinGW needs some extra
# symbols from libraries such as fmt, which are propagated by FEXCore_Base
# During generation of the import library (dll.a), MinGW needs some extra symbols from libraries
# such as fmt, which are propagated by FEXCore_Base. Wonderful.
if (MINGW)
target_link_libraries(${Name} PRIVATE FEXCore_Base)
endif()
@@ -315,8 +306,7 @@ if (ENABLE_FEX_ALLOCATOR)
target_include_directories(JemallocLibs PRIVATE "${PROJECT_SOURCE_DIR}/include/")
endif()
if (ENABLE_JEMALLOC_GLIBC_ALLOC)
set_source_files_properties(Interface/HLE/Thunks/Thunks.cpp
PROPERTIES COMPILE_DEFINITIONS ENABLE_JEMALLOC_GLIBC=1)
set_source_files_properties(Interface/HLE/Thunks/Thunks.cpp PROPERTIES COMPILE_DEFINITIONS ENABLE_JEMALLOC_GLIBC=1)
target_link_libraries(JemallocLibs INTERFACE FEX_jemalloc_glibc)
endif()
+7 -2
View File
@@ -15,9 +15,14 @@ JITSymbols::~JITSymbols() {
}
}
void JITSymbols::InitFile(uint32_t ProcessPID) {
void JITSymbols::InitFile() {
// We can't use FILE here since we must be robust against forking processes closing our FD from under us.
const auto PerfMap = fextl::fmt::format("/tmp/perf-{}.map", ProcessPID);
#ifdef __ANDROID__
// Android simpleperf looks in /data/local/tmp instead of /tmp
const auto PerfMap = fextl::fmt::format("/data/local/tmp/perf-{}.map", getpid());
#else
const auto PerfMap = fextl::fmt::format("/tmp/perf-{}.map", getpid());
#endif
fd = open(PerfMap.c_str(), O_CREAT | O_TRUNC | O_WRONLY | O_APPEND, 0644);
}
+1 -1
View File
@@ -35,7 +35,7 @@ public:
JITSymbols();
~JITSymbols();
void InitFile(uint32_t ProcessPID);
void InitFile();
void RegisterNamedRegion(const void* HostAddr, uint32_t CodeSize, std::string_view Name);
void RegisterJITSpace(const void* HostAddr, uint32_t CodeSize);
+2 -69
View File
@@ -42,19 +42,6 @@
"Enable expensive validation when loading code caches"
]
},
"JITFeatures": {
"Type": "strenum",
"Default": "FEXCore::Config::JITFeatures::OFF",
"AffectsCodeGen": "true",
"Enums": {
"ENABLEX87FIP": "enablex87fip"
},
"Desc": [
"Allows controlling JIT features in the unified location",
"\toff: Default JIT features.",
"\tenablex87fip: Turns on the x87 FIP emulation feature"
]
},
"HostFeatures": {
"Type": "strenum",
"Default": "FEXCore::Config::HostFeatures::OFF",
@@ -104,15 +91,7 @@
"ENABLESSE4A": "enablesse4a",
"DISABLESSE4A": "disablesse4a",
"ENABLEMOPS": "enablemops",
"DISABLEMOPS": "disablemops",
"ENABLEI8MM": "enablei8mm",
"DISABLEI8MM": "disablei8mm",
"ENABLEDOTPROD": "enabledotprod",
"DISABLEDOTPROD": "disabledotprod",
"ENABLEBF16": "enablebf16",
"DISABLEBF16": "disablebf16",
"ENABLETPIDRRO": "enabletpidrro",
"DISABLETPIDRRO": "disabletpidrro"
"DISABLEMOPS": "disablemops"
},
"Desc": [
"Allows controlling of the CPU features in the JIT.",
@@ -137,11 +116,7 @@
"\t{enable,disable}wfxt: Will force enable or disable wfxt even if the host doesn't support it",
"\t{enable,disable}3dnow: Will force enable or disable 3DNow! even if the host doesn't support it",
"\t{enable,disable}sse4a: Will force enable or disable SSE4a even if the host doesn't support it",
"\t{enable,disable}mops: Will force enable or disable FEAT_MOPS even if the host doesn't support it",
"\t{enable,disable}i8mm: Will force enable or disable i8mm even if the host doesn't support it",
"\t{enable,disable}dotprod: Will force enable or disable dotprod even if the host doesn't support it",
"\t{enable,disable}bf16: Will force enable or disable bf16 even if the host doesn't support it",
"\t{enable,disable}tpidrro: Will force enable or disable tpidrro for CPU index even if the host doesn't support it"
"\t{enable,disable}mops: Will force enable or disable FEAT_MOPS even if the host doesn't support it"
]
},
"SmallTSCScale": {
@@ -160,14 +135,6 @@
"Hides hybrid CPU core arrangement."
]
},
"SoftwareRNG": {
"Type": "bool",
"Default": "false",
"AffectsCodeGen": "true",
"Desc": [
"Emulates RDRAND and RDSEED in software when the host does not implement FEAT_RNG."
]
},
"CPUFeatureRegisters": {
"Type": "str",
"Default": "",
@@ -217,14 +184,6 @@
"Attempt to cache anonymous code"
]
},
"DiskCacheMemorySize": {
"Type": "uint32",
"Default": "0",
"AffectsCodeGen": "false",
"Desc": [
"How much memory, if any, to use as a cache for the cache"
]
},
"DiskCachePath": {
"Type": "str",
"Default": "",
@@ -240,23 +199,6 @@
"Desc": [
"Optional list of extra read-only disk cache DBs to consider"
]
},
"DiskCacheMaxFileSize": {
"Type": "uint64",
"Default": "1073741824",
"AffectsCodeGen": "false",
"Desc": [
"Size limit on the main cache file - index not included. Default 1G"
]
},
"DiskCachePruneStaleEntries": {
"Type": "bool",
"Default": "true",
"AffectsCodeGen": "false",
"Desc": [
"Allows FEX to prune stale disk cache entries on startup.",
"Frees space by removing cache entries associated with old FEX versions."
]
}
},
"Emulation": {
@@ -373,15 +315,6 @@
"Can potentially introduce more stutters, more likely the higher the number.",
"Don't have this number larger than the increase count!"
]
},
"UndefinedFlagAcidTest": {
"Type": "bool",
"Default": "false",
"AffectsCodeGen": "true",
"Desc": [
"Ensures that all undefined flags generate consistent results",
"This is a useful test to determine if undefined flags are causing problems"
]
}
},
"Debug": {
+2 -8
View File
@@ -4,13 +4,13 @@
#include "Common/JitSymbols.h"
#include "Interface/Core/CPUBackend.h"
#include "Interface/Core/CPUID.h"
#include "Interface/Core/DiskCache.h"
#include "Interface/Core/SharedCodeBufferManager.h"
#include <Interface/IR/IntrusiveIRList.h>
#include <FEXCore/Config/Config.h>
#include <FEXCore/Core/Context.h>
#include <FEXCore/Core/CoreState.h>
#include <FEXCore/Core/HostFeatures.h>
#include <FEXCore/Core/DiskCache.h>
#include <FEXCore/IR/IR.h>
#include <FEXCore/Utils/CompilerDefs.h>
#include <FEXCore/Utils/SignalScopeGuards.h>
@@ -83,6 +83,7 @@ public:
FEX_CONFIG_OPT(EnableLazyCodeCaching, ENABLELAZYCODECACHINGWIP);
FEX_CONFIG_OPT(EnableCodeCacheValidation, ENABLECODECACHEVALIDATION);
uint64_t ComputeCodeMapId(std::string_view Filename, int FD) override;
bool SaveData(Core::InternalThreadState&, int TargetFD, const ExecutableFileSectionInfo&, uint64_t SerializedBaseAddress) override;
fextl::unique_ptr<MappedCodeCacheFile> LoadCache(std::span<std::byte> CacheFile, const ExecutableFileInfo&, uint64_t FileStartVA) override;
@@ -368,17 +369,10 @@ public:
FEX_CONFIG_OPT(DisableTelemetry, DISABLETELEMETRY);
FEX_CONFIG_OPT(DisableVixlIndirectCalls, DISABLE_VIXL_INDIRECT_RUNTIME_CALLS);
FEX_CONFIG_OPT(SmallTSCScale, SMALLTSCSCALE);
FEX_CONFIG_OPT(SoftwareRNG, SOFTWARERNG);
FEX_CONFIG_OPT(StrictInProcessSplitLocks, STRICTINPROCESSSPLITLOCKS);
FEX_CONFIG_OPT(MonoHacks, MONOHACKS);
FEX_CONFIG_OPT(JITFeatures, JITFEATURES);
} Config;
bool SoftwareRNGEnabled() const {
return Config.SoftwareRNG() && HostRNGAvailable;
}
bool HostRNGAvailable {};
FEXCore::Utils::WritePriorityMutex::Mutex CodeInvalidationMutex {};
uint32_t StrictSplitLockMutex {};
@@ -665,11 +665,11 @@ void Arm64Emitter::FillSpecialRegs(ARMEmitter::Register TmpReg, ARMEmitter::Regi
// We don't bother spilling these in SpillStaticRegs,
// since all that matters is we restore them on a fill.
// It's not a concern if they get trounced by something else.
if (EmitterCTX->HostFeatures.SupportsSVE256()) {
if (EmitterCTX->HostFeatures.SupportsSVE256) {
ptrue(ARMEmitter::SubRegSize::i8Bit, PRED_TMP_32B, ARMEmitter::PredicatePattern::SVE_VL32);
}
if (EmitterCTX->HostFeatures.SupportsSVE128()) {
if (EmitterCTX->HostFeatures.SupportsSVE128) {
ptrue(ARMEmitter::SubRegSize::i8Bit, PRED_TMP_16B, ARMEmitter::PredicatePattern::SVE_VL16);
}
@@ -739,7 +739,7 @@ void Arm64Emitter::SpillStaticRegs(ARMEmitter::Register TmpReg, SpillStaticRegOp
}
if (Options.FPRs) {
if (EmitterCTX->HostFeatures.SupportsAVX() && EmitterCTX->HostFeatures.SupportsSVE256()) {
if (EmitterCTX->HostFeatures.SupportsAVX && EmitterCTX->HostFeatures.SupportsSVE256) {
for (size_t i = 0; i < StaticFPRegisters.size(); i++) {
const auto Reg = StaticFPRegisters[i];
@@ -826,7 +826,7 @@ void Arm64Emitter::FillStaticRegs(FillStaticRegOptions Options) {
FillSpecialRegs(TmpReg, TmpReg2, {.SetFIZ = true, .SetPredRegs = Options.FPRs});
if (Options.FPRs) {
if (EmitterCTX->HostFeatures.SupportsAVX() && EmitterCTX->HostFeatures.SupportsSVE256()) {
if (EmitterCTX->HostFeatures.SupportsAVX && EmitterCTX->HostFeatures.SupportsSVE256) {
for (size_t i = 0; i < StaticFPRegisters.size(); i++) {
const auto Reg = StaticFPRegisters[i];
if (((1U << Reg.Idx()) & Options.FPRFillMask) != 0) {
@@ -990,7 +990,7 @@ void Arm64Emitter::PopGeneralRegisters(std::span<const ARMEmitter::Register> Reg
}
size_t Arm64Emitter::PushDynamicRegs(ARMEmitter::Register TmpReg) {
const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256();
const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256;
const auto GPRSize = GeneralRegistersNotPreserved.size() * Core::CPUState::GPR_REG_SIZE;
const auto FPRRegSize = CanUseSVE256 ? 32 : 16;
const auto FPRSize = GeneralFPRegisters.size() * FPRRegSize;
@@ -1013,7 +1013,7 @@ size_t Arm64Emitter::PushDynamicRegs(ARMEmitter::Register TmpReg) {
}
void Arm64Emitter::PopDynamicRegs() {
const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256();
const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256;
// Pop vectors first
PopVectorRegisters(CanUseSVE256, GeneralFPRegisters);
@@ -1023,7 +1023,7 @@ void Arm64Emitter::PopDynamicRegs() {
}
size_t Arm64Emitter::SpillForPreserveAllABICall(ARMEmitter::Register TmpReg, bool FPRs) {
const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256();
const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256;
const auto FPRRegSize = CanUseSVE256 ? 32 : 16;
std::span<const ARMEmitter::Register> DynamicGPRs {};
@@ -1078,7 +1078,7 @@ size_t Arm64Emitter::SpillForPreserveAllABICall(ARMEmitter::Register TmpReg, boo
}
void Arm64Emitter::FillForPreserveAllABICall(bool FPRs) {
const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256();
const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256;
std::span<const ARMEmitter::Register> DynamicGPRs {};
std::span<const ARMEmitter::VRegister> DynamicFPRs {};
@@ -1129,9 +1129,4 @@ void Arm64Emitter::Align16B() {
}
}
ARMEmitter::Register Arm64Emitter::GetX86GPRMapping(FEXCore::X86State::X86Reg Reg) {
LOGMAN_THROW_A_FMT(Reg <= FEXCore::X86State::X86Reg::REG_R15, "Incorrect register");
return StaticRegisters[Reg];
}
} // namespace FEXCore::CPU
@@ -235,8 +235,6 @@ protected:
void Align16B();
ARMEmitter::Register GetX86GPRMapping(FEXCore::X86State::X86Reg Reg);
#ifdef VIXL_SIMULATOR
// Generates a vixl simulator runtime call.
//
@@ -16,7 +16,6 @@ namespace CPU {
constexpr static uint64_t NamedVectorConstants[FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_CONST_POOL_MAX][2] = {
{0x0003'0002'0001'0000ULL, 0x0007'0006'0005'0004ULL}, // NAMED_VECTOR_INCREMENTAL_U16_INDEX
{0x000B'000A'0009'0008ULL, 0x000F'000E'000D'000CULL}, // NAMED_VECTOR_INCREMENTAL_U16_INDEX_UPPER
{0x0706'0504'0302'0100ULL, 0x0F0E'0D0C'0B0A'0908ULL}, // NAMED_VECTOR_INCREMENTAL_U8_INDEX
{0x0000'0000'8000'0000ULL, 0x0000'0000'8000'0000ULL}, // NAMED_VECTOR_PADDSUBPS_INVERT
{0x0000'0000'8000'0000ULL, 0x0000'0000'8000'0000ULL}, // NAMED_VECTOR_PADDSUBPS_INVERT_UPPER
{0x8000'0000'0000'0000ULL, 0x0000'0000'0000'0000ULL}, // NAMED_VECTOR_PADDSUBPD_INVERT
+1 -1
View File
@@ -115,7 +115,7 @@ namespace CPU {
* @return Information about the compiled code block.
*/
[[nodiscard]]
virtual CompiledCode EmitJITCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
virtual CompiledCode CompileCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
FEXCore::Core::DebugData* DebugData, bool CheckTF) = 0;
virtual CompiledCode LoadCachedCode(std::span<const uint8_t> HostBytes) {
+97 -138
View File
@@ -89,18 +89,6 @@ namespace ProductNames {
static const char ARM_Blizzard_M2Pro[] = "Apple Blizzard (M2 Pro)";
static const char ARM_Avalanche_M2Max[] = "Apple Avalanche (M2 Max)";
static const char ARM_Blizzard_M2Max[] = "Apple Blizzard (M2 Max)";
static const char ARM_Sawtooth_M3[] = "Apple Sawtooth (M3)";
static const char ARM_Everest_M3[] = "Apple Everest (M3)";
static const char ARM_Sawtooth_M3Pro[] = "Apple Sawtooth (M3 Pro)";
static const char ARM_Everest_M3Pro[] = "Apple Everest (M3 Pro)";
static const char ARM_Sawtooth_M3Max[] = "Apple Sawtooth (M3 Max)";
static const char ARM_Everest_M3Max[] = "Apple Everest (M3 Max)";
static const char ARM_DonanP_M4[] = "Apple DonanP (M4)";
static const char ARM_DonanE_M4[] = "Apple DonanE (M4)";
static const char ARM_TahitiP_A18[] = "Apple TahitiP (A18)";
static const char ARM_TahitiE_A18[] = "Apple TahitiE (A18)";
static const char ARM_SotraP_M5[] = "Apple SotraP (M5)";
static const char ARM_SotraE_M5[] = "Apple SotraE (M5)";
static const char ARM_AppleSilicon[] = "Apple Silicon";
static const char ARM_ORYON_1[] = "Oryon-1";
@@ -199,17 +187,11 @@ void CPUIDEmu::SetupHostHybridFlag() {
// CPU priority order
// This is mostly arbitrary but will sort by some sort of CPU priority by performance
// Relative list so things they will commonly end up in big.little configurations sort of relate
static constexpr std::array<CPUMIDR, 80> CPUMIDRs = {{
static constexpr std::array<CPUMIDR, 68> CPUMIDRs = {{
// Typically big CPU cores
{0x51, 0x002, 1, ProductNames::ARM_ORYON_3}, // Qualcomm Oryon-3
{0x51, 0x001, 1, ProductNames::ARM_ORYON_1}, // Qualcomm Oryon-1
{0x61, 0x065, 1, ProductNames::ARM_SotraP_M5}, // Apple SotraP (M5)
{0x61, 0x061, 1, ProductNames::ARM_TahitiP_A18}, // Apple TahitiP (A18)
{0x61, 0x053, 1, ProductNames::ARM_DonanP_M4}, // Apple DonanP (M4)
{0x61, 0x048, 1, ProductNames::ARM_Sawtooth_M3Max}, // Apple Sawtooth (M3 Max)
{0x61, 0x044, 1, ProductNames::ARM_Sawtooth_M3Pro}, // Apple Sawtooth (M3 Pro)
{0x61, 0x042, 1, ProductNames::ARM_Sawtooth_M3}, // Apple Sawtooth (M3)
{0x61, 0x039, 1, ProductNames::ARM_Avalanche_M2Max}, // Apple Avalanche (M2 Max)
{0x61, 0x035, 1, ProductNames::ARM_Avalanche_M2Pro}, // Apple Avalanche (M2 Pro)
{0x61, 0x033, 1, ProductNames::ARM_Avalanche_M2}, // Apple Avalanche (M2)
@@ -266,12 +248,6 @@ void CPUIDEmu::SetupHostHybridFlag() {
{0x41, 0xd07, 1, ProductNames::ARM_A57}, // A57
// Typically Little CPU cores
{0x61, 0x064, 1, ProductNames::ARM_SotraE_M5}, // Apple SotraE (M5)
{0x61, 0x060, 1, ProductNames::ARM_TahitiE_A18}, // Apple TahitiE (A18)
{0x61, 0x052, 1, ProductNames::ARM_DonanE_M4}, // Apple DonanE (M4)
{0x61, 0x049, 0, ProductNames::ARM_Everest_M3Max}, // Apple Everest (M3 Max)
{0x61, 0x045, 0, ProductNames::ARM_Everest_M3Pro}, // Apple Everest (M3 Pro)
{0x61, 0x043, 0, ProductNames::ARM_Everest_M3}, // Apple Everest (M3)
{0x61, 0x038, 0, ProductNames::ARM_Blizzard_M2Max}, // Apple Blizzard (M2 Max)
{0x61, 0x034, 0, ProductNames::ARM_Blizzard_M2Pro}, // Apple Blizzard (M2 Pro)
{0x61, 0x032, 0, ProductNames::ARM_Blizzard_M2}, // Apple Blizzard (M2)
@@ -443,7 +419,7 @@ void CPUIDEmu::SetupHostHybridFlag() {}
void CPUIDEmu::SetupFeatures() {
if (CTX->HostFeatures.SupportsAVX()) {
if (CTX->HostFeatures.SupportsAVX) {
XCR0 |= XCR0_AVX;
}
@@ -482,8 +458,6 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_01h(uint32_t Leaf) const {
(Cores << 16) | // Number of addressable IDs for the logical cores in the physical CPU
(GetCPUID() << 24); // Local APIC ID
const uint32_t SupportsRAND = CTX->HostFeatures.SupportsRAND || CTX->SoftwareRNGEnabled();
Res.ecx = (1 << 0) | // SSE3
(CTX->HostFeatures.SupportsPMULL_128Bit << 1) | // PCLMULQDQ
(1 << 2) | // DS area supports 64bit layout
@@ -514,7 +488,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_01h(uint32_t Leaf) const {
(SupportsAVX() << 27) | // OSXSAVE
(SupportsAVX() << 28) | // AVX
(SupportsAVX() << 29) | // F16C
(SupportsRAND << 30) | // RDRAND
(CTX->HostFeatures.SupportsRAND << 30) | // RDRAND
(Hypervisor << 31);
Res.edx = (1 << 0) | // FPU
@@ -674,12 +648,6 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_06h(uint32_t Leaf) const {
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
FEXCore::CPUID::FunctionResults Res {};
// AVX-VNNI is only advertised when the CPU supports I8MM or Dot Product.
// Without these features the implementation is so slow that it is likely
// to harm performance.
const uint32_t SupportsAVXVNNI = SupportsAVX() && (CTX->HostFeatures.SupportsI8MM || CTX->HostFeatures.SupportsDotProd);
if (Leaf == 0) {
#ifndef _WIN32
constexpr uint32_t SUPPORTS_RDPID = 1;
@@ -696,47 +664,40 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
const uint32_t SupportsVPCLMULQDQ = CTX->HostFeatures.SupportsPMULL_128Bit && SupportsAVX();
const uint32_t SupportsWFXT = CTX->HostFeatures.SupportsWFXT;
const uint32_t SupportsRAND = CTX->HostFeatures.SupportsRAND || CTX->SoftwareRNGEnabled();
// When the X87 FIP feature is enabled, we update the state pointers on every instruction.
// We don't even update them on exception so this isn't quite true the other way anyway.
const uint32_t X87DataPtrExcept = (CTX->Config.JITFeatures & FEXCore::Config::JITFeatures::ENABLEX87FIP) == 0;
// Number of subfunctions
// TODO: For now, subfunction 1 only exposes AVX-VNNI so we make it conditional
// on AVX-VNNI support. We should revisit this if/when we add more to this leaf.
Res.eax = SupportsAVXVNNI;
Res.ebx = (1 << 0) | // FS/GS support
(0 << 1) | // TSC adjust MSR
(0 << 2) | // SGX
(SupportsAVX() << 3) | // BMI1
(0 << 4) | // Intel Hardware Lock Elison
(SupportsAVX() << 5) | // AVX2 support
(X87DataPtrExcept << 6) | // FPU data pointer updated only on exception
(1 << 7) | // SMEP support
(SupportsAVX() << 8) | // BMI2
(SupportsEnhancedREPMOVS << 9) | // Enhanced REP MOVSB/STOSB
(1 << 10) | // INVPCID for system software control of process-context
(0 << 11) | // Restricted transactional memory
(0 << 12) | // Intel resource directory technology Monitoring
(1 << 13) | // Deprecates FPU CS and DS
(0 << 14) | // Intel MPX
(0 << 15) | // Intel Resource Directory Technology Allocation
(0 << 16) | // AVX512-F
(0 << 17) | // AVX512-DQ
(SupportsRAND << 18) | // RDSEED
(1 << 19) | // ADCX and ADOX instructions
(0 << 20) | // SMAP Supervisor mode access prevention and CLAC/STAC instructions
(0 << 21) | // AVX512-IFMA
(0 << 22) | // PCOMMIT (deprecated?)
(1 << 23) | // CLFLUSHOPT instruction
(1 << 24) | // CLWB instruction
(0 << 25) | // Intel processor trace
(0 << 26) | // AVX512-PF
(0 << 27) | // AVX512-ER
(0 << 28) | // AVX512-CD
(Features.SHA << 29) | // SHA instructions
(0 << 30) | // AVX512-BW
(0 << 31); // AVX512-VL
Res.eax = 0x0;
Res.ebx = (1 << 0) | // FS/GS support
(0 << 1) | // TSC adjust MSR
(0 << 2) | // SGX
(SupportsAVX() << 3) | // BMI1
(0 << 4) | // Intel Hardware Lock Elison
(SupportsAVX() << 5) | // AVX2 support
(1 << 6) | // FPU data pointer updated only on exception
(1 << 7) | // SMEP support
(SupportsAVX() << 8) | // BMI2
(SupportsEnhancedREPMOVS << 9) | // Enhanced REP MOVSB/STOSB
(1 << 10) | // INVPCID for system software control of process-context
(0 << 11) | // Restricted transactional memory
(0 << 12) | // Intel resource directory technology Monitoring
(1 << 13) | // Deprecates FPU CS and DS
(0 << 14) | // Intel MPX
(0 << 15) | // Intel Resource Directory Technology Allocation
(0 << 16) | // AVX512-F
(0 << 17) | // AVX512-DQ
(CTX->HostFeatures.SupportsRAND << 18) | // RDSEED
(1 << 19) | // ADCX and ADOX instructions
(0 << 20) | // SMAP Supervisor mode access prevention and CLAC/STAC instructions
(0 << 21) | // AVX512-IFMA
(0 << 22) | // PCOMMIT (deprecated?)
(1 << 23) | // CLFLUSHOPT instruction
(1 << 24) | // CLWB instruction
(0 << 25) | // Intel processor trace
(0 << 26) | // AVX512-PF
(0 << 27) | // AVX512-ER
(0 << 28) | // AVX512-CD
(Features.SHA << 29) | // SHA instructions
(0 << 30) | // AVX512-BW
(0 << 31); // AVX512-VL
Res.ecx = (1 << 0) | // PREFETCHWT1
(0 << 1) | // AVX512VBMI
@@ -785,7 +746,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
(0 << 11) | // rtm-always-abort
(0 << 12) | // Reserved
(0 << 13) | // TSX Force Abort (TSX will force abort if attempted)
(1 << 14) | // SERIALIZE instruction
(0 << 14) | // SERIALIZE instruction
((Hybrid ? 1U : 0U) << 15) | // Hybrid
(0 << 16) | // TSXLDTRK (TSX Suspend load address tracking) - Allows untracked memory loads inside TSX region
(0 << 17) | // Reserved
@@ -804,40 +765,38 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
(0 << 30) | // Arch capabilities - MSR module specific
(0 << 31); // SSBD - Speculative Store Bypass Disable
} else if (Leaf == 1) {
const uint32_t SupportsAVXConvert = SupportsAVX() && CTX->HostFeatures.SupportsBF16;
Res.eax = (0U << 0) | // SHA512
(0U << 1) | // SM3
(0U << 2) | // SM4
(0U << 3) | // RAO_INT
(SupportsAVXVNNI << 4) | // AVX_VNNI
(0U << 5) | // AVX512_BF16
(0U << 6) | // LASS (Linear Address Space Separation)
(0U << 7) | // CMPCCXADD
(0U << 8) | // ARCH_PERFMON_EXT
(0U << 9) | // Reserved
(0U << 10) | // FAST_REP_MOVSB
(0U << 11) | // FAST_REP_STOSB
(0U << 12) | // FAST_REP_CMPSB_SCASB
(0U << 13) | // Reserved
(0U << 14) | // Reserved
(0U << 15) | // Reserved
(0U << 16) | // Reserved
(0U << 17) | // FRED (Flexible Return and Event Delivery)
(0U << 18) | // LKGS (Load into Kernel GS Base)
(0U << 19) | // WRMSRNS
(0U << 20) | // NMI_SRC
(0U << 21) | // AMX_FP16
(0U << 22) | // HRESET
(0U << 23) | // AVX_IFMA
(0U << 24) | // Reserved
(0U << 25) | // Reserved
(0U << 26) | // LAM (Linear Address Masking)
(0U << 27) | // MSRLIST
(0U << 28) | // Reserved
(0U << 29) | // Reserved
(0U << 30) | // INVD_DISABLE_POST_BIOS_DONE
(0U << 31); // MOVRS
Res.eax = (0U << 0) | // SHA512
(0U << 1) | // SM3
(0U << 2) | // SM4
(0U << 3) | // RAO_INT
(0U << 4) | // AVX_VNNI
(0U << 5) | // AVX512_BF16
(0U << 6) | // LASS (Linear Address Space Separation)
(0U << 7) | // CMPCCXADD
(0U << 8) | // ARCH_PERFMON_EXT
(0U << 9) | // Reserved
(0U << 10) | // FAST_REP_MOVSB
(0U << 11) | // FAST_REP_STOSB
(0U << 12) | // FAST_REP_CMPSB_SCASB
(0U << 13) | // Reserved
(0U << 14) | // Reserved
(0U << 15) | // Reserved
(0U << 16) | // Reserved
(0U << 17) | // FRED (Flexible Return and Event Delivery)
(0U << 18) | // LKGS (Load into Kernel GS Base)
(0U << 19) | // WRMSRNS
(0U << 20) | // NMI_SRC
(0U << 21) | // AMX_FP16
(0U << 22) | // HRESET
(0U << 23) | // AVX_IFMA
(0U << 24) | // Reserved
(0U << 25) | // Reserved
(0U << 26) | // LAM (Linear Address Masking)
(0U << 27) | // MSRLIST
(0U << 28) | // Reserved
(0U << 29) | // Reserved
(0U << 30) | // INVD_DISABLE_POST_BIOS_DONE
(0U << 31); // MOVRS
// Bits 4-31 currently reserved.
Res.ebx = (0U << 0) | // PPIN
@@ -854,31 +813,31 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
(0U << 5); // MSR_IMM
// Bits 25-31 also reserved.
Res.edx = (0U << 0) | // Reserved
(0U << 1) | // Reserved
(0U << 2) | // Reserved
(0U << 3) | // Reserved
(0U << 4) | // AVX_VNNI_INT8
(SupportsAVXConvert << 5) | // AVX_NE_CONVERT
(0U << 6) | // Reserved
(0U << 7) | // Reserved
(0U << 8) | // AMX_COMPLEX
(0U << 9) | // Reserved
(0U << 10) | // AVX_VNNI_INT16
(0U << 11) | // Reserved
(0U << 12) | // Reserved
(0U << 13) | // UTMR (User-timer events)
(0U << 14) | // PREFETCHI
(0U << 15) | // USER_MSR
(0U << 16) | // Reserved
(0U << 17) | // UIRET_UIF
(0U << 18) | // CET_SSS
(0U << 19) | // AVX10
(0U << 20) | // Reserved
(0U << 21) | // APX_F
(0U << 22) | // SEC-TEE_ATTESTATION
(0U << 23) | // MWAIT
(0U << 24); // SLSM (Static LSM)
Res.edx = (0U << 0) | // Reserved
(0U << 1) | // Reserved
(0U << 2) | // Reserved
(0U << 3) | // Reserved
(0U << 4) | // AVX_VNNI_INT8
(0U << 5) | // AVX_NE_CONVERT
(0U << 6) | // Reserved
(0U << 7) | // Reserved
(0U << 8) | // AMX_COMPLEX
(0U << 9) | // Reserved
(0U << 10) | // AVX_VNNI_INT16
(0U << 11) | // Reserved
(0U << 12) | // Reserved
(0U << 13) | // UTMR (User-timer events)
(0U << 14) | // PREFETCHI
(0U << 15) | // USER_MSR
(0U << 16) | // Reserved
(0U << 17) | // UIRET_UIF
(0U << 18) | // CET_SSS
(0U << 19) | // AVX10
(0U << 20) | // Reserved
(0U << 21) | // APX_F
(0U << 22) | // SEC-TEE_ATTESTATION
(0U << 23) | // MWAIT
(0U << 24); // SLSM (Static LSM)
} else if (Leaf == 2) {
// All bits are reserved except for EDX
Res.eax = 0;
@@ -1162,14 +1121,14 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0004h(uint32_t Leaf) con
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0002h(uint32_t Leaf, uint32_t CPU) const {
FEXCore::CPUID::FunctionResults Res {};
auto& Data = PerCPUData[std::min<uint32_t>(CPU, PerCPUData.size() - 1)];
auto& Data = PerCPUData[CPU];
memcpy(&Res, Data.ProductName, std::min(strlen(Data.ProductName), sizeof(FEXCore::CPUID::FunctionResults)));
return Res;
}
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0003h(uint32_t Leaf, uint32_t CPU) const {
FEXCore::CPUID::FunctionResults Res {};
auto& Data = PerCPUData[std::min<uint32_t>(CPU, PerCPUData.size() - 1)];
auto& Data = PerCPUData[CPU];
const auto RemainingStringSize = std::max<ssize_t>(0, strlen(Data.ProductName) - 16);
memcpy(&Res, Data.ProductName + 16, std::min<size_t>(RemainingStringSize, sizeof(FEXCore::CPUID::FunctionResults)));
return Res;
@@ -1177,7 +1136,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0003h(uint32_t Leaf, uin
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0004h(uint32_t Leaf, uint32_t CPU) const {
FEXCore::CPUID::FunctionResults Res {};
auto& Data = PerCPUData[std::min<uint32_t>(CPU, PerCPUData.size() - 1)];
auto& Data = PerCPUData[CPU];
const auto RemainingStringSize = std::max<ssize_t>(0, strlen(Data.ProductName) - 32);
memcpy(&Res, Data.ProductName + 32, std::min<size_t>(RemainingStringSize, sizeof(FEXCore::CPUID::FunctionResults)));
return Res;
-1
View File
@@ -119,7 +119,6 @@ private:
bool Hybrid {};
uint32_t Cores {};
FEX_CONFIG_OPT(HideHypervisorBit, HIDEHYPERVISORBIT);
FEX_CONFIG_OPT(JITFeatures, JITFEATURES);
// XFEATURE_ENABLED_MASK
// Mask that configures what features are enabled on the CPU.
+21 -24
View File
@@ -1,5 +1,4 @@
// SPDX-License-Identifier: MIT
#include "Utils/crc32.h"
#include "FEXCore/Utils/LogManager.h"
#include "FEXCore/Utils/MathUtils.h"
#include "FEXCore/Utils/TypeDefines.h"
@@ -273,6 +272,16 @@ CodeCache::CodeCache(ContextImpl& CTX_)
: CTX(CTX_) {}
CodeCache::~CodeCache() = default;
uint64_t CodeCache::ComputeCodeMapId(std::string_view Filename, int FD) {
if (Filename.empty()) {
return 0xffff'ffff'ffff'ffff;
}
// For now, we just use the file path as an identifier.
// TODO: Ensure the hash is unique enough to distinguish executables while remaining independent of the installation location
return XXH3_64bits(Filename.data(), Filename.size());
}
struct CodeCacheHeader {
std::array<char, 4> Magic = ExpectedMagic;
// Version history:
@@ -529,37 +538,30 @@ ApplyRIPMoveRelocation(ContextImpl& CTX, uint64_t GuestRIP, uint8_t RegisterInde
Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(RegisterIndex), Pointer, CPU::Arm64Emitter::PadType::DOPAD);
}
static inline int64_t ReadLiveGuestData(uint64_t SiteAddress, uint8_t ValueSize) {
static inline void ApplyPatchableDataRelocation(uint64_t SiteAddress, uint8_t ValueSize, uint8_t RegisterIndex, CPU::Arm64Emitter& Emitter) {
uint64_t Value = 0;
memcpy(&Value, reinterpret_cast<const void*>(SiteAddress), ValueSize);
Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(RegisterIndex), Value, CPU::Arm64Emitter::PadType::DOPAD);
}
static inline int64_t ReadLiveGuestDisplacement(uint64_t SiteAddress, uint8_t ValueSize) {
uint64_t Raw = 0;
memcpy(&Raw, reinterpret_cast<const void*>(SiteAddress), ValueSize);
// manual sign-extension from guest live bytes
if (ValueSize == 1) {
return (int8_t)Raw;
} else if (ValueSize == 2) {
return (int16_t)Raw;
} else if (ValueSize == 4) {
// 1/2 sizes not permitted in DetectDataMasks currently
if (ValueSize == 4) {
return (int32_t)Raw;
} else {
return (int64_t)Raw;
}
}
static inline void ApplyPatchableDataRelocation(uint64_t SiteAddress, uint8_t ValueSize, uint8_t RegisterIndex, CPU::Arm64Emitter& Emitter) {
Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(RegisterIndex), ReadLiveGuestData(SiteAddress, ValueSize),
CPU::Arm64Emitter::PadType::DOPAD);
}
static inline void ApplyPatchableRIPLiteralRelocation(uint64_t SiteAddress, uint8_t ValueSize, CPU::Arm64Emitter& Emitter) {
Emitter.dc64(SiteAddress + ValueSize + ReadLiveGuestData(SiteAddress, ValueSize));
Emitter.dc64(SiteAddress + ValueSize + ReadLiveGuestDisplacement(SiteAddress, ValueSize));
}
static inline void ApplyPatchableRIPMoveRelocation(uint64_t SiteAddress, uint8_t ValueSize, uint8_t RegisterIndex, CPU::Arm64Emitter& Emitter) {
const uint64_t Target = SiteAddress + ValueSize + ReadLiveGuestData(SiteAddress, ValueSize);
Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(RegisterIndex), Target, CPU::Arm64Emitter::PadType::DOPAD);
}
static inline void ApplyPatchableCRCMoveRelocation(uint64_t SiteAddress, uint8_t ValueSize, uint8_t RegisterIndex, CPU::Arm64Emitter& Emitter) {
const uint64_t Target = FEXCore::Utils::crc32(reinterpret_cast<const uint8_t*>(SiteAddress), ValueSize);
const uint64_t Target = SiteAddress + ValueSize + ReadLiveGuestDisplacement(SiteAddress, ValueSize);
Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(RegisterIndex), Target, CPU::Arm64Emitter::PadType::DOPAD);
}
@@ -597,11 +599,6 @@ bool CodeCache::ApplyPackedCodeRelocations(uint64_t GuestEntry, std::span<std::b
Reloc.PatchableData.RegisterIndex, Emitter);
break;
}
case FEXCore::CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_CRC_MOVE: {
ApplyPatchableCRCMoveRelocation(GuestEntry + Reloc.PatchableData.SiteOffset, Reloc.PatchableData.ValueSize,
Reloc.PatchableData.RegisterIndex, Emitter);
break;
}
default: ERROR_AND_DIE_FMT("Unknown packed relocation type {}", ToUnderlying((CPU::RelocationTypes)Reloc.Type));
}
}
+45 -37
View File
@@ -30,7 +30,6 @@ $end_info$
#include "Interface/IR/RegisterAllocationData.h"
#include "Utils/Allocator.h"
#include "Utils/Allocator/HostAllocator.h"
#include "Utils/crc32.h"
#include <FEXCore/Utils/SpinWaitLock.h>
#include "Utils/variable_length_integer.h"
@@ -77,6 +76,9 @@ $end_info$
#include <unordered_map>
#include <utility>
#include <xxhash.h>
#if defined(ARCHITECTURE_arm64)
#include <arm_acle.h>
#endif
namespace FEXCore::Context {
ContextImpl::ContextImpl(const FEXCore::HostFeatures& Features)
@@ -90,7 +92,7 @@ ContextImpl::ContextImpl(const FEXCore::HostFeatures& Features)
if (Config.BlockJITNaming() || Config.GlobalJITNaming() || Config.LibraryJITNaming()) {
// Only initialize symbols file if enabled. Ensures we don't pollute /tmp with empty files.
Symbols.InitFile(Features.ProcessPID);
Symbols.InitFile();
}
uint64_t FrequencyCounter = FEXCore::GetCycleCounterFrequency();
@@ -105,14 +107,6 @@ ContextImpl::ContextImpl(const FEXCore::HostFeatures& Features)
// Track atomic TSO emulation configuration.
UpdateAtomicTSOEmulationConfig();
#ifndef _WIN32
// Check if the kernel supports getrandom().
uint64_t Probe {};
HostRNGAvailable = FHU::Syscalls::getrandom(&Probe, sizeof(Probe), 0) == sizeof(Probe);
#else
HostRNGAvailable = true;
#endif
DiskCache.Init(this);
}
@@ -264,8 +258,8 @@ uint32_t ContextImpl::ReconstructCompactedEFLAGS(FEXCore::Core::InternalThreadSt
void ContextImpl::ReconstructXMMRegisters(const FEXCore::Core::InternalThreadState* Thread, __uint128_t* XMM_Low, __uint128_t* YMM_High) {
const size_t MaximumRegisters = Config.Is64BitMode ? FEXCore::Core::CPUState::NUM_XMMS : 8;
if (YMM_High != nullptr && HostFeatures.SupportsAVX()) {
const bool SupportsConvergedRegisters = HostFeatures.SupportsSVE256();
if (YMM_High != nullptr && HostFeatures.SupportsAVX) {
const bool SupportsConvergedRegisters = HostFeatures.SupportsSVE256;
if (SupportsConvergedRegisters) {
///< Output wants to de-interleave
@@ -288,8 +282,8 @@ void ContextImpl::ReconstructXMMRegisters(const FEXCore::Core::InternalThreadSta
void ContextImpl::SetXMMRegistersFromState(FEXCore::Core::InternalThreadState* Thread, const __uint128_t* XMM_Low, const __uint128_t* YMM_High) {
const size_t MaximumRegisters = Config.Is64BitMode ? FEXCore::Core::CPUState::NUM_XMMS : 8;
if (YMM_High != nullptr && HostFeatures.SupportsAVX()) {
const bool SupportsConvergedRegisters = HostFeatures.SupportsSVE256();
if (YMM_High != nullptr && HostFeatures.SupportsAVX) {
const bool SupportsConvergedRegisters = HostFeatures.SupportsSVE256;
if (SupportsConvergedRegisters) {
///< Output wants to de-interleave
@@ -364,6 +358,11 @@ bool ContextImpl::InitCore() {
// Set up the SignalDelegator config since core is initialized.
SignalDelegation->SetConfig(Dispatcher->MakeSignalDelegatorConfig());
#if defined(_WIN32) && !defined(ARCHITECTURE_arm64ec)
// WOW64 always needs the interrupt fault check to be enabled.
Config.NeedsPendingInterruptFaultCheck = true;
#endif
if (Config.GdbServer) {
// If gdbserver is enabled then this needs to be enabled.
Config.NeedsPendingInterruptFaultCheck = true;
@@ -529,8 +528,6 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
bool HasCustomIR {};
bool WantsDiskCachePatching = DiskCache.IsReadingDiskCache() || DiskCache.IsWritingDiskCache();
if (HasCustomIRHandlers.load(std::memory_order_relaxed)) {
std::shared_lock lk(CustomIRMutex);
auto Handler = CustomIRHandlers.find(GuestRIP);
@@ -645,11 +642,27 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
auto ExistingCodePtr = reinterpret_cast<uint8_t*>(Block.Entry + BlockInstructionsLength);
auto InstAddressReg = Thread->OpDispatcher->_EntrypointOffset(GPRSize, InstAddress - GuestRIP);
auto Value = FEXCore::Utils::crc32(ExistingCodePtr, DecodedInfo->InstSize);
auto CRC = WantsDiskCachePatching ?
Thread->OpDispatcher->_PatchableGuestCRC(IR::OpSize::i64Bit, Value, (int64_t)ExistingCodePtr, DecodedInfo->InstSize) :
Thread->OpDispatcher->Constant(Value);
auto CodeChanged = Thread->OpDispatcher->_ValidateCode(CRC, InstAddressReg, DecodedInfo->InstSize);
auto crc32 = [](const uint8_t* Ptr, size_t Size) -> uint32_t {
#if defined(ARCHITECTURE_arm64)
uint32_t Result {};
#define do_crc(type, suffix) \
while (Size >= sizeof(type)) { \
Result = __crc32##suffix(Result, *reinterpret_cast<const type*>(Ptr)); \
Ptr += sizeof(type); \
Size -= sizeof(type); \
}
do_crc(uint64_t, d);
do_crc(uint32_t, w);
do_crc(uint16_t, h);
do_crc(uint8_t, b);
return Result;
#else
// Unsupported on non-arm.
return 0;
#endif
};
auto CodeChanged = Thread->OpDispatcher->_ValidateCode(
Thread->OpDispatcher->Constant(crc32(ExistingCodePtr, DecodedInfo->InstSize)), InstAddressReg, DecodedInfo->InstSize);
auto InvalidateCodeCond = Thread->OpDispatcher->CondJump(CodeChanged);
@@ -661,10 +674,11 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
Thread->OpDispatcher->StartNewBlock();
// Generate a relocatable entry for invalidation purposes.
auto EntryToInvalidate = Thread->OpDispatcher->_EntrypointOffset(GPRSize, 0);
auto NewRIP = Thread->OpDispatcher->_EntrypointOffset(GPRSize, InstAddress - GuestRIP);
// Invalidate and exit the function
Thread->OpDispatcher->_ThreadRemoveCodeEntry(EntryToInvalidate, NewRIP);
auto EntryReg = Thread->OpDispatcher->_EntrypointOffset(GPRSize, 0);
Thread->OpDispatcher->_ThreadRemoveCodeEntry(EntryReg);
// Exit the function at this instruction after invalidation.
Thread->OpDispatcher->ExitFunction(Thread->OpDispatcher->_InlineEntrypointOffset(GPRSize, InstAddress - GuestRIP));
auto NextOpBlock = Thread->OpDispatcher->CreateNewCodeBlockAfter(CurrentBlock);
@@ -715,19 +729,13 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
LogMan::Msg::EFmt("Invalid or Unknown instruction: {} 0x{:x}", TableInfo->Name ?: "UND", Block.Entry - GuestRIP);
}
if (Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::BREAKPOINT_0 ||
Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::BREAKPOINT_1 ||
Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::BREAKPOINT_2 ||
Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::BREAKPOINT_3) {
Thread->OpDispatcher->BreakpointOp(DecodedInfo, FEXCore::ToUnderlying(Block.BlockStatus) -
FEXCore::ToUnderlying(Frontend::Decoder::DecodedBlockStatus::BREAKPOINT_0));
} else if (Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::INVALID_INST ||
Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::BAD_RELOCATION) {
if (Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::INVALID_INST ||
Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::BAD_RELOCATION) {
Thread->OpDispatcher->InvalidOp(DecodedInfo);
} else if (Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::UNIMPLEMENTED_INST) {
Thread->OpDispatcher->UnimplementedOp(DecodedInfo);
} else {
Thread->OpDispatcher->NoExecOrPartialOp(DecodedInfo);
Thread->OpDispatcher->NoExecOp(DecodedInfo);
}
}
@@ -765,9 +773,9 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
#endif
Thread->OpDispatcher->Finalize();
}
Thread->FrontendDecoder->DelayedDisownBuffer();
Thread->FrontendDecoder->DelayedDisownBuffer();
}
IR::IREmitter* IREmitter = Thread->OpDispatcher.get();
@@ -838,7 +846,7 @@ ContextImpl::CompileCodeResult ContextImpl::CompileCode(FEXCore::Core::InternalT
// If the trap flag is set we generate single instruction blocks that each check to generate a single step exception.
bool TFSet = Thread->CurrentFrame->State.flags[X86State::RFLAG_TF_RAW_LOC];
auto CompiledCode = Thread->CPUBackend->EmitJITCode(GuestRIP, Length, TotalInstructions == 1, &*IRView, DebugData.get(), TFSet);
auto CompiledCode = Thread->CPUBackend->CompileCode(GuestRIP, Length, TotalInstructions == 1, &*IRView, DebugData.get(), TFSet);
// Release the IR
Thread->OpDispatcher->DelayedDisownBuffer();
+154 -363
View File
@@ -2,22 +2,18 @@
#define XXH_STATIC_LINKING_ONLY
#include "FEXCore/Utils/TypeDefines.h"
#include "Interface/Core/Frontend.h"
#include "FEXCore/Config/Config.h"
#include "FEXCore/fextl/string.h"
#include "FEXHeaderUtils/Filesystem.h"
#include "FEXCore/Core/DiskCache.h"
#include "FEXCore/Core/DiskCacheFileMapper.h"
#include "FEXCore/Utils/LogManager.h"
#include "Interface/Context/Context.h"
#include <FEXCore/Config/Config.h>
#include <FEXCore/Core/DiskCache.h>
#include <FEXCore/Core/DiskCacheFileMapper.h>
#include <FEXCore/fextl/memory.h>
#include <FEXCore/fextl/string.h>
#include <FEXCore/HLE/SyscallHandler.h>
#include <FEXCore/Utils/File.h>
#include <FEXCore/Utils/FileUtils.h>
#include <FEXCore/Utils/LogManager.h>
#include <FEXCore/Utils/Profiler.h>
#include <FEXCore/Utils/TypeDefines.h>
#include <FEXHeaderUtils/Filesystem.h>
#include "FEXCore/HLE/SyscallHandler.h"
#include "FEXCore/Utils/File.h"
#include "FEXCore/fextl/memory.h"
#include <cstdint>
#include <cstring>
#include <atomic>
@@ -273,16 +269,37 @@ namespace DiskCache {
}
}
bool IndexedDB::StoreCacheBlob(const MesaFOZ::foz_payload_key& UniqueKey, uint64_t LookupKey, std::span<const uint8_t> Blob,
MesaFOZ::mesa_index_db_file_entry& IndexEntry, std::span<const uint8_t> IndexBlob) {
bool IndexedDB::StoreCacheBlob(const MesaFOZ::foz_payload_key& UniqueKey, uint64_t LookupKey, std::span<const uint8_t> Blob, Index& Index,
std::mutex& IndexMutex, std::span<const uint8_t> IndexBlob) {
if (ReadOnly) {
// shouldn't happen
return false;
}
if (MaxSizeReached || CacheFileSize + Blob.size() >= MaxFileSize) {
MaxSizeReached = true;
return false;
{
std::lock_guard Guard(IndexMutex);
auto IndexIt = Index.find(LookupKey);
bool Dupe = false;
if (IndexIt != Index.end()) {
if (IndexIt->second.MoreEntries.get()) {
for (auto& [Key, Elem] : *IndexIt->second.MoreEntries) {
if (XXH128_isEqual(Elem.GuestHash, *(const XXH128_hash_t*)(IndexBlob.data()))) {
Dupe = true;
break;
}
}
}
if (!Dupe && XXH128_isEqual(IndexIt->second.MainEntry.GuestHash, *(const XXH128_hash_t*)(IndexBlob.data()))) {
Dupe = true;
}
// could happen if it's seen again while in flight in the store queue
if (Dupe) {
return true;
}
if (IndexIt->second.MoreEntries.get() && IndexIt->second.MoreEntries->size() >= LOOKUP_KEY_MAX_BUCKET_DEPTH) {
return true;
}
}
}
if (!CacheFOZ.Lock(STORE_LOCK_TIMEOUT_MS) || !IndexFOZ.Lock(STORE_LOCK_TIMEOUT_MS)) {
@@ -300,10 +317,10 @@ namespace DiskCache {
return false;
}
IndexEntry = {.hash = LookupKey,
.size = (uint32_t)Blob.size(),
.last_access_time = 0, // todo..
.cache_db_file_offset = BlobOffset};
MesaFOZ::mesa_index_db_file_entry IndexEntry {.hash = LookupKey,
.size = (uint32_t)Blob.size(),
.last_access_time = 0, // todo..
.cache_db_file_offset = BlobOffset};
std::span<const uint8_t> IndexBlobChunks[] = {
{(const uint8_t*)&IndexEntry, sizeof(IndexEntry)},
@@ -324,6 +341,22 @@ namespace DiskCache {
CacheFileSize = BlobOffset + Blob.size();
}
const IndexExtraBlobHeader* IndexBlobHeader = reinterpret_cast<const IndexExtraBlobHeader*>(IndexBlob.data());
struct IndexEntry NewEntry {this, BlobOffset, (uint32_t)Blob.size(), IndexBlobHeader->GuestSize, IndexBlobHeader->GuestHash};
NewEntry.GuestExtents.resize(IndexBlobHeader->GuestExtentsCount);
memcpy(NewEntry.GuestExtents.data(), reinterpret_cast<const uint32_t*>(IndexBlob.data() + sizeof(IndexExtraBlobHeader)),
IndexBlobHeader->GuestExtentsCount * sizeof(uint32_t));
std::lock_guard Guard(IndexMutex);
auto It = Index.find(LookupKey);
if (It == Index.end()) {
Index.emplace(LookupKey, IndexCacheHead {std::move(NewEntry), IndexBlobHeader->GuestFootprint, nullptr});
} else {
if (!It->second.MoreEntries.get()) {
It->second.MoreEntries = fextl::make_unique<fextl::multimap<uint64_t, struct IndexEntry>>();
}
It->second.MoreEntries->insert({IndexBlobHeader->GuestFootprint, std::move(NewEntry)});
}
return true;
}
@@ -360,39 +393,6 @@ namespace DiskCache {
FileMapper = Func;
}
static inline void PruneStaleEntries(std::string_view CacheBase, std::string_view MachineBucketHash) {
FEXCORE_PROFILE_SCOPED("DiskCache::PruneStaleEntries");
FEX_CONFIG_OPT(DiskCachePruneStaleEntries, DISKCACHEPRUNESTALEENTRIES);
if (!DiskCachePruneStaleEntries) {
return;
}
struct SimpleCapture {
std::string_view CacheBase, MachineBucketHash;
} const SimpleCapture {
.CacheBase = CacheBase,
.MachineBucketHash = MachineBucketHash,
};
FEXCore::FileUtils::WalkDirectory(
CacheBase,
[](std::string_view name, bool is_dir, const void* user_data) {
if (!is_dir) {
return;
}
auto capture = reinterpret_cast<const struct SimpleCapture*>(user_data);
// Current behaviour is to remove entries that no longer match the MachineBucketHash.
// This means that if the Disk Cache version no longer matches, or the FEXCore::HostFeatures differ, then they get removed.
if (name != capture->MachineBucketHash) {
FEXCore::FileUtils::RecursiveRemoveDirectory(fextl::fmt::format("{}/{}", capture->CacheBase, name));
}
},
&SimpleCapture);
}
void DiskCache::Init(FEXCore::Context::ContextImpl* CTX) {
this->CTX = CTX;
@@ -402,50 +402,30 @@ namespace DiskCache {
fextl::string SerializedConfig = FEXCore::Config::SerializeForCache();
const auto HostFeatureHash = CTX->HostFeatures.HashForCaching();
struct __attribute__((packed)) {
uint16_t FormatVersion;
uint8_t Is64BitMode;
uint64_t HostFeaturesHash;
} MachineBucketData = {FormatVersion, HostFeatureHash.HostFeaturesHash};
} BucketHeader = {FormatVersion, CTX->Config.Is64BitMode, CTX->HostFeatures.HashForCaching()};
uint8_t ProcessFeaturesHash {};
memcpy(&ProcessFeaturesHash, &HostFeatureHash.ProcessFeaturesHash, sizeof(ProcessFeaturesHash));
LogMan::Msg::IFmt("Disk Cache features:");
LogMan::Msg::IFmt("HostFeatures Hash: 0x{:016x}", HostFeatureHash.HostFeaturesHash);
LogMan::Msg::IFmt("ProcessFeaturesHash: 0x{:02x}", ProcessFeaturesHash);
fextl::vector<uint8_t> BucketBytes(sizeof(MachineBucketData) + sizeof(HostFeatureHash.ProcessFeaturesHash) + SerializedConfig.size());
memcpy(BucketBytes.data(), &MachineBucketData, sizeof(MachineBucketData));
// 64-bit mode and HostType is in the ProcessBucket hash only instead of the MachineBucketHash
// These effect code-gen, but they aren't part of the MachineBucketHash, as it comes from Process state.
memcpy(&BucketBytes[sizeof(MachineBucketData)], &HostFeatureHash.ProcessFeaturesHash, sizeof(HostFeatureHash.ProcessFeaturesHash));
memcpy(BucketBytes.data() + sizeof(MachineBucketData) + sizeof(HostFeatureHash.ProcessFeaturesHash), SerializedConfig.data(),
SerializedConfig.size());
uint64_t MachineBucketHash = XXH3_64bits(BucketBytes.data(), sizeof(MachineBucketData));
uint64_t ProcessBucketHash =
XXH3_64bits(BucketBytes.data() + sizeof(MachineBucketData), sizeof(HostFeatureHash.ProcessFeaturesHash) + SerializedConfig.size());
BucketHash.high64 = MachineBucketHash;
BucketHash.low64 = ProcessBucketHash;
fextl::vector<uint8_t> BucketBytes;
BucketBytes.resize(sizeof(BucketHeader) + SerializedConfig.size());
memcpy(BucketBytes.data(), &BucketHeader, sizeof(BucketHeader));
memcpy(BucketBytes.data() + sizeof(BucketHeader), SerializedConfig.data(), SerializedConfig.size());
BucketHash = XXH3_128bits(BucketBytes.data(), BucketBytes.size());
fextl::string BasePath = BasePathOverride();
if (BasePath.empty()) {
BasePath = FEXCore::Config::GetCacheDirectory() + "DiskCache/";
BasePath += fextl::fmt::format("{:016x}{:016x}", BucketHash.high64, BucketHash.low64) + "/";
}
const auto MachineBucketHashAsString = fextl::fmt::format("{:016x}", MachineBucketHash);
PruneStaleEntries(BasePath, MachineBucketHashAsString);
BasePath += MachineBucketHashAsString + "/";
FHU::Filesystem::CreateDirectories(BasePath);
if (!MapDiskCacheFiles) {
FileMapper = nullptr;
}
const auto RWDBBasePath = fextl::fmt::format("{}RWCacheDB_{:016x}", BasePath, ProcessBucketHash);
fextl::string RWDBBasePath = BasePath + "RWCacheDB";
OpenCacheDB(RWDBBasePath, false);
if (RWCacheDB && !FoundMetadata) {
@@ -453,8 +433,7 @@ namespace DiskCache {
MesaFOZ::foz_payload_key MetadataKey;
memset(MetadataKey.bytes, 0xFF, sizeof(MetadataKey));
IndexExtraBlobHeader MetaDataHeader = {};
MesaFOZ::mesa_index_db_file_entry IndexEntry = {};
RWCacheDB->StoreCacheBlob(MetadataKey, ~0, {BucketBytes.data(), BucketBytes.size()}, IndexEntry,
RWCacheDB->StoreCacheBlob(MetadataKey, ~0, {BucketBytes.data(), BucketBytes.size()}, Index, IndexLock,
{reinterpret_cast<uint8_t*>(&MetaDataHeader), sizeof(MetaDataHeader)});
Index.erase(~0);
}
@@ -480,7 +459,7 @@ namespace DiskCache {
if (IsWritingDiskCache()) {
FEXCore::Threads::Flags WriterThreadFlags = {.LowPriority = true, .Internal = true};
Writer = fextl::make_unique<WorkQueueThread>(WriterThreadFlags, "FEX:DiskCache");
Writer = fextl::make_unique<WorkQueueThread>(WriterThreadFlags);
}
}
@@ -507,49 +486,6 @@ namespace DiskCache {
return XXH3_64bits(&BlobKeyBytes, sizeof(BlobKeyBytes));
}
struct DiskCache::PruneMemoryLRUWorkItem final : WorkQueueThread::WorkItem {
DiskCache* Self;
PruneMemoryLRUWorkItem(DiskCache* Self)
: Self(Self) {}
void Run() override {
if (Self->MemoryLRUCurrentSize <= Self->MemoryLRUMaxSize + Self->MemoryLRUEvictThreshold) {
return;
}
std::lock_guard IndexGuard(Self->IndexLock);
std::lock_guard LRUGuard(Self->MemoryLRULock);
if (Self->MemoryLRU.empty()) {
return;
}
auto Last = std::prev(Self->MemoryLRU.end());
while (Self->MemoryLRUCurrentSize > Self->MemoryLRUMaxSize && !Self->MemoryLRU.empty()) {
auto LastKey = *Last;
auto IndexEntry = Self->LookupLocked(LastKey.LookupKey, LastKey.GuestHash, LastKey.GuestFootprint);
bool AtFront = (Last == Self->MemoryLRU.begin());
if (IndexEntry && IndexEntry->MemoryBlob.use_count() > 1) {
// being read rn, keep moving
if (AtFront) {
break;
}
Last--;
continue;
}
if (IndexEntry) {
IndexEntry->MemoryBlob.reset();
IndexEntry->LRUEntry.reset();
}
Self->MemoryLRUCurrentSize -= LastKey.Size;
auto Deleted = Last;
if (AtFront) {
Self->MemoryLRU.erase(Deleted);
break;
}
Last--;
Self->MemoryLRU.erase(Deleted);
}
}
};
std::optional<CodeHitData> DiskCache::Lookup(Core::InternalThreadState* Thread, std::optional<ExecutableFileSectionInfo> Region,
uint64_t GuestRIP, std::optional<uint64_t>& GuestCodeKey) {
if (!IsReadingDiskCache()) {
@@ -610,8 +546,6 @@ namespace DiskCache {
fextl::multimap<uint64_t, IndexEntry>::iterator MoreEntriesIt;
fextl::multimap<uint64_t, IndexEntry>* MapPointer = nullptr;
IndexEntry* EntryUnderReview;
fextl::shared_ptr<fextl::vector<uint8_t>> BlobRef;
std::optional<fextl::list<MemoryLRUKey>::iterator> LRUIter;
uint64_t CurrentFootprint = 0;
{
std::lock_guard Guard(IndexLock);
@@ -629,13 +563,9 @@ namespace DiskCache {
}
if (MapPointer && MoreEntriesIt != It->second.MoreEntries->end()) {
EntryUnderReview = &MoreEntriesIt->second;
BlobRef = EntryUnderReview->MemoryBlob;
LRUIter = EntryUnderReview->LRUEntry;
CurrentFootprint = MoreEntriesIt->first;
} else {
EntryUnderReview = &MainEntry;
BlobRef = EntryUnderReview->MemoryBlob;
LRUIter = EntryUnderReview->LRUEntry;
CurrentFootprint = MainEntryFootprint;
TriedMainEntry = true;
}
@@ -652,8 +582,6 @@ namespace DiskCache {
// if the current footprint is also the main entry's footprint, give main entry a shot next
if (!TriedMainEntry && MoreEntriesIt->first == MainEntryFootprint) {
EntryUnderReview = &MainEntry;
BlobRef = EntryUnderReview->MemoryBlob;
LRUIter = EntryUnderReview->LRUEntry;
CurrentFootprint = MainEntryFootprint;
TriedMainEntry = true;
} else {
@@ -666,27 +594,17 @@ namespace DiskCache {
break;
} else {
EntryUnderReview = &MainEntry;
BlobRef = EntryUnderReview->MemoryBlob;
LRUIter = EntryUnderReview->LRUEntry;
CurrentFootprint = MainEntryFootprint;
TriedMainEntry = true;
}
}
if (!EntryUnderReview && MapPointer && MoreEntriesIt != MapPointer->end()) {
EntryUnderReview = &MoreEntriesIt->second;
BlobRef = EntryUnderReview->MemoryBlob;
LRUIter = EntryUnderReview->LRUEntry;
CurrentFootprint = MoreEntriesIt->first;
}
}
Advance = true;
// entry not backed by anything right now - todo prune..
if (!EntryUnderReview->DB && !BlobRef) {
continue;
}
// do we have enough room in our live code to even hash GuestSize worth?
if (Available < EntryUnderReview->GuestSize) {
continue;
@@ -717,7 +635,7 @@ namespace DiskCache {
break;
} else if (Validation) {
fextl::vector<uint8_t> GuestCode(Entry.GuestSize);
if (Entry.Size >= Entry.GuestSize && Entry.DB && Entry.DB->ReadCacheBlob(Entry.Offset + Entry.Size - Entry.GuestSize, GuestCode)) {
if (Entry.Size >= Entry.GuestSize && Entry.DB->ReadCacheBlob(Entry.Offset + Entry.Size - Entry.GuestSize, GuestCode)) {
const uint8_t* CachedGuest = GuestCode.data();
const uint8_t* LiveGuest = reinterpret_cast<const uint8_t*>(GuestRIP);
uint64_t DiffCount = 0;
@@ -781,17 +699,7 @@ namespace DiskCache {
HitData.Blob.resize(GuestPages.size() * sizeof(uint64_t) + EntrySizeWithoutGuestCode);
memcpy(HitData.Blob.data(), GuestPages.data(), GuestPages.size() * sizeof(uint64_t));
uint32_t BlobOffset = GuestPages.size() * sizeof(uint64_t);
bool FoundInLRU = false;
if (BlobRef && BlobRef->size() >= EntrySizeWithoutGuestCode) {
FoundInLRU = true;
memcpy(HitData.Blob.data() + BlobOffset, BlobRef->data(), EntrySizeWithoutGuestCode);
if (LRUIter) {
std::lock_guard Guard(MemoryLRULock);
// avoided disk by nabbing from lru, bump to front
// LogMan::Msg::IFmt("lru hit! {}", MemoryLRUCurrentSize);
MemoryLRU.splice(MemoryLRU.begin(), MemoryLRU, *LRUIter);
}
} else if (!Entry.DB->ReadCacheBlob(Entry.Offset, {HitData.Blob.data() + BlobOffset, EntrySizeWithoutGuestCode})) {
if (!Entry.DB->ReadCacheBlob(Entry.Offset, {HitData.Blob.data() + BlobOffset, EntrySizeWithoutGuestCode})) {
return std::nullopt;
}
@@ -812,29 +720,6 @@ namespace DiskCache {
return std::nullopt;
}
bool StoreInMemory = !FoundInLRU;
if (EntrySizeWithoutGuestCode > MemoryLRUMaxSize) {
StoreInMemory = false;
}
if (StoreInMemory) {
auto NewBlobRef = fextl::make_shared<fextl::vector<uint8_t>>(EntrySizeWithoutGuestCode);
memcpy(NewBlobRef->data(), HitData.Blob.data() + BlobOffset, EntrySizeWithoutGuestCode);
std::lock_guard Guard(IndexLock);
auto CurrentIndexEntry = LookupLocked(LookupKey, Header.GuestHash, LastFootprintHashed);
if (CurrentIndexEntry && !CurrentIndexEntry->MemoryBlob) {
std::lock_guard LRUGuard(MemoryLRULock);
MemoryLRU.push_front({LookupKey, Header.GuestHash, LastFootprintHashed, EntrySizeWithoutGuestCode});
CurrentIndexEntry->LRUEntry = MemoryLRU.begin();
CurrentIndexEntry->MemoryBlob = NewBlobRef;
MemoryLRUCurrentSize += EntrySizeWithoutGuestCode;
}
}
if (StoreInMemory && MemoryLRUCurrentSize > MemoryLRUMaxSize + MemoryLRUEvictThreshold) {
Writer->QueueWork(fextl::make_unique<PruneMemoryLRUWorkItem>(this));
}
HitData.HostCode = {HitData.Blob.data() + BlobOffset, Header.HostSize};
BlobOffset += Header.HostSize;
HitData.EntryPointRIPs = {reinterpret_cast<uint64_t*>(HitData.Blob.data() + BlobOffset), Header.EntryPointCount};
@@ -900,81 +785,23 @@ namespace DiskCache {
}
}
IndexEntry* DiskCache::LookupLocked(const uint64_t LookupKey, const XXH128_hash_t& GuestHash, const uint64_t GuestFootprint) {
auto It = Index.find(LookupKey);
if (It != Index.end()) {
if (XXH128_isEqual(It->second.MainEntry.GuestHash, GuestHash)) {
return &It->second.MainEntry;
}
if (It->second.MoreEntries.get()) {
auto Range = It->second.MoreEntries->equal_range(GuestFootprint);
for (auto MoreEntriesIt = Range.first; MoreEntriesIt != Range.second; MoreEntriesIt++) {
if (XXH128_isEqual(MoreEntriesIt->second.GuestHash, GuestHash)) {
return &MoreEntriesIt->second;
}
}
}
}
return nullptr;
}
struct DiskCache::CacheStoreWorkItem final : WorkQueueThread::WorkItem {
DiskCache* Self;
IndexedDB* DB;
MesaFOZ::foz_payload_key UniqueKey;
uint64_t LookupKey;
std::span<uint8_t> Blob;
fextl::vector<uint8_t> Blob;
fextl::vector<uint8_t> IndexBlob;
bool StoreDisk;
CacheStoreWorkItem(DiskCache* Self, IndexedDB* DB, const MesaFOZ::foz_payload_key& UniqueKey, uint64_t LookupKey,
std::span<uint8_t> Blob, fextl::vector<uint8_t>&& IndexBlob, bool StoreDisk)
fextl::vector<uint8_t>&& Blob, fextl::vector<uint8_t>&& IndexBlob)
: Self(Self)
, DB(DB)
, UniqueKey(UniqueKey)
, LookupKey(LookupKey)
, Blob(Blob)
, IndexBlob(std::move(IndexBlob))
, StoreDisk(StoreDisk) {}
, Blob(std::move(Blob))
, IndexBlob(std::move(IndexBlob)) {}
void Run() override {
struct MesaFOZ::mesa_index_db_file_entry IndexHeader;
bool DiskSuccess = !StoreDisk || DB->StoreCacheBlob(UniqueKey, LookupKey, Blob, IndexHeader, IndexBlob);
bool KeepEntryInMemory = true;
// todo possible other lru condition here like entry size?
if (Blob.size() > Self->MemoryLRUMaxSize) {
KeepEntryInMemory = false;
} else {
Self->MemoryLRUCurrentSize += Blob.size();
}
const IndexExtraBlobHeader* IndexAfterHeader = reinterpret_cast<const IndexExtraBlobHeader*>(IndexBlob.data());
fextl::list<MemoryLRUKey>::iterator NewLRUEntry;
if (KeepEntryInMemory) {
std::lock_guard Guard(Self->MemoryLRULock);
Self->MemoryLRU.push_front({LookupKey, IndexAfterHeader->GuestHash, IndexAfterHeader->GuestFootprint, (uint32_t)Blob.size()});
NewLRUEntry = Self->MemoryLRU.begin();
}
{
std::lock_guard Guard(Self->IndexLock);
auto IndexEntry = Self->LookupLocked(LookupKey, IndexAfterHeader->GuestHash, IndexAfterHeader->GuestFootprint);
LOGMAN_THROW_A_FMT(IndexEntry != nullptr, "Stored Index entry not found?");
if (IndexEntry) {
if (StoreDisk && DiskSuccess) {
IndexEntry->DB = DB;
IndexEntry->Offset = IndexHeader.cache_db_file_offset;
}
if (!KeepEntryInMemory) {
IndexEntry->MemoryBlob.reset();
} else {
IndexEntry->LRUEntry = NewLRUEntry;
}
}
}
if (KeepEntryInMemory && Self->MemoryLRUCurrentSize > Self->MemoryLRUMaxSize + Self->MemoryLRUEvictThreshold) {
Self->Writer->QueueWork(fextl::make_unique<PruneMemoryLRUWorkItem>(Self));
}
DB->StoreCacheBlob(UniqueKey, LookupKey, Blob, Self->Index, Self->IndexLock, IndexBlob);
}
};
@@ -998,11 +825,6 @@ namespace DiskCache {
if (Target >= Region->BeginVA && Target < Region->EndVA) {
continue;
}
// let it through if it's inside the same ELF image? (like bss)
if (Region->FileInfo.MappedSize && Target >= Region->FileStartVA && Target < Region->FileStartVA + Region->FileInfo.MappedSize) {
continue;
}
auto TargetSection = CTX->SyscallHandler->LookupExecutableFileSection(Thread, Target);
if (!TargetSection || TargetSection->FileInfo.FileId != Region->FileInfo.FileId) {
// we don't know where it's pointing, so we don't know how to encode the offset, so we can't cache atm
@@ -1021,6 +843,61 @@ namespace DiskCache {
}
}
fextl::set<uint64_t> DataMaskAddresses;
fextl::vector<uint32_t> ExactGuestCodeExtents;
uint64_t CurStartExtent = 0, CurEndExtent = 0;
const Frontend::Decoder::DecodedBlocks* LastBlock = nullptr;
for (auto& SubBlock : DecodedBlockInfo->Blocks) {
if (SubBlock.BlockStatus != Frontend::Decoder::DecodedBlockStatus::SUCCESS) {
return false;
}
if (!CurStartExtent) {
CurStartExtent = SubBlock.Entry;
CurEndExtent = SubBlock.Entry + SubBlock.Size;
} else {
LOGMAN_THROW_A_FMT(SubBlock.Entry >= CurEndExtent, "DecodedBlocks not sorted or overlapping?");
if (SubBlock.Entry == CurEndExtent) {
CurEndExtent = SubBlock.Entry + SubBlock.Size;
} else {
ExactGuestCodeExtents.push_back(CurStartExtent - GuestRIP);
ExactGuestCodeExtents.push_back(CurEndExtent - CurStartExtent);
CurStartExtent = SubBlock.Entry;
CurEndExtent = SubBlock.Entry + SubBlock.Size;
}
}
// split extents according to data masks as well
for (auto& Mask : SubBlock.DataMasks) {
if (Mask.FieldAddress > CurStartExtent) {
ExactGuestCodeExtents.push_back(CurStartExtent - GuestRIP);
ExactGuestCodeExtents.push_back(Mask.FieldAddress - CurStartExtent);
}
CurStartExtent = Mask.FieldAddress + Mask.ValueSize;
DataMaskAddresses.insert(Mask.FieldAddress);
}
LastBlock = &SubBlock;
}
if (LastBlock && (CurStartExtent != GuestRIP || CurEndExtent != GuestRIP + GuestCode.size())) {
ExactGuestCodeExtents.push_back(CurStartExtent - GuestRIP);
ExactGuestCodeExtents.push_back(CurEndExtent - CurStartExtent);
}
if (ExactGuestCodeExtents.size() == 0) {
ExactGuestCodeExtents.reserve(2);
ExactGuestCodeExtents.push_back(0);
ExactGuestCodeExtents.push_back(GuestCode.size());
}
uint64_t GuestFootprint = XXH3_64bits(ExactGuestCodeExtents.data(), ExactGuestCodeExtents.size() * sizeof(uint32_t));
// if (ExactGuestCodeExtents.size()) {
// LogMan::Msg::IFmt("store! length {:d}", GuestCode.size());
// for(uint32_t i = 0; i < ExactGuestCodeExtents.size(); i+=2 ) {
// LogMan::Msg::IFmt("extent {} {}", ExactGuestCodeExtents[i], ExactGuestCodeExtents[i]+ExactGuestCodeExtents[i+1]);
// }
// }
const uint32_t EntryPointCount = (uint32_t)CompiledCode.EntryPoints.size();
const size_t HeaderOffset = 0;
@@ -1033,8 +910,9 @@ namespace DiskCache {
const size_t TotalSize = GuestCodeOffset + GuestCode.size();
// we'll copy everything into here and pass it to the Writer, then return to caller quickly
auto BlobRef = fextl::make_shared<fextl::vector<uint8_t>>(TotalSize);
uint8_t* BlobData = BlobRef->data();
fextl::vector<uint8_t> Blob;
Blob.resize(TotalSize);
uint8_t* BlobData = Blob.data();
BlobFixedHeader Header {
.GuestSize = (uint32_t)GuestCode.size(),
@@ -1044,6 +922,15 @@ namespace DiskCache {
.ThunkRelocCount = ThunkRelocCount,
};
{
XXH3_state_t HashState;
XXH3_128bits_reset(&HashState);
for (uint32_t i = 0; i < ExactGuestCodeExtents.size(); i += 2) {
XXH3_128bits_update(&HashState, GuestCode.data() + ExactGuestCodeExtents[i], ExactGuestCodeExtents[i + 1]);
}
Header.GuestHash = XXH3_128bits_digest(&HashState);
}
memcpy(BlobData + HeaderOffset, &Header, sizeof(Header));
memcpy(BlobData + HostCodeOffset, CompiledCode.BlockBegin, CompiledCode.Size);
// pack and relocate entrypoints
@@ -1056,8 +943,6 @@ namespace DiskCache {
EntryIdx++;
}
fextl::set<uint64_t> DataMasksConsumed;
// pack relocations
auto* SmallRelocs = reinterpret_cast<BlobSmallRelocation*>(BlobData + SmallRelocsOffset);
auto* ThunkRelocs = reinterpret_cast<BlobThunkRelocation*>(BlobData + ThunkRelocsOffset);
@@ -1094,8 +979,7 @@ namespace DiskCache {
}
case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_DATA_MOVE:
case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_MOVE:
case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_LITERAL:
case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_CRC_MOVE: {
case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_LITERAL: {
// same data for all, relative vs. not and register vs. literal will depend on type on apply
BlobSmallRelocation SmallReloc = {};
SmallReloc.Offset = Reloc.Header.Offset;
@@ -1105,8 +989,12 @@ namespace DiskCache {
SmallReloc.PatchableData.SiteOffset = uint32_t(Reloc.GuestPatchableData.SiteAddress - GuestRIP);
SmallRelocs[SmallIdx++] = SmallReloc;
// mark the corresponding data mask consumed
DataMasksConsumed.insert(Reloc.GuestPatchableData.SiteAddress);
// mark the corresponding data mask consumed - we might not find one if they got removed due to the smc workaround
// the hash will just fail on lookup later
auto It = DataMaskAddresses.find(Reloc.GuestPatchableData.SiteAddress);
if (It != DataMaskAddresses.end()) {
DataMaskAddresses.erase(It);
}
break;
}
@@ -1121,108 +1009,18 @@ namespace DiskCache {
}
}
fextl::vector<uint32_t> ExactGuestCodeExtents;
uint64_t CurStartExtent = 0, CurEndExtent = 0;
const Frontend::Decoder::DecodedBlocks* LastBlock = nullptr;
for (auto& SubBlock : DecodedBlockInfo->Blocks) {
if (SubBlock.BlockStatus != Frontend::Decoder::DecodedBlockStatus::SUCCESS) {
return false;
}
if (!CurStartExtent) {
CurStartExtent = SubBlock.Entry;
CurEndExtent = SubBlock.Entry + SubBlock.Size;
} else {
LOGMAN_THROW_A_FMT(SubBlock.Entry >= CurEndExtent, "DecodedBlocks not sorted or overlapping?");
if (SubBlock.Entry == CurEndExtent) {
CurEndExtent = SubBlock.Entry + SubBlock.Size;
} else {
ExactGuestCodeExtents.push_back(CurStartExtent - GuestRIP);
ExactGuestCodeExtents.push_back(CurEndExtent - CurStartExtent);
CurStartExtent = SubBlock.Entry;
CurEndExtent = SubBlock.Entry + SubBlock.Size;
}
}
// split extents according to data masks as well
for (auto& Mask : SubBlock.DataMasks) {
if (Mask.Type == Frontend::Decoder::DataMaskType::NOP || DataMasksConsumed.find(Mask.FieldAddress) != DataMasksConsumed.end()) {
if (Mask.FieldAddress > CurStartExtent) {
ExactGuestCodeExtents.push_back(CurStartExtent - GuestRIP);
ExactGuestCodeExtents.push_back(Mask.FieldAddress - CurStartExtent);
}
CurStartExtent = Mask.FieldAddress + Mask.ValueSize;
}
}
LastBlock = &SubBlock;
}
if (LastBlock && (CurStartExtent != GuestRIP || CurEndExtent != GuestRIP + GuestCode.size())) {
ExactGuestCodeExtents.push_back(CurStartExtent - GuestRIP);
ExactGuestCodeExtents.push_back(CurEndExtent - CurStartExtent);
if (!DataMaskAddresses.empty()) {
LogMan::Msg::IFmt("DiskCache: DataMask unaccounted for! {:x}", GuestCodeKey);
// this would mean we omitted contents in the hash that we're not going to patch, which would be loading corrupt code
return false;
}
if (ExactGuestCodeExtents.size() == 0) {
ExactGuestCodeExtents.reserve(2);
ExactGuestCodeExtents.push_back(0);
ExactGuestCodeExtents.push_back(GuestCode.size());
}
uint64_t GuestFootprint = XXH3_64bits(ExactGuestCodeExtents.data(), ExactGuestCodeExtents.size() * sizeof(uint32_t));
// if (ExactGuestCodeExtents.size()) {
// LogMan::Msg::IFmt("store! length {:d}", GuestCode.size());
// for(uint32_t i = 0; i < ExactGuestCodeExtents.size(); i+=2 ) {
// LogMan::Msg::IFmt("extent {} {}", ExactGuestCodeExtents[i], ExactGuestCodeExtents[i]+ExactGuestCodeExtents[i+1]);
// }
// }
{
XXH3_state_t HashState;
XXH3_128bits_reset(&HashState);
for (uint32_t i = 0; i < ExactGuestCodeExtents.size(); i += 2) {
XXH3_128bits_update(&HashState, GuestCode.data() + ExactGuestCodeExtents[i], ExactGuestCodeExtents[i + 1]);
}
Header.GuestHash = XXH3_128bits_digest(&HashState);
}
memcpy(BlobData + HeaderOffset, &Header, sizeof(Header));
memcpy(BlobData + GuestCodeOffset, GuestCode.data(), GuestCode.size());
auto RangeInfo = CTX->SyscallHandler->QueryGuestExecutableRange(Thread, GuestRIP);
uint64_t LookupKey =
MakeLookupKey(Thread, GuestCodeKey, RangeInfo.Writable, GuestRIP == CTX->GetMonoBackPatcherBlock().load(std::memory_order_relaxed));
// blob done, publish to index as in-memory for now, flush to disk below
struct IndexEntry NewEntry {nullptr, 0, (uint32_t)TotalSize, Header.GuestSize, Header.GuestHash, BlobRef};
NewEntry.GuestExtents = ExactGuestCodeExtents;
{
std::lock_guard Guard(IndexLock);
auto It = Index.find(LookupKey);
if (It == Index.end()) {
Index.emplace(LookupKey, IndexCacheHead {std::move(NewEntry), GuestFootprint, nullptr});
} else {
bool Dupe = false;
if (It->second.MoreEntries.get()) {
for (auto& [Key, Elem] : *It->second.MoreEntries) {
if (XXH128_isEqual(Elem.GuestHash, Header.GuestHash)) {
Dupe = true;
break;
}
}
}
if (!Dupe && XXH128_isEqual(It->second.MainEntry.GuestHash, Header.GuestHash)) {
Dupe = true;
}
// could happen if it's seen again while in flight in the store queue
if (Dupe) {
return true;
}
if (!It->second.MoreEntries.get()) {
It->second.MoreEntries = fextl::make_unique<fextl::multimap<uint64_t, struct IndexEntry>>();
} else if (It->second.MoreEntries->size() >= LOOKUP_KEY_MAX_BUCKET_DEPTH) {
return false;
}
It->second.MoreEntries->insert({GuestFootprint, std::move(NewEntry)});
}
}
memcpy(BlobData + GuestCodeOffset, GuestCode.data(), GuestCode.size());
MesaFOZ::foz_payload_key Key = {};
{
XXH3_state_t HashState;
@@ -1241,15 +1039,8 @@ namespace DiskCache {
memcpy(IndexBlob.data(), &IndexBlobHeader, sizeof(IndexExtraBlobHeader));
memcpy(IndexBlob.data() + sizeof(IndexExtraBlobHeader), ExactGuestCodeExtents.data(), ExactGuestCodeExtents.size() * sizeof(uint32_t));
bool StoreDisk = true;
if (RWCacheDB->Full()) {
StoreDisk = false;
}
// hand the rest off to the writer thread
Writer->QueueWork(fextl::make_unique<CacheStoreWorkItem>(this, RWCacheDB.get(), Key, LookupKey, std::span(BlobData, TotalSize),
std::move(IndexBlob), StoreDisk));
Writer->QueueWork(fextl::make_unique<CacheStoreWorkItem>(this, RWCacheDB.get(), Key, LookupKey, std::move(Blob), std::move(IndexBlob)));
return true;
}
-258
View File
@@ -1,258 +0,0 @@
// SPDX-License-Identifier: MIT
#pragma once
#include "Interface/Core/JIT/Relocations.h"
#include "Interface/Core/Frontend.h"
#include "Interface/Core/CPUBackend.h"
#include <FEXCore/Core/CodeCache.h>
#include <FEXCore/Core/Context.h>
#include <FEXCore/Config/Config.h>
#include <FEXCore/Utils/File.h>
#include <FEXCore/Utils/WorkQueueThread.h>
#include <FEXCore/fextl/memory.h>
#include <FEXCore/fextl/string.h>
#include <FEXCore/fextl/unordered_set.h>
#include <FEXCore/fextl/robin_map.h>
#include <FEXCore/fextl/map.h>
#include <FEXCore/fextl/vector.h>
#include <stdint.h>
#include <mutex>
#include <optional>
#include <span>
#include <xxhash.h>
namespace FEXCore {
namespace Context {
class ContextImpl;
}
namespace DiskCache {
namespace MesaFOZ {
#define FOSSILIZE_BLOB_HASH_LENGTH 40 /* SHA1 hexadecimal string length */
struct __attribute__((packed)) foz_payload_key {
uint8_t bytes[FOSSILIZE_BLOB_HASH_LENGTH];
};
struct __attribute__((packed)) foz_payload_header {
uint32_t payload_size;
uint32_t format;
uint32_t crc;
uint32_t uncompressed_size;
};
struct mesa_index_db_file_entry;
} // namespace MesaFOZ
class IndexedDB;
struct MemoryLRUKey {
uint64_t LookupKey;
XXH128_hash_t GuestHash;
uint64_t GuestFootprint;
uint32_t Size;
};
struct IndexEntry {
IndexedDB* DB;
uint64_t Offset;
uint32_t Size;
uint32_t GuestSize;
XXH128_hash_t GuestHash;
fextl::shared_ptr<fextl::vector<uint8_t>> MemoryBlob;
fextl::vector<uint32_t> GuestExtents;
std::optional<fextl::list<MemoryLRUKey>::iterator> LRUEntry;
};
struct IndexCacheHead {
struct IndexEntry MainEntry;
uint64_t MainEntryFootprint;
fextl::unique_ptr<fextl::multimap<uint64_t, IndexEntry>> MoreEntries; // sorted by guest footprint
};
struct __attribute__((packed)) BlobFixedHeader {
uint32_t GuestSize;
uint32_t HostSize;
uint32_t EntryPointCount;
uint32_t SmallRelocCount;
uint32_t ThunkRelocCount;
XXH128_hash_t GuestHash;
};
// packed struct for types 0, 2 and 3. type 1 is bigger and separate below
struct __attribute__((packed)) BlobSmallRelocation {
uint32_t Offset;
uint8_t Type;
union {
struct __attribute__((packed)) {
uint32_t Symbol;
} Named;
struct __attribute__((packed)) {
uint64_t GuestRIP;
} RIPLiteral;
struct __attribute__((packed)) {
uint8_t RegisterIndex;
uint64_t GuestRIP;
} RIPMove;
struct __attribute__((packed)) {
uint8_t RegisterIndex;
uint8_t ValueSize;
uint32_t SiteOffset;
} PatchableData;
};
};
// type 1, implicit
struct __attribute__((packed)) BlobThunkRelocation {
uint32_t Offset;
uint8_t RegisterIndex;
uint8_t SymbolHash[32]; // sha256sum in the real RelocNamedThunkMove
};
struct CodeHitData {
fextl::vector<uint8_t> Blob;
std::span<uint8_t> HostCode;
std::span<const uint64_t> GuestPages;
std::span<uint64_t> EntryPointRIPs;
std::span<const uint32_t> EntryPointHostOffsets;
// the spans above point to memory owned by the Blob vec, so it's important this can't be copied
CodeHitData() = default;
CodeHitData(CodeHitData&&) = default;
CodeHitData& operator=(CodeHitData&&) = default;
CodeHitData(const CodeHitData&) = delete;
CodeHitData& operator=(const CodeHitData&) = delete;
};
using Index = fextl::robin_map<uint64_t, IndexCacheHead>;
class FOZFile {
public:
bool Open(const fextl::string& CacheFileName, bool ReadOnly);
bool Lock(uint32_t TimeoutMS) {
if (!FD) {
return false;
}
return FD->Lock(TimeoutMS);
}
bool Unlock() {
if (!FD) {
return false;
}
return FD->Unlock();
}
File::File::FileHandleType GetHandle() {
return FD ? FD->GetHandle() : (File::File::FileHandleType)-1;
}
ssize_t Size();
bool ReadAll(fextl::vector<uint8_t>& Out); // from first blob
bool ReadBlob(uint64_t Offset, std::span<uint8_t> OutBlob);
bool WriteBlob(const MesaFOZ::foz_payload_key& Key, std::span<const std::span<const uint8_t>> BlobChunks, uint64_t& OutBlobOffset);
private:
static constexpr uint32_t OPEN_LOCK_TIMEOUT_MS = 100;
fextl::string FileName;
fextl::unique_ptr<File::File> FD;
bool ReadOnly = false;
};
class IndexedDB {
public:
bool Open(const fextl::string& CacheDBName, bool ReadOnly);
void PopulateIndex(Index& CacheIndex, bool& FoundMetadata);
bool ReadCacheBlob(uint64_t Offset, std::span<uint8_t> OutBlob);
bool StoreCacheBlob(const MesaFOZ::foz_payload_key& UniqueKey, uint64_t LookupKey, std::span<const uint8_t> Blob,
MesaFOZ::mesa_index_db_file_entry& IndexEntry, std::span<const uint8_t> IndexBlob);
bool Full() const {
return MaxSizeReached;
}
private:
// stores run on the Writer, so returning quick isn't as important
static constexpr uint32_t STORE_LOCK_TIMEOUT_MS = 1000;
static constexpr uint64_t BIG_MAPPING_SIZE = 1ULL << 33;
FOZFile CacheFOZ;
uint8_t* CacheFileMapping = nullptr;
std::atomic<uint64_t> CacheFileSize;
FOZFile IndexFOZ;
bool ReadOnly = false;
bool MaxSizeReached = false;
FEX_CONFIG_OPT(MaxFileSize, DISKCACHEMAXFILESIZE);
};
class DiskCache {
public:
void Init(FEXCore::Context::ContextImpl* CTX);
std::optional<CodeHitData> Lookup(Core::InternalThreadState* Thread, std::optional<ExecutableFileSectionInfo> Region, uint64_t GuestRIP,
std::optional<uint64_t>& GuestCodeKey);
void Validate(uint64_t GuestCodeKey, const CodeHitData& Hit, const CPU::CPUBackend::CompiledCode& CompiledCode,
std::optional<ExecutableFileSectionInfo> Region);
bool Store(Core::InternalThreadState* Thread, std::optional<ExecutableFileSectionInfo> Region, uint64_t GuestRIP, uint64_t GuestCodeKey,
std::span<const uint8_t> GuestCode, const CPU::CPUBackend::CompiledCode& CompiledCode,
std::span<const FEXCore::CPU::Relocation> Relocations, const Frontend::Decoder::DecodedBlockInformation* DecodedBlockInfo);
bool IsWritingDiskCache() const {
return WritingDiskCache;
}
bool IsReadingDiskCache() const {
return ReadingDiskCache;
}
bool IsValidating() const {
return Validation;
}
private:
bool OpenCacheDB(const fextl::string& CacheDBName, bool ReadOnly);
uint64_t MakeLookupKey(Core::InternalThreadState* Thread, const uint64_t ModuleOffset, bool Writable, bool MonoBackpatcher);
IndexEntry* LookupLocked(const uint64_t LookupKey, const XXH128_hash_t& GuestHash, const uint64_t GuestFootprint);
bool ReadingDiskCache {};
bool WritingDiskCache {};
FEXCore::Context::ContextImpl* CTX;
XXH128_hash_t BucketHash;
fextl::vector<fextl::unique_ptr<IndexedDB>> ROCacheDBs;
fextl::unique_ptr<IndexedDB> RWCacheDB;
Index Index;
std::mutex IndexLock;
bool FoundMetadata = false;
struct CacheStoreWorkItem;
struct PruneMemoryLRUWorkItem;
std::atomic<uint64_t> MemoryLRUCurrentSize {};
std::mutex MemoryLRULock;
fextl::list<MemoryLRUKey> MemoryLRU;
// the Writer holds references to all this stuff above and needs to be last
fextl::unique_ptr<WorkQueueThread> Writer;
FEX_CONFIG_OPT(EnableDiskCache, DISKCACHE);
FEX_CONFIG_OPT(Validation, DISKCACHEVALIDATION);
FEX_CONFIG_OPT(MapDiskCacheFiles, DISKCACHEFILEMAPPING);
FEX_CONFIG_OPT(RelocationFilter, DISKCACHERELOCATIONFILTER);
FEX_CONFIG_OPT(AnonCaching, DISKCACHEANONCACHING);
FEX_CONFIG_OPT(BasePathOverride, DISKCACHEPATH);
FEX_CONFIG_OPT(RODBNames, DISKCACHERODBNAMES);
FEX_CONFIG_OPT(MemoryLRUMaxSize, DISKCACHEMEMORYSIZE);
uint64_t MemoryLRUEvictThreshold = MemoryLRUMaxSize / 25;
};
static constexpr uint16_t AnonPrefixGuestBytes = 64;
// The current version of the diskcache.
// This must be changed any time codegen changes occur!
// Be aware of the impact of changing this frequently!
static constexpr uint16_t FormatVersion = 44;
static constexpr uint32_t LOOKUP_KEY_MAX_BUCKET_DEPTH = 500;
} // namespace DiskCache
} // namespace FEXCore
@@ -276,15 +276,8 @@ void Dispatcher::EmitDispatcher() {
str(TMP2, STATE, offsetof(FEXCore::Core::CPUState, DeferredSignalRefCount));
// Trigger segfault if any deferred signals are pending
constexpr size_t InterruptPageOffset =
offsetof(FEXCore::Core::InternalThreadState, InterruptFaultPage) - offsetof(FEXCore::Core::InternalThreadState, BaseFrameState);
if constexpr (InterruptPageOffset <= 32760) {
str(ARMEmitter::XReg::zr, STATE, InterruptPageOffset);
} else {
// Need to use vector 128-bit store for this range.
// Doesn't matter which register we use to store.
str(ARMEmitter::QReg::q0, STATE, InterruptPageOffset);
}
strb(ARMEmitter::XReg::zr, STATE,
offsetof(FEXCore::Core::InternalThreadState, InterruptFaultPage) - offsetof(FEXCore::Core::InternalThreadState, BaseFrameState));
#endif
};
@@ -417,7 +410,6 @@ void Dispatcher::EmitDispatcher() {
brk(0);
}
ARMEmitter::BackwardLabel GuestSIGSEGVLabelPostSpill;
{
// Guest Overflow handler
// Needs to be distinct from the SignalHandlerReturnAddress
@@ -425,8 +417,6 @@ void Dispatcher::EmitDispatcher() {
SpillStaticRegs(TMP1);
(void)Bind(&GuestSIGSEGVLabelPostSpill);
// hlt/udf = SIGILL
// brk = SIGTRAP
// ??? = SIGSEGV
@@ -442,26 +432,6 @@ void Dispatcher::EmitDispatcher() {
}
}
{
// Faultless synchronous signal.
// Some games abuse SIGILL/SIGTRAP/SIGSEGV heavily and the fault overhead is visible.
// Instead of double faulting, clean up state and jump to the frontend to handle this.
// `FEXCore::Core::CpuStateFrame::SynchronousFaultData` needs to be setup at this point.
GuestSignal_SynchronousFault = GetCursorAddress<uint64_t>();
SpillStaticRegs(TMP1);
ldr(ARMEmitter::XReg::x0, &l_CTX);
mov(ARMEmitter::XReg::x1, STATE);
ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.SynchronousExceptionHandler));
if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
GenerateIndirectRuntimeCall<void, void*, void*>(ARMEmitter::Reg::r2);
} else {
br(ARMEmitter::Reg::r2);
}
}
{
ThreadPauseHandlerAddressSpillSRA = GetCursorAddress<uint64_t>();
SpillStaticRegs(TMP1);
@@ -537,174 +507,43 @@ void Dispatcher::EmitDispatcher() {
(void)b(&LoopTop);
}
auto EmitL1Lookup = [&](int rip_offset = 0) {
// L1 lookup after syscall. Mirrors the implementation in `ExitFunction`.
// Most syscall instructions just return to the following RIP, so it is highly likely to be in the L1 in tight loops.
ldr(TMP3, STATE_PTR(CpuStateFrame, State.rip));
auto EmitLongALUOpHandler = [&](auto R, auto Offset) {
auto Address = GetCursorAddress<uint64_t>();
// TF may be set: a single-stepped SYSCALL or CPUID. An L1 hit would jump into a block compiled without the
// per-instruction single-step trap, and the trap would only be raised at the next dispatcher entry, instructions
// later. The loop top checks TF and compiles single-step blocks.
ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
(void)cbnz(ARMEmitter::Size::i32Bit, TMP1, &LoopTop);
ldp<ARMEmitter::IndexType::OFFSET>(TMP1, TMP2, STATE, offsetof(FEXCore::Core::CpuStateFrame, State.L1Pointer));
if (rip_offset) {
add(ARMEmitter::Size::i64Bit, TMP3, TMP3, rip_offset);
}
// Calculate (tmp1 + ((ripreg & L1_ENTRIES_MASK) << 4)) for the address
// L1Mask is pre-shifted.
and_(ARMEmitter::Size::i64Bit, TMP2, TMP2, TMP3, ARMEmitter::ShiftType::LSL, FEXCore::ilog2(sizeof(LookupCache::LookupCacheEntry)));
add(TMP1, TMP1, TMP2);
ldp<ARMEmitter::IndexType::OFFSET>(TMP2, TMP1, TMP1, 0);
sub(TMP1, TMP1, TMP3);
(void)cbnz(ARMEmitter::Size::i64Bit, TMP1, &LoopTop);
br(TMP2);
};
{
// All dynamic and static registers are spilled coming in to this handler.
// It's also the end of block and RIP might have changed, so we jump directly to the top of the loop.
ThreadDispatchSyscallHandler = GetCursorAddress<uint64_t>();
// Store in the state that we are in a syscall
// 16bit LoadConstant to be a single instruction
// This gives the signal handler a value to check to see if we are in a syscall at all
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, 0xFFFF);
str(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo));
ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.SyscallHandlerObj));
ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.SyscallHandlerFunc));
mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, STATE.R());
if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
GenerateIndirectRuntimeCall<uint64_t, void*, void*, void*>(ARMEmitter::Reg::r3);
} else {
blr(ARMEmitter::Reg::r3);
}
// Fix the stack and any values that were stepped on
// Syscall result is in any static register that the frontend desired.
FillStaticRegs({
.OptionalReg = ARMEmitter::Reg::r1,
.OptionalReg2 = ARMEmitter::Reg::r2,
});
// Now the registers we've spilled are back in their original host registers
// We can safely claim we are no longer in a syscall
str(ARMEmitter::XReg::zr, STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo));
EmitL1Lookup();
}
{
// All dynamic and static registers are spilled coming in to this handler.
// It's also the end of block and RIP might have changed, so we jump directly to the top of the loop.
ThreadDispatchCPUIDHandler = GetCursorAddress<uint64_t>();
// CPUID behaves like an ISB.
isb();
auto RAXReg = GetX86GPRMapping(FEXCore::X86State::REG_RAX);
auto RBXReg = GetX86GPRMapping(FEXCore::X86State::REG_RBX);
auto RCXReg = GetX86GPRMapping(FEXCore::X86State::REG_RCX);
auto RDXReg = GetX86GPRMapping(FEXCore::X86State::REG_RDX);
ARMEmitter::ForwardLabel SetupCPUIDFault {};
ldr(TMP2, STATE, offsetof(FEXCore::Core::CPUState, CoreFlags));
(void)tbz(TMP2, FEXCore::ilog2(FEXCore::ToUnderlying(FEXCore::Core::CoreFlagsEnum::CPUID_ENABLED)), &SetupCPUIDFault);
// TMP1 passes the instruction size. Usually only 2-bytes but prefixes can make it 15.
ldr(TMP2, STATE_PTR(CpuStateFrame, State.rip));
add(TMP2, TMP2, TMP1);
str(TMP2, STATE_PTR(CpuStateFrame, State.rip));
mov(ARMEmitter::XReg::x1, RAXReg.X());
mov(ARMEmitter::XReg::x2, RCXReg.X());
// x0 = CPUID Handler
// x1 = CPUID Function (RAX)
// x2 = CPUID Leaf (RCX)
ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.CPUIDObj));
ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.CPUIDFunction));
if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
GenerateIndirectRuntimeCall<__uint128_t, void*, uint64_t, uint64_t>(ARMEmitter::Reg::r3);
} else {
blr(ARMEmitter::Reg::r3);
}
// Results are in x0, x1
auto Res0 = ARMEmitter::Reg::r0;
auto Res1 = ARMEmitter::Reg::r1;
PushDynamicRegs(TMP4);
SpillStaticRegs(TMP4);
if (!TMP_ABIARGS) {
// SRA Overlaps, need to move.
mov(ARMEmitter::Size::i64Bit, TMP1, Res0);
mov(ARMEmitter::Size::i64Bit, TMP2, Res1);
Res0 = TMP1;
Res1 = TMP2;
mov(ARMEmitter::XReg::x0, TMP1);
mov(ARMEmitter::XReg::x1, TMP2);
mov(ARMEmitter::XReg::x2, TMP3);
}
ldr(ARMEmitter::XReg::x3, R, Offset);
if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
GenerateIndirectRuntimeCall<__uint128_t, uint64_t, uint64_t, uint64_t>(ARMEmitter::Reg::r3);
} else {
blr(ARMEmitter::Reg::r3);
}
// Result is now in x0, x1
if (!TMP_ABIARGS) {
mov(TMP1, ARMEmitter::XReg::x0);
mov(TMP2, ARMEmitter::XReg::x1);
}
FillStaticRegs();
// Results need to be moved to 32-bit EAX, ECX, EBX, EDX.
mov(ARMEmitter::Size::i32Bit, RAXReg, Res0);
mov(ARMEmitter::Size::i32Bit, RCXReg, Res1);
ubfx(ARMEmitter::Size::i64Bit, RBXReg, Res0, 32, 32);
ubfx(ARMEmitter::Size::i64Bit, RDXReg, Res1, 32, 32);
// Fix the stack and any values that were stepped on
PopDynamicRegs();
EmitL1Lookup();
// Go back to our code block
ret();
return Address;
};
// CPUID fault handling
(void)Bind(&SetupCPUIDFault);
constexpr Core::CpuStateFrame::SynchronousFaultDataStruct State = {
.FaultToTopAndGeneratedException = 1,
.Signal = Core::FAULT_SIGSEGV,
.TrapNo = X86State::X86_TRAPNO_GP,
.si_code = 0x80,
.err_code = 0,
.FaultAddress = 0, // Comes from register.
};
uint64_t Constant {};
memcpy(&Constant, &State, sizeof(uint64_t));
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, static_cast<uint64_t>(Constant));
str(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData));
ldr(TMP2, STATE_PTR(CpuStateFrame, State.rip));
str(TMP2, STATE, offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData) + 8);
(void)b(&GuestSIGSEGVLabelPostSpill);
}
{
// All dynamic and static registers are spilled coming in to this handler.
// It's also the end of block and RIP might have changed, so we jump directly to the top of the loop.
ThreadDispatchRemoveCodeEntry = GetCursorAddress<uint64_t>();
// Arguments are already in x0, x1. Just jump to the handler.
ldr(ARMEmitter::XReg::x2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.ThreadRemoveCodeEntryFromJIT));
if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
GenerateIndirectRuntimeCall<void, void*, void*>(ARMEmitter::Reg::r2);
} else {
blr(ARMEmitter::Reg::r2);
}
FillStaticRegs({
.OptionalReg = ARMEmitter::Reg::r1,
.OptionalReg2 = ARMEmitter::Reg::r2,
});
// Now go back to the regular dispatcher loop
(void)b(&LoopTop);
}
LUDIVHandlerAddress = EmitLongALUOpHandler(STATE_PTR(CpuStateFrame, Pointers.LUDIV));
LDIVHandlerAddress = EmitLongALUOpHandler(STATE_PTR(CpuStateFrame, Pointers.LDIV));
EmitF64Sin();
EmitF64Cos();
@@ -2781,15 +2620,13 @@ void Dispatcher::InitThreadPointers(FEXCore::Core::InternalThreadState* Thread)
Ptrs.ExitFunctionLinker = ExitFunctionLinkerAddress;
Ptrs.ThreadStopHandlerSpillSRA = ThreadStopHandlerAddressSpillSRA;
Ptrs.ThreadPauseHandlerSpillSRA = ThreadPauseHandlerAddressSpillSRA;
Ptrs.ThreadDispatchSyscallHandler = ThreadDispatchSyscallHandler;
Ptrs.ThreadDispatchCPUIDHandler = ThreadDispatchCPUIDHandler;
Ptrs.ThreadDispatchRemoveCodeEntry = ThreadDispatchRemoveCodeEntry;
Ptrs.GuestSignal_SIGILL = GuestSignal_SIGILL;
Ptrs.GuestSignal_SIGTRAP = GuestSignal_SIGTRAP;
Ptrs.GuestSignal_SIGSEGV = GuestSignal_SIGSEGV;
Ptrs.GuestSignal_SynchronousFault = GuestSignal_SynchronousFault;
Ptrs.SignalReturnHandler = SignalHandlerReturnAddress;
Ptrs.SignalReturnHandlerRT = SignalHandlerReturnAddressRT;
Ptrs.LUDIVHandler = LUDIVHandlerAddress;
Ptrs.LDIVHandler = LDIVHandlerAddress;
Ptrs.F64SinHandler = F64SinHandlerAddress;
Ptrs.F64CosHandler = F64CosHandlerAddress;
Ptrs.F64TanHandler = F64TanHandlerAddress;
@@ -81,16 +81,12 @@ private:
uint64_t AbsoluteLoopTopAddressEnterECFillSRA {};
uint64_t ThreadPauseHandlerAddress {};
uint64_t ThreadPauseHandlerAddressSpillSRA {};
uint64_t ThreadDispatchSyscallHandler {};
uint64_t ThreadDispatchCPUIDHandler {};
uint64_t ThreadDispatchRemoveCodeEntry {};
uint64_t ExitFunctionLinkerAddress {};
uint64_t SignalHandlerReturnAddress {};
uint64_t SignalHandlerReturnAddressRT {};
uint64_t GuestSignal_SIGILL {};
uint64_t GuestSignal_SIGTRAP {};
uint64_t GuestSignal_SIGSEGV {};
uint64_t GuestSignal_SynchronousFault {};
uint64_t PauseReturnInstruction {};
std::array<uint64_t, FallbackABI::FABI_UNKNOWN> ABIPointers {};
@@ -99,6 +95,10 @@ private:
uint64_t Start {};
uint64_t End {};
// Long division helpers
uint64_t LUDIVHandlerAddress {};
uint64_t LDIVHandlerAddress {};
// F64 reduced-precision shared handlers
uint64_t F64SinHandlerAddress {};
uint64_t F64CosHandlerAddress {};
+68 -168
View File
@@ -8,7 +8,6 @@ $end_info$
#include "Interface/Context/Context.h"
#include "Interface/Core/Frontend.h"
#include "Interface/Core/OpcodeDispatcher.h"
#include "Interface/Core/X86Tables/X86Tables.h"
#include "Interface/Core/LookupCache.h"
@@ -79,56 +78,39 @@ Decoder::Decoder(FEXCore::Core::InternalThreadState* Thread)
X87Table = &FEXCore::X86Tables::X87F80Ops;
}
if (CTX->HostFeatures.SupportsAVX() && CTX->HostFeatures.SupportsSVE256()) {
if (CTX->HostFeatures.SupportsAVX && CTX->HostFeatures.SupportsSVE256) {
VEXTable = &FEXCore::X86Tables::VEXTableOps;
VEXTableGroup = &FEXCore::X86Tables::VEXTableGroupOps;
} else if (CTX->HostFeatures.SupportsAVX()) {
} else if (CTX->HostFeatures.SupportsAVX) {
VEXTable = &FEXCore::X86Tables::VEXTableOps_AVX128;
VEXTableGroup = &FEXCore::X86Tables::VEXTableGroupOps_AVX128;
}
}
int32_t Decoder::CheckRangeExecutable(uint64_t Address, uint64_t Size) {
bool Decoder::CheckRangeExecutable(uint64_t Address, uint64_t Size) {
// Check for wraparound
if (Address + Size < Address) {
// Say that any bytes that wrapped around are unreadable
return -(Address + Size);
return false;
}
if (Thread->CurrentFrame->State.Dr7) [[unlikely]] {
// If Dr7 has anything enabled then check it.
for (size_t i = 0; i < Core::CPUState::NUM_DEBUG_ADDRESSES; ++i) {
if (Core::CPUState::IsInstructionBPEnabled(i, Thread->CurrentFrame->State.Dr7)) {
size_t DrSize = Core::CPUState::BreakpointSize(i, Thread->CurrentFrame->State.Dr7);
if (Address >= Thread->CurrentFrame->State.Dr[i] && (Address + Size - 1) < (Thread->CurrentFrame->State.Dr[i] + DrSize)) {
HitBreakpoint = i;
return false;
}
}
}
}
uint64_t RemainingSize = Size;
while (Address < ExecutableRangeBase || Address + RemainingSize > ExecutableRangeEnd) {
while (Address < ExecutableRangeBase || Address + Size > ExecutableRangeEnd) {
auto RangeInfo = CTX->SyscallHandler->QueryGuestExecutableRange(Thread, Address);
ExecutableRangeBase = RangeInfo.Base;
ExecutableRangeEnd = RangeInfo.Base + RangeInfo.Size;
ExecutableRangeWritable = RangeInfo.Writable;
if (RangeInfo.Size == 0) {
// Return the number of bytes that couldn't be decoded.
return -(Size - RemainingSize);
return false;
}
uint64_t RangeRemainingSize = ExecutableRangeEnd - Address;
if (RemainingSize > RangeRemainingSize) {
RemainingSize -= RangeRemainingSize;
if (Size > RangeRemainingSize) {
Size -= RangeRemainingSize;
Address += RangeRemainingSize;
}
}
// Say all bytes are readable.
return Size;
return true;
}
uint8_t Decoder::ReadByte() {
@@ -146,7 +128,7 @@ uint8_t Decoder::ReadByte() {
std::optional<uint8_t> Decoder::PeekByte(uint8_t Offset) {
uint64_t ByteAddress = reinterpret_cast<uint64_t>(InstStream.InstStream + InstructionSize + Offset);
if (CheckRangeExecutable(ByteAddress, 1) == 1) {
if (CheckRangeExecutable(ByteAddress, 1)) {
return InstStream.AdjustedInstStream[InstructionSize + Offset];
} else {
return std::nullopt;
@@ -160,8 +142,7 @@ std::pair<uint64_t, bool> Decoder::ReadData(uint8_t Size) {
uint64_t Address = reinterpret_cast<uint64_t>(InstStream.InstStream + InstructionSize);
LastFieldReadOffset = (uint8_t)InstructionSize;
LastFieldReadSize = Size;
auto ReadableSize = CheckRangeExecutable(Address, Size);
if (ReadableSize == Size) {
if (CheckRangeExecutable(Address, Size)) {
std::memcpy(&Res, &InstStream.AdjustedInstStream[InstructionSize], Size);
} else {
HitNonExecutableRange = true;
@@ -169,14 +150,13 @@ std::pair<uint64_t, bool> Decoder::ReadData(uint8_t Size) {
Res = 0;
}
// Skip the number of decoded bytes regardless to get instruction size.
// Even partially decoded instruction.
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
for (size_t i = 0; i < std::abs(ReadableSize); ++i) {
for (size_t i = 0; i < Size; ++i) {
ReadByte();
}
#else
SkipBytes(std::abs(ReadableSize));
SkipBytes(Size);
#endif
if (Relocations) {
@@ -236,7 +216,6 @@ void Decoder::DecodeModRM_16(X86Tables::DecodedOperand* Operand, X86Tables::ModR
Operand->Type = DecodedOperand::OpType::SIB;
Operand->Data.SIB.Scale = 1;
Operand->Data.SIB.Offset = Literal;
Operand->Data.SIB.PatchableDisp = false;
// Only called when ModRM.mod != 0b11
struct Encodings {
@@ -312,7 +291,6 @@ void Decoder::DecodeModRM_64(X86Tables::DecodedOperand* Operand, X86Tables::ModR
// SIB
Operand->Type = DecodedOperand::OpType::SIB;
Operand->Data.SIB.Scale = 1 << SIB.scale;
Operand->Data.SIB.PatchableDisp = false;
// The invalid encoding types are described at Table 1-12. "promoted nsigned is always non-zero"
{
@@ -351,7 +329,6 @@ void Decoder::DecodeModRM_64(X86Tables::DecodedOperand* Operand, X86Tables::ModR
auto [Literal, IsRelocation] = ReadData(4);
Operand->Type = IsRelocation ? DecodedOperand::OpType::RIPRelativeRelocation : DecodedOperand::OpType::RIPRelative;
Operand->Data.RIPLiteral.Value = Literal;
Operand->Data.RIPLiteral.PatchableDisp = false;
} else {
// Register-direct addressing
Operand->Type = DecodedOperand::OpType::GPRDirect;
@@ -367,7 +344,6 @@ void Decoder::DecodeModRM_64(X86Tables::DecodedOperand* Operand, X86Tables::ModR
Operand->Type = IsRelocation ? DecodedOperand::OpType::GPRIndirectRelocation : DecodedOperand::OpType::GPRIndirect;
Operand->Data.GPRIndirect.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, ModRM.rm, false, false, false, false);
Operand->Data.GPRIndirect.Displacement = Literal;
Operand->Data.GPRIndirect.PatchableDisp = false;
}
}
@@ -390,10 +366,6 @@ Decoder::DecodedBlockStatus Decoder::NormalOp(const FEXCore::X86Tables::X86InstI
return DecodedBlockStatus::INVALID_INST;
}
if (Info->Type == FEXCore::X86Tables::TYPE_UNIMPLEMENTED) {
return DecodedBlockStatus::UNIMPLEMENTED_INST;
}
if (!(Info->Flags & FEXCore::X86Tables::InstFlags::FLAGS_SUPPORTS_LOCK) && (DecodeInst->Flags & DecodeFlags::FLAG_LOCK)) {
// Instruction has lock prefix but doesn't support lock.
return DecodedBlockStatus::UNIMPLEMENTED_INST;
@@ -524,7 +496,15 @@ Decoder::DecodedBlockStatus Decoder::NormalOp(const FEXCore::X86Tables::X86InstI
auto* CurrentDest = &DecodeInst->Dest;
if (HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_REX_IN_BYTE)) {
if (HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_DST_RAX) ||
HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_DST_RDX)) {
// Some instructions hardcode their destination as RAX
CurrentDest->Type = DecodedOperand::OpType::GPR;
CurrentDest->Data.GPR.HighBits = false;
CurrentDest->Data.GPR.GPR =
HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_DST_RAX) ? FEXCore::X86State::REG_RAX : FEXCore::X86State::REG_RDX;
CurrentDest = &DecodeInst->Src[0];
} else if (HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_REX_IN_BYTE)) {
LOGMAN_THROW_A_FMT(!HasMODRM, "This instruction shouldn't have ModRM!");
// If the REX is in the byte that means the lower nibble of the OP contains the destination GPR
@@ -641,6 +621,18 @@ Decoder::DecodedBlockStatus Decoder::NormalOp(const FEXCore::X86Tables::X86InstI
++CurrentSrc;
}
if (HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_SRC_RAX)) {
DecodeInst->Src[CurrentSrc].Type = DecodedOperand::OpType::GPR;
DecodeInst->Src[CurrentSrc].Data.GPR.HighBits = false;
DecodeInst->Src[CurrentSrc].Data.GPR.GPR = FEXCore::X86State::REG_RAX;
++CurrentSrc;
} else if (HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_SRC_RCX)) {
DecodeInst->Src[CurrentSrc].Type = DecodedOperand::OpType::GPR;
DecodeInst->Src[CurrentSrc].Data.GPR.HighBits = false;
DecodeInst->Src[CurrentSrc].Data.GPR.GPR = FEXCore::X86State::REG_RCX;
++CurrentSrc;
}
if (VEXOperand == FEXCore::X86Tables::InstFlags::FLAGS_VEX_DST) {
CurrentDest->Type = DecodedOperand::OpType::GPR;
CurrentDest->Data.GPR.HighBits = false;
@@ -703,10 +695,6 @@ Decoder::DecodedBlockStatus Decoder::NormalOpHeader(const FEXCore::X86Tables::X8
return DecodedBlockStatus::INVALID_INST;
}
if (Info->Type == FEXCore::X86Tables::TYPE_UNIMPLEMENTED) {
return DecodedBlockStatus::UNIMPLEMENTED_INST;
}
LOGMAN_THROW_A_FMT(Info->Type != FEXCore::X86Tables::TYPE_REX_PREFIX, "REX PREFIX should have been decoded before this!");
// A normal instruction is the most likely.
@@ -757,7 +745,7 @@ Decoder::DecodedBlockStatus Decoder::NormalOpHeader(const FEXCore::X86Tables::X8
if (LocalInfo->Type == FEXCore::X86Tables::TYPE_SECOND_GROUP_MODRM && ModRM.mod == 0b11) {
// Everything in this group is privileged instructions aside from XGETBV
constexpr std::array<uint8_t, 8> RegToField = {
255, 0, 1, 2, 255, 3, 255, 4,
255, 0, 1, 2, 255, 255, 255, 3,
};
uint8_t Field = RegToField[ModRM.reg];
if (Field == 255) {
@@ -766,9 +754,6 @@ Decoder::DecodedBlockStatus Decoder::NormalOpHeader(const FEXCore::X86Tables::X8
LocalOp = (Field << 3) | ModRM.rm;
return NormalOp(&SecondModRMTableOps[LocalOp], LocalOp);
} else if (LocalInfo->Type == FEXCore::X86Tables::TYPE_SECOND_GROUP_MODRM && ModRM.reg == 5) {
// Special case 0x0F 0x01 /5 group as they are currently all reserved.
return DecodedBlockStatus::UNIMPLEMENTED_INST;
} else {
return NormalOp(&SecondInstGroupOps[LocalOp], LocalOp);
}
@@ -972,24 +957,23 @@ Decoder::DecodedBlockStatus Decoder::DecodeInstructionImpl(uint64_t PC) {
// If any of these three prefixes are used then it falls down the subtable
// Additionally: If you hit repeat of differnt prefixes then only the LAST one before this one works for subtable selection
const bool NoOverlay = (FEXCore::X86Tables::SecondBaseOps[EscapeOp].Flags & InstFlags::FLAGS_NO_OVERLAY) != 0;
const bool NoOverlay66 = (FEXCore::X86Tables::SecondBaseOps[EscapeOp].Flags & InstFlags::FLAGS_NO_OVERLAY66) != 0;
const bool NoPrefixOrder = (FEXCore::X86Tables::SecondBaseOps[EscapeOp].Flags & InstFlags::FLAGS_NO_PREFIX_ORDER) != 0;
bool NoOverlay = (FEXCore::X86Tables::SecondBaseOps[EscapeOp].Flags & InstFlags::FLAGS_NO_OVERLAY) != 0;
bool NoOverlay66 = (FEXCore::X86Tables::SecondBaseOps[EscapeOp].Flags & InstFlags::FLAGS_NO_OVERLAY66) != 0;
DecodeREXIfValid(-2);
if (NoOverlay) { // This section of the table ignores prefix extention
return NormalOpHeader(&FEXCore::X86Tables::SecondBaseOps[EscapeOp], EscapeOp);
} else if (LastEscapePrefix == 0xF3 || (NoPrefixOrder && DecodeInst->Flags & DecodeFlags::FLAG_REP_PREFIX)) { // REP
} else if (LastEscapePrefix == 0xF3) { // REP
// Remove prefix so it doesn't effect calculations.
// This is only an escape prefix rather tan modifier now
DecodeInst->Flags &= ~DecodeFlags::FLAG_REP_PREFIX;
return NormalOpHeader(&FEXCore::X86Tables::RepModOps[EscapeOp], EscapeOp);
} else if (LastEscapePrefix == 0xF2 || (NoPrefixOrder && DecodeInst->Flags & DecodeFlags::FLAG_REPNE_PREFIX)) { // REPNE
} else if (LastEscapePrefix == 0xF2) { // REPNE
// Remove prefix so it doesn't effect calculations.
// This is only an escape prefix rather tan modifier now
DecodeInst->Flags &= ~DecodeFlags::FLAG_REPNE_PREFIX;
return NormalOpHeader(&FEXCore::X86Tables::RepNEModOps[EscapeOp], EscapeOp);
} else if ((LastEscapePrefix == 0x66 || (NoPrefixOrder && DecodeInst->Flags & DecodeFlags::FLAG_OPERAND_SIZE)) && !NoOverlay66) { // Operand Size
} else if (LastEscapePrefix == 0x66 && !NoOverlay66) { // Operand Size
// Remove prefix so it doesn't effect calculations.
// This is only an escape prefix rather tan modifier now
DecodeInst->Flags &= ~DecodeFlags::FLAG_OPERAND_SIZE;
@@ -1109,27 +1093,21 @@ Decoder::DecodedBlockStatus Decoder::DecodeInstruction(uint64_t PC) {
// Will be set if DecodeInstructionImpl tries to read non-executable memory
HitNonExecutableRange = false;
HitBadRelocation = false;
HitBreakpoint = ~0U;
auto ErrorDuringDecoding = DecodeInstructionImpl(PC);
if (ErrorDuringDecoding != DecodedBlockStatus::SUCCESS || HitNonExecutableRange || HitBadRelocation) [[unlikely]] {
// Put an invalid instruction in the stream so the core can raise SIGILL if hit
// Error while decoding instruction. We don't know the table or instruction size
const auto InstSize = DecodeInst->InstSize;
DecodeInst->TableInfo = nullptr;
if (HitBreakpoint != ~0U) {
return static_cast<Decoder::DecodedBlockStatus>(FEXCore::ToUnderlying(DecodedBlockStatus::BREAKPOINT_0) + HitBreakpoint);
}
DecodeInst->InstSize = 0;
// A decode error can be caused by substituting zero for an inaccessible
// instruction byte, so the instruction fetch fault takes priority.
if (HitNonExecutableRange) {
return DecodeInst->InstSize ? DecodedBlockStatus::PARTIAL_DECODE_INST : DecodedBlockStatus::NOEXEC_INST;
return InstSize ? DecodedBlockStatus::PARTIAL_DECODE_INST : DecodedBlockStatus::NOEXEC_INST;
}
// Keep InstSize around for PARTIAL_DECODE_INST so we can get the offset where it partial decoded.
DecodeInst->InstSize = 0;
if (HitBadRelocation) {
return DecodedBlockStatus::BAD_RELOCATION;
}
@@ -1389,8 +1367,8 @@ const Decoder::DecodeStream Decoder::AdjustAddrForSpecialRegion(const uint8_t* _
constexpr uint64_t VSyscall_Base = 0xFFFF'FFFF'FF60'0000ULL;
constexpr uint64_t VSyscall_End = VSyscall_Base + 0x1000;
if (BlockInfo.Is64BitMode && CTX->HostFeatures.ProcessFeatures.HostType == FEXCore::HostFeatures::HostTypeEnum::Linux &&
RIP >= VSyscall_Base && RIP < VSyscall_End) {
if (BlockInfo.Is64BitMode && CTX->HostFeatures.HostType == FEXCore::HostFeatures::HostTypeEnum::Linux && RIP >= VSyscall_Base &&
RIP < VSyscall_End) {
// VSyscall
// This doesn't exist on AArch64 and on x86_64 hosts this is emulated with faults to a region mapped with --xp permissions
// Offset 0: vgettimeofday
@@ -1418,126 +1396,45 @@ bool Decoder::CheckIfCacheable(FEXCore::Core::InternalThreadState& Thread, const
}
void Decoder::DetectDataMasks(uint64_t OpAddress, DecodedBlocks& Block) {
FEXCore::X86Tables::DecodedOperand* LiteralToPatch = nullptr;
// cmp *, imm8 - seen varying in mono jitted code
{
FEXCore::X86Tables::ModRMDecoded ModRM;
ModRM.Hex = DecodeInst->ModRM;
if ((DecodeInst->OPRaw == 0x80 || DecodeInst->OPRaw == 0x83) && ModRM.reg == 7 && LastFieldReadSize == 1) {
for (auto& Src : DecodeInst->Src) {
if (Src.IsLiteral()) {
LiteralToPatch = &Src;
break;
}
}
if (LiteralToPatch && LiteralToPatch->Literal() != 0) {
Block.DataMasks.push_back({OpAddress + LastFieldReadOffset, DataMaskType::MOV, LastFieldReadSize});
LiteralToPatch->Type = X86Tables::DecodedOperand::OpType::LiteralPatchable;
LiteralToPatch->Data.LiteralPatchable.FieldOffset = LastFieldReadOffset;
LiteralToPatch->Data.LiteralPatchable.Width = LastFieldReadSize;
}
}
}
if (LastFieldReadSize < 4) {
return;
}
DataMaskType Type = DataMaskType::NONE;
FEXCore::X86Tables::DecodedOperand* LiteralToPatch = nullptr;
DataMaskType Type;
// imm32 or imm64 at the end type instructions
if (DecodeInst->TableInfo->Flags & X86Tables::InstFlags::FLAGS_LITERAL_PATCHABLE) {
// mov reg,imm
if (DecodeInst->OP >= 0xB8 && DecodeInst->OP <= 0xBF) {
for (auto& Src : DecodeInst->Src) {
if (Src.IsLiteral()) {
LiteralToPatch = &Src;
break;
}
}
if (DecodeInst->Dest.IsLiteral()) {
LiteralToPatch = &DecodeInst->Dest;
}
// heuristic: if it's a small value, assume it's more likely to be part of the code around it
bool IsMOV = (DecodeInst->OPRaw >= 0xB8 && DecodeInst->OPRaw <= 0xBF) || DecodeInst->OPRaw == 0xC7;
if (LiteralToPatch && IsMOV && LiteralToPatch->Literal() < 0x10000ULL) {
LiteralToPatch = nullptr;
}
if (LiteralToPatch && LiteralToPatch->Literal() != 0) {
Type = DataMaskType::MOV;
}
// we could filter to certain high values that are more likely to be pointers/etc?
// const uint64_t Value = Lit->Data.Literal.Value;
// if (LiteralToPatch && Value < 0x1000000ULL) {
// LiteralToPatch = nullptr;
// }
Type = DataMaskType::MOV;
}
// anything that has a patchable disp32
bool TryDisp = DecodeInst->Flags & X86Tables::DecodeFlags::FLAG_DECODED_MODRM;
if (TryDisp) {
FEXCore::X86Tables::ModRMDecoded ModRM;
ModRM.Hex = DecodeInst->ModRM;
{
// todo we could handle both imm + disp with more load tracking
bool FoundLiteral = false;
FEXCore::X86Tables::DecodedOperand* OpToPatch = nullptr;
for (auto& Src : DecodeInst->Src) {
if (Src.IsLiteral()) {
FoundLiteral = true;
break;
}
if (Src.IsRIPRelative() || Src.IsGPRIndirect() || Src.IsSIB()) {
OpToPatch = &Src;
}
}
if (DecodeInst->Dest.IsLiteral()) {
FoundLiteral = true;
}
if (DecodeInst->Dest.IsRIPRelative() || DecodeInst->Dest.IsGPRIndirect() || DecodeInst->Dest.IsSIB()) {
OpToPatch = &DecodeInst->Dest;
}
if (!FoundLiteral && OpToPatch) {
if (DecodeInst->TableInfo->OpcodeDispatcher.OpDispatch == &IR::OpDispatchBuilder::NOPOp) {
// if it's a nop disp, only mask data out, no other action required
Type = DataMaskType::NOP;
} else if (OpToPatch->IsRIPRelative()) {
Type = DataMaskType::DISP;
OpToPatch->Data.RIPLiteral.PatchableDisp = true;
OpToPatch->Data.RIPLiteral.DispOffset = LastFieldReadOffset;
} else if (OpToPatch->IsGPRIndirect()) {
// filter out disp8
if (!(ModRM.mod == 1)) {
Type = DataMaskType::DISP;
OpToPatch->Data.GPRIndirect.PatchableDisp = true;
OpToPatch->Data.GPRIndirect.DispOffset = LastFieldReadOffset;
}
} else if (OpToPatch->IsSIB()) {
FEXCore::X86Tables::SIBDecoded SIB;
SIB.Hex = DecodeInst->SIB;
// two disp32 cases here
if (ModRM.mod == 0b10 || (ModRM.mod == 0 && SIB.base == 0b101)) {
Type = DataMaskType::DISP;
OpToPatch->Data.SIB.PatchableDisp = true;
OpToPatch->Data.SIB.DispOffset = LastFieldReadOffset;
}
}
}
}
}
// jmp/call branches that use a literal rip-relative offset
// some of those may be inlined by multiblock and will be cleaned up at decode end
if (DecodeInst->TableInfo->Flags & X86Tables::InstFlags::FLAGS_SETS_RIP && DecodeInst->Src[0].IsLiteral() && DecodeInst->Src[0].Literal() != 0) {
if (DecodeInst->TableInfo->Flags & X86Tables::InstFlags::FLAGS_SETS_RIP && DecodeInst->Src[0].IsLiteral()) {
LiteralToPatch = &DecodeInst->Src[0];
Type = DataMaskType::BRANCH;
}
// todo add a bunch more
if (Type != DataMaskType::NONE) {
if (LiteralToPatch) {
Block.DataMasks.push_back({OpAddress + LastFieldReadOffset, Type, LastFieldReadSize});
if (LiteralToPatch) {
LiteralToPatch->Type = X86Tables::DecodedOperand::OpType::LiteralPatchable;
LiteralToPatch->Data.LiteralPatchable.FieldOffset = LastFieldReadOffset;
LiteralToPatch->Data.LiteralPatchable.Width = LastFieldReadSize;
}
LiteralToPatch->Type = X86Tables::DecodedOperand::OpType::LiteralPatchable;
LiteralToPatch->Data.LiteralPatchable.FieldOffset = LastFieldReadOffset;
LiteralToPatch->Data.LiteralPatchable.Width = LastFieldReadSize;
}
}
@@ -1567,6 +1464,10 @@ void Decoder::PruneInlinedBranchDataMasks() {
void Decoder::DecodeLoop(const uint8_t* _InstStream, uint64_t GuestSizePause) {
// counter-intuitively, the masks are also needed for lookup on anon prefix decodes, not just stores
bool WantsDataMasks = CTX->DiskCache.IsReadingDiskCache() || CTX->DiskCache.IsWritingDiskCache();
// remove this if we ever fixup ValidateCode crc constant after relocations
if (CTX->Config.SMCChecks == FEXCore::Config::CONFIG_SMC_FULL) {
WantsDataMasks = false;
}
while (!FinalInstruction && (Paused || !BlocksToDecode.empty())) {
bool Pausing = false;
@@ -1687,10 +1588,6 @@ void Decoder::DecodeLoop(const uint8_t* _InstStream, uint64_t GuestSizePause) {
EraseBlock = true;
} else {
LogMan::Msg::EFmt("{} instruction in entry block: {:X}",
BlockIt->BlockStatus == DecodedBlockStatus::BREAKPOINT_0 ? "Breakpoint" :
BlockIt->BlockStatus == DecodedBlockStatus::BREAKPOINT_1 ? "Breakpoint" :
BlockIt->BlockStatus == DecodedBlockStatus::BREAKPOINT_2 ? "Breakpoint" :
BlockIt->BlockStatus == DecodedBlockStatus::BREAKPOINT_3 ? "Breakpoint" :
BlockIt->BlockStatus == DecodedBlockStatus::INVALID_INST ? "Invalid" :
BlockIt->BlockStatus == DecodedBlockStatus::NOEXEC_INST ? "NoExec" :
BlockIt->BlockStatus == DecodedBlockStatus::BAD_RELOCATION ? "BadRelocation" :
@@ -1724,6 +1621,9 @@ void Decoder::DecodeLoop(const uint8_t* _InstStream, uint64_t GuestSizePause) {
// NOTE: This will invalidate BlockIt, this is fine as we immediately break from the loop and EraseBlock cannot be true
if (CTX->AreMonoHacksActive() && IsBranchMonoTailcall(BlockIt->NumInstructions)) {
BlockIt->ForceFullSMCDetection = true;
// todo abandon patching this for now, as the crc will fail and it will lock up redoing it over and over
// we should fix the crc at relocation if this is important
BlockIt->DataMasks.clear();
}
BranchTargetInMultiblockRange();
}
+2 -10
View File
@@ -30,13 +30,9 @@ public:
PARTIAL_DECODE_INST,
BAD_RELOCATION,
UNIMPLEMENTED_INST,
BREAKPOINT_0,
BREAKPOINT_1,
BREAKPOINT_2,
BREAKPOINT_3,
};
enum class DataMaskType : uint8_t { NONE, MOV, BRANCH, DISP, NOP };
enum class DataMaskType : uint8_t { MOV, BRANCH };
struct DataMask final {
uint64_t FieldAddress;
@@ -119,10 +115,7 @@ private:
void DetectDataMasks(uint64_t OpAddress, DecodedBlocks& Block);
void PruneInlinedBranchDataMasks();
// Check for the number of bytes that are able to be decoded as executable
// Size: Returns Size if able to decode all bytes.
// Negative: Number of bytes that failed to decode.
int32_t CheckRangeExecutable(uint64_t Address, uint64_t Size);
bool CheckRangeExecutable(uint64_t Address, uint64_t Size);
uint8_t ReadByte();
std::optional<uint8_t> PeekByte(uint8_t Offset);
@@ -160,7 +153,6 @@ private:
bool ExecutableRangeWritable {};
bool HitNonExecutableRange {};
bool HitBadRelocation {};
uint32_t HitBreakpoint {~0U};
struct DecodeStream {
// Original instruction stream RIP location.
@@ -100,7 +100,6 @@ void InterpreterOps::FillFallbackIndexPointers(Core::FallbackABIInfo* Info, uint
reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64SCALE>::handle)};
// SSE4.2 string instructions
// NOTE: Currently unused. See VectorFallbacks.h.
Info[Core::OPINDEX_VPCMPESTRX] = {ABIHandlers[FABI_I32_I64_I64_V128_V128_I16],
reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_VPCMPESTRX>::handle)};
Info[Core::OPINDEX_VPCMPISTRX] = {ABIHandlers[FABI_I32_V128_V128_I16],
@@ -8,7 +8,6 @@
#include <cstring>
// NOTE: Currently unused. See VectorFallbacks.h.
namespace FEXCore::CPU {
#ifdef ARCHITECTURE_arm64
FEXCORE_PRESERVE_ALL_ATTR static int32_t GetImplicitLength(FEXCore::VectorRegType data, uint16_t control) {
@@ -13,33 +13,29 @@
namespace FEXCore::CPU {
// PCMPXSTRX control byte fields
enum class AggregationOp {
EqualAny = 0b00,
Ranges = 0b01,
EqualEach = 0b10,
EqualOrdered = 0b11,
};
enum class SourceData {
U8,
U16,
S8,
S16,
};
enum class Polarity {
Positive,
Negative,
PositiveMasked,
NegativeMasked,
};
// NB: The fallback handlers for the *PMCP*STR* instructions
// are no longer used since we now emit inline ASM for them.
// We preserve this as a reference implementation.
template<>
struct OpHandlers<IR::OP_VPCMPESTRX> {
enum class AggregationOp {
EqualAny = 0b00,
Ranges = 0b01,
EqualEach = 0b10,
EqualOrdered = 0b11,
};
enum class SourceData {
U8,
U16,
S8,
S16,
};
enum class Polarity {
Positive,
Negative,
PositiveMasked,
NegativeMasked,
};
FEXCORE_PRESERVE_ALL_ATTR static uint32_t handle(uint64_t RAX, uint64_t RDX, VectorRegType lhs_v, VectorRegType rhs_v, uint16_t control) {
__uint128_t lhs;
memcpy(&lhs, &lhs_v, sizeof(lhs));
+21 -211
View File
@@ -9,7 +9,6 @@ $end_info$
#include "FEXCore/IR/IR.h"
#include "Interface/Context/Context.h"
#include "Interface/Core/JIT/JITClass.h"
#include "Interface/Core/JIT/Relocations.h"
#include "Interface/IR/Passes/RegisterAllocationPass.h"
namespace FEXCore::CPU {
@@ -67,17 +66,7 @@ DEF_OP(EntrypointOffset) {
DEF_OP(PatchableGuestData) {
auto Op = IROp->C<IR::IROp_PatchableGuestData>();
InsertGuestPatchableMove(RelocationTypes::RELOC_GUEST_PATCHABLE_DATA_MOVE, GetReg(Node), Op->Value, Op->SiteAddress, (uint8_t)Op->SiteSize);
}
DEF_OP(PatchableGuestRIP) {
auto Op = IROp->C<IR::IROp_PatchableGuestRIP>();
InsertGuestPatchableMove(RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_MOVE, GetReg(Node), Op->Value, Op->SiteAddress, (uint8_t)Op->SiteSize);
}
DEF_OP(PatchableGuestCRC) {
auto Op = IROp->C<IR::IROp_PatchableGuestRIP>();
InsertGuestPatchableMove(RelocationTypes::RELOC_GUEST_PATCHABLE_CRC_MOVE, GetReg(Node), Op->Value, Op->SiteAddress, (uint8_t)Op->SiteSize);
InsertGuestPatchableDataMove(GetReg(Node), Op->Value, Op->SiteAddress, (uint8_t)Op->SiteSize);
}
DEF_OP(InlineConstant) {
@@ -865,159 +854,6 @@ DEF_OP(PExt) {
}
}
// Divide a uint128_t by a uint64_t, producing quotient and remainder.
//
// Adapted from divllu() in libdivide: https://github.com/ridiculousfish/libdivide/blob/master/doc/divlu.c
// divllu() specifically is licensed as public domain, despite the zlib license that covers the rest of libdivide.
// divllu() is itself an adaptation of Knuth's classic Algorithm D for long division, with some optimizations.
// Where possible we have followed divllu() to the letter, including comments, but some changes were needed
// due to our limited set of temp registers. Some comments below represent the original C code from divllu()
// which is intended to match the assembly we emit.
//
// Additional reading:
// https://ridiculousfish.com/blog/posts/labor-of-division-episode-iv.html
// https://ridiculousfish.com/blog/posts/labor-of-division-episode-v.html
void Arm64JITCore::EmitDivllu(ARMEmitter::Register Quotient, ARMEmitter::Register Remainder, ARMEmitter::Register Upper,
ARMEmitter::Register Lower, ARMEmitter::Register Divisor) {
const auto EmitSize = ARMEmitter::Size::i64Bit;
ARMEmitter::ForwardLabel UpperReduced {};
ARMEmitter::ForwardLabel HighDigitDone {};
ARMEmitter::ForwardLabel LowDigitDone {};
// We work in base 2**32.
// A uint32 holds a single digit. A uint64 holds two digits.
// Our numerator is conceptually [num3, num2, num1, num0].
// Our denominator is [den1, den0].
// The normalization shift factor.
const auto Shift = TMP1;
// The divisor, and the high and low words of the dividend.
const auto Den = TMP2;
const auto NumHi = TMP3;
const auto NumLo = TMP4;
// A partial remainder.
const auto Rem = TMP3;
// Check for overflow and divide by 0.
// TODO: IF FEX handled #DE, we could simplify this I think.
if (NumHi != Upper) {
mov(EmitSize, NumHi, Upper);
}
cmp(EmitSize, Upper, Divisor);
(void)b(ARMEmitter::Condition::CC_LO, &UpperReduced);
udiv(EmitSize, Quotient, Upper, Divisor);
msub(EmitSize, NumHi, Quotient, Divisor, Upper);
(void)Bind(&UpperReduced);
// Determine the normalization factor. We multiply den by this, so that its leading digit is at
// least half b. In binary this means just shifting left by the number of leading zeros, so that
// there's a 1 in the MSB.
// We also shift numer by the same amount. This cannot overflow because numhi < den.
// The expression (-shift & 63) is the same as (64 - shift), except it avoids the UB of shifting
// by 64. The funny bitwise 'and' ensures that numlo does not get shifted into numhi if shift is 0.
// shift = divlu_count_leading_zeros64(den);
clz(EmitSize, Shift, Divisor);
// den <<= shift;
lslv(EmitSize, Den, Divisor, Shift);
// numhi <<= shift;
lslv(EmitSize, NumHi, NumHi, Shift);
// numhi |= (numlo >> (-shift & 63)) & (-(int64_t)shift >> 63);
mvn(EmitSize, Quotient, Shift);
lsr(EmitSize, Remainder, Lower, 1);
lsrv(EmitSize, Remainder, Remainder, Quotient);
orr(EmitSize, NumHi, NumHi, Remainder);
// numlo <<= shift;
lslv(EmitSize, NumLo, Lower, Shift);
// We wish to compute q1 = [n3 n2 n1] / [d1 d0].
// Estimate q1 as [n3 n2] / [d1], and then correct it.
// Note while qhat may be 2 digits, q1 is always 1 digit.
{
const auto Den1 = TMP1;
const auto Den0 = TMP1;
// The estimated quotient, and estimated remainder.
const auto QHat = Quotient;
const auto RHat = Remainder;
// Variables used to correct the estimated quotient.
const auto C1 = TMP1;
const auto C2 = Remainder;
// qhat = numhi / den1;
lsr(EmitSize, Den1, Den, 32);
udiv(EmitSize, QHat, NumHi, Den1);
// rhat = numhi % den1;
msub(EmitSize, RHat, QHat, Den1, NumHi);
// c1 = qhat * den0;
mov(ARMEmitter::Size::i32Bit, Den0, Den);
mul(EmitSize, C1, QHat, Den0);
// c2 = rhat * b + num1;
extr(EmitSize, C2, RHat, NumLo, 32);
// if (c1 > c2)
// qhat -= (c1 - c2 > den) ? 2 : 1;
// C1 holds c1 - c2 after the subs.
subs(EmitSize, C1, C1, C2);
(void)b(ARMEmitter::Condition::CC_LS, &HighDigitDone);
sub(EmitSize, QHat, QHat, 1);
cmp(EmitSize, C1, Den);
(void)b(ARMEmitter::Condition::CC_LS, &HighDigitDone);
sub(EmitSize, QHat, QHat, 1);
(void)Bind(&HighDigitDone);
// q1 = (uint32_t)qhat;
// This move is a nop for us.
}
// Compute the true (partial) remainder.
// rem = numhi * b + num1 - q1 * den;
extr(EmitSize, Rem, NumHi, NumLo, 32);
msub(EmitSize, Rem, Quotient, Den, Rem);
// We wish to compute q0 = [rem1 rem0 n0] / [d1 d0].
// Estimate q0 as [rem1 rem0] / [d1] and correct it.
{
const auto Den1 = TMP1;
const auto Den0 = TMP3;
// The estimated quotient, and its corresponding remainder
const auto QHat = Remainder;
const auto RHat = TMP1;
// Variables used to correct the estimated quotient.
const auto C1 = TMP3;
const auto C2 = NumLo;
// qhat = rem / den1;
lsr(EmitSize, Den1, Den, 32);
udiv(EmitSize, QHat, Rem, Den1);
// rhat = rem % den1;
msub(EmitSize, RHat, QHat, Den1, Rem);
// c1 = qhat * den0;
mov(ARMEmitter::Size::i32Bit, Den0, Den);
mul(EmitSize, C1, QHat, Den0);
// c2 = rhat * b + num0;
bfi(EmitSize, C2, RHat, 32, 32);
// if (c1 > c2)
// qhat -= (c1 - c2 > den) ? 2 : 1;
// C1 holds c1 - c2 after the subs.
subs(EmitSize, C1, C1, C2);
(void)b(ARMEmitter::Condition::CC_LS, &LowDigitDone);
sub(EmitSize, QHat, QHat, 1);
cmp(EmitSize, C1, Den);
(void)b(ARMEmitter::Condition::CC_LS, &LowDigitDone);
sub(EmitSize, QHat, QHat, 1);
(void)Bind(&LowDigitDone);
// q0 = (uint32_t)qhat;
// This move is a nop for us.
}
// q = ((uint64_t)q1 << 32) | q0;
orr(EmitSize, Quotient, Remainder, Quotient, ARMEmitter::ShiftType::LSL, 32);
}
DEF_OP(Div) {
auto Op = IROp->C<IR::IROp_Div>();
const auto OpSize = IROp->Size;
@@ -1083,48 +919,22 @@ DEF_OP(Div) {
// If the sign bit matches then the result is zero
(void)cbz(EmitSize, TMP1, &Only64Bit);
// The x86 instruction raises #DE when the divisor is zero which we don't do.
// TODO: #DE
(void)cbz(EmitSize, Divisor, &Only64Bit);
// Long divide
// EmitDivllu() is the unsigned case, so we need to "cast" to unsigned first,
// and then apply the correct sign back to the results afterwards.
{
const auto Sign = TMP1;
const auto AbsUpper = TMP3;
const auto AbsLower = TMP4;
const auto AbsDivisor = TMP2;
mov(EmitSize, TMP1, Upper);
mov(EmitSize, TMP2, Lower);
mov(EmitSize, TMP3, Divisor);
mrs(TMP1, ARMEmitter::SystemRegister::NZCV);
str(TMP1.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[X86State::RFLAG_NZCV_LOC]));
ldr(TMP4, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.LDIVHandler));
// Extract sign bit of the dividend.
asr(EmitSize, Sign, Upper, 63);
str<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
blr(TMP4);
ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
// Compute absolute value of the 128 bit dividend.
eor(EmitSize, AbsLower, Lower, Sign);
eor(EmitSize, AbsUpper, Upper, Sign);
subs(EmitSize, AbsLower, AbsLower, Sign);
sbc(EmitSize, AbsUpper, AbsUpper, Sign);
// Absolute value of the divisor.
cmp(EmitSize, Divisor, 0);
cneg(EmitSize, AbsDivisor, Divisor, ARMEmitter::Condition::CC_LT);
// Now do the unsigned divide.
EmitDivllu(Quotient, Remainder, AbsUpper, AbsLower, AbsDivisor);
// The quotient is negative when the dividend and the divisor have different signs.
eor(EmitSize, TMP1, Upper, Divisor);
cmp(EmitSize, TMP1, 0);
cneg(EmitSize, Quotient, Quotient, ARMEmitter::Condition::CC_LT);
ldr(TMP1.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[X86State::RFLAG_NZCV_LOC]));
msr(ARMEmitter::SystemRegister::NZCV, TMP1);
// Compute the remainder.
msub(EmitSize, Remainder, Quotient, Divisor, Lower);
// Move results to the destination registers
mov(EmitSize, Quotient, TMP1);
mov(EmitSize, Remainder, TMP2);
// Skip 64-bit path
(void)b(&LongDIVRet);
@@ -1192,22 +1002,22 @@ DEF_OP(UDiv) {
// Check the upper bits for zero
// If the upper bits are zero then we can do a 64-bit divide
(void)cbz(EmitSize, Upper, &Only64Bit);
// The x86 instruction raises #DE when the divisor is zero which we don't do.
// TODO: #DE
(void)cbz(EmitSize, Divisor, &Only64Bit);
// Long divide
{
mrs(TMP1, ARMEmitter::SystemRegister::NZCV);
str(TMP1.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[X86State::RFLAG_NZCV_LOC]));
mov(EmitSize, TMP1, Upper);
mov(EmitSize, TMP2, Lower);
mov(EmitSize, TMP3, Divisor);
EmitDivllu(Quotient, Remainder, Upper, Lower, Divisor);
ldr(TMP4, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.LUDIVHandler));
ldr(TMP1.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[X86State::RFLAG_NZCV_LOC]));
msr(ARMEmitter::SystemRegister::NZCV, TMP1);
str<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
blr(TMP4);
ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
// Compute the remainder.
msub(EmitSize, Remainder, Quotient, Divisor, Lower);
// Move results to the destination registers
mov(EmitSize, Quotient, TMP1);
mov(EmitSize, Remainder, TMP2);
// Skip 64-bit path
(void)b(&LongDIVRet);
@@ -121,10 +121,21 @@ auto Arm64JITCore::InsertGuestPatchableRIPLiteral(uint64_t GuestRIP, uint64_t Si
};
}
void Arm64JITCore::InsertGuestPatchableMove(FEXCore::CPU::RelocationTypes Type, ARMEmitter::Register Reg, uint64_t Value,
uint64_t SiteAddress, uint8_t ValueSize) {
void Arm64JITCore::InsertGuestPatchableDataMove(ARMEmitter::Register Reg, uint64_t Value, uint64_t SiteAddress, uint8_t ValueSize) {
Relocation MoveABI = Relocation::Default();
MoveABI.GuestPatchableData.Header = {.Offset = GetCursorOffset(), .Type = Type};
MoveABI.GuestPatchableData.Header = {.Offset = GetCursorOffset(), .Type = FEXCore::CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_DATA_MOVE};
MoveABI.GuestPatchableData.RegisterIndex = Reg.Idx();
MoveABI.GuestPatchableData.ValueSize = ValueSize;
MoveABI.GuestPatchableData.SiteAddress = SiteAddress;
// this might get patched on disk cache load
LoadConstant(ARMEmitter::Size::i64Bit, Reg, Value, FEXCore::CPU::Arm64Emitter::PadType::DOPAD);
Relocations.emplace_back(MoveABI);
}
void Arm64JITCore::InsertGuestPatchableRIPMove(ARMEmitter::Register Reg, uint64_t Value, uint64_t SiteAddress, uint8_t ValueSize) {
Relocation MoveABI = Relocation::Default();
MoveABI.GuestPatchableData.Header = {.Offset = GetCursorOffset(), .Type = FEXCore::CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_MOVE};
MoveABI.GuestPatchableData.RegisterIndex = Reg.Idx();
MoveABI.GuestPatchableData.ValueSize = ValueSize;
MoveABI.GuestPatchableData.SiteAddress = SiteAddress;
+98 -21
View File
@@ -84,8 +84,7 @@ DEF_OP(ExitFunction) {
str(REG_CALLRET_SP, STATE_PTR(CpuStateFrame, State.callret_sp));
add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, StaticRegisters[X86State::REG_RSP], 0);
if (Op->PatchSiteAddress) {
InsertGuestPatchableMove(RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_MOVE, EC_CALL_CHECKER_PC_REG, NewRIP, Op->PatchSiteAddress,
Op->PatchSiteSize);
InsertGuestPatchableRIPMove(EC_CALL_CHECKER_PC_REG, NewRIP, Op->PatchSiteAddress, Op->PatchSiteSize);
} else {
InsertGuestRIPMove(EC_CALL_CHECKER_PC_REG, NewRIP);
}
@@ -283,11 +282,51 @@ DEF_OP(CondJump) {
}
DEF_OP(Syscall) {
SpillStaticRegs(TMP1);
// Arguments are passed as follows:
// X0: SyscallHandler
// X1: ThreadState
// Jump to the syscall dispatch handler. We won't return after this.
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.ThreadDispatchSyscallHandler));
br(TMP1);
PushDynamicRegs(TMP1);
uint32_t GPRSpillMask = ~0U;
uint32_t FPRSpillMask = ~0U;
SpillStaticRegs(TMP1, {
.GPRSpillMask = GPRSpillMask,
.FPRSpillMask = FPRSpillMask,
});
// Now that we are spilled, store in the state that we are in a syscall
// Still without overwriting registers that matter
// 16bit LoadConstant to be a single instruction
// This gives the signal handler a value to check to see if we are in a syscall at all
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, GPRSpillMask & 0xFFFF);
str(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo));
ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.SyscallHandlerObj));
ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.SyscallHandlerFunc));
mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, STATE.R());
if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
GenerateIndirectRuntimeCall<uint64_t, void*, void*, void*>(ARMEmitter::Reg::r3);
} else {
blr(ARMEmitter::Reg::r3);
}
// Fix the stack and any values that were stepped on
// Syscall result is in any static register that the frontend desired.
FillStaticRegs({
.OptionalReg = ARMEmitter::Reg::r1,
.OptionalReg2 = ARMEmitter::Reg::r2,
.GPRFillMask = GPRSpillMask,
.FPRFillMask = FPRSpillMask,
});
// Now the registers we've spilled are back in their original host registers
// We can safely claim we are no longer in a syscall
str(ARMEmitter::XReg::zr, STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo));
PopDynamicRegs();
}
DEF_OP(Thunk) {
@@ -374,33 +413,71 @@ DEF_OP(ValidateCode) {
DEF_OP(ThreadRemoveCodeEntry) {
auto Op = IROp->C<IR::IROp_ThreadRemoveCodeEntry>();
SpillStaticRegs(TMP1);
// Store the new RIP to go to.
str(GetReg(Op->NewRIP).X(), STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip));
// Move the entry to ABI before saving state.
mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, GetReg(Op->EntryToInvalidate));
mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, GetReg(Op->Entry));
PushDynamicRegs(TMP4);
SpillStaticRegs(TMP4);
// Arguments are passed as follows:
// X0: Thread
// X1: RIPToInvalidate
// X1: RIP
mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, STATE.R());
// Jump to the invalidate dispatch handler. We won't return after this.
ldr(ARMEmitter::XReg::x2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.ThreadDispatchRemoveCodeEntry));
br(ARMEmitter::XReg::x2);
ldr(ARMEmitter::XReg::x2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.ThreadRemoveCodeEntryFromJIT));
if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
GenerateIndirectRuntimeCall<void, void*, void*>(ARMEmitter::Reg::r2);
} else {
blr(ARMEmitter::Reg::r2);
}
FillStaticRegs();
// Fix the stack and any values that were stepped on
PopDynamicRegs();
}
DEF_OP(CPUID) {
auto Op = IROp->C<IR::IROp_CPUID>();
SpillStaticRegs(TMP1);
isb();
mov(ARMEmitter::Size::i64Bit, TMP2, GetReg(Op->Function));
mov(ARMEmitter::Size::i64Bit, TMP3, GetReg(Op->Leaf));
LoadConstant(ARMEmitter::Size::i32Bit, TMP1, Op->InstSize);
// Jump to the syscall dispatch handler. We won't return after this.
ldr(TMP2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.ThreadDispatchCPUIDHandler));
br(TMP2);
PushDynamicRegs(TMP4);
SpillStaticRegs(TMP4);
// x0 = CPUID Handler
// x1 = CPUID Function
// x2 = CPUID Leaf
ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.CPUIDObj));
ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.CPUIDFunction));
if (!TMP_ABIARGS) {
mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, TMP2);
mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, TMP3);
}
if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
GenerateIndirectRuntimeCall<__uint128_t, void*, uint64_t, uint64_t>(ARMEmitter::Reg::r3);
} else {
blr(ARMEmitter::Reg::r3);
}
if (!TMP_ABIARGS) {
mov(ARMEmitter::Size::i64Bit, TMP1, ARMEmitter::Reg::r0);
mov(ARMEmitter::Size::i64Bit, TMP2, ARMEmitter::Reg::r1);
}
FillStaticRegs();
PopDynamicRegs();
// Results are in x0, x1
// Results want to be 4xi32 scalars
mov(ARMEmitter::Size::i32Bit, GetReg(Op->OutEAX), TMP1);
mov(ARMEmitter::Size::i32Bit, GetReg(Op->OutECX), TMP2);
ubfx(ARMEmitter::Size::i64Bit, GetReg(Op->OutEBX), TMP1, 32, 32);
ubfx(ARMEmitter::Size::i64Bit, GetReg(Op->OutEDX), TMP2, 32, 32);
}
DEF_OP(XGetBV) {
@@ -405,37 +405,6 @@ DEF_OP(VFCVTN2) {
}
}
DEF_OP(VBFCVTN) {
LOGMAN_THROW_A_FMT(CTX->HostFeatures.SupportsBF16, "Need BF16 for VBFCVTN");
const auto Op = IROp->C<IR::IROp_VBFCVTN>();
const auto Dst = GetVReg(Node);
const auto Vector = GetVReg(Op->Vector);
bfcvtn(Dst.Q(), Vector.Q());
}
DEF_OP(VBFCVTN2) {
LOGMAN_THROW_A_FMT(CTX->HostFeatures.SupportsBF16, "Need BF16 for VBFCVTN");
const auto Op = IROp->C<IR::IROp_VBFCVTN2>();
const auto Dst = GetVReg(Node);
const auto VectorLower = GetVReg(Op->VectorLower);
const auto VectorUpper = GetVReg(Op->VectorUpper);
auto Lower = VectorLower;
if (Dst != VectorLower) {
mov(VTMP1.Q(), VectorLower.Q());
Lower = VTMP1;
}
bfcvtn2(Lower.Q(), VectorUpper.Q());
if (Dst != VectorLower) {
mov(Dst.Q(), Lower.Q());
}
}
DEF_OP(Vector_FToI) {
const auto Op = IROp->C<IR::IROp_Vector_FToI>();
const auto OpSize = IROp->Size;
+31 -43
View File
@@ -36,44 +36,35 @@ $end_info$
#include <cstdio>
#include <cstring>
#include <optional>
#include <type_traits>
#include <unistd.h>
#ifdef _WIN32
#include <atomic>
#else
#include <FEXHeaderUtils/Syscalls.h>
#endif
namespace {
#ifndef _WIN32
struct DivRem {
uint64_t Quotient;
uint64_t Remainder;
};
static std::optional<uint64_t> RDRANDFallback(uint64_t Reseed) {
uint64_t Value {};
FHU::Syscalls::getrandom(&Value, sizeof(Value), 0);
return Value;
static struct DivRem LUDIV(uint64_t SrcHigh, uint64_t SrcLow, uint64_t Divisor) {
__uint128_t Source = (static_cast<__uint128_t>(SrcHigh) << 64) | SrcLow;
return {
.Quotient = (uint64_t)(Source / Divisor),
.Remainder = (uint64_t)(Source % Divisor),
};
}
#else
static struct DivRem
LDIV(uint64_t SrcHigh, uint64_t SrcLow, int64_t Divisor) {
__int128_t Source = (static_cast<__uint128_t>(SrcHigh) << 64) | SrcLow;
// Windows does not have an equivalent to getrandom() without dynamically linking
// bcrypt et al. Since this fallback is just for compat it does not need to be
// cryptographic, so instead we vendor SplitMix64 as a naive fallback.
// Reference implementation by Sebastiano Vigna, public domain (CC0)
// https://prng.di.unimi.it/splitmix64.c
static std::atomic<uint64_t> RNGState {static_cast<uint64_t>(__builtin_readcyclecounter())};
static std::optional<uint64_t> RDRANDFallback(uint64_t Reseed) {
const uint64_t State = RNGState.load(std::memory_order_relaxed) + 0x9E3779B97F4A7C15ULL;
RNGState.store(State, std::memory_order_relaxed);
uint64_t Value = (State ^ (State >> 30)) * 0xBF58476D1CE4E5B9ULL;
Value = (Value ^ (Value >> 27)) * 0x94D049BB133111EBULL;
return Value ^ (Value >> 31);
return {
.Quotient = (uint64_t)(Source / Divisor),
.Remainder = (uint64_t)(Source % Divisor),
};
}
#endif
static void PrintValue(uint64_t Value) {
static void
PrintValue(uint64_t Value) {
LogMan::Msg::DFmt("Value: 0x{:x}", Value);
}
@@ -625,9 +616,9 @@ void Arm64JITCore::Op_NoOp(const IR::IROp_Header* IROp, IR::Ref Node) {}
Arm64JITCore::Arm64JITCore(FEXCore::Context::ContextImpl* ctx, FEXCore::Core::InternalThreadState* Thread)
: CPUBackend(*ctx, Thread)
, Arm64Emitter(ctx)
, HostSupportsSVE128 {ctx->HostFeatures.SupportsSVE128()}
, HostSupportsSVE256 {ctx->HostFeatures.SupportsSVE256()}
, HostSupportsAVX256 {ctx->HostFeatures.SupportsAVX() && ctx->HostFeatures.SupportsSVE256()}
, HostSupportsSVE128 {ctx->HostFeatures.SupportsSVE128 != 0}
, HostSupportsSVE256 {ctx->HostFeatures.SupportsSVE256 != 0}
, HostSupportsAVX256 {ctx->HostFeatures.SupportsAVX && ctx->HostFeatures.SupportsSVE256}
, HostSupportsRPRES {ctx->HostFeatures.SupportsRPRES != 0}
, HostSupportsAFP {ctx->HostFeatures.SupportsAFP != 0}
, CTX {ctx}
@@ -671,7 +662,8 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::ContextImpl* ctx, FEXCore::Core::In
Ptrs.SyscallHandlerFunc = PMF.GetVTableEntry(CTX->SyscallHandler);
}
Ptrs.ExitFunctionLink = reinterpret_cast<uintptr_t>(&Arm64JITCore::ExitFunctionLink);
Ptrs.RDRANDFallback = reinterpret_cast<uint64_t>(RDRANDFallback);
Ptrs.LUDIV = reinterpret_cast<uint64_t>(LUDIV);
Ptrs.LDIV = reinterpret_cast<uint64_t>(LDIV);
}
CurrentCodeBuffer = SharedCodeBuffers.GetLatest();
@@ -754,17 +746,13 @@ void Arm64JITCore::EmitTFCheck() {
.TrapNo = X86State::X86_TRAPNO_DB,
.si_code = 2,
.err_code = 0,
.FaultAddress = 0,
};
uint64_t Constant {};
memcpy(&Constant, &State, sizeof(uint64_t));
memcpy(&Constant, &State, sizeof(State));
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, static_cast<uint64_t>(Constant));
// Can't use stp, offset is too large.
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, Constant);
str(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData));
str(ARMEmitter::XReg::zr, STATE, offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData) + 8);
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGTRAP));
br(TMP1);
@@ -778,7 +766,7 @@ void Arm64JITCore::EmitTFCheck() {
void Arm64JITCore::EmitSuspendInterruptCheck() {
if (CTX->Config.NeedsPendingInterruptFaultCheck) {
// Trigger a fault if there are any pending interrupts
// Used only for gdbserver at the moment
// Used only for suspend on WIN32 at the moment
constexpr size_t InterruptPageOffset =
offsetof(FEXCore::Core::InternalThreadState, InterruptFaultPage) - offsetof(FEXCore::Core::InternalThreadState, BaseFrameState);
if constexpr (InterruptPageOffset <= 32760) {
@@ -790,7 +778,7 @@ void Arm64JITCore::EmitSuspendInterruptCheck() {
}
}
#ifdef _WIN32
#ifdef ARCHITECTURE_arm64ec
static constexpr uint16_t SuspendMagic {0xCAFE};
ldr(TMP2.W(), STATE_PTR(CpuStateFrame, SuspendDoorbell));
@@ -852,9 +840,9 @@ CodeBuffer::CodeBufferAllocation Arm64JITCore::AllocateCodeBufferInSharedCache(s
return AllocatedInfo;
}
CPUBackend::CompiledCode Arm64JITCore::EmitJITCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
FEXCore::Core::DebugData* DebugData, bool CheckTF) {
FEXCORE_PROFILE_SCOPED("Arm64::EmitJITCode");
FEXCORE_PROFILE_SCOPED("Arm64::CompileCode");
const auto PrevNumAllocations = Relocations.size();
+4 -8
View File
@@ -51,7 +51,7 @@ public:
~Arm64JITCore() override;
[[nodiscard]]
CPUBackend::CompiledCode EmitJITCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
CPUBackend::CompiledCode CompileCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
FEXCore::Core::DebugData* DebugData, bool CheckTF) override;
[[nodiscard]]
@@ -564,8 +564,8 @@ private:
*/
void InsertGuestRIPMove(ARMEmitter::Register Reg, uint64_t Constant);
void InsertGuestPatchableMove(FEXCore::CPU::RelocationTypes Type, ARMEmitter::Register Reg, uint64_t Value, uint64_t SiteAddress,
uint8_t ValueSize);
void InsertGuestPatchableDataMove(ARMEmitter::Register Reg, uint64_t Value, uint64_t SiteAddress, uint8_t ValueSize);
void InsertGuestPatchableRIPMove(ARMEmitter::Register Reg, uint64_t Value, uint64_t SiteAddress, uint8_t ValueSize);
/**
* @brief Inserts a named symbol as a literal in memory
@@ -629,14 +629,10 @@ private:
void Emulate128BitGather(IR::OpSize Size, IR::OpSize ElementSize, ARMEmitter::VRegister Dst, ARMEmitter::VRegister IncomingDst,
std::optional<ARMEmitter::Register> BaseAddr, ARMEmitter::VRegister VectorIndexLow,
std::optional<ARMEmitter::VRegister> VectorIndexHigh, ARMEmitter::VRegister MaskReg, IR::OpSize VectorIndexSize,
size_t DataElementOffsetStart, size_t IndexElementOffsetStart, uint8_t OffsetScale, IR::OpSize AddrSize,
bool MaskAllOnes);
size_t DataElementOffsetStart, size_t IndexElementOffsetStart, uint8_t OffsetScale, IR::OpSize AddrSize);
void EmitTFCheck();
void EmitDivllu(ARMEmitter::Register Quotient, ARMEmitter::Register Remainder, ARMEmitter::Register Upper, ARMEmitter::Register Lower,
ARMEmitter::Register Divisor);
void EmitSuspendInterruptCheck();
void EmitEntryPoint(ARMEmitter::BackwardLabel& HeaderLabel, bool CheckTF);
+24 -37
View File
@@ -1058,7 +1058,7 @@ void Arm64JITCore::Emulate128BitGather(IR::OpSize Size, IR::OpSize ElementSize,
ARMEmitter::VRegister IncomingDst, std::optional<ARMEmitter::Register> BaseAddr,
ARMEmitter::VRegister VectorIndexLow, std::optional<ARMEmitter::VRegister> VectorIndexHigh,
ARMEmitter::VRegister MaskReg, IR::OpSize VectorIndexSize, size_t DataElementOffsetStart,
size_t IndexElementOffsetStart, uint8_t OffsetScale, IR::OpSize AddrSize, bool MaskAllOnes) {
size_t IndexElementOffsetStart, uint8_t OffsetScale, IR::OpSize AddrSize) {
LOGMAN_THROW_A_FMT(ElementSize >= IR::OpSize::i8Bit && ElementSize <= IR::OpSize::i64Bit, "Invalid element size");
const auto PerformSMove = [this](IR::OpSize ElementSize, const ARMEmitter::Register Dst, const ARMEmitter::VRegister Vector, int index) {
@@ -1116,13 +1116,11 @@ void Arm64JITCore::Emulate128BitGather(IR::OpSize Size, IR::OpSize ElementSize,
for (size_t i = DataElementOffsetStart, IndexElement = IndexElementOffsetStart; i < NumDataElements; ++i, ++IndexElement) {
ARMEmitter::ForwardLabel Skip {};
if (!MaskAllOnes) {
// Extract mask element
PerformMove(ElementSize, WorkingReg, MaskReg, i);
// Extract mask element
PerformMove(ElementSize, WorkingReg, MaskReg, i);
// Skip if the mask's sign bit isn't set
(void)tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
}
// Skip if the mask's sign bit isn't set
(void)tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
// Extract Index Element
if ((IndexElement * IR::OpSizeToSize(VectorIndexSize)) >= 16) {
@@ -1218,15 +1216,12 @@ DEF_OP(VLoadVectorGatherMasked) {
const auto SubRegSize = ConvertSubRegSize8(IROp);
const auto CMPPredicate = ARMEmitter::PReg::p0;
const auto GoverningPredicate = Is256Bit ? PRED_TMP_32B : PRED_TMP_16B;
// A mask known to be all-ones loads every element straight into the destination under the governing predicate.
const ARMEmitter::PRegister CMPPredicate = Op->MaskAllOnes ? GoverningPredicate : ARMEmitter::PReg::p0;
auto TempDst = Op->MaskAllOnes ? Dst : VTMP1;
if (!Op->MaskAllOnes) {
// Check if the sign bit is set for the given element size.
cmplt(SubRegSize, CMPPredicate, GoverningPredicate.Zeroing(), MaskReg.Z(), 0);
}
// Check if the sign bit is set for the given element size.
cmplt(SubRegSize, CMPPredicate, GoverningPredicate.Zeroing(), MaskReg.Z(), 0);
auto TempDst = VTMP1;
// No need to load a temporary register in the case that we weren't provided a base address and there is no scaling.
ARMEmitter::SVEMemOperand MemDst {ARMEmitter::SVEMemOperand(VectorIndexLow.Z(), 0)};
@@ -1261,14 +1256,12 @@ DEF_OP(VLoadVectorGatherMasked) {
default: break;
}
if (!Op->MaskAllOnes) {
///< Merge elements based on predicate.
sel(SubRegSize, Dst.Z(), CMPPredicate, TempDst.Z(), IncomingDst.Z());
}
///< Merge elements based on predicate.
sel(SubRegSize, Dst.Z(), CMPPredicate, TempDst.Z(), IncomingDst.Z());
} else {
LOGMAN_THROW_A_FMT(!Is256Bit, "Can't emulate this gather load in the backend! Programming error!");
Emulate128BitGather(IROp->Size, IROp->ElementSize, Dst, IncomingDst, BaseAddr, VectorIndexLow, VectorIndexHigh, MaskReg,
VectorIndexSize, DataElementOffsetStart, IndexElementOffsetStart, OffsetScale, Op->AddrSize, Op->MaskAllOnes);
VectorIndexSize, DataElementOffsetStart, IndexElementOffsetStart, OffsetScale, Op->AddrSize);
}
}
@@ -1302,20 +1295,16 @@ DEF_OP(VLoadVectorGatherMaskedQPS) {
}
const auto CMPPredicate = ARMEmitter::PReg::p0;
// A mask known to be all-ones loads every element under the governing predicate and narrows straight into the destination.
const ARMEmitter::PRegister CMPPredicate2 = Op->MaskAllOnes ? PRED_TMP_16B : ARMEmitter::PReg::p1;
const auto CMPPredicate2 = ARMEmitter::PReg::p1;
const auto GoverningPredicate = PRED_TMP_16B;
if (!Op->MaskAllOnes) {
// Check if the sign bit is set for the given element size.
// This will set the predicate bits for elements [0, 1, 2, 3]
// We then use punpklo to extend the low results to be for 64-bit elements.
cmplt(ARMEmitter::SubRegSize::i32Bit, CMPPredicate, GoverningPredicate.Zeroing(), MaskReg.Z(), 0);
punpklo(CMPPredicate2, CMPPredicate);
}
// Check if the sign bit is set for the given element size.
// This will set the predicate bits for elements [0, 1, 2, 3]
// We then use punpklo to extend the low results to be for 64-bit elements.
cmplt(ARMEmitter::SubRegSize::i32Bit, CMPPredicate, GoverningPredicate.Zeroing(), MaskReg.Z(), 0);
punpklo(CMPPredicate2, CMPPredicate);
auto TempDst = VTMP1;
const auto NarrowDst = Op->MaskAllOnes ? Dst : TempDst;
auto GatherExtend = [this](ARMEmitter::VRegister Dst, std::optional<ARMEmitter::Register> BaseAddr, ARMEmitter::VRegister VectorIndex,
ARMEmitter::PRegister CMPPredicate, ARMEmitter::SVEModType ModType, uint8_t OffsetScale) {
@@ -1339,23 +1328,21 @@ DEF_OP(VLoadVectorGatherMaskedQPS) {
GatherExtend(TempDst, BaseAddr, VectorIndexLow, CMPPredicate2, ModType, OffsetScale);
if (VectorIndexHigh.has_value()) {
if (!Op->MaskAllOnes) {
punpkhi(CMPPredicate2, CMPPredicate);
}
punpkhi(CMPPredicate2, CMPPredicate);
GatherExtend(VTMP2, BaseAddr, *VectorIndexHigh, CMPPredicate2, ModType, OffsetScale);
// Move elements to the lower half.
uzp1(ARMEmitter::SubRegSize::i32Bit, NarrowDst.Q(), TempDst.Q(), VTMP2.Q());
uzp1(ARMEmitter::SubRegSize::i32Bit, TempDst.Q(), TempDst.Q(), VTMP2.Q());
///< Merge elements based on predicate.
sel(ARMEmitter::SubRegSize::i32Bit, Dst.Z(), CMPPredicate, TempDst.Z(), IncomingDst.Z());
} else {
// Move elements to the lower half.
xtn(ARMEmitter::SubRegSize::i32Bit, NarrowDst.Q(), TempDst.Q());
}
if (!Op->MaskAllOnes) {
xtn(ARMEmitter::SubRegSize::i32Bit, TempDst.Q(), TempDst.Q());
///< Merge elements based on predicate.
sel(ARMEmitter::SubRegSize::i32Bit, Dst.Z(), CMPPredicate, TempDst.Z(), IncomingDst.Z());
}
} else {
Emulate128BitGather(IR::OpSize::i128Bit, IR::OpSize::i32Bit, Dst, IncomingDst, BaseAddr, VectorIndexLow, VectorIndexHigh, MaskReg,
IR::OpSize::i64Bit, 0, 0, OffsetScale, Op->AddrSize, Op->MaskAllOnes);
IR::OpSize::i64Bit, 0, 0, OffsetScale, Op->AddrSize);
}
}
+19 -62
View File
@@ -68,42 +68,31 @@ DEF_OP(Break) {
.TrapNo = Op->Reason.TrapNumber,
.si_code = Op->Reason.si_code,
.err_code = Op->Reason.ErrorRegister,
.FaultAddress = 0, // Comes from SSA argument.
};
uint64_t Constant {};
memcpy(&Constant, &State, sizeof(uint64_t));
memcpy(&Constant, &State, sizeof(State));
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, static_cast<uint64_t>(Constant));
auto FaultGPR = GetZeroableReg(Op->FaultAddress);
// Can't use stp, offset is too large.
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, Constant);
str(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData));
str(FaultGPR.X(), STATE, offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData) + 8);
if (CTX->HostFeatures.ProcessFeatures.SupportsFaultFreeSynchronousFaults) {
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SynchronousFault));
switch (Op->Reason.Signal) {
case Core::FAULT_SIGILL:
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGILL));
br(TMP1);
} else {
switch (Op->Reason.Signal) {
case Core::FAULT_SIGILL:
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGILL));
br(TMP1);
break;
case Core::FAULT_SIGTRAP:
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGTRAP));
br(TMP1);
break;
case Core::FAULT_SIGSEGV:
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGSEGV));
br(TMP1);
break;
default:
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGTRAP));
br(TMP1);
break;
}
break;
case Core::FAULT_SIGTRAP:
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGTRAP));
br(TMP1);
break;
case Core::FAULT_SIGSEGV:
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGSEGV));
br(TMP1);
break;
default:
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGTRAP));
br(TMP1);
break;
}
}
@@ -320,39 +309,7 @@ DEF_OP(ProcessorID) {
DEF_OP(RDRAND) {
auto Op = IROp->C<IR::IROp_RDRAND>();
if (CTX->HostFeatures.SupportsRAND) {
mrs(GetReg(Node), Op->GetReseeded ? ARMEmitter::SystemRegister::RNDRRS : ARMEmitter::SystemRegister::RNDR);
return;
}
// Software fallback, call the host RNG generator.
PushDynamicRegs(TMP4);
SpillStaticRegs(TMP4);
// x0 = Reseed
// x1 = Generator
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, Op->GetReseeded ? 1 : 0);
ldr(ARMEmitter::XReg::x1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.RDRANDFallback));
if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
GenerateIndirectRuntimeCall<__uint128_t, uint64_t>(ARMEmitter::Reg::r1);
} else {
blr(ARMEmitter::Reg::r1);
}
if (!TMP_ABIARGS) {
mov(ARMEmitter::Size::i64Bit, TMP1, ARMEmitter::Reg::r0);
mov(ARMEmitter::Size::i64Bit, TMP2, ARMEmitter::Reg::r1);
}
FillStaticRegs();
PopDynamicRegs();
// Results are in x0, x1
// std::optional<uint64_t>: value in x0, engaged flag in the low byte of x1. Match the hardware behaviour of setting Z when
// no number was produced.
mov(ARMEmitter::Size::i64Bit, GetReg(Node), TMP1);
tst(ARMEmitter::Size::i64Bit, TMP2, 0xFF);
mrs(GetReg(Node), Op->GetReseeded ? ARMEmitter::SystemRegister::RNDRRS : ARMEmitter::SystemRegister::RNDR);
}
DEF_OP(Yield) {
@@ -37,10 +37,6 @@ enum class RelocationTypes : uint32_t {
// Like PATCHABLE_RIP_LITERAL but puts it in a register
// Aligned to struct RelocGuestPatchableData
RELOC_GUEST_PATCHABLE_RIP_MOVE,
// Patchable guest CRC
// Aligned to struct RelocGuestPatchableData
RELOC_GUEST_PATCHABLE_CRC_MOVE,
};
struct FEX_PACKED RelocationHeader final {
+3 -223
View File
@@ -930,9 +930,6 @@ DEF_OP(VectorImm) {
const auto Is256Bit = OpSize == IR::OpSize::i256Bit;
LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
if (Op->Do64BitExpansion) {
LOGMAN_THROW_A_FMT(!Is256Bit, "64-bit expansion unsupported on 256-bit operations.");
}
const auto ElementSize = Op->Header.ElementSize;
const auto SubRegSize = ConvertSubRegSize8(IROp);
@@ -948,22 +945,12 @@ DEF_OP(VectorImm) {
dup_imm(SubRegSize, Dst.Z(), static_cast<int8_t>(Op->Immediate));
}
} else {
if (!Op->Do64BitExpansion && ElementSize == IR::OpSize::i64Bit) {
if (ElementSize == IR::OpSize::i64Bit) {
// movi with 64bit element size doesn't do what we want here
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, static_cast<uint64_t>(Op->Immediate) << Op->ShiftAmount);
dup(SubRegSize, Dst.Q(), TMP1.R());
} else {
uint64_t Imm = Op->Immediate;
if (Op->Do64BitExpansion) {
// Need to manually expand the immediate for this case.
Imm = 0;
for (size_t i = 0; i < 8; ++i) {
if ((Op->Immediate >> i) & 1) {
Imm |= (0xFFULL << (i * 8));
}
}
}
movi(SubRegSize, Dst.Q(), Imm, Op->ShiftAmount);
movi(SubRegSize, Dst.Q(), Op->Immediate, Op->ShiftAmount);
}
}
}
@@ -1125,10 +1112,8 @@ DEF_OP(VAddP) {
// pairwise addition, the SVE version actually interleaves the
// results of the pairwise addition (gross!), so we need to undo that.
addp(SubRegSize, LHS.Z(), Pred, LHS.Z(), VectorUpper.Z());
// Extract the upper half first, since Dst may alias the LHS.
uzp2(SubRegSize, VTMP2.Z(), LHS.Z(), LHS.Z());
uzp1(SubRegSize, Dst.Z(), LHS.Z(), LHS.Z());
uzp2(SubRegSize, VTMP2.Z(), LHS.Z(), LHS.Z());
// Merge upper half with lower half.
splice<ARMEmitter::OpType::Destructive>(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), PRED_TMP_16B, Dst.Z(), VTMP2.Z());
@@ -2182,46 +2167,6 @@ DEF_OP(VCMPGT) {
}
}
DEF_OP(VUCMPGT) {
const auto Op = IROp->C<IR::IROp_VUCMPGT>();
const auto OpSize = IROp->Size;
const auto ElementSize = Op->Header.ElementSize;
const auto SubRegSize = ConvertSubRegSizePair16(IROp);
const auto IsScalar = ElementSize == OpSize;
const auto Is256Bit = OpSize == IR::OpSize::i256Bit;
LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
const auto Dst = GetVReg(Node);
const auto Vector1 = GetVReg(Op->Vector1);
const auto Vector2 = GetVReg(Op->Vector2);
if (HostSupportsSVE256 && Is256Bit) {
const auto Mask = PRED_TMP_32B.Zeroing();
const auto ComparePred = ARMEmitter::PReg::p0;
// FIXME: We should rework this op to avoid the NZCV spill/fill dance.
mrs(TMP1, ARMEmitter::SystemRegister::NZCV);
// General idea is to compare for unsigned greater-than, bitwise NOT
// the valid values, then ORR the NOTed values with the original
// values to form entries that are all 1s.
cmphi(SubRegSize.Vector, ComparePred, Mask, Vector1.Z(), Vector2.Z());
not_(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), Vector1.Z());
movprfx(SubRegSize.Vector, Dst.Z(), ComparePred.Zeroing(), Vector1.Z());
orr(SubRegSize.Vector, Dst.Z(), ComparePred.Merging(), Dst.Z(), VTMP1.Z());
// Restore NZCV
msr(ARMEmitter::SystemRegister::NZCV, TMP1);
} else {
if (IsScalar) {
cmhi(SubRegSize.Scalar, Dst, Vector1, Vector2);
} else {
cmhi(SubRegSize.Vector, Dst.Q(), Vector1.Q(), Vector2.Q());
}
}
}
DEF_OP(VCMPGTZ) {
const auto Op = IROp->C<IR::IROp_VCMPGTZ>();
const auto OpSize = IROp->Size;
@@ -3888,171 +3833,6 @@ DEF_OP(VMul) {
}
}
DEF_OP(VUSDot) {
///< Dest = Acc + dot(Vector1 (unsigned 8-bit), Vector2 (signed 8-bit))
// Matches:
// - SVE - USDOT
// - ASIMD - USDOT
const auto Op = IROp->C<IR::IROp_VUSDot>();
const auto OpSize = IROp->Size;
const auto Is256Bit = OpSize == IR::OpSize::i256Bit;
LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
const auto Dst = GetVReg(Node);
const auto Acc = GetVReg(Op->Acc);
const auto Vector1 = GetVReg(Op->Vector1);
const auto Vector2 = GetVReg(Op->Vector2);
// USDOT accumulates in to its destination register,
// so we need to emit a move if Acc != Dst
ARMEmitter::VRegister DestTmp = Dst;
if (Dst != Acc) {
if (Dst != Vector1 && Dst != Vector2) {
DestTmp = Dst;
} else {
DestTmp = VTMP1;
}
}
if (HostSupportsSVE256 && Is256Bit) {
if (Dst != Acc) {
mov(DestTmp.Z(), Acc.Z());
}
usdot(DestTmp.Z(), Vector1.Z(), Vector2.Z());
if (Dst != DestTmp) {
mov(Dst.Z(), DestTmp.Z());
}
} else {
if (Dst != Acc) {
mov(DestTmp.Q(), Acc.Q());
}
usdot(DestTmp.Q(), Vector1.Q(), Vector2.Q());
if (Dst != DestTmp) {
mov(Dst.Q(), DestTmp.Q());
}
}
}
DEF_OP(VSDot) {
///< Dest = Acc + dot(Vector1 (signed 8-bit), Vector2 (signed 8-bit))
// Matches:
// - SVE - SDOT
// - ASIMD - SDOT
const auto Op = IROp->C<IR::IROp_VSDot>();
const auto OpSize = IROp->Size;
const auto Is256Bit = OpSize == IR::OpSize::i256Bit;
LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
const auto Dst = GetVReg(Node);
const auto Acc = GetVReg(Op->Acc);
const auto Vector1 = GetVReg(Op->Vector1);
const auto Vector2 = GetVReg(Op->Vector2);
// SDOT accumulates in to its destination register,
// so we need to emit a move if Acc != Dst
ARMEmitter::VRegister DestTmp = Dst;
if (Dst != Acc) {
if (Dst != Vector1 && Dst != Vector2) {
DestTmp = Dst;
} else {
DestTmp = VTMP1;
}
}
if (HostSupportsSVE256 && Is256Bit) {
if (Dst != Acc) {
mov(DestTmp.Z(), Acc.Z());
}
sdot(ARMEmitter::SubRegSize::i32Bit, DestTmp.Z(), Vector1.Z(), Vector2.Z());
if (Dst != DestTmp) {
mov(Dst.Z(), DestTmp.Z());
}
} else {
if (Dst != Acc) {
mov(DestTmp.Q(), Acc.Q());
}
sdot(ARMEmitter::SubRegSize::i32Bit, DestTmp.Q(), Vector1.Q(), Vector2.Q());
if (Dst != DestTmp) {
mov(Dst.Q(), DestTmp.Q());
}
}
}
DEF_OP(VSAddLP) {
const auto Op = IROp->C<IR::IROp_VSAddLP>();
const auto OpSize = IROp->Size;
const auto SubRegSize = ConvertSubRegSize248(IROp);
const auto Is256Bit = OpSize == IR::OpSize::i256Bit;
LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
const auto Dst = GetVReg(Node);
const auto Vector = GetVReg(Op->Vector);
if (HostSupportsSVE256 && Is256Bit) {
// SVE only has the accumulating form, so accumulate in to a zeroed register.
// Zero a temporary instead if Dst aliases the source.
const auto DestTmp = Dst == Vector ? VTMP1 : Dst;
dup_imm(SubRegSize, DestTmp.Z(), 0);
sadalp(SubRegSize, DestTmp.Z(), PRED_TMP_32B.Merging(), Vector.Z());
if (Dst != DestTmp) {
mov(Dst.Z(), DestTmp.Z());
}
} else {
saddlp(SubRegSize, Dst.Q(), Vector.Q());
}
}
DEF_OP(VSAdALP) {
const auto Op = IROp->C<IR::IROp_VSAdALP>();
const auto OpSize = IROp->Size;
const auto SubRegSize = ConvertSubRegSize248(IROp);
const auto Is256Bit = OpSize == IR::OpSize::i256Bit;
LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
const auto Dst = GetVReg(Node);
const auto Acc = GetVReg(Op->Acc);
const auto Vector = GetVReg(Op->Vector);
// SADALP accumulates in to its destination register,
// so we need to emit a move if Acc != Dst
ARMEmitter::VRegister DestTmp = Dst;
if (Dst != Acc) {
DestTmp = Dst != Vector ? Dst : VTMP1;
}
if (HostSupportsSVE256 && Is256Bit) {
if (Dst != Acc) {
mov(DestTmp.Z(), Acc.Z());
}
sadalp(SubRegSize, DestTmp.Z(), PRED_TMP_32B.Merging(), Vector.Z());
if (Dst != DestTmp) {
mov(Dst.Z(), DestTmp.Z());
}
} else {
if (Dst == Vector) {
// ASIMD has the non-accumulating form, which is cheaper than shuffling through a temporary.
saddlp(SubRegSize, VTMP1.Q(), Vector.Q());
add(SubRegSize, Dst.Q(), Acc.Q(), VTMP1.Q());
return;
}
if (Dst != Acc) {
mov(Dst.Q(), Acc.Q());
}
sadalp(SubRegSize, Dst.Q(), Vector.Q());
}
}
DEF_OP(VUMull) {
const auto Op = IROp->C<IR::IROp_VUMull>();
const auto OpSize = IROp->Size;
File diff suppressed because it is too large. Load diff
+21 -160
View File
@@ -207,12 +207,9 @@ public:
FlushRegisterCache();
return _ExitFunction(GetOpSize(NewRIP), NewRIP, Hint, CallReturnAddress, CallReturnBlock, PatchSiteAddress, PatchSiteSize);
}
IRPair<IROp_Break> Break(BreakDefinition Reason, Ref RelocatedPC = nullptr) {
IRPair<IROp_Break> Break(BreakDefinition Reason) {
FlushRegisterCache();
if (!RelocatedPC) {
RelocatedPC = Constant(0);
}
return _Break(Reason, RelocatedPC);
return _Break(Reason);
}
IRPair<IROp_Thunk> Thunk(Ref ArgPtr, SHA256Sum ThunkNameHash) {
FlushRegisterCache();
@@ -220,8 +217,6 @@ public:
}
bool FinishOp(uint64_t NextRIP, bool LastOp) {
++RegCache.Index;
// If we are switching to a new block and this current block has yet to set a RIP
// Then we need to insert an unconditional jump from the current block to the one we are going to
// This happens most frequently when an instruction jumps backwards to another location
@@ -368,8 +363,7 @@ public:
void MOVVectorUnalignedOp(OpcodeArgs);
void MOVVectorUnalignedNoNopOp(OpcodeArgs);
void MOVVectorNTOp(OpcodeArgs, bool IsAVX);
void ALURAXOp(OpcodeArgs, FEXCore::IR::IROps ALUIROp);
void ALUOp(OpcodeArgs, FEXCore::IR::IROps ALUIROp, FEXCore::IR::IROps AtomicFetchOp, unsigned SrcIdx, bool DestRAX);
void ALUOp(OpcodeArgs, FEXCore::IR::IROps ALUIROp, FEXCore::IR::IROps AtomicFetchOp, unsigned SrcIdx);
void LSLOp(OpcodeArgs);
void INTOp(OpcodeArgs);
void SyscallOp(OpcodeArgs, bool IsSyscallInst);
@@ -380,8 +374,8 @@ public:
void IRETOp(OpcodeArgs);
void CallbackReturnOp(OpcodeArgs);
void SecondaryALUOp(OpcodeArgs);
void ADCOp(OpcodeArgs, uint32_t SrcIndex, bool DestRAX);
void SBBOp(OpcodeArgs, uint32_t SrcIndex, bool DestRAX);
void ADCOp(OpcodeArgs, uint32_t SrcIndex);
void SBBOp(OpcodeArgs, uint32_t SrcIndex);
void SALCOp(OpcodeArgs);
void PUSHOp(OpcodeArgs);
void PUSHREGOp(OpcodeArgs);
@@ -401,18 +395,16 @@ public:
void JUMPFARIndirectOp(OpcodeArgs);
void CALLFARIndirectOp(OpcodeArgs);
void RETFARIndirectOp(OpcodeArgs);
void TESTOp(OpcodeArgs, uint32_t SrcIndex, bool DestRAX);
void TESTOp(OpcodeArgs, uint32_t SrcIndex);
void ARPLOp(OpcodeArgs);
void MOVSXDOp(OpcodeArgs);
void MOVSXOp(OpcodeArgs);
void MOVZXOp(OpcodeArgs);
void CMPOp(OpcodeArgs, uint32_t SrcIndex, bool DestRAX);
void CMPOp(OpcodeArgs, uint32_t SrcIndex);
void SETccOp(OpcodeArgs);
void CQOOp(OpcodeArgs);
void CDQOp(OpcodeArgs);
std::optional<Ref> XCHGOpImpl(OpcodeArgs, Ref Src);
void XCHGOp(OpcodeArgs);
void XCHGRAXOp(OpcodeArgs);
void SAHFOp(OpcodeArgs);
void LAHFOp(OpcodeArgs);
void MOVSegOp(OpcodeArgs, bool ToSeg);
@@ -434,11 +426,11 @@ public:
void RotateOp(OpcodeArgs, bool Left, bool IsImmediate, bool Is1Bit);
void RCROp1Bit(OpcodeArgs);
void RCROp8x1Bit(OpcodeArgs);
void RCROp(OpcodeArgs, bool UseRCX);
void RCRSmallerOp(OpcodeArgs, bool UseRCX);
void RCROp(OpcodeArgs);
void RCRSmallerOp(OpcodeArgs);
void RCLOp1Bit(OpcodeArgs);
void RCLOp(OpcodeArgs, bool UseRCX);
void RCLSmallerOp(OpcodeArgs, bool UseRCX);
void RCLOp(OpcodeArgs);
void RCLSmallerOp(OpcodeArgs);
void BTOp(OpcodeArgs, uint32_t SrcIndex, enum BTAction Action);
@@ -493,7 +485,6 @@ public:
void SGDTOp(OpcodeArgs);
void SIDTOp(OpcodeArgs);
void SMSWOp(OpcodeArgs);
void SLDTOp(OpcodeArgs);
enum class VectorOpType {
MMX,
@@ -674,8 +665,6 @@ public:
void VPMADDUBSWOp(OpcodeArgs);
void VPMADDWDOp(OpcodeArgs);
void VPDPBUSDOp(OpcodeArgs, bool Saturating);
void VPDPWSSDOp(OpcodeArgs, bool Saturating);
void VPMASKMOVOp(OpcodeArgs, bool IsStore);
@@ -752,7 +741,7 @@ public:
void X87FLDCW(OpcodeArgs);
void X87FNSAVE(OpcodeArgs);
void X87FNSTENV(OpcodeArgs);
void X87FNSTSW(OpcodeArgs, bool DestRAX);
void X87FNSTSW(OpcodeArgs);
void X87FRSTOR(OpcodeArgs);
void X87FSTCW(OpcodeArgs);
void X87FXAM(OpcodeArgs);
@@ -761,7 +750,6 @@ public:
void X87LDENV(OpcodeArgs);
void X87ModifySTP(OpcodeArgs, bool Inc);
void X87OpHelper(OpcodeArgs, FEXCore::IR::IROps IROp, bool ZeroC2);
void X87NOP(OpcodeArgs);
enum class FCOMIFlags {
FLAGS_X87,
@@ -835,7 +823,6 @@ public:
void LoadFenceOrXRSTOR(OpcodeArgs);
void MemFenceOrXSAVEOPT(OpcodeArgs);
void StoreFenceOrCLFlush(OpcodeArgs);
void SerializeOp(OpcodeArgs);
void UMonitorOrCLRSSBSY(OpcodeArgs);
void UMWaitOp(OpcodeArgs);
void CLZeroOp(OpcodeArgs);
@@ -902,8 +889,7 @@ public:
void Extrq(OpcodeArgs);
void Insertq(OpcodeArgs);
void BreakOp(OpcodeArgs, FEXCore::IR::BreakDefinition BreakDefinition, Ref RelocatedPC = nullptr);
void BreakpointOp(OpcodeArgs, uint32_t BP);
void BreakOp(OpcodeArgs, FEXCore::IR::BreakDefinition BreakDefinition);
void UnimplementedOp(OpcodeArgs);
void PermissionRestrictedOp(OpcodeArgs);
@@ -1050,9 +1036,6 @@ public:
void AVX128_VPMADDUBSW(OpcodeArgs);
void AVX128_VPMADDWD(OpcodeArgs);
void AVX128_VPDPImpl(OpcodeArgs, std::function<Ref(Ref Acc, Ref Src1, Ref Src2)> Helper);
void AVX128_VPDPBUSD(OpcodeArgs, bool Saturating);
void AVX128_VPDPWSSD(OpcodeArgs, bool Saturating);
void AVX128_VBLEND(OpcodeArgs, IR::OpSize ElementSize);
@@ -1093,19 +1076,13 @@ public:
void AVX128_VFMAScalarImpl(OpcodeArgs, IROps IROp, uint8_t Src1Idx, uint8_t Src2Idx, uint8_t AddendIdx);
void AVX128_VFMAddSubImpl(OpcodeArgs, bool AddSub, uint8_t Src1Idx, uint8_t Src2Idx, uint8_t AddendIdx);
RefPair AVX128_VPGatherQPSImpl(OpcodeArgs, Ref Dest, Ref Mask, RefVSIB VSIB, bool MaskAllOnes);
RefPair AVX128_VPGatherImpl(OpcodeArgs, OpSize Size, OpSize ElementLoadSize, OpSize AddrElementSize, RefPair Dest, RefPair Mask,
RefVSIB VSIB, bool MaskAllOnesLow = false, bool MaskAllOnesHigh = false);
RefPair AVX128_VPGatherQPSImpl(OpcodeArgs, Ref Dest, Ref Mask, RefVSIB VSIB);
RefPair AVX128_VPGatherImpl(OpcodeArgs, OpSize Size, OpSize ElementLoadSize, OpSize AddrElementSize, RefPair Dest, RefPair Mask, RefVSIB VSIB);
void AVX128_VPGATHER(OpcodeArgs, OpSize AddrElementSize);
void AVX128_VCVTPH2PS(OpcodeArgs);
void AVX128_VCVTPS2PH(OpcodeArgs);
void AVX128_VBCSTNEBF162PS(OpcodeArgs);
void AVX128_VBCSTNESH2PS(OpcodeArgs);
void AVX128_VCVTNEEBF162PS(OpcodeArgs, bool Even);
void AVX128_VCVTNEEPH2PS(OpcodeArgs, bool Even);
void AVX128_VCVTNEPS2BF16(OpcodeArgs);
// End of AVX 128-bit implementation
@@ -1140,16 +1117,10 @@ public:
void AVXVectorUnaryOp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);
void AVXVectorVariableBlend(OpcodeArgs, IR::OpSize ElementSize);
void AVXVBCSTNEBF162PS(OpcodeArgs);
void AVXVBCSTNESH2PS(OpcodeArgs);
void AVXVCVTNEEBF162PS(OpcodeArgs, bool Even);
void AVXVCVTNEEPH2PS(OpcodeArgs, bool Even);
void AVXVCVTNEPS2BF16(OpcodeArgs);
// End of AVX 256-bit implementation
void InvalidOp(OpcodeArgs);
void NoExecOrPartialOp(OpcodeArgs);
void NoExecOp(OpcodeArgs);
void SetPackedRFLAG(bool Lower8, Ref Src);
Ref GetPackedRFLAG(uint32_t FlagsMask = ~0U);
@@ -1199,21 +1170,6 @@ public:
// At block boundaries, fix up the carry flag.
if (!SRAOnly) {
RectifyCarryInvert(CFInvertedABI);
// Also we can't trust the LastDef or PrevDef arrays accross block boundaries, so clear them as well.
for (auto& Def : RegCache.LastDef) {
Def = nullptr;
}
for (auto& Def : RegCache.PrevDef) {
Def = nullptr;
}
for (auto& DefIndex : RegCache.LastDefIndex) {
DefIndex = 0;
}
for (auto& DefIndex : RegCache.PrevDefIndex) {
DefIndex = 0;
}
RegCache.Index = 0;
}
if (!MMXOnly) {
@@ -1330,7 +1286,7 @@ protected:
/* On ASIMD platforms, the emulation happens to preserve NZCV, unlike the
* more optimal SVE implementation that clobbers.
*/
if (!CTX->HostFeatures.SupportsSVE()) {
if (!CTX->HostFeatures.SupportsSVE128 && !CTX->HostFeatures.SupportsSVE256) {
return;
}
@@ -1351,7 +1307,6 @@ protected:
private:
FEX_CONFIG_OPT(ReducedPrecisionMode, X87REDUCEDPRECISION);
FEX_CONFIG_OPT(UndefinedFlagAcidTest, UNDEFINEDFLAGACIDTEST);
struct JumpTargetInfo {
Ref BlockEntry;
@@ -1429,11 +1384,6 @@ private:
const X86Tables::DecodedOperand& Imm, bool IsAVX);
void PCMPXSTRXOpImpl(OpcodeArgs, bool IsExplicit, bool IsMask, bool IsAVX);
Ref PCMPXSTRXSaturateExplicitLength(IR::OpSize ElementSize, IR::OpSize LengthSize, Ref RawLength);
Ref PCMPXSTRXEqualAny(IR::OpSize ElementSize, Ref Src1, Ref Src2, Ref Src1ValidElements, Ref Src2ValidElements);
Ref PCMPXSTRXRanges(IR::OpSize ElementSize, Ref Src1, Ref Src2, Ref Src1ValidElements, Ref Src2ValidElements, bool IsSigned);
Ref PCMPXSTRXEqualEach(IR::OpSize ElementSize, Ref Src1, Ref Src2, Ref Src1ValidElements, Ref Src2ValidElements);
Ref PCMPXSTRXEqualOrdered(IR::OpSize ElementSize, Ref Src1, Ref Src2, Ref Src1Length, Ref Src2Length, Ref Src1ValidElements, Ref Indices);
Ref PHADDSOpImpl(OpSize Size, Ref Src1, Ref Src2);
@@ -1450,10 +1400,6 @@ private:
Ref PMADDUBSWOpImpl(IR::OpSize Size, Ref Src1, Ref Src2);
Ref VPDPBUSDOpImpl(IR::OpSize Size, Ref Acc, Ref Src1, Ref Src2, bool Saturating);
Ref VPDPWSSDOpImpl(IR::OpSize Size, Ref Acc, Ref Src1, Ref Src2, bool Saturating);
Ref PMULHRSWOpImpl(OpSize Size, Ref Src1, Ref Src2);
Ref PMULHWOpImpl(OpcodeArgs, bool Signed, Ref Src1, Ref Src2);
@@ -1534,7 +1480,7 @@ private:
void SaveAVXState(Ref MemBase);
void XRstorOpImpl(OpcodeArgs);
void RestoreX87State(OpcodeArgs, Ref MemBase);
void RestoreX87State(Ref MemBase);
void RestoreSSEState(Ref MemBase);
void RestoreMXCSRState(Ref MXCSR);
void RestoreAVXState(Ref MemBase);
@@ -1555,20 +1501,6 @@ private:
void StoreGPRRegister(uint32_t GPR, const Ref Src, IR::OpSize Size = OpSize::iInvalid, uint8_t Offset = 0);
void StoreXMMRegister(uint32_t XMM, const Ref Src);
// Matches semantics around GPR storing that matches `StoreResult_WithOpSize` behaviour.
void StoreGPRResultWithZExtSemantics(uint32_t GPR, Ref Src, IR::OpSize OpSize) {
const auto GPRSize = GetGPROpSize();
if (GPRSize == OpSize::i64Bit && OpSize == OpSize::i32Bit) {
// If the Source IR op is 64 bits, we need to zext the upper bits
// For all other sizes, the upper bits are guaranteed to already be zero
Src = GetOpSize(Src) == OpSize::i64Bit ? ARef(Src).Bfe(0, 32).Ref() : Src;
StoreGPRRegister(GPR, Src, GPRSize);
} else {
StoreGPRRegister(GPR, Src, std::min(GPRSize, OpSize));
}
}
Ref _GetRelocatedPC(const FEXCore::X86Tables::DecodedOp& Op, int64_t Offset, bool Inline) {
const auto GPRSize = GetGPROpSize();
const auto Offs = Op->PC + Op->InstSize + Offset - Entry;
@@ -1677,7 +1609,7 @@ private:
}
constexpr OpSize GetGuestVectorLength() const {
return (CTX->HostFeatures.SupportsSVE256() && CTX->HostFeatures.SupportsAVX()) ? OpSize::i256Bit : OpSize::i128Bit;
return (CTX->HostFeatures.SupportsSVE256 && CTX->HostFeatures.SupportsAVX) ? OpSize::i256Bit : OpSize::i128Bit;
}
[[nodiscard]]
@@ -1935,55 +1867,6 @@ private:
InvalidateReg(Core::CPUState::AF_AS_GREG);
}
// Sets the desired flags to zero when acid test is enabled.
// Not meant to be fast, just meant to produce consistent results.
void AcidTestFlags(unsigned BitOffset = 0, unsigned BitOffset2 = 0, unsigned BitOffset3 = 0, unsigned BitOffset4 = 0,
unsigned BitOffset5 = 0, unsigned BitOffset6 = 0) {
if (!UndefinedFlagAcidTest) [[likely]] {
return;
}
unsigned BitFlags {};
if (BitOffset) {
BitFlags |= 1U << BitOffset;
}
if (BitOffset2) {
BitFlags |= 1U << BitOffset2;
}
if (BitOffset3) {
BitFlags |= 1U << BitOffset3;
}
if (BitOffset4) {
BitFlags |= 1U << BitOffset4;
}
if (BitOffset5) {
BitFlags |= 1U << BitOffset5;
}
if (BitOffset6) {
BitFlags |= 1U << BitOffset6;
}
auto Flag = Constant(0);
if (BitFlags & (1U << X86State::RFLAG_SF_RAW_LOC)) {
SetRFLAG(Flag, FEXCore::X86State::RFLAG_SF_RAW_LOC);
}
if (BitFlags & (1U << X86State::RFLAG_ZF_RAW_LOC)) {
SetRFLAG(Flag, FEXCore::X86State::RFLAG_ZF_RAW_LOC);
}
if (BitFlags & (1U << X86State::RFLAG_CF_RAW_LOC)) {
SetRFLAG(Flag, FEXCore::X86State::RFLAG_CF_RAW_LOC);
}
if (BitFlags & (1U << X86State::RFLAG_OF_RAW_LOC)) {
SetRFLAG(Flag, FEXCore::X86State::RFLAG_OF_RAW_LOC);
}
if (BitFlags & (1U << X86State::RFLAG_AF_RAW_LOC)) {
SetRFLAG(Flag, FEXCore::X86State::RFLAG_AF_RAW_LOC);
}
if (BitFlags & (1U << X86State::RFLAG_AF_RAW_LOC)) {
SetRFLAG(Flag, FEXCore::X86State::RFLAG_AF_RAW_LOC);
}
}
[[nodiscard]]
static CondClass CondForNZCVBit(unsigned BitOffset, bool Invert) {
switch (BitOffset) {
@@ -2049,25 +1932,12 @@ private:
uint64_t Partial;
Ref Value[64];
// Keep track of where each register was last written to within the current block,
// for use in peephole optimizations. LastDefIndex is a parallel structure,
// which tracks the index of each Op in LastDef.
Ref LastDef[64];
uint32_t LastDefIndex[64];
// Similar to LastDef, but tracks the write before LastDef. Think of it as LastDef^2.
Ref PrevDef[64];
uint32_t PrevDefIndex[64];
// The index of the current instruction in the current block.
uint32_t Index;
} RegCache {};
void InvalidateReg(uint8_t Index) {
uint64_t Bit = (1ull << (uint64_t)Index);
RegCache.Cached &= ~Bit;
RegCache.Written &= ~Bit;
RegCache.LastDef[Index] = nullptr;
}
Ref LoadRegCache(uint64_t Offset, uint8_t Index, RegClass Class, IR::OpSize Size) {
@@ -2183,10 +2053,6 @@ private:
RegCache.Value[Index] = Value;
RegCache.Cached |= Bit;
RegCache.Written |= Bit;
RegCache.PrevDef[Index] = RegCache.LastDef[Index];
RegCache.PrevDefIndex[Index] = RegCache.LastDefIndex[Index];
RegCache.LastDef[Index] = Value;
RegCache.LastDefIndex[Index] = RegCache.Index;
}
void InvalidateHighAVXRegisters() {
@@ -2298,7 +2164,8 @@ private:
Ref V_inv = GetRFLAG(FEXCore::X86State::RFLAG_OF_RAW_LOC, true);
SetRFLAG<FEXCore::X86State::RFLAG_PF_RAW_LOC>(V_inv);
// Both Intel and AMD manual state that OF, SF, and AF are set to zero.
// Intel: OF, SF, and AF set to zero
// AMD: no mention of OF, SF and AF but actual hardware seems to always zero
//
// Zero AF. Note that the comparison sets the raw PF to 0/1 above, so
// PF[4] is 0 so the XOR with PF will have no effect, so setting the AF
@@ -2529,12 +2396,6 @@ private:
void SetX87FTW(Ref FTW);
Ref GetX87FTW_Helper();
void SetX87Top(Ref Value);
void StoreX87FIP(FEXCore::X86Tables::DecodedOp Op) {
if (CTX->Config.JITFeatures & FEXCore::Config::JITFeatures::ENABLEX87FIP) {
// Only store FIP if the feature is enabled.
_StoreContextGPR(OpSize::i64Bit, GetRelocatedPC(Op, -Op->InstSize), offsetof(FEXCore::Core::CPUState, FIP));
}
}
void ChgStateX87_MMX() override {
LOGMAN_THROW_A_FMT(MMXState == MMXState_X87, "Expected state to be x87");
@@ -1470,35 +1470,6 @@ void OpDispatchBuilder::AVX128_VPMADDWD(OpcodeArgs) {
[this](IR::OpSize _ElementSize, Ref Src1, Ref Src2) { return PMADDWDOpImpl(OpSize::i128Bit, Src1, Src2); });
}
void OpDispatchBuilder::AVX128_VPDPImpl(OpcodeArgs, std::function<Ref(Ref Acc, Ref Src1, Ref Src2)> Helper) {
const auto Size = OpSizeFromDst(Op);
const auto Is128Bit = Size == OpSize::i128Bit;
auto Acc = AVX128_LoadSource_WithOpSize(Op, Op->Dest, Op->Flags, !Is128Bit);
auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);
auto Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, !Is128Bit);
RefPair Result {};
Result.Low = Helper(Acc.Low, Src1.Low, Src2.Low);
if (Is128Bit) {
Result.High = LoadZeroVector(OpSize::i128Bit);
} else {
Result.High = Helper(Acc.High, Src1.High, Src2.High);
}
AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);
}
void OpDispatchBuilder::AVX128_VPDPBUSD(OpcodeArgs, bool Saturating) {
AVX128_VPDPImpl(Op,
[this, Saturating](Ref Acc, Ref Src1, Ref Src2) { return VPDPBUSDOpImpl(OpSize::i128Bit, Acc, Src1, Src2, Saturating); });
}
void OpDispatchBuilder::AVX128_VPDPWSSD(OpcodeArgs, bool Saturating) {
AVX128_VPDPImpl(Op,
[this, Saturating](Ref Acc, Ref Src1, Ref Src2) { return VPDPWSSDOpImpl(OpSize::i128Bit, Acc, Src1, Src2, Saturating); });
}
void OpDispatchBuilder::AVX128_VBLEND(OpcodeArgs, IR::OpSize ElementSize) {
const auto SrcSize = OpSizeFromSrc(Op);
const auto Is128Bit = SrcSize == OpSize::i128Bit;
@@ -2052,8 +2023,7 @@ void OpDispatchBuilder::AVX128_VFMAddSubImpl(OpcodeArgs, bool AddSub, uint8_t Sr
}
OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherImpl(OpcodeArgs, OpSize Size, OpSize ElementLoadSize, OpSize AddrElementSize,
RefPair Dest, RefPair Mask, RefVSIB VSIB, bool MaskAllOnesLow,
bool MaskAllOnesHigh) {
RefPair Dest, RefPair Mask, RefVSIB VSIB) {
LOGMAN_THROW_A_FMT(AddrElementSize == OpSize::i32Bit || AddrElementSize == OpSize::i64Bit, "Unknown address element size");
const auto Is128Bit = Size == OpSize::i128Bit;
@@ -2067,7 +2037,7 @@ OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherImpl(OpcodeArgs, Op
BaseAddr = Invalid();
}
if (CTX->HostFeatures.SupportsSVE128()) {
if (CTX->HostFeatures.SupportsSVE128) {
if (ElementLoadSize == OpSize::i64Bit && AddrElementSize == OpSize::i32Bit) {
// In the case that FEX is loading double the amount of data than the number of address bits then we can optimize this case.
// For 256-bits of data we need to sign extend all four 32-bit address elements to be 64-bit.
@@ -2103,7 +2073,7 @@ OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherImpl(OpcodeArgs, Op
RefPair Result {};
///< Calculate the low-half.
Result.Low = _VLoadVectorGatherMasked(OpSize::i128Bit, ElementLoadSize, Dest.Low, Mask.Low, BaseAddr, VSIB.Low, VSIB.High,
AddrElementSize, VSIB.Scale, 0, 0, AddrSize, MaskAllOnesLow);
AddrElementSize, VSIB.Scale, 0, 0, AddrSize);
if (Is128Bit) {
Result.High = LoadZeroVector(OpSize::i128Bit);
@@ -2118,7 +2088,6 @@ OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherImpl(OpcodeArgs, Op
Ref DestReg = Dest.High;
Ref MaskReg = Mask.High;
bool MaskAllOnes = MaskAllOnesHigh;
uint8_t IndexElementOffset {};
uint8_t DataElementOffset {};
if (AddrElementSize == ElementLoadSize) {
@@ -2135,15 +2104,13 @@ OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherImpl(OpcodeArgs, Op
AddrAddressing.High = Invalid();
DestReg = Result.Low; ///< Start mixing with the low register.
MaskReg = Mask.Low; ///< Mask starts with the low mask here.
MaskAllOnes = MaskAllOnesLow;
IndexElementOffset = 0;
DataElementOffset = IR::NumElements(OpSize::i128Bit, ElementLoadSize) / 2;
}
///< Calculate the high-half.
auto ResultHigh =
_VLoadVectorGatherMasked(OpSize::i128Bit, ElementLoadSize, DestReg, MaskReg, BaseAddr, AddrAddressing.Low, AddrAddressing.High,
AddrElementSize, VSIB.Scale, DataElementOffset, IndexElementOffset, AddrSize, MaskAllOnes);
auto ResultHigh = _VLoadVectorGatherMasked(OpSize::i128Bit, ElementLoadSize, DestReg, MaskReg, BaseAddr, AddrAddressing.Low,
AddrAddressing.High, AddrElementSize, VSIB.Scale, DataElementOffset, IndexElementOffset, AddrSize);
if (AddrElementSize == OpSize::i64Bit && ElementLoadSize == OpSize::i32Bit) {
// If we only fetched 128-bits worth of data then the upper-result is all zero.
@@ -2156,7 +2123,7 @@ OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherImpl(OpcodeArgs, Op
return Result;
}
OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherQPSImpl(OpcodeArgs, Ref Dest, Ref Mask, RefVSIB VSIB, bool MaskAllOnes) {
OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherQPSImpl(OpcodeArgs, Ref Dest, Ref Mask, RefVSIB VSIB) {
///< BaseAddr doesn't need to exist, calculate that here.
Ref BaseAddr = VSIB.BaseAddr;
@@ -2170,7 +2137,7 @@ OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherQPSImpl(OpcodeArgs,
bool NeedsSVEScale = (VSIB.Scale == 2 || VSIB.Scale == 8) || (BaseAddr == Invalid() && VSIB.Scale != 1);
if (CTX->HostFeatures.SupportsSVE128() && NeedsSVEScale) {
if (CTX->HostFeatures.SupportsSVE128 && NeedsSVEScale) {
// SVE gather instructions don't support scaling their vector elements by anything other than 1 or the address element size.
// Pre-scale 64-bit addresses in the case that scale doesn't match in-order to hit SVE code paths more frequently.
// Only hit this path if the host supports SVE. Otherwise it's a degradation for the ASIMD codepath.
@@ -2188,8 +2155,7 @@ OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherQPSImpl(OpcodeArgs,
auto AddrSize = (Op->Flags & X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) != 0 ? (GPRSize >> 1) : GPRSize;
///< Calculate the low-half.
Result.Low =
_VLoadVectorGatherMaskedQPS(OpSize::i128Bit, OpSize::i32Bit, Dest, Mask, BaseAddr, VSIB.Low, VSIB.High, VSIB.Scale, AddrSize, MaskAllOnes);
Result.Low = _VLoadVectorGatherMaskedQPS(OpSize::i128Bit, OpSize::i32Bit, Dest, Mask, BaseAddr, VSIB.Low, VSIB.High, VSIB.Scale, AddrSize);
Result.High = LoadZeroVector(OpSize::i128Bit);
if (VSIB.High == Invalid()) {
// Special case for only loading two floats.
@@ -2218,25 +2184,10 @@ void OpDispatchBuilder::AVX128_VPGATHER(OpcodeArgs, OpSize AddrElementSize) {
auto VSIB = AVX128_LoadVSIB(Op, Op->Src[0], Op->Flags, NeedsHighAddrBytes);
auto Mask = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, !Is128Bit);
// Check if the mask was computed from a self comparison, which indicates an all-ones mask.
// This is a compiler idiom for an unmasked gather. In this case, we can apply a peephole and skip
// all mask checks in the codegen, reducing codesize substantially.
bool MaskAllOnesLow = false;
bool MaskAllOnesHigh = false;
if (Op->Src[1].IsGPR()) {
const auto MaskXMM = Op->Src[1].Data.GPR.GPR - X86State::REG_XMM_0;
const Ref LowDef = RegCache.LastDef[FPR0Index + MaskXMM];
const Ref HighDef = RegCache.LastDef[AVXHigh0Index + MaskXMM];
MaskAllOnesLow = LowDef && IsValueVectorAllOnes(WrapNode(LowDef));
MaskAllOnesHigh = !Is128Bit && HighDef && IsValueVectorAllOnes(WrapNode(HighDef));
}
bool NeedsSVEScale = (VSIB.Scale == 2 || VSIB.Scale == 8) || (VSIB.BaseAddr == Invalid() && VSIB.Scale != 1);
const bool NeedsExplicitSVEPath =
CTX->HostFeatures.SupportsSVE128() && AddrElementSize == OpSize::i32Bit && ElementLoadSize == OpSize::i32Bit && NeedsSVEScale;
CTX->HostFeatures.SupportsSVE128 && AddrElementSize == OpSize::i32Bit && ElementLoadSize == OpSize::i32Bit && NeedsSVEScale;
RefPair Result {};
if (NeedsExplicitSVEPath) {
@@ -2263,15 +2214,15 @@ void OpDispatchBuilder::AVX128_VPGATHER(OpcodeArgs, OpSize AddrElementSize) {
}
///< AddressElementSize is now OpSize::i64Bit
Result = AVX128_VPGatherQPSImpl(Op, Dest.Low, Mask.Low, VSIBLow, MaskAllOnesLow);
Result = AVX128_VPGatherQPSImpl(Op, Dest.Low, Mask.Low, VSIBLow);
if (NeedsHighAddrBytes) {
auto Res = AVX128_VPGatherQPSImpl(Op, Dest.High, Mask.High, VSIBHigh, MaskAllOnesHigh);
auto Res = AVX128_VPGatherQPSImpl(Op, Dest.High, Mask.High, VSIBHigh);
Result.High = Res.Low;
}
} else if (AddrElementSize == OpSize::i64Bit && ElementLoadSize == OpSize::i32Bit) {
Result = AVX128_VPGatherQPSImpl(Op, Dest.Low, Mask.Low, VSIB, MaskAllOnesLow);
Result = AVX128_VPGatherQPSImpl(Op, Dest.Low, Mask.Low, VSIB);
} else {
Result = AVX128_VPGatherImpl(Op, Size, ElementLoadSize, AddrElementSize, Dest, Mask, VSIB, MaskAllOnesLow, MaskAllOnesHigh);
Result = AVX128_VPGatherImpl(Op, Size, ElementLoadSize, AddrElementSize, Dest, Mask, VSIB);
}
AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);
@@ -2355,124 +2306,4 @@ void OpDispatchBuilder::AVX128_VCVTPS2PH(OpcodeArgs) {
}
}
void OpDispatchBuilder::AVX128_VBCSTNEBF162PS(OpcodeArgs) {
const auto DstSize = GetDstSize(Op);
const auto Is128Bit = DstSize == Core::CPUState::XMM_SSE_REG_SIZE;
auto Src1 = LoadSourceFPR_WithOpSize(Op, Op->Src[0], OpSize::i16Bit, Op->Flags);
// bfloat16 is converted to fp32 and broadcast through the vector.
// bfloat16->float32 is left-shift by 16.
Src1 = _VShlI(OpSize::i128Bit, OpSize::i32Bit, Src1, 16);
Src1 = _VDupElement(OpSize::i128Bit, OpSize::i32Bit, Src1, 0);
RefPair Result {};
Result.Low = Src1;
if (Is128Bit) {
Result.High = LoadZeroVector(OpSize::i128Bit);
} else {
Result.High = Result.Low;
}
AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);
}
void OpDispatchBuilder::AVX128_VBCSTNESH2PS(OpcodeArgs) {
const auto DstSize = GetDstSize(Op);
const auto Is128Bit = DstSize == Core::CPUState::XMM_SSE_REG_SIZE;
auto Src1 = LoadSourceFPR_WithOpSize(Op, Op->Src[0], OpSize::i16Bit, Op->Flags);
// float16 is converted to fp32 and broadcast through the vector.
Src1 = _Float_FToF(OpSize::i32Bit, OpSize::i16Bit, Src1);
Src1 = _VDupElement(OpSize::i128Bit, OpSize::i32Bit, Src1, 0);
RefPair Result {};
Result.Low = Src1;
if (Is128Bit) {
Result.High = LoadZeroVector(OpSize::i128Bit);
} else {
Result.High = Result.Low;
}
AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);
}
void OpDispatchBuilder::AVX128_VCVTNEEBF162PS(OpcodeArgs, bool Even) {
const auto DstSize = GetDstSize(Op);
const auto Is128Bit = DstSize == Core::CPUState::XMM_SSE_REG_SIZE;
auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);
RefPair Result {};
auto Convert = [&](auto Src, bool Even) -> Ref {
// Even or odd elements of each 128-bit register is converted from bfloat16 to float32.
// For even, just left shift the elements.
if (Even) {
return _VShlI(OpSize::i128Bit, OpSize::i32Bit, Src, 16);
}
// For Odds, we just need to mask off the lower 16-bits of each 32-bit element.
// Easiest way to do this is with a right then left shift.
auto Tmp = _VUShrI(OpSize::i128Bit, OpSize::i32Bit, Src, 16);
return _VShlI(OpSize::i128Bit, OpSize::i32Bit, Tmp, 16);
};
Result.Low = Convert(Src.Low, Even);
if (Is128Bit) {
Result.High = LoadZeroVector(OpSize::i128Bit);
} else {
Result.High = Convert(Src.High, Even);
}
AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);
}
void OpDispatchBuilder::AVX128_VCVTNEEPH2PS(OpcodeArgs, bool Even) {
const auto DstSize = GetDstSize(Op);
const auto Is128Bit = DstSize == Core::CPUState::XMM_SSE_REG_SIZE;
auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);
RefPair Result {};
auto Convert = [&](auto Src, bool Even) -> Ref {
// Transpose even or odd elements and then fcvtl.
if (Even) {
Src = _VUnZip(OpSize::i128Bit, OpSize::i16Bit, Src, Src);
} else {
Src = _VUnZip2(OpSize::i128Bit, OpSize::i16Bit, Src, Src);
}
return _Vector_FToF(OpSize::i128Bit, OpSize::i32Bit, Src, OpSize::i16Bit);
};
Result.Low = Convert(Src.Low, Even);
if (Is128Bit) {
Result.High = LoadZeroVector(OpSize::i128Bit);
} else {
Result.High = Convert(Src.High, Even);
}
AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);
}
void OpDispatchBuilder::AVX128_VCVTNEPS2BF16(OpcodeArgs) {
const auto SrcSize = OpSizeFromSrc(Op);
const auto Is128BitSrc = SrcSize == OpSize::i128Bit;
auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128BitSrc);
// Convert 4 or 8 32-bits floats in to bfloat16.
// This instruction only operates with rounding mode set to `Rounds to Nearest (even)`.
RefPair Result {};
Result.Low = _VBFCVTN(OpSize::i128Bit, Src.Low);
if (!Is128BitSrc) {
Result.Low = _VBFCVTN2(OpSize::i128Bit, Result.Low, Src.High);
}
Result.High = LoadZeroVector(OpSize::i128Bit);
AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);
}
} // namespace FEXCore::IR
@@ -5,30 +5,21 @@
namespace FEXCore::IR {
constexpr inline DispatchTableEntry OpDispatch_BaseOpTable[] = {
// Instructions
{0x00, 4, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_ADD, FEXCore::IR::IROps::OP_ATOMICFETCHADD, 0, false>},
{0x04, 2, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_ADD, FEXCore::IR::IROps::OP_ATOMICFETCHADD, 0, true>},
{0x00, 6, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_ADD, FEXCore::IR::IROps::OP_ATOMICFETCHADD, 0>},
{0x08, 4, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_OR, FEXCore::IR::IROps::OP_ATOMICFETCHOR, 0, false>},
{0x0c, 2, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_OR, FEXCore::IR::IROps::OP_ATOMICFETCHOR, 0, true>},
{0x08, 6, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_OR, FEXCore::IR::IROps::OP_ATOMICFETCHOR, 0>},
{0x10, 4, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ADCOp, 0, false>},
{0x14, 2, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ADCOp, 0, true>},
{0x10, 6, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ADCOp, 0>},
{0x18, 4, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SBBOp, 0, false>},
{0x1c, 2, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SBBOp, 0, true>},
{0x18, 6, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SBBOp, 0>},
{0x20, 4, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_ANDWITHFLAGS, FEXCore::IR::IROps::OP_ATOMICFETCHAND, 0, false>},
{0x24, 2, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_ANDWITHFLAGS, FEXCore::IR::IROps::OP_ATOMICFETCHAND, 0, true>},
{0x20, 6, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_ANDWITHFLAGS, FEXCore::IR::IROps::OP_ATOMICFETCHAND, 0>},
{0x28, 4, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_SUB, FEXCore::IR::IROps::OP_ATOMICFETCHSUB, 0, false>},
{0x2c, 2, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_SUB, FEXCore::IR::IROps::OP_ATOMICFETCHSUB, 0, true>},
{0x28, 6, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_SUB, FEXCore::IR::IROps::OP_ATOMICFETCHSUB, 0>},
{0x30, 4, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_XOR, FEXCore::IR::IROps::OP_ATOMICFETCHXOR, 0, false>},
{0x34, 2, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_XOR, FEXCore::IR::IROps::OP_ATOMICFETCHXOR, 0, true>},
{0x38, 4, &OpDispatchBuilder::Bind<&OpDispatchBuilder::CMPOp, 0, false>},
{0x3c, 2, &OpDispatchBuilder::Bind<&OpDispatchBuilder::CMPOp, 0, true>},
{0x30, 6, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_XOR, FEXCore::IR::IROps::OP_ATOMICFETCHXOR, 0>},
{0x38, 6, &OpDispatchBuilder::Bind<&OpDispatchBuilder::CMPOp, 0>},
{0x50, 8, &OpDispatchBuilder::PUSHREGOp},
{0x58, 8, &OpDispatchBuilder::POPOp},
{0x68, 1, &OpDispatchBuilder::PUSHOp},
@@ -38,7 +29,7 @@ constexpr inline DispatchTableEntry OpDispatch_BaseOpTable[] = {
{0x6C, 4, &OpDispatchBuilder::PermissionRestrictedOp},
{0x70, 16, &OpDispatchBuilder::CondJUMPOp},
{0x84, 2, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 0, false>},
{0x84, 2, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 0>},
{0x86, 2, &OpDispatchBuilder::XCHGOp},
{0x88, 4, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVGPROp, 0>},
@@ -46,7 +37,7 @@ constexpr inline DispatchTableEntry OpDispatch_BaseOpTable[] = {
{0x8D, 1, &OpDispatchBuilder::LEAOp},
{0x8E, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVSegOp, true>},
{0x8F, 1, &OpDispatchBuilder::POPOp},
{0x90, 8, &OpDispatchBuilder::XCHGRAXOp},
{0x90, 8, &OpDispatchBuilder::XCHGOp},
{0x98, 1, &OpDispatchBuilder::CDQOp},
{0x99, 1, &OpDispatchBuilder::CQOOp},
@@ -58,7 +49,7 @@ constexpr inline DispatchTableEntry OpDispatch_BaseOpTable[] = {
{0xA4, 2, &OpDispatchBuilder::MOVSOp},
{0xA6, 2, &OpDispatchBuilder::CMPSOp},
{0xA8, 2, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 0, true>},
{0xA8, 2, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 0>},
{0xAA, 2, &OpDispatchBuilder::STOSOp},
{0xAC, 2, &OpDispatchBuilder::LODSOp},
{0xAE, 2, &OpDispatchBuilder::SCASOp},
@@ -27,7 +27,7 @@ void OpDispatchBuilder::SHA1NEXTEOp(OpcodeArgs) {
Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
Ref Result {};
if (CTX->HostFeatures.SupportsSVE128()) {
if (CTX->HostFeatures.SupportsSVE128) {
auto ZeroVec = LoadZeroVector(OpSize::i128Bit);
auto Tmp = _VInsElement(OpSize::i128Bit, OpSize::i32Bit, 3, 3, ZeroVec, Dest);
auto Xar = _VXar(OpSize::i128Bit, OpSize::i32Bit, ZeroVec, Tmp, 2);
@@ -158,6 +158,11 @@ void OpDispatchBuilder::SHA256RNDS2Op(OpcodeArgs) {
UnimplementedOp(Op);
return;
}
Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);
Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
// Hardcoded to XMM0
auto XMM0 = LoadXMMRegister(0);
auto shuffle_abcd = [this](Ref Src1, Ref Src2) -> Ref {
// Generates a suitable SHA256 `abcd` configuration from x86 format.
auto Tmp = _VZip2(OpSize::i128Bit, OpSize::i64Bit, Src2, Src1);
@@ -170,99 +175,14 @@ void OpDispatchBuilder::SHA256RNDS2Op(OpcodeArgs) {
return _VRev64(OpSize::i128Bit, OpSize::i32Bit, Tmp);
};
// Peephole: Attempt to identify this pattern:
// sha256rnds2 xmm1, xmm2
// pshufd xmm0, xmm0, 0xe (or psrldq xmm0, 8)
// sha256rnds2 xmm2, xmm1
// This is an idiom used by various crypto libraries (OpenSSL etc).
// Since the ARM64 SHA256 instructions compute all four rounds,
// we can elide the second SHA256RNDS2 and the key shuffle.
// Since this matching is done on IR nodes, this is somewhat fragile,
// since changes to the translation of the constitiuent instructions
// would break the pattern.
//
// NB: xmm1 and xmm2 are used here as examples, but note that
// xmm0 is hardcoded as the key register in SHA256RNDS2.
if (Op->Dest.IsGPR() && Op->Src[0].IsGPR()) {
// xmm1 in our example above.
const auto X = Op->Src[0].Data.GPR.GPR - X86State::REG_XMM_0;
// xmm2 in our example above.
const auto Y = Op->Dest.Data.GPR.GPR - X86State::REG_XMM_0;
const int KeyIndex = FPR0Index;
const int XIndex = FPR0Index + X;
const int YIndex = FPR0Index + Y;
// It is helpful to remember that the current Op
// is the "second" SHA256RNDS2 in the example, so we are looking
// backwards to find the rest of the pattern.
const Ref XLastWritten = RegCache.LastDef[XIndex];
const auto XLastWrittenIndex = RegCache.LastDefIndex[XIndex];
const Ref KeyLastWritten = RegCache.LastDef[KeyIndex];
// xmm0 and X must have been written to in in this block.
if (!XLastWritten || !KeyLastWritten) {
goto no_peephole;
}
// True if no instruction wrote Y after the first SHA256RNDS2.
const bool YUnchanged = !RegCache.LastDef[YIndex] || RegCache.LastDefIndex[YIndex] < XLastWrittenIndex;
// True if the key shift is after the first SHA256RNDS2, and no instruction wrote XMM0 afterwards.
const bool KeyUnchanged = RegCache.LastDefIndex[KeyIndex] > XLastWrittenIndex &&
(!RegCache.PrevDef[KeyIndex] || RegCache.PrevDefIndex[KeyIndex] < XLastWrittenIndex);
if (!YUnchanged || !KeyUnchanged) {
goto no_peephole;
}
// Now make sure that the op that wrote the key is what we expect.
// This should match the translation of a `pshufd xmm0, xmm0, 0xe` or `psrldq xmm0, 8` instruction.
const auto KeyShift = GetOpHeader(WrapNode(KeyLastWritten));
const bool IsVExtr = KeyShift->Op == OP_VEXTR && KeyShift->Size == OpSize::i128Bit;
if (!IsVExtr) {
goto no_peephole;
}
// Compute shift in bytes, since pshufd and psrldq use different element sizes.
const auto ShiftInBytes = KeyShift->C<IR::IROp_VExtr>()->Index * IR::OpSizeToSize(KeyShift->ElementSize);
const bool IsKeyShift = ShiftInBytes == 8 && IsValueLoadRegister(KeyShift->C<IR::IROp_VExtr>()->VectorUpper, RegClass::FPR, 0);
if (!IsKeyShift) {
goto no_peephole;
}
// Now check that the rest of the OPs match the pattern.
const Ref FirstZip = GetArgIfOp(XLastWritten, OP_VREV64, 0);
// A and B are the SHA256H and SHA256H2 output of the first SHA256RNDS2 instruction
const Ref A = GetArgIfOp(FirstZip, OP_VZIP2, 1);
const Ref B = GetArgIfOp(FirstZip, OP_VZIP2, 0);
// Figure out based on A what the source of the first SHA256RNDS2 was.
const Ref FirstSrc = GetArgIfOp(GetArgIfOp(GetArgIfOp(A, OP_VSHA256H, 0), OP_VREV64, 0), OP_VZIP2, 0);
// True if X holds the result of the first SHA256RNDS2, and its source register was Y.
const bool IsFirstResult = FirstSrc && IsValueLoadRegister(WrapNode(FirstSrc), RegClass::FPR, Y);
if (!IsFirstResult) {
goto no_peephole;
}
// The pattern matched, so we can elide the second SHA256RNDS2,
// and instead use the result of the first pair of SHA256H and SHA256H2 instructions
// to populate the final result.
StoreResult_WithAVXInsert(VectorOpType::SSE, RegClass::FPR, Op, shuffle_efgh(A, B));
return;
}
no_peephole:
Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);
Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
// Hardcoded to XMM0
auto XMM0 = LoadXMMRegister(0);
auto ABCD = shuffle_abcd(Dest, Src);
auto EFGH = shuffle_efgh(Dest, Src);
// x86 uses only the lower 64 bits of the key for two rounds.
// ARM64 does four rounds, so in order to support the peephole optimization,
// we use the full key always.
auto A = _VSha256H(ABCD, EFGH, XMM0);
auto B = _VSha256H2(EFGH, ABCD, XMM0);
// x86 uses only the bottom 64-bits of the key, so duplicate to match ARM64 semantics.
auto Key = _VDupElement(OpSize::i128Bit, OpSize::i64Bit, XMM0, 0);
auto A = _VSha256H(ABCD, EFGH, Key);
auto B = _VSha256H2(EFGH, ABCD, Key);
auto Result = shuffle_abcd(A, B);
StoreResult_WithAVXInsert(VectorOpType::SSE, RegClass::FPR, Op, Result);
@@ -9,36 +9,36 @@ constexpr DispatchTableEntry OpDispatch_PrimaryGroupTables[] = {
// GROUP 1
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 0), 1, &OpDispatchBuilder::SecondaryALUOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 1), 1, &OpDispatchBuilder::SecondaryALUOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ADCOp, 1, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SBBOp, 1, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ADCOp, 1>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SBBOp, 1>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 4), 1, &OpDispatchBuilder::SecondaryALUOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 5), 1, &OpDispatchBuilder::SecondaryALUOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 6), 1, &OpDispatchBuilder::SecondaryALUOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::CMPOp, 1, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::CMPOp, 1>}, // CMP
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 0), 1, &OpDispatchBuilder::SecondaryALUOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 1), 1, &OpDispatchBuilder::SecondaryALUOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ADCOp, 1, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SBBOp, 1, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ADCOp, 1>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SBBOp, 1>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 4), 1, &OpDispatchBuilder::SecondaryALUOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 5), 1, &OpDispatchBuilder::SecondaryALUOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 6), 1, &OpDispatchBuilder::SecondaryALUOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::CMPOp, 1, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::CMPOp, 1>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 0), 1, &OpDispatchBuilder::SecondaryALUOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 1), 1, &OpDispatchBuilder::SecondaryALUOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ADCOp, 1, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SBBOp, 1, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ADCOp, 1>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SBBOp, 1>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 4), 1, &OpDispatchBuilder::SecondaryALUOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 5), 1, &OpDispatchBuilder::SecondaryALUOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 6), 1, &OpDispatchBuilder::SecondaryALUOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::CMPOp, 1, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::CMPOp, 1>},
// GROUP 2
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, true, true, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, false, true, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RCLOp, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RCROp, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 2), 1, &OpDispatchBuilder::RCLOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 3), 1, &OpDispatchBuilder::RCROp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 4), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SHLImmediateOp, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 5), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SHRImmediateOp, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SHLImmediateOp, false>}, // SAL
@@ -46,8 +46,8 @@ constexpr DispatchTableEntry OpDispatch_PrimaryGroupTables[] = {
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, true, true, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, false, true, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RCLOp, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RCROp, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 2), 1, &OpDispatchBuilder::RCLOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 3), 1, &OpDispatchBuilder::RCROp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 4), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SHLImmediateOp, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 5), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SHRImmediateOp, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SHLImmediateOp, false>}, // SAL
@@ -73,8 +73,8 @@ constexpr DispatchTableEntry OpDispatch_PrimaryGroupTables[] = {
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, true, false, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, false, false, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RCLSmallerOp, true>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RCRSmallerOp, true>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 2), 1, &OpDispatchBuilder::RCLSmallerOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 3), 1, &OpDispatchBuilder::RCRSmallerOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 4), 1, &OpDispatchBuilder::SHLOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 5), 1, &OpDispatchBuilder::SHROp}, // SHR by CL
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 6), 1, &OpDispatchBuilder::SHLOp}, // SAL
@@ -82,16 +82,16 @@ constexpr DispatchTableEntry OpDispatch_PrimaryGroupTables[] = {
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, true, false, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, false, false, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RCLOp, true>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RCROp, true>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 2), 1, &OpDispatchBuilder::RCLOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 3), 1, &OpDispatchBuilder::RCROp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 4), 1, &OpDispatchBuilder::SHLOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 5), 1, &OpDispatchBuilder::SHROp}, // SHR by CL
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 6), 1, &OpDispatchBuilder::SHLOp}, // SAL
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ASHROp, false, false>}, // SAR
// GROUP 3
{OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF6), 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 1, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF6), 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 1, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF6), 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 1>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF6), 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 1>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF6), 2), 1, &OpDispatchBuilder::NOTOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF6), 3), 1, &OpDispatchBuilder::NEGOp}, // NEG
{OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF6), 4), 1, &OpDispatchBuilder::MULOp},
@@ -99,8 +99,8 @@ constexpr DispatchTableEntry OpDispatch_PrimaryGroupTables[] = {
{OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF6), 6), 1, &OpDispatchBuilder::DIVOp}, // DIV
{OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF6), 7), 1, &OpDispatchBuilder::IDIVOp}, // IDIV
{OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF7), 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 1, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF7), 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 1, false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF7), 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 1>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF7), 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 1>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF7), 2), 1, &OpDispatchBuilder::NOTOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF7), 3), 1, &OpDispatchBuilder::NEGOp}, // NEG
@@ -10,11 +10,6 @@ constexpr uint16_t PF_66 = 2;
constexpr uint16_t PF_F2 = 3;
constexpr DispatchTableEntry OpDispatch_SecondaryGroupTables[] = {
// GROUP 6
{OPD(FEXCore::X86Tables::TYPE_GROUP_6, PF_NONE, 0), 1, &OpDispatchBuilder::SLDTOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_6, PF_F3, 0), 1, &OpDispatchBuilder::SLDTOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_6, PF_66, 0), 1, &OpDispatchBuilder::SLDTOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_6, PF_F2, 0), 1, &OpDispatchBuilder::SLDTOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_6, PF_NONE, 3), 1, &OpDispatchBuilder::PermissionRestrictedOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_6, PF_F3, 3), 1, &OpDispatchBuilder::PermissionRestrictedOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_6, PF_66, 3), 1, &OpDispatchBuilder::PermissionRestrictedOp},
@@ -10,17 +10,14 @@ constexpr DispatchTableEntry OpDispatch_SecondaryModRMTables[] = {
// REG /2
{((1 << 3) | 0), 1, &OpDispatchBuilder::XGetBVOp},
{((1 << 3) | 1), 1, &OpDispatchBuilder::PermissionRestrictedOp},
// REG /3
{((2 << 3) | 7), 1, &OpDispatchBuilder::PermissionRestrictedOp},
// REG /5
{((3 << 3) | 0), 1, &OpDispatchBuilder::SerializeOp},
// REG /7
{((4 << 3) | 1), 1, &OpDispatchBuilder::RDTSCPOp},
{((4 << 3) | 4), 1, &OpDispatchBuilder::CLZeroOp},
{((3 << 3) | 0), 1, &OpDispatchBuilder::PermissionRestrictedOp},
{((3 << 3) | 1), 1, &OpDispatchBuilder::RDTSCPOp},
{((3 << 3) | 4), 1, &OpDispatchBuilder::CLZeroOp},
};
} // namespace FEXCore::IR
@@ -8,7 +8,6 @@ $end_info$
#include "Interface/Context/Context.h"
#include "Interface/Core/OpcodeDispatcher.h"
#include "Interface/Core/Interpreter/Fallbacks/VectorFallbacks.h"
#include "Interface/Core/X86Tables/X86Tables.h"
#include "Interface/IR/IR.h"
@@ -3025,7 +3024,7 @@ void OpDispatchBuilder::XSaveOpImpl(OpcodeArgs) {
StoreIfFlagSet(1, [this, Op] { SaveSSEState(XSaveBase(Op)); });
}
// AVX
if (CTX->HostFeatures.SupportsAVX()) {
if (CTX->HostFeatures.SupportsAVX) {
StoreIfFlagSet(2, [this, Op] { std::invoke(SaveAVXStateFunc, this, XSaveBase(Op)); });
}
@@ -3072,12 +3071,6 @@ void OpDispatchBuilder::SaveX87State(OpcodeArgs, Ref MemBase) {
{ _StoreMemGPR(OpSize::i16Bit, ReconstructFSW_Helper(), MemBase, Constant(2), OpSize::i16Bit, MemOffsetType::SXTX, 1); }
{
auto FIPSize = Op->Flags & X86Tables::DecodeFlags::FLAG_REX_WIDENING ? OpSize::i64Bit : OpSize::i32Bit;
auto FIP = _LoadContextGPR(OpSize::i64Bit, offsetof(FEXCore::Core::CPUState, FIP));
_StoreMemGPR(FIPSize, FIP, MemBase, Constant(8), FIPSize, MemOffsetType::SXTX, 1);
}
{
// Abridged FTW
auto FTW = _LoadContextGPR(OpSize::i8Bit, offsetof(FEXCore::Core::CPUState, AbridgedFTW));
@@ -3179,7 +3172,7 @@ Ref OpDispatchBuilder::GetMXCSR() {
void OpDispatchBuilder::FXRStoreOp(OpcodeArgs) {
Ref Mem = MakeSegmentAddress(Op, Op->Src[0]);
RestoreX87State(Op, Mem);
RestoreX87State(Mem);
RestoreSSEState(Mem);
Ref MXCSR = _LoadMemGPR(OpSize::i32Bit, Mem, Constant(24), OpSize::i32Bit, MemOffsetType::SXTX, 1);
@@ -3225,14 +3218,14 @@ void OpDispatchBuilder::XRstorOpImpl(OpcodeArgs) {
// x87
{
RestoreIfFlagSetOrDefault(0, [this, Op] { RestoreX87State(Op, XSaveBase(Op)); }, [this, Op] { DefaultX87State(Op); });
RestoreIfFlagSetOrDefault(0, [this, Op] { RestoreX87State(XSaveBase(Op)); }, [this, Op] { DefaultX87State(Op); });
}
// SSE
{
RestoreIfFlagSetOrDefault(1, [this, Op] { RestoreSSEState(XSaveBase(Op)); }, [this] { DefaultSSEState(); });
}
// AVX
if (CTX->HostFeatures.SupportsAVX()) {
if (CTX->HostFeatures.SupportsAVX) {
RestoreIfFlagSetOrDefault(
2, [this, Op] { std::invoke(RestoreAVXStateFunc, this, XSaveBase(Op)); }, [this] { std::invoke(DefaultAVXStateFunc, this); });
}
@@ -3250,22 +3243,15 @@ void OpDispatchBuilder::XRstorOpImpl(OpcodeArgs) {
}
}
void OpDispatchBuilder::RestoreX87State(OpcodeArgs, Ref MemBase) {
void OpDispatchBuilder::RestoreX87State(Ref MemBase) {
_StackForceSlow();
auto NewFCW = _LoadMemGPR(OpSize::i16Bit, MemBase, OpSize::i16Bit);
_StoreContextGPR(OpSize::i16Bit, NewFCW, offsetof(FEXCore::Core::CPUState, FCW));
Ref Top {};
{
auto NewFSW = _LoadMemGPR(OpSize::i16Bit, MemBase, Constant(2), OpSize::i16Bit, MemOffsetType::SXTX, 1);
Top = ReconstructX87StateFromFSW_Helper(NewFSW);
}
{
auto FIPSize = Op->Flags & X86Tables::DecodeFlags::FLAG_REX_WIDENING ? OpSize::i64Bit : OpSize::i32Bit;
auto FIP = _LoadMemGPR(FIPSize, MemBase, Constant(8), FIPSize, MemOffsetType::SXTX, 1);
_StoreContextGPR(OpSize::i64Bit, FIP, offsetof(FEXCore::Core::CPUState, FIP));
ReconstructX87StateFromFSW_Helper(NewFSW);
}
{
@@ -3274,21 +3260,10 @@ void OpDispatchBuilder::RestoreX87State(OpcodeArgs, Ref MemBase) {
_StoreContextGPR(OpSize::i8Bit, NewFTW, offsetof(FEXCore::Core::CPUState, AbridgedFTW));
}
auto SevenConst = Constant(7);
auto low = Constant(~0ULL);
auto high = Constant(0xFFFF);
Ref Mask = _VLoadTwoGPRs(low, high);
const auto StoreSize = ReducedPrecisionMode ? OpSize::i64Bit : OpSize::i128Bit;
for (uint32_t i = 0; i < Core::CPUState::NUM_MMS; ++i) {
Ref Reg = _LoadMemFPR(OpSize::i128Bit, MemBase, Constant(16 * i + 32), OpSize::i8Bit, MemOffsetType::SXTX, 1);
// Mask off the top bits
Reg = _VAnd(OpSize::i128Bit, Reg, Mask);
if (ReducedPrecisionMode) {
// Convert to double precision
Reg = _F80CVT(OpSize::i64Bit, Reg);
}
_StoreContextFPRIndexed(Reg, Top, StoreSize, MMBaseOffset(), IR::OpSizeToSize(OpSize::i128Bit));
Top = _And(OpSize::i32Bit, Add(OpSize::i32Bit, Top, 1), SevenConst);
for (uint32_t i = 0; i < Core::CPUState::NUM_MMS; i += 2) {
auto MMRegs = LoadMemPairFPR(OpSize::i128Bit, MemBase, i * 16 + 32);
_StoreContextFPR(OpSize::i128Bit, MMRegs.Low, MMBaseOffset() + i * 16);
_StoreContextFPR(OpSize::i128Bit, MMRegs.High, MMBaseOffset() + (i + 1) * 16);
}
}
@@ -3785,84 +3760,6 @@ void OpDispatchBuilder::VPMADDUBSWOp(OpcodeArgs) {
StoreResultFPR(Op, Result);
}
Ref OpDispatchBuilder::VPDPBUSDOpImpl(IR::OpSize Size, Ref Acc, Ref Src1, Ref Src2, bool Saturating) {
// Does four 8-bit unsigned * signed byte multiplies per 32-bit element, sums them and accumulates in to the destination
if (CTX->HostFeatures.SupportsI8MM) {
// The I8MM extension maps onto VPDP* almost directly.
if (!Saturating) {
return _VUSDot(Size, Acc, Src1, Src2);
}
auto DotProduct = _VUSDot(Size, LoadZeroVector(Size), Src1, Src2);
return _VSQAdd(Size, OpSize::i32Bit, Acc, DotProduct);
}
if (CTX->HostFeatures.SupportsDotProd) {
// VSDOT assumes signed input, so we need to convert Src1 to signed, and then
// perform correction afterwards using 0x40 to account for the unsigned input.
auto Src1Signed = _VXor(Size, Src1, _VectorImm(Size, OpSize::i8Bit, 0x80));
auto SixtyFour = _VectorImm(Size, OpSize::i8Bit, 0x40);
auto DotProduct = _VSDot(Size, Saturating ? LoadZeroVector(Size) : Acc, Src2, SixtyFour);
DotProduct = _VSDot(Size, DotProduct, Src2, SixtyFour);
DotProduct = _VSDot(Size, DotProduct, Src1Signed, Src2);
if (Saturating) {
return _VSQAdd(Size, OpSize::i32Bit, Acc, DotProduct);
}
return DotProduct;
}
// Naive software implementation.
auto Even = _VUnZip(Size, OpSize::i16Bit, Src1, Src2);
auto Even1_16b = _VUXTL(Size, OpSize::i8Bit, Even);
auto Even2_16b = _VSXTL2(Size, OpSize::i8Bit, Even);
auto ResMul_Even = _VMul(Size, OpSize::i16Bit, Even1_16b, Even2_16b);
auto Odd = _VUnZip2(Size, OpSize::i16Bit, Src1, Src2);
auto Odd1_16b = _VUXTL(Size, OpSize::i8Bit, Odd);
auto Odd2_16b = _VSXTL2(Size, OpSize::i8Bit, Odd);
auto ResMul_Odd = _VMul(Size, OpSize::i16Bit, Odd1_16b, Odd2_16b);
auto DotProduct = _VSAdALP(Size, OpSize::i16Bit, _VSAddLP(Size, OpSize::i16Bit, ResMul_Even), ResMul_Odd);
if (Saturating) {
return _VSQAdd(Size, OpSize::i32Bit, Acc, DotProduct);
}
return _VAdd(Size, OpSize::i32Bit, Acc, DotProduct);
}
Ref OpDispatchBuilder::VPDPWSSDOpImpl(IR::OpSize Size, Ref Acc, Ref Src1, Ref Src2, bool Saturating) {
auto DotProduct = PMADDWDOpImpl(Size, Src1, Src2);
if (!Saturating) {
return _VAdd(Size, OpSize::i32Bit, Acc, DotProduct);
}
auto NegDotProduct = _VNeg(Size, OpSize::i32Bit, DotProduct);
return _VSQSub(Size, OpSize::i32Bit, Acc, NegDotProduct);
}
void OpDispatchBuilder::VPDPBUSDOp(OpcodeArgs, bool Saturating) {
const auto Size = OpSizeFromSrc(Op);
Ref Acc = LoadSourceFPR(Op, Op->Dest, Op->Flags);
Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);
Ref Result = VPDPBUSDOpImpl(Size, Acc, Src1, Src2, Saturating);
StoreResultFPR(Op, Result);
}
void OpDispatchBuilder::VPDPWSSDOp(OpcodeArgs, bool Saturating) {
const auto Size = OpSizeFromSrc(Op);
Ref Acc = LoadSourceFPR(Op, Op->Dest, Op->Flags);
Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);
Ref Result = VPDPWSSDOpImpl(Size, Acc, Src1, Src2, Saturating);
StoreResultFPR(Op, Result);
}
Ref OpDispatchBuilder::PMULHWOpImpl(OpcodeArgs, bool Signed, Ref Src1, Ref Src2) {
const auto Size = OpSizeFromSrc(Op);
if (Signed) {
@@ -5463,132 +5360,6 @@ void OpDispatchBuilder::VPERMILRegOp(OpcodeArgs, IR::OpSize ElementSize) {
StoreResultFPR(Op, Result);
}
// Computes the number of valid elements in a string for the explicit case,
// where "valid" means that the character is not after the Nth character in the string.
Ref OpDispatchBuilder::PCMPXSTRXSaturateExplicitLength(OpSize ElementSize, OpSize LengthSize, Ref RawLength) {
const uint32_t NumElements = IR::NumElements(OpSize::i128Bit, ElementSize);
SaveNZCV();
_SubNZCV(LengthSize, RawLength, Constant(0));
Ref AbsLength = _Neg(LengthSize, RawLength, CondClass::MI);
return _Select(OpSize::i32Bit, LengthSize, CondClass::ULT, AbsLength, Constant(NumElements), AbsLength, Constant(NumElements));
}
// Equal Any is a "is character in set" operation.
// The set of characters is passed in Src1, and we then check if every
// character in Src2 is in that set.
Ref OpDispatchBuilder::PCMPXSTRXEqualAny(OpSize ElementSize, Ref Src1, Ref Src2, Ref Src1ValidElements, Ref Src2ValidElements) {
const uint32_t NumElements = IR::NumElements(OpSize::i128Bit, ElementSize);
// First we sanitize Src1, and zero out elements after the end of the string,
// removing them from the matching set.
Ref Src1Element0 = _VDupElement(OpSize::i128Bit, ElementSize, Src1, 0);
Ref SanitizedSrc1 = _VBSL(OpSize::i128Bit, Src1ValidElements, Src1, Src1Element0);
// Now walk through every element in Src1, broadcast it into a temporary vector,
// and compare it against every element in Src2, then OR the results
// together to get the final match vector.
Ref Matches = _VCMPEQ(OpSize::i128Bit, ElementSize, Src2, Src1Element0);
for (uint32_t i = 1; i < NumElements; i++) {
Ref Src1Element = _VDupElement(OpSize::i128Bit, ElementSize, SanitizedSrc1, i);
Ref ElementMatches = _VCMPEQ(OpSize::i128Bit, ElementSize, Src2, Src1Element);
Matches = _VOr(OpSize::i128Bit, Matches, ElementMatches);
}
// Handle the case where Src1 was entirely empty, and also sanitize the results
// for any NULLs in Src2, since those don't count towards matching.
Ref Src1NotEmpty = _VDupElement(OpSize::i128Bit, ElementSize, Src1ValidElements, 0);
Matches = _VAnd(OpSize::i128Bit, Matches, Src1NotEmpty);
return _VAnd(OpSize::i128Bit, Matches, Src2ValidElements);
}
// The ranges aggregation is a weird one. It checks if every character in Src2 is
// within a character range (ie. a-z or A-Z) similar to regex.
// Ranges are passed in Src1 as pairs of characters in adjacent lanes.
Ref OpDispatchBuilder::PCMPXSTRXRanges(OpSize ElementSize, Ref Src1, Ref Src2, Ref Src1ValidElements, Ref Src2ValidElements, bool IsSigned) {
const uint32_t NumElements = IR::NumElements(OpSize::i128Bit, ElementSize);
// Signed or unsigned greater than comparison, based on the Imm8 value
const auto GreaterThan = [&](Ref Lhs, Ref Rhs) {
return IsSigned ? _VCMPGT(OpSize::i128Bit, ElementSize, Lhs, Rhs) : _VUCMPGT(OpSize::i128Bit, ElementSize, Lhs, Rhs);
};
// First we walk through the ranges from Src1, and construct
// temporary vectors to represent the lower and upper bounds
// of the comparison. Then check if both comparisons are true,
// zero out invalid lanes, and OR the result into the final result vector.
Ref Result {};
for (uint32_t i = 0; i < NumElements; i += 2) {
Ref LowerBound = _VDupElement(OpSize::i128Bit, ElementSize, Src1, i);
Ref UpperBound = _VDupElement(OpSize::i128Bit, ElementSize, Src1, i + 1);
Ref BelowLower = GreaterThan(LowerBound, Src2);
Ref AboveUpper = GreaterThan(Src2, UpperBound);
// A range is only valid if its upper bound is a valid Src1 element.
Ref RangeValid = _VDupElement(OpSize::i128Bit, ElementSize, Src1ValidElements, i + 1);
// RangeValid & ~BelowLower & ~AboveUpper
Ref InRange = _VAndn(OpSize::i128Bit, RangeValid, BelowLower);
InRange = _VAndn(OpSize::i128Bit, InRange, AboveUpper);
Result = Result ? _VOr(OpSize::i128Bit, Result, InRange) : InRange;
}
// Invalid (ie. NULL) Src2 characters don't count towards matching.
return _VAnd(OpSize::i128Bit, Result, Src2ValidElements);
}
// This aggregation is the simplest, and is the most similar
// to strcmp(). It Just compares lanewise which characters are
// equal in each string.
Ref OpDispatchBuilder::PCMPXSTRXEqualEach(OpSize ElementSize, Ref Src1, Ref Src2, Ref Src1ValidElements, Ref Src2ValidElements) {
// Elements match when both are valid and equal, or when both are invalid.
Ref Equal = _VCMPEQ(OpSize::i128Bit, ElementSize, Src1, Src2);
Ref BothValid = _VAnd(OpSize::i128Bit, Src1ValidElements, Src2ValidElements);
Ref EitherValid = _VOr(OpSize::i128Bit, Src1ValidElements, Src2ValidElements);
Ref ValidMatches = _VAnd(OpSize::i128Bit, Equal, BothValid);
return _VOrn(OpSize::i128Bit, ValidMatches, EitherValid);
}
// EqualOrdered implements a strstr()-like semantic finding all instances
// of the string Src1 in Src2.
Ref OpDispatchBuilder::PCMPXSTRXEqualOrdered(OpSize ElementSize, Ref Src1, Ref Src2, Ref Src1Length, Ref Src2Length, Ref Src1ValidElements,
Ref Indices) {
const uint32_t NumElements = IR::NumElements(OpSize::i128Bit, ElementSize);
// Broadcast needle[i] into every lane of a temp vector, then compare it to
// the haystack vector. On succesive iterations, we shift the haystack over
// by one element, and compare it against the next needle element.
// AND together the results of all comparisons, and what is left should
// be a vector where the only 'true' elements are lanes where the full
// needle was found in the haystack, or those positions after the end of the string.
Ref Result {};
for (uint32_t i = 0; i < NumElements; i++) {
Ref Needle = _VDupElement(OpSize::i128Bit, ElementSize, Src1, i);
Ref Haystack = i == 0 ? Src2 : _VExtr(OpSize::i128Bit, ElementSize, Needle, Src2, i);
Ref ElementsEqual = _VCMPEQ(OpSize::i128Bit, ElementSize, Haystack, Needle);
// TODO: I think a more optimal version of this is possible, perhaps using
// MATCH from SVE2?
Ref ElementsValid = _VDupElement(OpSize::i128Bit, ElementSize, Src1ValidElements, i);
Ref ElementsEqualValid = _VOrn(OpSize::i128Bit, ElementsEqual, ElementsValid);
// Ternary to handle the first row case
Result = Result ? _VAnd(OpSize::i128Bit, Result, ElementsEqualValid) : ElementsEqualValid;
}
// Clear positions in the result after the end of the string.
Ref FirstClearedPosition = Add(OpSize::i32Bit, _Sub(OpSize::i32Bit, Src2Length, Src1Length), 1);
FirstClearedPosition =
_Select(OpSize::i32Bit, OpSize::i32Bit, CondClass::ULT, Src2Length, Constant(NumElements), FirstClearedPosition, Constant(NumElements));
FirstClearedPosition =
_Select(OpSize::i32Bit, OpSize::i32Bit, CondClass::EQ, Src1Length, Constant(0), Constant(NumElements), FirstClearedPosition);
Ref FirstClearedPositionVector = _VDupFromGPR(OpSize::i128Bit, ElementSize, FirstClearedPosition);
Ref KeptPositions = _VCMPGT(OpSize::i128Bit, ElementSize, FirstClearedPositionVector, Indices);
return _VAnd(OpSize::i128Bit, Result, KeptPositions);
}
void OpDispatchBuilder::PCMPXSTRXOpImpl(OpcodeArgs, bool IsExplicit, bool IsMask, bool IsAVX) {
const uint16_t Control = Op->Src[1].Literal();
@@ -5601,166 +5372,84 @@ void OpDispatchBuilder::PCMPXSTRXOpImpl(OpcodeArgs, bool IsExplicit, bool IsMask
Ref Src1 = LoadSourceFPR_WithOpSize(Op, Op->Dest, OpSize::i128Bit, Op->Flags);
Ref Src2 = LoadSourceFPR_WithOpSize(Op, Op->Src[0], OpSize::i128Bit, Op->Flags, {.Align = OpSize::i8Bit});
// Parse the immediate control bits.
// See section 4.1 in Intel SDM for details.
const auto Format = static_cast<CPU::SourceData>(Control & 0b11);
const auto Aggregation = static_cast<CPU::AggregationOp>((Control >> 2) & 0b11);
const auto Polarity = static_cast<CPU::Polarity>((Control >> 4) & 0b11);
const bool OutputSelection = (Control & (1 << 6)) != 0;
// Helper constants
const bool IsWords = Format == CPU::SourceData::U16 || Format == CPU::SourceData::S16;
const bool IsSigned = Format == CPU::SourceData::S8 || Format == CPU::SourceData::S16;
const auto ElementSize = IsWords ? OpSize::i16Bit : OpSize::i8Bit;
const bool IsEqualOrdered = Aggregation == CPU::AggregationOp::EqualOrdered;
const bool NeedsSrc1ValidElements = true;
const bool NeedsSrc2ValidElements = !IsEqualOrdered || Polarity == CPU::Polarity::NegativeMasked;
const uint32_t NumElements = IR::NumElements(OpSize::i128Bit, ElementSize);
// Load a vector where each element contains the value of its own index
Ref Indices = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, IsWords ? NAMED_VECTOR_INCREMENTAL_U16_INDEX : NAMED_VECTOR_INCREMENTAL_U8_INDEX);
// Helper lambda that computes the lowest index element set "all high" in mask.
Ref VecNumElements {};
const auto LowestIndex = [&](Ref Mask) {
if (!VecNumElements) {
VecNumElements = _VectorImm(OpSize::i128Bit, ElementSize, NumElements);
}
Ref Candidates = _VBSL(OpSize::i128Bit, Mask, Indices, VecNumElements);
return _VUMinV(OpSize::i128Bit, ElementSize, Candidates);
};
Ref Src1Length {};
Ref Src2Length {};
Ref Src2LengthVector {};
Ref Src1ValidElementsMask {};
// Compute the valid element masks for the two source strings.
Ref IntermediateResult {};
if (IsExplicit) {
const auto LengthSize = OpSizeFromSrc(Op);
// Will be 4 in the absence of a REX.W bit and 8 in the presence of a REX.W bit.
//
// While the control bit immediate for the instruction itself is only ever 8 bits
// in size, we use it as a 16-bit value so that we can use the 8th bit to signify
// whether or not RAX and RDX should be interpreted as a 64-bit value.
const auto SrcSize = OpSizeFromSrc(Op);
const auto Is64Bit = SrcSize == OpSize::i64Bit;
const auto NewControl = uint16_t(Control | (uint16_t(Is64Bit) << 8));
Src1Length = PCMPXSTRXSaturateExplicitLength(ElementSize, LengthSize, LoadGPRRegister(X86State::REG_RAX));
Src2Length = PCMPXSTRXSaturateExplicitLength(ElementSize, LengthSize, LoadGPRRegister(X86State::REG_RDX));
Ref SrcRAX = LoadGPRRegister(X86State::REG_RAX);
Ref SrcRDX = LoadGPRRegister(X86State::REG_RDX);
if (NeedsSrc1ValidElements) {
Src1ValidElementsMask = _VCMPGT(OpSize::i128Bit, ElementSize, _VDupFromGPR(OpSize::i128Bit, ElementSize, Src1Length), Indices);
}
if (NeedsSrc2ValidElements) {
Src2LengthVector = _VDupFromGPR(OpSize::i128Bit, ElementSize, Src2Length);
}
IntermediateResult = _VPCMPESTRX(Src1, Src2, SrcRAX, SrcRDX, NewControl);
} else {
// Get index of the first NUL element, to determine length of input strings.
Ref Src1FirstNull = LowestIndex(_VCMPEQZ(OpSize::i128Bit, ElementSize, Src1));
Ref Src2FirstNull = LowestIndex(_VCMPEQZ(OpSize::i128Bit, ElementSize, Src2));
Src1Length = _VExtractToGPR(OpSize::i128Bit, ElementSize, Src1FirstNull, 0);
Src2Length = _VExtractToGPR(OpSize::i128Bit, ElementSize, Src2FirstNull, 0);
if (NeedsSrc1ValidElements) {
// If the index in the lane is less than the index of the first NUL, then it's a valid element.
Src1ValidElementsMask = _VCMPGT(OpSize::i128Bit, ElementSize, _VDupElement(OpSize::i128Bit, ElementSize, Src1FirstNull, 0), Indices);
}
if (NeedsSrc2ValidElements) {
Src2LengthVector = _VDupElement(OpSize::i128Bit, ElementSize, Src2FirstNull, 0);
}
IntermediateResult = _VPCMPISTRX(Src1, Src2, Control);
}
Ref Src1HasInvalidElements = Select01(OpSize::i32Bit, CondClass::ULT, Src1Length, Constant(NumElements));
Ref Src2HasInvalidElements = Select01(OpSize::i32Bit, CondClass::ULT, Src2Length, Constant(NumElements));
Ref Src2ValidElementsMask = NeedsSrc2ValidElements ? _VCMPGT(OpSize::i128Bit, ElementSize, Src2LengthVector, Indices) : nullptr;
// Perform the aggregation operations, see section 4.1.3 in the Intel SDM.
Ref Matches {};
switch (Aggregation) {
case CPU::AggregationOp::EqualAny:
Matches = PCMPXSTRXEqualAny(ElementSize, Src1, Src2, Src1ValidElementsMask, Src2ValidElementsMask);
break;
case CPU::AggregationOp::Ranges:
Matches = PCMPXSTRXRanges(ElementSize, Src1, Src2, Src1ValidElementsMask, Src2ValidElementsMask, IsSigned);
break;
case CPU::AggregationOp::EqualEach:
Matches = PCMPXSTRXEqualEach(ElementSize, Src1, Src2, Src1ValidElementsMask, Src2ValidElementsMask);
break;
case CPU::AggregationOp::EqualOrdered:
Matches = PCMPXSTRXEqualOrdered(ElementSize, Src1, Src2, Src1Length, Src2Length, Src1ValidElementsMask, Indices);
break;
}
// Invert the matches if the polarity is negative or negative masked,
// see section 4.1.4 in the Intel SDM.
if (Polarity == CPU::Polarity::Negative) {
Matches = _VNot(OpSize::i128Bit, Matches);
} else if (Polarity == CPU::Polarity::NegativeMasked) {
Matches = _VXor(OpSize::i128Bit, Matches, Src2ValidElementsMask);
}
// Compute the flags and result
Ref CF {};
Ref OF {};
// PCMPESTRM and PCMPISTRM
// These instructions return a mask of the matching elements.
if (IsMask) {
Ref Result = Matches;
// Byte/word mask rather than a bit mask.
if (OutputSelection) {
// Result is already a byte/word mask, so just compute flags.
// CF set when no elements matched
// OF is set when the first element matched.
Ref AnyMatch = _VExtractToGPR(OpSize::i128Bit, ElementSize, _VUMaxV(OpSize::i128Bit, ElementSize, Matches), 0);
CF = Select01(OpSize::i32Bit, CondClass::EQ, AnyMatch, Constant(0));
OF = _And(OpSize::i64Bit, _VExtractToGPR(OpSize::i128Bit, ElementSize, Matches, 0), Constant(1));
// For the masked variant of the instructions, if control[6] is set, then we
// need to expand the intermediate result into a byte or word mask (depending
// on data size specified in control[1]) along the entire length of XMM0,
// where set bits in the intermediate result set the corresponding entry
// in XMM0 to all 1s and unset bits set the corresponding entry to all 0s.
//
// If control[6] is not set, then we just store the intermediate result as-is
// into the least significant bits of XMM0 and zero extend it.
const auto IsExpandedMask = (Control & 0b0100'0000) != 0;
if (IsExpandedMask) {
// We need to iterate over the intermediate result and
// expand the mask into XMM0 elements.
const auto ElementSize = 1U << (Control & 1);
const auto NumElements = 16U >> (Control & 1);
Ref Result = LoadZeroVector(OpSize::i128Bit);
for (uint32_t i = 0; i < NumElements; i++) {
Ref SignBit = _Sbfe(OpSize::i64Bit, 1, i, IntermediateResult);
Result = _VInsGPR(OpSize::i128Bit, IR::SizeToOpSize(ElementSize), i, Result, SignBit);
}
if (IsAVX) {
StoreXMMRegister(0, Result);
} else {
StoreXMMRegister_WithAVXInsert(VectorOpType::SSE, 0, Result);
}
} else {
// Convert per element vector to a bit mask with one bit per lane.
Ref BitPositions = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, NAMED_VECTOR_MOVMASKB);
Ref Zero = LoadZeroVector(OpSize::i128Bit);
// We insert the intermediate result as-is.
Ref Result = _VCastFromGPR(OpSize::i128Bit, OpSize::i16Bit, IntermediateResult);
// If the elements are words, truncate to bytes first.
Result = IsWords ? _VUnZip(OpSize::i128Bit, OpSize::i8Bit, Matches, Zero) : Matches;
// Now repeat this pattern until we collapse each byte into a single bit.
Result = _VAnd(OpSize::i128Bit, Result, BitPositions);
Result = _VAddP(OpSize::i128Bit, OpSize::i8Bit, Result, Zero);
Result = _VAddP(OpSize::i128Bit, OpSize::i8Bit, Result, Zero);
Result = _VAddP(OpSize::i128Bit, OpSize::i8Bit, Result, Zero);
// Compute the flags from the bit mask in Result
Ref BitMask = _VExtractToGPR(OpSize::i128Bit, IsWords ? OpSize::i8Bit : OpSize::i16Bit, Result, 0);
CF = Select01(OpSize::i32Bit, CondClass::EQ, BitMask, Constant(0));
OF = _And(OpSize::i64Bit, BitMask, Constant(1));
if (IsAVX) {
StoreXMMRegister(0, Result);
} else {
StoreXMMRegister_WithAVXInsert(VectorOpType::SSE, 0, Result);
}
}
StoreXMMRegister_WithAVXInsert(IsAVX ? VectorOpType::AVX : VectorOpType::SSE, 0, Result);
// PCMPESTRI and PCMPISTRI
// These instructions return either the index of the first match
// or the number of elements in the string if no match.
} else {
Ref LowestMatch = _VExtractToGPR(OpSize::i128Bit, ElementSize, LowestIndex(Matches), 0);
CF = Select01(OpSize::i32Bit, CondClass::EQ, LowestMatch, Constant(NumElements));
OF = Select01(OpSize::i32Bit, CondClass::EQ, LowestMatch, Constant(0));
Ref ZeroConst = Constant(0);
Ref Result = LowestMatch;
// Most significant rather than least significant index.
if (OutputSelection) {
// The max is also 0 when nothing matched, LowestMatch tells those apart.
Ref Candidates = _VAnd(OpSize::i128Bit, Indices, Matches);
Ref HighestMatchVector = _VUMaxV(OpSize::i128Bit, ElementSize, Candidates);
Ref HighestMatch = _VExtractToGPR(OpSize::i128Bit, ElementSize, HighestMatchVector, 0);
Result = _Select(OpSize::i32Bit, OpSize::i32Bit, CondClass::EQ, LowestMatch, Constant(NumElements), Constant(NumElements), HighestMatch);
}
// For the indexed variant of the instructions, if control[6] is set, then we
// store the index of the most significant bit into ECX. If it's not set,
// then we store the least significant bit.
const auto UseMSBIndex = (Control & 0b0100'0000) != 0;
Ref ResultNoFlags = _Bfe(OpSize::i32Bit, 16, 0, IntermediateResult);
Ref IfZero = Constant(16 >> (Control & 1));
Ref IfNotZero = UseMSBIndex ? _FindMSB(IR::OpSize::i32Bit, ResultNoFlags) : _FindLSB(IR::OpSize::i32Bit, ResultNoFlags);
Ref Result = _Select(OpSize::i64Bit, OpSize::i64Bit, CondClass::EQ, ResultNoFlags, ZeroConst, IfZero, IfNotZero);
// Store the result, it is already zero-extended to 64-bit implicitly.
StoreGPRRegister(X86State::REG_RCX, Result);
}
// SF/ZF are set when Src1/Src2 respectively have invalid elements.
Ref SF = Src1HasInvalidElements;
Ref ZF = Src2HasInvalidElements;
Ref NZCV = _Orlshl(OpSize::i64Bit, _Lshl(OpSize::i64Bit, OF, Constant(28)), CF, 29);
NZCV = _Orlshl(OpSize::i64Bit, NZCV, ZF, 30);
NZCV = _Orlshl(OpSize::i64Bit, NZCV, SF, 31);
SetNZCV(NZCV);
CFInverted = true;
// Set all of the necessary flags. NZCV stored in bits 28...31 like the hw op.
SetNZCV(IntermediateResult);
CFInverted = false;
ZeroPF_AF();
}
@@ -6068,86 +5757,4 @@ void OpDispatchBuilder::Insertq(OpcodeArgs) {
StoreResult_WithAVXInsert(VectorOpType::SSE, RegClass::FPR, Op, Result);
}
void OpDispatchBuilder::AVXVBCSTNEBF162PS(OpcodeArgs) {
const auto DstSize = OpSizeFromDst(Op);
auto Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
// bfloat16 is converted to fp32 and broadcast through the vector.
// bfloat16->float32 is left-shift by 16.
Src = _VShlI(DstSize, OpSize::i32Bit, Src, 16);
Src = _VDupElement(DstSize, OpSize::i32Bit, Src, 0);
StoreResultFPR(Op, Src);
}
void OpDispatchBuilder::AVXVBCSTNESH2PS(OpcodeArgs) {
const auto DstSize = OpSizeFromDst(Op);
auto Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
// float16 is converted to fp32 and broadcast through the vector.
Src = _Float_FToF(OpSize::i32Bit, OpSize::i16Bit, Src);
Src = _VDupElement(DstSize, OpSize::i32Bit, Src, 0);
StoreResultFPR(Op, Src);
}
void OpDispatchBuilder::AVXVCVTNEEPH2PS(OpcodeArgs, bool Even) {
const auto DstSize = OpSizeFromDst(Op);
auto Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
// float16 is converted to fp32 and broadcast through the vector.
if (Even) {
Src = _VUnZip(DstSize, OpSize::i16Bit, Src, Src);
} else {
Src = _VUnZip2(DstSize, OpSize::i16Bit, Src, Src);
}
Src = _Vector_FToF(DstSize, OpSize::i32Bit, Src, OpSize::i16Bit);
StoreResultFPR(Op, Src);
}
void OpDispatchBuilder::AVXVCVTNEEBF162PS(OpcodeArgs, bool Even) {
const auto DstSize = OpSizeFromDst(Op);
auto Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
auto Convert = [&](auto Src, bool Even) -> Ref {
// Even or odd elements of each 128-bit register is converted from bfloat16 to float32.
// For even, just left shift the elements.
if (Even) {
return _VShlI(DstSize, OpSize::i32Bit, Src, 16);
}
// For Odds, we just need to mask off the lower 16-bits of each 32-bit element.
// Can't do the same movi+vbsl instruction as ASIMD. Just use shifts instead.
auto Tmp = _VUShrI(DstSize, OpSize::i32Bit, Src, 16);
return _VShlI(DstSize, OpSize::i32Bit, Tmp, 16);
};
StoreResultFPR(Op, Convert(Src, Even));
}
void OpDispatchBuilder::AVXVCVTNEPS2BF16(OpcodeArgs) {
const auto DstSize = OpSizeFromDst(Op);
const auto Is128Bit = DstSize == OpSize::i128Bit;
auto Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
// Convert 4 or 8 32-bits floats in to bfloat16.
// This instruction only operates with rounding mode set to `Rounds to Nearest (even)`.
Ref Result {};
if (Is128Bit) {
// Just do the operation in the lower half of the register.
Result = _VBFCVTN(OpSize::i128Bit, Src);
} else {
// Split the operation in to two and recombine.
Ref UpperLane = _VDupElement(OpSize::i256Bit, OpSize::i128Bit, Src, 1);
auto ResultLow = _VBFCVTN(OpSize::i128Bit, Src);
Result = _VBFCVTN2(OpSize::i128Bit, ResultLow, UpperLane);
}
StoreResultFPR(Op, Result);
}
} // namespace FEXCore::IR
@@ -67,13 +67,11 @@ void OpDispatchBuilder::FLD(OpcodeArgs, IR::OpSize Width) {
ConvertedData = _F80CVTTo(Data, Width);
}
_PushStack(ConvertedData, Data, Width);
StoreX87FIP(Op);
}
// Float LoaD operation with memory operand
void OpDispatchBuilder::FLDFromStack(OpcodeArgs) {
_CopyPushStack(Op->OP & 7);
StoreX87FIP(Op);
}
void OpDispatchBuilder::FBLD(OpcodeArgs) {
@@ -81,21 +79,18 @@ void OpDispatchBuilder::FBLD(OpcodeArgs) {
Ref Data = LoadSourceFPR_WithOpSize(Op, Op->Src[0], OpSize::f80Bit, Op->Flags);
Ref ConvertedData = _F80BCDLoad(Data);
_PushStack(ConvertedData, Invalid(), OpSize::iInvalid);
StoreX87FIP(Op);
}
void OpDispatchBuilder::FBSTP(OpcodeArgs) {
Ref converted = _F80BCDStore(_ReadStackValue(0));
StoreResultFPR_WithOpSize(Op, Op->Dest, converted, OpSize::f80Bit, OpSize::i8Bit);
_PopStackDestroy();
StoreX87FIP(Op);
}
void OpDispatchBuilder::FLD_Const(OpcodeArgs, NamedVectorConstant K) {
// Update TOP
Ref Data = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, K);
_PushStack(Data, Data, OpSize::f80Bit);
StoreX87FIP(Op);
}
void OpDispatchBuilder::FILD(OpcodeArgs) {
@@ -127,7 +122,6 @@ void OpDispatchBuilder::FILD(OpcodeArgs) {
Ref ConvertedData = _VLoadTwoGPRs(shifted, upper);
_PushStack(ConvertedData, Invalid(), OpSize::iInvalid);
StoreX87FIP(Op);
}
void OpDispatchBuilder::FST(OpcodeArgs, IR::OpSize Width) {
@@ -141,7 +135,6 @@ void OpDispatchBuilder::FST(OpcodeArgs, IR::OpSize Width) {
if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
_PopStackDestroy();
}
StoreX87FIP(Op);
}
void OpDispatchBuilder::FSTToStack(OpcodeArgs) {
@@ -151,7 +144,6 @@ void OpDispatchBuilder::FSTToStack(OpcodeArgs) {
if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
_PopStackDestroy();
}
StoreX87FIP(Op);
}
// Store integer to memory (possibly with truncation)
@@ -191,7 +183,6 @@ void OpDispatchBuilder::FIST(OpcodeArgs, bool Truncate) {
if ((Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) != 0) {
_PopStackDestroy();
}
StoreX87FIP(Op);
}
void OpDispatchBuilder::FADD(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispatchBuilder::OpResult ResInST0) {
@@ -206,7 +197,6 @@ void OpDispatchBuilder::FADD(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispa
if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
_PopStackDestroy();
}
StoreX87FIP(Op);
return;
}
@@ -223,7 +213,6 @@ void OpDispatchBuilder::FADD(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispa
// top of stack is at offset zero
_F80AddValue(0, Arg);
StoreX87FIP(Op);
}
void OpDispatchBuilder::FMUL(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispatchBuilder::OpResult ResInST0) {
@@ -238,7 +227,6 @@ void OpDispatchBuilder::FMUL(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispa
if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
_PopStackDestroy();
}
StoreX87FIP(Op);
return;
}
@@ -259,7 +247,6 @@ void OpDispatchBuilder::FMUL(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispa
if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
_PopStackDestroy();
}
StoreX87FIP(Op);
}
void OpDispatchBuilder::FDIV(OpcodeArgs, IR::OpSize Width, bool Integer, bool Reverse, OpDispatchBuilder::OpResult ResInST0) {
@@ -277,7 +264,6 @@ void OpDispatchBuilder::FDIV(OpcodeArgs, IR::OpSize Width, bool Integer, bool Re
if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
_PopStackDestroy();
}
StoreX87FIP(Op);
return;
}
@@ -302,7 +288,6 @@ void OpDispatchBuilder::FDIV(OpcodeArgs, IR::OpSize Width, bool Integer, bool Re
if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
_PopStackDestroy();
}
StoreX87FIP(Op);
}
void OpDispatchBuilder::FSUB(OpcodeArgs, IR::OpSize Width, bool Integer, bool Reverse, OpDispatchBuilder::OpResult ResInST0) {
@@ -320,7 +305,6 @@ void OpDispatchBuilder::FSUB(OpcodeArgs, IR::OpSize Width, bool Integer, bool Re
if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
_PopStackDestroy();
}
StoreX87FIP(Op);
return;
}
@@ -345,7 +329,6 @@ void OpDispatchBuilder::FSUB(OpcodeArgs, IR::OpSize Width, bool Integer, bool Re
if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
_PopStackDestroy();
}
StoreX87FIP(Op);
}
Ref OpDispatchBuilder::GetX87FTW_Helper() {
@@ -415,8 +398,7 @@ void OpDispatchBuilder::X87FNSTENV(OpcodeArgs) {
{
// Instruction Offset
auto FIP = _LoadContextGPR(OpSize::i64Bit, offsetof(FEXCore::Core::CPUState, FIP));
_StoreMemGPR(Size, FIP, Mem, Constant(IR::OpSizeToSize(Size) * 3), Size, MemOffsetType::SXTX, 1);
_StoreMemGPR(Size, ZeroConst, Mem, Constant(IR::OpSizeToSize(Size) * 3), Size, MemOffsetType::SXTX, 1);
}
{
@@ -514,8 +496,7 @@ void OpDispatchBuilder::X87FNSAVE(OpcodeArgs) {
{
// Instruction Offset
auto FIP = _LoadContextGPR(OpSize::i64Bit, offsetof(FEXCore::Core::CPUState, FIP));
_StoreMemGPR(Size, FIP, Mem, Constant(IR::OpSizeToSize(Size) * 3), Size, MemOffsetType::SXTX, 1);
_StoreMemGPR(Size, ZeroConst, Mem, Constant(IR::OpSizeToSize(Size) * 3), Size, MemOffsetType::SXTX, 1);
}
{
@@ -585,12 +566,6 @@ void OpDispatchBuilder::X87FRSTOR(OpcodeArgs) {
SetX87FTW(_LoadMemGPR(Size, Mem, Constant(IR::OpSizeToSize(Size) * 2), Size, MemOffsetType::SXTX, 1));
}
{
// FIP
auto FIP = _LoadMemGPR(Size, Mem, Constant(IR::OpSizeToSize(Size) * 3), Size, MemOffsetType::SXTX, 1);
_StoreContextGPR(OpSize::i64Bit, FIP, offsetof(FEXCore::Core::CPUState, FIP));
}
auto SevenConst = Constant(7);
auto low = Constant(~0ULL);
auto high = Constant(0xFFFF);
@@ -644,7 +619,6 @@ void OpDispatchBuilder::FXCH(OpcodeArgs) {
_F80StackXchange(Offset);
}
SetRFLAG<FEXCore::X86State::X87FLAG_C1_LOC>(Constant(0));
StoreX87FIP(Op);
}
void OpDispatchBuilder::X87FYL2X(OpcodeArgs, bool IsFYL2XP1) {
@@ -653,7 +627,6 @@ void OpDispatchBuilder::X87FYL2X(OpcodeArgs, bool IsFYL2XP1) {
} else {
_F80FYL2XStack();
}
StoreX87FIP(Op);
}
void OpDispatchBuilder::FCOMI(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispatchBuilder::FCOMIFlags WhichFlags, bool PopTwice) {
@@ -718,7 +691,6 @@ void OpDispatchBuilder::FCOMI(OpcodeArgs, IR::OpSize Width, bool Integer, OpDisp
} else if ((Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) != 0) {
_PopStackDestroy();
}
StoreX87FIP(Op);
}
void OpDispatchBuilder::FTST(OpcodeArgs) {
@@ -738,7 +710,6 @@ void OpDispatchBuilder::FTST(OpcodeArgs) {
// Set Invalid Operation flag when unordered (NaN comparison)
// The x87 exception flags are sticky. Preserve earlier result
SetRFLAG<FEXCore::X86State::X87FLAG_IE_LOC>(_Or(OpSize::i32Bit, GetRFLAG(FEXCore::X86State::X87FLAG_IE_LOC), HostFlag_Unordered));
StoreX87FIP(Op);
}
void OpDispatchBuilder::X87OpHelper(OpcodeArgs, FEXCore::IR::IROps IROp, bool ZeroC2) {
@@ -746,7 +717,6 @@ void OpDispatchBuilder::X87OpHelper(OpcodeArgs, FEXCore::IR::IROps IROp, bool Ze
if (ZeroC2) {
SetRFLAG<FEXCore::X86State::X87FLAG_C2_LOC>(Constant(0));
}
StoreX87FIP(Op);
}
void OpDispatchBuilder::X87ModifySTP(OpcodeArgs, bool Inc) {
@@ -758,7 +728,6 @@ void OpDispatchBuilder::X87ModifySTP(OpcodeArgs, bool Inc) {
// C1 set to 0
SetRFLAG<FEXCore::X86State::X87FLAG_C1_LOC>(Constant(0));
StoreX87FIP(Op);
}
// Operations dealing with loading and storing environment pieces
@@ -799,14 +768,10 @@ Ref OpDispatchBuilder::ReconstructFSW_Helper(Ref T) {
// Store Status Word
// There's no load Status Word instruction but you can load it through frstor
// or fldenv.
void OpDispatchBuilder::X87FNSTSW(OpcodeArgs, bool DestRAX) {
void OpDispatchBuilder::X87FNSTSW(OpcodeArgs) {
Ref TopValue = _SyncStackToSlow();
Ref StatusWord = ReconstructFSW_Helper(TopValue);
if (DestRAX) {
StoreGPRRegister(X86State::REG_RAX, StatusWord, OpSize::i16Bit);
} else {
StoreResultGPR(Op, StatusWord);
}
StoreResultGPR(Op, StatusWord);
}
void OpDispatchBuilder::FNCLEX(OpcodeArgs) {
@@ -840,20 +805,15 @@ void OpDispatchBuilder::FNINIT(OpcodeArgs) {
SetRFLAG<FEXCore::X86State::X87FLAG_C2_LOC>(Zero);
SetRFLAG<FEXCore::X86State::X87FLAG_C3_LOC>(Zero);
SetRFLAG<FEXCore::X86State::X87FLAG_IE_LOC>(Zero);
// fninit resets the FIP.
_StoreContextGPR(OpSize::i64Bit, _Constant(0), offsetof(FEXCore::Core::CPUState, FIP));
}
void OpDispatchBuilder::X87FFREE(OpcodeArgs) {
_InvalidateStack(Op->OP & 7);
StoreX87FIP(Op);
}
void OpDispatchBuilder::X87EMMS(OpcodeArgs) {
// Tags all get set to 0b11
_InvalidateStack(0xff);
StoreX87FIP(Op);
}
void OpDispatchBuilder::X87FCMOV(OpcodeArgs) {
@@ -892,112 +852,29 @@ void OpDispatchBuilder::X87FCMOV(OpcodeArgs) {
Ref VecCond = _VDupFromGPR(OpSize::i128Bit, OpSize::i64Bit, SelectCC0All1(CC));
_F80VBSLStack(OpSize::i128Bit, VecCond, Op->OP & 7, 0);
StoreX87FIP(Op);
}
void OpDispatchBuilder::X87FXAM(OpcodeArgs) {
auto a = _ReadStackValue(0);
Ref Value = ReducedPrecisionMode ? _VExtractToGPR(OpSize::i64Bit, OpSize::i64Bit, a, 0) : _VExtractToGPR(OpSize::i128Bit, OpSize::i64Bit, a, 1);
Ref Result =
ReducedPrecisionMode ? _VExtractToGPR(OpSize::i64Bit, OpSize::i64Bit, a, 0) : _VExtractToGPR(OpSize::i128Bit, OpSize::i64Bit, a, 1);
// Extract the sign bit, which goes in C1
Ref Result = ReducedPrecisionMode ? _Bfe(OpSize::i64Bit, 1, 63, Value) : _Bfe(OpSize::i64Bit, 1, 15, Value);
// Extract the sign bit
Result = ReducedPrecisionMode ? _Bfe(OpSize::i64Bit, 1, 63, Result) : _Bfe(OpSize::i64Bit, 1, 15, Result);
SetRFLAG<FEXCore::X86State::X87FLAG_C1_LOC>(Result);
auto NotEmpty = _StackValidTag(0);
Ref IsEmpty = _Xor(OpSize::i64Bit, NotEmpty, Constant(1));
Ref IsNaN {};
Ref IsDenormal {};
Ref IsInf {};
Ref IsZero {};
Ref IsUnsupported {};
Ref NoSignBit {};
// Claim this is a normal number
// We don't support anything else
auto TopValid = _StackValidTag(0);
// TODO: The codegen for this is not optimal, and can probably be improved
// if FXAM ends up on the hot path for some workload.
if (ReducedPrecisionMode) {
constexpr uint64_t ExponentMask = 0x7FF0'0000'0000'0000ULL;
NoSignBit = _Bfe(OpSize::i64Bit, 63, 0, Value);
IsInf = Select01(OpSize::i64Bit, CondClass::EQ, NoSignBit, Constant(ExponentMask));
IsNaN = Select01(OpSize::i64Bit, CondClass::UGT, NoSignBit, Constant(ExponentMask));
IsZero = Select01(OpSize::i64Bit, CondClass::EQ, NoSignBit, Constant(0));
// 64 bit floats can't represent an x87 denormal, nor any of the
// unsupported encodings.
IsDenormal = Constant(0);
IsUnsupported = Constant(0);
} else {
Ref Mantissa = _VExtractToGPR(OpSize::i128Bit, OpSize::i64Bit, a, 0);
// "J" is the name given to the msb of the mantissa in the SDM.
Ref JBit = _Bfe(OpSize::i64Bit, 1, 63, Mantissa);
Ref Exponent = _Bfe(OpSize::i64Bit, 15, 0, Value);
Ref IsExponentZero = Select01(OpSize::i64Bit, CondClass::EQ, Exponent, Constant(0));
Ref IsExponentMax = Select01(OpSize::i64Bit, CondClass::EQ, Exponent, Constant(0x7FFF));
// Inf is when mantissa only has the J bit set, exponent is all 1's.
Ref IsOnlyJBit = Select01(OpSize::i64Bit, CondClass::EQ, Mantissa, Constant(1ULL << 63));
IsInf = _And(OpSize::i64Bit, IsExponentMax, IsOnlyJBit);
// NaN is when the low 63 bits of the mantissa are non-zero
// and exponent is max, and the J bit is set.
Ref Fraction = _Bfe(OpSize::i64Bit, 63, 0, Mantissa);
Ref FractionNonZero = Select01(OpSize::i64Bit, CondClass::NEQ, Fraction, Constant(0));
Ref IsExponentMaxWithJBit = _And(OpSize::i64Bit, IsExponentMax, JBit);
IsNaN = _And(OpSize::i64Bit, IsExponentMaxWithJBit, FractionNonZero);
// Zero and Denormal are basically the same as the 64-bit case.
Ref MantissaNonZero = Select01(OpSize::i64Bit, CondClass::NEQ, Mantissa, Constant(0));
Ref MantissaZero = _Xor(OpSize::i64Bit, MantissaNonZero, Constant(1));
IsZero = _And(OpSize::i64Bit, IsExponentZero, MantissaZero);
IsDenormal = _And(OpSize::i64Bit, IsExponentZero, MantissaNonZero);
// This is where things are weird. If the J bit is not set
// and the exponent is non-zero, then this is an "unsupported"
// encoding, which I believe is left in for legacy reasons.
Ref IsSupported = _Or(OpSize::i64Bit, IsExponentZero, JBit);
IsUnsupported = _Xor(OpSize::i64Bit, IsSupported, Constant(1));
}
// NormalFiniteNumber = !Zero && !Denormal && !Inf && !NaN && !Empty && !Unsupported
Ref temp1 = _Or(OpSize::i64Bit, IsZero, IsDenormal);
Ref temp2 = _Or(OpSize::i64Bit, IsInf, IsNaN);
Ref temp3 = _Or(OpSize::i64Bit, IsUnsupported, IsEmpty);
temp1 = _Or(OpSize::i64Bit, temp1, temp2);
temp2 = _Or(OpSize::i64Bit, temp1, temp3);
Ref NormalFiniteNumber = _Xor(OpSize::i64Bit, temp2, Constant(1));
// Set C3, C2, C0 based on the class of the FP value
// Table is from "FXAM" page in the SDM.
// +----------------------+----+----+----+
// | Class | C3 | C2 | C0 |
// +----------------------+----+----+----+
// | Unsupported | 0 | 0 | 0 |
// | NaN | 0 | 0 | 1 |
// | Normal finite number | 0 | 1 | 0 |
// | Infinity | 0 | 1 | 1 |
// | Zero | 1 | 0 | 0 |
// | Empty | 1 | 0 | 1 |
// | Denormal number | 1 | 1 | 0 |
// +----------------------+----+----+----+
// C0 = IsNaN || IsInf || IsEmpty
Ref C0 = _Or(OpSize::i64Bit, IsNaN, IsInf);
C0 = _Or(OpSize::i64Bit, C0, IsEmpty);
// C2 = (IsInf || Denormal || NormalFiniteNumber) && !IsEmpty
Ref C2 = _Or(OpSize::i64Bit, IsInf, IsDenormal);
C2 = _Or(OpSize::i64Bit, C2, NormalFiniteNumber);
C2 = _And(OpSize::i64Bit, C2, NotEmpty);
// C3 = Zero || IsEmpty || Denormal
Ref C3 = _Or(OpSize::i64Bit, IsZero, IsEmpty);
C3 = _Or(OpSize::i64Bit, C3, IsDenormal);
// In the case of top being invalid then C3:C2:C0 is 0b101
auto C3 = Select01(OpSize::i32Bit, CondClass::NEQ, TopValid, Constant(1));
auto C2 = TopValid;
auto C0 = C3; // Mirror C3 until something other than zero is supported
SetRFLAG<FEXCore::X86State::X87FLAG_C0_LOC>(C0);
SetRFLAG<FEXCore::X86State::X87FLAG_C2_LOC>(C2);
SetRFLAG<FEXCore::X86State::X87FLAG_C3_LOC>(C3);
StoreX87FIP(Op);
}
void OpDispatchBuilder::X87FXTRACT(OpcodeArgs) {
@@ -1008,11 +885,6 @@ void OpDispatchBuilder::X87FXTRACT(OpcodeArgs) {
auto Sig = _F80XTRACT_SIG(Top);
_PushStack(Exp, Invalid(), OpSize::iInvalid);
_PushStack(Sig, Invalid(), OpSize::iInvalid);
StoreX87FIP(Op);
}
void OpDispatchBuilder::X87NOP(OpcodeArgs) {
StoreX87FIP(Op);
}
} // namespace FEXCore::IR
@@ -68,7 +68,6 @@ void OpDispatchBuilder::FLDF64(OpcodeArgs, IR::OpSize Width) {
ConvertedData = _F80CVT(OpSize::i64Bit, Data);
}
_PushStack(ConvertedData, Data, Width);
StoreX87FIP(Op);
}
void OpDispatchBuilder::FBLDF64(OpcodeArgs) {
@@ -77,7 +76,6 @@ void OpDispatchBuilder::FBLDF64(OpcodeArgs) {
Ref ConvertedData = _F80BCDLoad(Data);
ConvertedData = _F80CVT(OpSize::i64Bit, ConvertedData);
_PushStack(ConvertedData, Invalid(), OpSize::iInvalid);
StoreX87FIP(Op);
}
void OpDispatchBuilder::FBSTPF64(OpcodeArgs) {
@@ -85,13 +83,11 @@ void OpDispatchBuilder::FBSTPF64(OpcodeArgs) {
converted = _F80BCDStore(converted);
StoreResultFPR_WithOpSize(Op, Op->Dest, converted, OpSize::f80Bit, OpSize::i8Bit);
_PopStackDestroy();
StoreX87FIP(Op);
}
void OpDispatchBuilder::FLDF64_Const(OpcodeArgs, uint64_t Num) {
auto Data = _VCastFromGPR(OpSize::i64Bit, OpSize::i64Bit, Constant(Num));
_PushStack(Data, Data, OpSize::i64Bit);
StoreX87FIP(Op);
}
void OpDispatchBuilder::FILDF64(OpcodeArgs) {
@@ -104,7 +100,6 @@ void OpDispatchBuilder::FILDF64(OpcodeArgs) {
}
auto ConvertedData = _Float_FromGPR_S(OpSize::i64Bit, ReadWidth == OpSize::i32Bit ? OpSize::i32Bit : OpSize::i64Bit, Data);
_PushStack(ConvertedData, Invalid(), OpSize::iInvalid);
StoreX87FIP(Op);
}
void OpDispatchBuilder::FISTF64(OpcodeArgs, bool Truncate) {
@@ -133,7 +128,6 @@ void OpDispatchBuilder::FISTF64(OpcodeArgs, bool Truncate) {
if ((Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) != 0) {
_PopStackDestroy();
}
StoreX87FIP(Op);
}
void OpDispatchBuilder::FADDF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispatchBuilder::OpResult ResInST0) {
@@ -148,7 +142,6 @@ void OpDispatchBuilder::FADDF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpDi
if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
_PopStackDestroy();
}
StoreX87FIP(Op);
return;
}
@@ -172,7 +165,6 @@ void OpDispatchBuilder::FADDF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpDi
// top of stack is at offset zero
_F80AddValue(0, arg);
StoreX87FIP(Op);
}
// FIXME: following is very similar to FADDF64
@@ -188,7 +180,6 @@ void OpDispatchBuilder::FMULF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpDi
if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
_PopStackDestroy();
}
StoreX87FIP(Op);
return;
}
@@ -216,7 +207,6 @@ void OpDispatchBuilder::FMULF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpDi
if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
_PopStackDestroy();
}
StoreX87FIP(Op);
}
void OpDispatchBuilder::FDIVF64(OpcodeArgs, IR::OpSize Width, bool Integer, bool Reverse, OpDispatchBuilder::OpResult ResInST0) {
@@ -241,7 +231,6 @@ void OpDispatchBuilder::FDIVF64(OpcodeArgs, IR::OpSize Width, bool Integer, bool
if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
_PopStackDestroy();
}
StoreX87FIP(Op);
return;
}
@@ -275,7 +264,6 @@ void OpDispatchBuilder::FDIVF64(OpcodeArgs, IR::OpSize Width, bool Integer, bool
if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
_PopStackDestroy();
}
StoreX87FIP(Op);
}
void OpDispatchBuilder::FSUBF64(OpcodeArgs, IR::OpSize Width, bool Integer, bool Reverse, OpDispatchBuilder::OpResult ResInST0) {
@@ -300,8 +288,6 @@ void OpDispatchBuilder::FSUBF64(OpcodeArgs, IR::OpSize Width, bool Integer, bool
if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
_PopStackDestroy();
}
StoreX87FIP(Op);
return;
}
@@ -335,7 +321,6 @@ void OpDispatchBuilder::FSUBF64(OpcodeArgs, IR::OpSize Width, bool Integer, bool
if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
_PopStackDestroy();
}
StoreX87FIP(Op);
}
void OpDispatchBuilder::FTSTF64(OpcodeArgs) {
@@ -345,7 +330,6 @@ void OpDispatchBuilder::FTSTF64(OpcodeArgs) {
// Now we do our comparison.
_F80StackTest(0);
ConvertNZCVToX87();
StoreX87FIP(Op);
}
void OpDispatchBuilder::FCOMIF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispatchBuilder::FCOMIFlags WhichFlags, bool PopTwice) {
@@ -392,7 +376,6 @@ void OpDispatchBuilder::FCOMIF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpD
} else if ((Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) != 0) {
_PopStackDestroy();
}
StoreX87FIP(Op);
}
void OpDispatchBuilder::X87FXTRACTF64(OpcodeArgs) {
@@ -446,6 +429,5 @@ void OpDispatchBuilder::X87FXTRACTF64(OpcodeArgs) {
_PopStackDestroy();
_PushStack(Exp, Invalid(), OpSize::iInvalid);
_PushStack(Sig, Invalid(), OpSize::iInvalid);
StoreX87FIP(Op);
}
} // namespace FEXCore::IR
@@ -16,7 +16,7 @@ static constexpr size_t INITIAL_CODE_SIZE = 1024 * 1024 * 16;
// We don't want to move above 128MB atm because that means we will have to encode longer jumps
static constexpr size_t MAX_CODE_SIZE = 1024 * 1024 * 128;
CodeBuffer::CodeBuffer(size_t Size, bool ShouldBeNamed)
CodeBuffer::CodeBuffer(size_t Size)
: AllocatedSize(Size) {
Ptr = static_cast<uint8_t*>(FEXCore::Allocator::VirtualAlloc(Size, true));
LOGMAN_THROW_A_FMT(!!Ptr, "Couldn't allocate code buffer");
@@ -28,9 +28,7 @@ CodeBuffer::CodeBuffer(size_t Size, bool ShouldBeNamed)
LogMan::Msg::EFmt("Failed to mprotect last page of code buffer.");
}
if (ShouldBeNamed) {
FEXCore::Allocator::VirtualName("FEXMemJIT", Ptr, Size);
}
FEXCore::Allocator::VirtualName("FEXMemJIT", Ptr, Size);
// Huge-pages reduce the amount of iTLB misses dramatically when it works.
FEXCore::Allocator::VirtualTHPControl(Ptr, Size, FEXCore::Allocator::THPControl::Enable);
@@ -45,17 +43,6 @@ CodeBuffer::~CodeBuffer() {
FEXCore::Allocator::VirtualFree(Ptr, AllocatedSize);
}
SharedCodeBufferManager::SharedCodeBufferManager() {
FEX_CONFIG_OPT(GlobalJITNaming, GLOBALJITNAMING);
FEX_CONFIG_OPT(LibraryJITNaming, LIBRARYJITNAMING);
FEX_CONFIG_OPT(BlockJITNaming, BLOCKJITNAMING);
// Only name the JIT buffers if perf JIT naming is disabled.
// `perf top` prefers VMA names over the JIT symbols file for some reason.
// Breaks memory tracking when naming is enabled, but it's a debug feature so it isn't expected to be enabled by default.
NameJITBuffers = !(GlobalJITNaming || LibraryJITNaming || BlockJITNaming);
}
fextl::shared_ptr<CodeBuffer> SharedCodeBufferManager::AllocateNew(size_t Size) {
#ifndef _WIN32
// MDWE (Memory-Deny-Write-Execute) is a new Linux 6.3 feature.
@@ -79,7 +66,7 @@ fextl::shared_ptr<CodeBuffer> SharedCodeBufferManager::AllocateNew(size_t Size)
}
#endif
auto Buffer = fextl::make_shared<CodeBuffer>(Size, NameJITBuffers);
auto Buffer = fextl::make_shared<CodeBuffer>(Size);
Latest = Buffer;
@@ -23,7 +23,7 @@ namespace FEXCore::CPU {
struct CodeBuffer {
fextl::unique_ptr<GuestToHostMap> LookupCache;
CodeBuffer(size_t Size, bool ShouldBeNamed);
CodeBuffer(size_t Size);
CodeBuffer(const CodeBuffer&) = delete;
CodeBuffer& operator=(const CodeBuffer&) = delete;
CodeBuffer(CodeBuffer&& oth) = delete;
@@ -111,7 +111,6 @@ private:
*/
class SharedCodeBufferManager {
public:
SharedCodeBufferManager();
virtual ~SharedCodeBufferManager() = default;
// Get the CodeBuffer that was most recently allocated.
@@ -132,7 +131,5 @@ private:
fextl::shared_ptr<CodeBuffer> Latest;
fextl::shared_ptr<CodeBuffer> AllocateNew(size_t Size);
bool NameJITBuffers {true};
};
} // namespace FEXCore::CPU
@@ -83,22 +83,22 @@ constexpr std::array<X86InstInfo[2], ENTRY_MAX> Primary_ArchSelect_LUT = {{
},
// ENTRY_27
{
{"DAA", TYPE_INST, GenFlagsDstSize(SIZE_8BIT), 0, { .OpDispatch = &IR::OpDispatchBuilder::DAAOp } },
{"DAA", TYPE_INST, GenFlagsDstSize(SIZE_8BIT) | FLAGS_SF_DST_RAX, 0, { .OpDispatch = &IR::OpDispatchBuilder::DAAOp } },
{"", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },
},
// ENTRY_2F
{
{"DAS", TYPE_INST, GenFlagsDstSize(SIZE_8BIT), 0, { .OpDispatch = &IR::OpDispatchBuilder::DASOp } },
{"DAS", TYPE_INST, GenFlagsDstSize(SIZE_8BIT) | FLAGS_SF_DST_RAX, 0, { .OpDispatch = &IR::OpDispatchBuilder::DASOp } },
{"", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },
},
// ENTRY_37
{
{"AAA", TYPE_INST, GenFlagsDstSize(SIZE_16BIT), 0, { .OpDispatch = &IR::OpDispatchBuilder::AAAOp } },
{"AAA", TYPE_INST, GenFlagsDstSize(SIZE_16BIT) | FLAGS_SF_DST_RAX, 0, { .OpDispatch = &IR::OpDispatchBuilder::AAAOp } },
{"", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },
},
// ENTRY_3F
{
{"AAS", TYPE_INST, GenFlagsDstSize(SIZE_16BIT), 0, { .OpDispatch = &IR::OpDispatchBuilder::AASOp } },
{"AAS", TYPE_INST, GenFlagsDstSize(SIZE_16BIT) | FLAGS_SF_DST_RAX, 0, { .OpDispatch = &IR::OpDispatchBuilder::AASOp } },
{"", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },
},
// ENTRY_40
@@ -134,23 +134,23 @@ constexpr std::array<X86InstInfo[2], ENTRY_MAX> Primary_ArchSelect_LUT = {{
},
// ENTRY_A0
{
{"MOV", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MEM_OFFSET, 4, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
{"MOV", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MEM_OFFSET, 8, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
{"MOV", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX | FLAGS_MEM_OFFSET, 4, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
{"MOV", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX | FLAGS_MEM_OFFSET, 8, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
},
// ENTRY_A1
{
{"MOV", TYPE_INST, FLAGS_MEM_OFFSET | FLAGS_LITERAL_PATCHABLE, 4, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
{"MOV", TYPE_INST, FLAGS_MEM_OFFSET | FLAGS_LITERAL_PATCHABLE, 8, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
{"MOV", TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_MEM_OFFSET, 4, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
{"MOV", TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_MEM_OFFSET, 8, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
},
// ENTRY_A2
{
{"MOV", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MEM_OFFSET, 4, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
{"MOV", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MEM_OFFSET, 8, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
{"MOV", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_SRC_RAX | FLAGS_MEM_OFFSET, 4, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
{"MOV", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_SRC_RAX | FLAGS_MEM_OFFSET, 8, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
},
// ENTRY_A3
{
{"MOV", TYPE_INST, FLAGS_MEM_OFFSET | FLAGS_LITERAL_PATCHABLE, 4, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
{"MOV", TYPE_INST, FLAGS_MEM_OFFSET | FLAGS_LITERAL_PATCHABLE, 8, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
{"MOV", TYPE_INST, FLAGS_SF_SRC_RAX | FLAGS_MEM_OFFSET, 4, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
{"MOV", TYPE_INST, FLAGS_SF_SRC_RAX | FLAGS_MEM_OFFSET, 8, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
},
// ENTRY_CE
{
@@ -159,17 +159,17 @@ constexpr std::array<X86InstInfo[2], ENTRY_MAX> Primary_ArchSelect_LUT = {{
},
// ENTRY_D4
{
{"AAM", TYPE_INST, GenFlagsSameSize(SIZE_8BIT), 1, { .OpDispatch = &IR::OpDispatchBuilder::AAMOp } },
{"AAM", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX, 1, { .OpDispatch = &IR::OpDispatchBuilder::AAMOp } },
{"", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },
},
// ENTRY_D5
{
{"AAD", TYPE_INST, GenFlagsSameSize(SIZE_8BIT), 1, { .OpDispatch = &IR::OpDispatchBuilder::AADOp } },
{"REX2", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },
{"AAD", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX, 1, { .OpDispatch = &IR::OpDispatchBuilder::AADOp } },
{"", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },
},
// ENTRY_D6
{
{"SALC", TYPE_INST, GenFlagsSameSize(SIZE_8BIT), 0, { .OpDispatch = &IR::OpDispatchBuilder::SALCOp } },
{"SALC", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX | FLAGS_SF_SRC_RAX, 0, { .OpDispatch = &IR::OpDispatchBuilder::SALCOp } },
{"", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },
},
// ENTRY_EA
@@ -204,8 +204,8 @@ const std::array<X86InstInfo, MAX_PRIMARY_TABLE_SIZE> BaseOps = []() consteval {
{0x01, 1, X86InstInfo{"ADD", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK, 0}},
{0x02, 1, X86InstInfo{"ADD", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM, 0}},
{0x03, 1, X86InstInfo{"ADD", TYPE_INST, FLAGS_MODRM, 0}},
{0x04, 1, X86InstInfo{"ADD", TYPE_INST, GenFlagsSameSize(SIZE_8BIT), 1}},
{0x05, 1, X86InstInfo{"ADD", TYPE_INST, FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_LITERAL_PATCHABLE, 4}},
{0x04, 1, X86InstInfo{"ADD", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX , 1}},
{0x05, 1, X86InstInfo{"ADD", TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},
{0x06, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_06] }}},
{0x07, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_07] }}},
@@ -214,16 +214,16 @@ const std::array<X86InstInfo, MAX_PRIMARY_TABLE_SIZE> BaseOps = []() consteval {
{0x09, 1, X86InstInfo{"OR", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK, 0}},
{0x0A, 1, X86InstInfo{"OR", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM, 0}},
{0x0B, 1, X86InstInfo{"OR", TYPE_INST, FLAGS_MODRM, 0}},
{0x0C, 1, X86InstInfo{"OR", TYPE_INST, GenFlagsSameSize(SIZE_8BIT), 1}},
{0x0D, 1, X86InstInfo{"OR", TYPE_INST, FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_LITERAL_PATCHABLE, 4}},
{0x0C, 1, X86InstInfo{"OR", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX , 1}},
{0x0D, 1, X86InstInfo{"OR", TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},
{0x0E, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_0E] }}},
{0x10, 1, X86InstInfo{"ADC", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK, 0}},
{0x11, 1, X86InstInfo{"ADC", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK, 0}},
{0x12, 1, X86InstInfo{"ADC", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM, 0}},
{0x13, 1, X86InstInfo{"ADC", TYPE_INST, FLAGS_MODRM, 0}},
{0x14, 1, X86InstInfo{"ADC", TYPE_INST, GenFlagsSameSize(SIZE_8BIT), 1}},
{0x15, 1, X86InstInfo{"ADC", TYPE_INST, FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_LITERAL_PATCHABLE, 4}},
{0x14, 1, X86InstInfo{"ADC", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX , 1}},
{0x15, 1, X86InstInfo{"ADC", TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},
{0x16, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_16] }}},
{0x17, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_17] }}},
@@ -231,8 +231,8 @@ const std::array<X86InstInfo, MAX_PRIMARY_TABLE_SIZE> BaseOps = []() consteval {
{0x19, 1, X86InstInfo{"SBB", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK, 0}},
{0x1A, 1, X86InstInfo{"SBB", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM, 0}},
{0x1B, 1, X86InstInfo{"SBB", TYPE_INST, FLAGS_MODRM, 0}},
{0x1C, 1, X86InstInfo{"SBB", TYPE_INST, GenFlagsSameSize(SIZE_8BIT), 1}},
{0x1D, 1, X86InstInfo{"SBB", TYPE_INST, FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_LITERAL_PATCHABLE, 4}},
{0x1C, 1, X86InstInfo{"SBB", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX , 1}},
{0x1D, 1, X86InstInfo{"SBB", TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},
{0x1E, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_1E] }}},
{0x1F, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_1F] }}},
@@ -240,32 +240,32 @@ const std::array<X86InstInfo, MAX_PRIMARY_TABLE_SIZE> BaseOps = []() consteval {
{0x21, 1, X86InstInfo{"AND", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK, 0}},
{0x22, 1, X86InstInfo{"AND", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM, 0}},
{0x23, 1, X86InstInfo{"AND", TYPE_INST, FLAGS_MODRM, 0}},
{0x24, 1, X86InstInfo{"AND", TYPE_INST, GenFlagsSameSize(SIZE_8BIT), 1}},
{0x25, 1, X86InstInfo{"AND", TYPE_INST, FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_LITERAL_PATCHABLE, 4}},
{0x24, 1, X86InstInfo{"AND", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX , 1}},
{0x25, 1, X86InstInfo{"AND", TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},
{0x27, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_27] }}},
{0x28, 1, X86InstInfo{"SUB", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK, 0}},
{0x29, 1, X86InstInfo{"SUB", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK, 0}},
{0x2A, 1, X86InstInfo{"SUB", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM, 0}},
{0x2B, 1, X86InstInfo{"SUB", TYPE_INST, FLAGS_MODRM, 0}},
{0x2C, 1, X86InstInfo{"SUB", TYPE_INST, GenFlagsSameSize(SIZE_8BIT), 1}},
{0x2D, 1, X86InstInfo{"SUB", TYPE_INST, FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_LITERAL_PATCHABLE, 4}},
{0x2C, 1, X86InstInfo{"SUB", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX , 1}},
{0x2D, 1, X86InstInfo{"SUB", TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},
{0x2F, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_2F] }}},
{0x30, 1, X86InstInfo{"XOR", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK, 0}},
{0x31, 1, X86InstInfo{"XOR", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK, 0}},
{0x32, 1, X86InstInfo{"XOR", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM, 0}},
{0x33, 1, X86InstInfo{"XOR", TYPE_INST, FLAGS_MODRM, 0}},
{0x34, 1, X86InstInfo{"XOR", TYPE_INST, GenFlagsSameSize(SIZE_8BIT), 1}},
{0x35, 1, X86InstInfo{"XOR", TYPE_INST, FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_LITERAL_PATCHABLE, 4}},
{0x34, 1, X86InstInfo{"XOR", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX , 1}},
{0x35, 1, X86InstInfo{"XOR", TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},
{0x37, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_37] }}},
{0x38, 1, X86InstInfo{"CMP", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{0x39, 1, X86InstInfo{"CMP", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{0x3A, 1, X86InstInfo{"CMP", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM, 0}},
{0x3B, 1, X86InstInfo{"CMP", TYPE_INST, FLAGS_MODRM, 0}},
{0x3C, 1, X86InstInfo{"CMP", TYPE_INST, GenFlagsSameSize(SIZE_8BIT), 1}},
{0x3D, 1, X86InstInfo{"CMP", TYPE_INST, FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_LITERAL_PATCHABLE, 4}},
{0x3C, 1, X86InstInfo{"CMP", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX , 1}},
{0x3D, 1, X86InstInfo{"CMP", TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},
{0x3F, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_3F] }}},
{0x40, 8, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_40] }}},
@@ -321,9 +321,9 @@ const std::array<X86InstInfo, MAX_PRIMARY_TABLE_SIZE> BaseOps = []() consteval {
{0x8D, 1, X86InstInfo{"LEA", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},
{0x8E, 1, X86InstInfo{"MOV", TYPE_INST, GenFlagsSameSize(SIZE_16BIT) | FLAGS_MODRM, 0}},
{0x8F, 1, X86InstInfo{"POP", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_ZERO_REG | FLAGS_DEBUG_MEM_ACCESS, 0}},
{0x90, 8, X86InstInfo{"XCHG", TYPE_INST, FLAGS_SF_REX_IN_BYTE, 0}},
{0x98, 1, X86InstInfo{"CDQE", TYPE_INST, FLAGS_NONE, 0}},
{0x99, 1, X86InstInfo{"CQO", TYPE_INST, FLAGS_NONE, 0}},
{0x90, 8, X86InstInfo{"XCHG", TYPE_INST, FLAGS_SF_REX_IN_BYTE | FLAGS_SF_SRC_RAX, 0}},
{0x98, 1, X86InstInfo{"CDQE", TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SF_SRC_RAX, 0}},
{0x99, 1, X86InstInfo{"CQO", TYPE_INST, FLAGS_SF_DST_RDX | FLAGS_SF_SRC_RAX, 0}},
// These three are all X87 instructions
{0x9B, 1, X86InstInfo{"FWAIT", TYPE_INST, FLAGS_NONE, 0}},
@@ -343,17 +343,17 @@ const std::array<X86InstInfo, MAX_PRIMARY_TABLE_SIZE> BaseOps = []() consteval {
{0xA6, 1, X86InstInfo{"CMPSB", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_DEBUG_MEM_ACCESS, 0}},
{0xA7, 1, X86InstInfo{"CMPS", TYPE_INST, FLAGS_DEBUG_MEM_ACCESS, 0}},
{0xA8, 1, X86InstInfo{"TEST", TYPE_INST, GenFlagsSameSize(SIZE_8BIT), 1}},
{0xA9, 1, X86InstInfo{"TEST", TYPE_INST, FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_LITERAL_PATCHABLE, 4}},
{0xAA, 1, X86InstInfo{"STOS", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_DEBUG_MEM_ACCESS, 0}},
{0xAB, 1, X86InstInfo{"STOS", TYPE_INST, FLAGS_DEBUG_MEM_ACCESS, 0}},
{0xAC, 1, X86InstInfo{"LODS", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_DEBUG_MEM_ACCESS, 0}},
{0xAD, 1, X86InstInfo{"LODS", TYPE_INST, FLAGS_DEBUG_MEM_ACCESS, 0}},
{0xAE, 1, X86InstInfo{"SCAS", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_DEBUG_MEM_ACCESS, 0}},
{0xAF, 1, X86InstInfo{"SCAS", TYPE_INST, FLAGS_DEBUG_MEM_ACCESS, 0}},
{0xA8, 1, X86InstInfo{"TEST", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX , 1}},
{0xA9, 1, X86InstInfo{"TEST", TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},
{0xAA, 1, X86InstInfo{"STOS", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_DEBUG_MEM_ACCESS | FLAGS_SF_SRC_RAX, 0}},
{0xAB, 1, X86InstInfo{"STOS", TYPE_INST, FLAGS_DEBUG_MEM_ACCESS | FLAGS_SF_SRC_RAX, 0}},
{0xAC, 1, X86InstInfo{"LODS", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX | FLAGS_DEBUG_MEM_ACCESS, 0}},
{0xAD, 1, X86InstInfo{"LODS", TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_DEBUG_MEM_ACCESS, 0}},
{0xAE, 1, X86InstInfo{"SCAS", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_DEBUG_MEM_ACCESS | FLAGS_SF_SRC_RAX, 0}},
{0xAF, 1, X86InstInfo{"SCAS", TYPE_INST, FLAGS_DEBUG_MEM_ACCESS | FLAGS_SF_SRC_RAX, 0}},
{0xB0, 8, X86InstInfo{"MOV", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_REX_IN_BYTE , 1}},
{0xB8, 8, X86InstInfo{"MOV", TYPE_INST, FLAGS_SF_REX_IN_BYTE | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_DISPLACE_SIZE_MUL_2 | FLAGS_LITERAL_PATCHABLE, 4}},
{0xB8, 8, X86InstInfo{"MOV", TYPE_INST, FLAGS_SF_REX_IN_BYTE | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_DISPLACE_SIZE_MUL_2, 4}},
{0xC2, 1, X86InstInfo{"RET", TYPE_INST, FLAGS_SETS_RIP | FLAGS_BLOCK_END, 2}},
{0xC3, 1, X86InstInfo{"RET", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_BLOCK_END , 0}},
@@ -371,9 +371,9 @@ const std::array<X86InstInfo, MAX_PRIMARY_TABLE_SIZE> BaseOps = []() consteval {
{0xD6, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_D6] }}},
{0xD7, 1, X86InstInfo{"XLAT", TYPE_INST, FLAGS_DEBUG_MEM_ACCESS, 0}},
{0xE0, 1, X86InstInfo{"LOOPNE", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT, 1}},
{0xE1, 1, X86InstInfo{"LOOPE", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT, 1}},
{0xE2, 1, X86InstInfo{"LOOP", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT, 1}},
{0xE0, 1, X86InstInfo{"LOOPNE", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_SF_SRC_RCX, 1}},
{0xE1, 1, X86InstInfo{"LOOPE", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_SF_SRC_RCX, 1}},
{0xE2, 1, X86InstInfo{"LOOP", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_SF_SRC_RCX, 1}},
{0xE3, 1, X86InstInfo{"JrCXZ", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT , 1}},
// Should just throw GP
@@ -36,11 +36,11 @@ constexpr std::array<X86InstInfo[2], ENTRY_MAX> PrimaryGroup_ArchSelect_LUT = {{
{"", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },
},
{
{"ADC", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK, 1, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::ADCOp, 1, false> }},
{"ADC", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK, 1, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::ADCOp, 1> }},
{"", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },
},
{
{"SBB", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK, 1, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::SBBOp, 1, false> }},
{"SBB", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK, 1, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::SBBOp, 1> }},
{"", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },
},
{
@@ -56,7 +56,7 @@ constexpr std::array<X86InstInfo[2], ENTRY_MAX> PrimaryGroup_ArchSelect_LUT = {{
{"", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },
},
{
{"CMP", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::CMPOp, 1, false> }},
{"CMP", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::CMPOp, 1> }},
{"", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },
},
}};
@@ -75,14 +75,14 @@ constexpr std::array<X86InstInfo, MAX_INST_GROUP_TABLE_SIZE> PrimaryInstGroupOps
{OPD(TYPE_GROUP_1, OpToIndex(0x80), 6), 1, X86InstInfo{"XOR", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK, 1}},
{OPD(TYPE_GROUP_1, OpToIndex(0x80), 7), 1, X86InstInfo{"CMP", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK, 1}},
{OPD(TYPE_GROUP_1, OpToIndex(0x81), 0), 1, X86InstInfo{"ADD", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK | FLAGS_LITERAL_PATCHABLE, 4}},
{OPD(TYPE_GROUP_1, OpToIndex(0x81), 1), 1, X86InstInfo{"OR", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK | FLAGS_LITERAL_PATCHABLE, 4}},
{OPD(TYPE_GROUP_1, OpToIndex(0x81), 2), 1, X86InstInfo{"ADC", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK | FLAGS_LITERAL_PATCHABLE, 4}},
{OPD(TYPE_GROUP_1, OpToIndex(0x81), 3), 1, X86InstInfo{"SBB", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK | FLAGS_LITERAL_PATCHABLE, 4}},
{OPD(TYPE_GROUP_1, OpToIndex(0x81), 4), 1, X86InstInfo{"AND", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK | FLAGS_LITERAL_PATCHABLE, 4}},
{OPD(TYPE_GROUP_1, OpToIndex(0x81), 5), 1, X86InstInfo{"SUB", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK | FLAGS_LITERAL_PATCHABLE, 4}},
{OPD(TYPE_GROUP_1, OpToIndex(0x81), 6), 1, X86InstInfo{"XOR", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK | FLAGS_LITERAL_PATCHABLE, 4}},
{OPD(TYPE_GROUP_1, OpToIndex(0x81), 7), 1, X86InstInfo{"CMP", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK | FLAGS_LITERAL_PATCHABLE, 4}},
{OPD(TYPE_GROUP_1, OpToIndex(0x81), 0), 1, X86InstInfo{"ADD", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK, 4}},
{OPD(TYPE_GROUP_1, OpToIndex(0x81), 1), 1, X86InstInfo{"OR", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK, 4}},
{OPD(TYPE_GROUP_1, OpToIndex(0x81), 2), 1, X86InstInfo{"ADC", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK, 4}},
{OPD(TYPE_GROUP_1, OpToIndex(0x81), 3), 1, X86InstInfo{"SBB", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK, 4}},
{OPD(TYPE_GROUP_1, OpToIndex(0x81), 4), 1, X86InstInfo{"AND", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK, 4}},
{OPD(TYPE_GROUP_1, OpToIndex(0x81), 5), 1, X86InstInfo{"SUB", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK, 4}},
{OPD(TYPE_GROUP_1, OpToIndex(0x81), 6), 1, X86InstInfo{"XOR", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK, 4}},
{OPD(TYPE_GROUP_1, OpToIndex(0x81), 7), 1, X86InstInfo{"CMP", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK, 4}},
// Duplicates the 0x80 opcode group
{OPD(TYPE_GROUP_1, OpToIndex(0x82), 0), 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = PrimaryGroup_ArchSelect_LUT[ENTRY_1_82_0] }}},
@@ -140,23 +140,23 @@ constexpr std::array<X86InstInfo, MAX_INST_GROUP_TABLE_SIZE> PrimaryInstGroupOps
{OPD(TYPE_GROUP_2, OpToIndex(0xD1), 6), 1, X86InstInfo{"SHL", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD1), 7), 1, X86InstInfo{"SAR", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD2), 0), 1, X86InstInfo{"ROL", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD2), 1), 1, X86InstInfo{"ROR", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD2), 2), 1, X86InstInfo{"RCL", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD2), 3), 1, X86InstInfo{"RCR", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD2), 4), 1, X86InstInfo{"SHL", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD2), 5), 1, X86InstInfo{"SHR", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD2), 6), 1, X86InstInfo{"SHL", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD2), 7), 1, X86InstInfo{"SAR", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD2), 0), 1, X86InstInfo{"ROL", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD2), 1), 1, X86InstInfo{"ROR", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD2), 2), 1, X86InstInfo{"RCL", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD2), 3), 1, X86InstInfo{"RCR", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD2), 4), 1, X86InstInfo{"SHL", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD2), 5), 1, X86InstInfo{"SHR", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD2), 6), 1, X86InstInfo{"SHL", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD2), 7), 1, X86InstInfo{"SAR", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD3), 0), 1, X86InstInfo{"ROL", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD3), 1), 1, X86InstInfo{"ROR", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD3), 2), 1, X86InstInfo{"RCL", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD3), 3), 1, X86InstInfo{"RCR", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD3), 4), 1, X86InstInfo{"SHL", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD3), 5), 1, X86InstInfo{"SHR", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD3), 6), 1, X86InstInfo{"SHL", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD3), 7), 1, X86InstInfo{"SAR", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD3), 0), 1, X86InstInfo{"ROL", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD3), 1), 1, X86InstInfo{"ROR", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD3), 2), 1, X86InstInfo{"RCL", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD3), 3), 1, X86InstInfo{"RCR", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD3), 4), 1, X86InstInfo{"SHL", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD3), 5), 1, X86InstInfo{"SHR", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD3), 6), 1, X86InstInfo{"SHL", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},
{OPD(TYPE_GROUP_2, OpToIndex(0xD3), 7), 1, X86InstInfo{"SAR", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX, 0}},
// GROUP 3
{OPD(TYPE_GROUP_3, OpToIndex(0xF6), 0), 1, X86InstInfo{"TEST", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1}},
@@ -168,7 +168,7 @@ constexpr std::array<X86InstInfo, MAX_INST_GROUP_TABLE_SIZE> PrimaryInstGroupOps
{OPD(TYPE_GROUP_3, OpToIndex(0xF6), 6), 1, X86InstInfo{"DIV", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_3, OpToIndex(0xF6), 7), 1, X86InstInfo{"IDIV", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_3, OpToIndex(0xF7), 0), 1, X86InstInfo{"TEST", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_LITERAL_PATCHABLE, 4}},
{OPD(TYPE_GROUP_3, OpToIndex(0xF7), 0), 1, X86InstInfo{"TEST", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},
{OPD(TYPE_GROUP_3, OpToIndex(0xF7), 1), 1, X86InstInfo{"TEST", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},
{OPD(TYPE_GROUP_3, OpToIndex(0xF7), 2), 1, X86InstInfo{"NOT", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK, 0}},
{OPD(TYPE_GROUP_3, OpToIndex(0xF7), 3), 1, X86InstInfo{"NEG", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK, 0}},
@@ -196,7 +196,7 @@ constexpr std::array<X86InstInfo, MAX_INST_GROUP_TABLE_SIZE> PrimaryInstGroupOps
{OPD(TYPE_GROUP_11, OpToIndex(0xC6), 0), 1, X86InstInfo{"MOV", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT, 1}},
{OPD(TYPE_GROUP_11, OpToIndex(0xC6), 1), 5, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_11, OpToIndex(0xC6), 7), 1, X86InstInfo{"XABORT", TYPE_INST, FLAGS_MODRM, 1}},
{OPD(TYPE_GROUP_11, OpToIndex(0xC7), 0), 1, X86InstInfo{"MOV", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_LITERAL_PATCHABLE, 4}},
{OPD(TYPE_GROUP_11, OpToIndex(0xC7), 0), 1, X86InstInfo{"MOV", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},
{OPD(TYPE_GROUP_11, OpToIndex(0xC7), 1), 5, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_11, OpToIndex(0xC7), 7), 1, X86InstInfo{"XBEGIN", TYPE_INST, FLAGS_MODRM | FLAGS_SRC_SEXT | FLAGS_SETS_RIP | FLAGS_DISPLACE_SIZE_DIV_2, 4}},
};
@@ -61,7 +61,7 @@ constexpr std::array<X86InstInfo, MAX_INST_SECOND_GROUP_TABLE_SIZE> SecondInstGr
// Pulls from other MODRM table
// GROUP 6
{OPD(TYPE_GROUP_6, PF_NONE, 0), 1, X86InstInfo{"SLDT", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_6, PF_NONE, 0), 1, X86InstInfo{"SLDT", TYPE_UNDEC, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_6, PF_NONE, 1), 1, X86InstInfo{"STR", TYPE_PRIV, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_6, PF_NONE, 2), 1, X86InstInfo{"LLDT", TYPE_PRIV, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_6, PF_NONE, 3), 1, X86InstInfo{"LTR", TYPE_INST, FLAGS_NONE, 0}},
@@ -70,7 +70,7 @@ constexpr std::array<X86InstInfo, MAX_INST_SECOND_GROUP_TABLE_SIZE> SecondInstGr
{OPD(TYPE_GROUP_6, PF_NONE, 6), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_6, PF_NONE, 7), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_6, PF_F3, 0), 1, X86InstInfo{"SLDT", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_6, PF_F3, 0), 1, X86InstInfo{"SLDT", TYPE_UNDEC, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_6, PF_F3, 1), 1, X86InstInfo{"STR", TYPE_PRIV, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_6, PF_F3, 2), 1, X86InstInfo{"LLDT", TYPE_PRIV, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_6, PF_F3, 3), 1, X86InstInfo{"LTR", TYPE_INST, FLAGS_NONE, 0}},
@@ -79,7 +79,7 @@ constexpr std::array<X86InstInfo, MAX_INST_SECOND_GROUP_TABLE_SIZE> SecondInstGr
{OPD(TYPE_GROUP_6, PF_F3, 6), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_6, PF_F3, 7), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_6, PF_66, 0), 1, X86InstInfo{"SLDT", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_6, PF_66, 0), 1, X86InstInfo{"SLDT", TYPE_UNDEC, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_6, PF_66, 1), 1, X86InstInfo{"STR", TYPE_PRIV, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_6, PF_66, 2), 1, X86InstInfo{"LLDT", TYPE_PRIV, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_6, PF_66, 3), 1, X86InstInfo{"LTR", TYPE_INST, FLAGS_NONE, 0}},
@@ -88,7 +88,7 @@ constexpr std::array<X86InstInfo, MAX_INST_SECOND_GROUP_TABLE_SIZE> SecondInstGr
{OPD(TYPE_GROUP_6, PF_66, 6), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_6, PF_66, 7), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_6, PF_F2, 0), 1, X86InstInfo{"SLDT", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_6, PF_F2, 0), 1, X86InstInfo{"SLDT", TYPE_UNDEC, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_6, PF_F2, 1), 1, X86InstInfo{"STR", TYPE_PRIV, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_6, PF_F2, 2), 1, X86InstInfo{"LLDT", TYPE_PRIV, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_6, PF_F2, 3), 1, X86InstInfo{"LTR", TYPE_INST, FLAGS_NONE, 0}},
@@ -98,13 +98,13 @@ constexpr std::array<X86InstInfo, MAX_INST_SECOND_GROUP_TABLE_SIZE> SecondInstGr
{OPD(TYPE_GROUP_6, PF_F2, 7), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0}},
// GROUP 7
{OPD(TYPE_GROUP_7, PF_NONE, 0), 1, X86InstInfo{"SGDT", TYPE_SECOND_GROUP_MODRM, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_7, PF_NONE, 1), 1, X86InstInfo{"SIDT", TYPE_SECOND_GROUP_MODRM, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_7, PF_NONE, 2), 1, X86InstInfo{"", TYPE_SECOND_GROUP_MODRM, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_7, PF_NONE, 3), 1, X86InstInfo{"", TYPE_SECOND_GROUP_MODRM, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_7, PF_NONE, 4), 1, X86InstInfo{"SMSW", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_7, PF_NONE, 5), 1, X86InstInfo{"", TYPE_SECOND_GROUP_MODRM, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_7, PF_NONE, 6), 1, X86InstInfo{"LMSW", TYPE_INST, FLAGS_MODRM, 0}},
{OPD(TYPE_GROUP_7, PF_NONE, 0), 1, X86InstInfo{"SGDT", TYPE_SECOND_GROUP_MODRM, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_7, PF_NONE, 1), 1, X86InstInfo{"SIDT", TYPE_SECOND_GROUP_MODRM, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_7, PF_NONE, 2), 1, X86InstInfo{"", TYPE_SECOND_GROUP_MODRM, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_7, PF_NONE, 3), 1, X86InstInfo{"", TYPE_SECOND_GROUP_MODRM, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_7, PF_NONE, 4), 1, X86InstInfo{"SMSW", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_7, PF_NONE, 5), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_7, PF_NONE, 6), 1, X86InstInfo{"LMSW", TYPE_INST, FLAGS_MODRM, 0}},
{OPD(TYPE_GROUP_7, PF_NONE, 7), 1, X86InstInfo{"INVLPG", TYPE_SECOND_GROUP_MODRM, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_7, PF_F3, 0), 1, X86InstInfo{"SGDT", TYPE_SECOND_GROUP_MODRM, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
@@ -112,7 +112,7 @@ constexpr std::array<X86InstInfo, MAX_INST_SECOND_GROUP_TABLE_SIZE> SecondInstGr
{OPD(TYPE_GROUP_7, PF_F3, 2), 1, X86InstInfo{"", TYPE_SECOND_GROUP_MODRM, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_7, PF_F3, 3), 1, X86InstInfo{"", TYPE_SECOND_GROUP_MODRM, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_7, PF_F3, 4), 1, X86InstInfo{"SMSW", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_7, PF_F3, 5), 1, X86InstInfo{"", TYPE_SECOND_GROUP_MODRM, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_7, PF_F3, 5), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_7, PF_F3, 6), 1, X86InstInfo{"LMSW", TYPE_INST, FLAGS_MODRM, 0}},
{OPD(TYPE_GROUP_7, PF_F3, 7), 1, X86InstInfo{"INVLPG", TYPE_SECOND_GROUP_MODRM, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
@@ -121,7 +121,7 @@ constexpr std::array<X86InstInfo, MAX_INST_SECOND_GROUP_TABLE_SIZE> SecondInstGr
{OPD(TYPE_GROUP_7, PF_66, 2), 1, X86InstInfo{"", TYPE_SECOND_GROUP_MODRM, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_7, PF_66, 3), 1, X86InstInfo{"", TYPE_SECOND_GROUP_MODRM, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_7, PF_66, 4), 1, X86InstInfo{"SMSW", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_7, PF_66, 5), 1, X86InstInfo{"", TYPE_SECOND_GROUP_MODRM, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_7, PF_66, 5), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_7, PF_66, 6), 1, X86InstInfo{"LMSW", TYPE_INST, FLAGS_MODRM, 0}},
{OPD(TYPE_GROUP_7, PF_66, 7), 1, X86InstInfo{"INVLPG", TYPE_SECOND_GROUP_MODRM, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
@@ -130,7 +130,7 @@ constexpr std::array<X86InstInfo, MAX_INST_SECOND_GROUP_TABLE_SIZE> SecondInstGr
{OPD(TYPE_GROUP_7, PF_F2, 2), 1, X86InstInfo{"", TYPE_SECOND_GROUP_MODRM, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_7, PF_F2, 3), 1, X86InstInfo{"", TYPE_SECOND_GROUP_MODRM, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_7, PF_F2, 4), 1, X86InstInfo{"SMSW", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
{OPD(TYPE_GROUP_7, PF_F2, 5), 1, X86InstInfo{"", TYPE_SECOND_GROUP_MODRM, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_7, PF_F2, 5), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0}},
{OPD(TYPE_GROUP_7, PF_F2, 6), 1, X86InstInfo{"LMSW", TYPE_INST, FLAGS_MODRM, 0}},
{OPD(TYPE_GROUP_7, PF_F2, 7), 1, X86InstInfo{"INVLPG", TYPE_SECOND_GROUP_MODRM, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
@@ -12,19 +12,6 @@ $end_info$
namespace FEXCore::X86Tables {
using namespace InstFlags;
enum ModRM_LUT {
ENTRY_7_0,
ENTRY_MAX,
};
constexpr std::array<X86InstInfo[2], ENTRY_MAX> ModRM_ArchSelect_LUT = {{
// ENTRY_7_0
{
{"SWAPGS", TYPE_INST, FLAGS_NONE, 0, { .OpDispatch = &IR::OpDispatchBuilder::UnimplementedOp }},
{"SWAPGS", TYPE_INST, FLAGS_NONE, 0, { .OpDispatch = &IR::OpDispatchBuilder::PermissionRestrictedOp }},
},
}};
constexpr std::array<X86InstInfo, MAX_SECOND_MODRM_TABLE_SIZE> SecondModRMTableOps = []() consteval {
std::array<X86InstInfo, MAX_SECOND_MODRM_TABLE_SIZE> Table{};
constexpr U8U8InfoStruct SecondaryModRMExtensionOpTable[] = {
@@ -40,7 +27,7 @@ constexpr std::array<X86InstInfo, MAX_SECOND_MODRM_TABLE_SIZE> SecondModRMTableO
// REG /2
{((1 << 3) | 0), 1, X86InstInfo{"XGETBV", TYPE_INST, FLAGS_NONE, 0}},
{((1 << 3) | 1), 1, X86InstInfo{"XSETBV", TYPE_INST, FLAGS_NONE, 0}},
{((1 << 3) | 1), 1, X86InstInfo{"XSETBV", TYPE_PRIV, FLAGS_NONE, 0}},
{((1 << 3) | 2), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0}},
{((1 << 3) | 3), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0}},
{((1 << 3) | 4), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0}},
@@ -58,25 +45,15 @@ constexpr std::array<X86InstInfo, MAX_SECOND_MODRM_TABLE_SIZE> SecondModRMTableO
{((2 << 3) | 6), 1, X86InstInfo{"SKINIT", TYPE_PRIV, FLAGS_NONE, 0}},
{((2 << 3) | 7), 1, X86InstInfo{"INVLPGA", TYPE_INST, FLAGS_NONE, 0}},
// REG /5
{((3 << 3) | 0), 1, X86InstInfo{"SERIALIZE", TYPE_INST, FLAGS_NONE, 0}},
{((3 << 3) | 1), 1, X86InstInfo{"", TYPE_UNIMPLEMENTED, FLAGS_NONE, 0}},
{((3 << 3) | 2), 1, X86InstInfo{"", TYPE_UNIMPLEMENTED, FLAGS_NONE, 0}},
{((3 << 3) | 3), 1, X86InstInfo{"", TYPE_UNIMPLEMENTED, FLAGS_NONE, 0}},
{((3 << 3) | 4), 1, X86InstInfo{"", TYPE_UNIMPLEMENTED, FLAGS_NONE, 0}},
{((3 << 3) | 5), 1, X86InstInfo{"", TYPE_UNIMPLEMENTED, FLAGS_NONE, 0}},
{((3 << 3) | 6), 1, X86InstInfo{"RDPKRU", TYPE_UNIMPLEMENTED, FLAGS_NONE, 0}},
{((3 << 3) | 7), 1, X86InstInfo{"WRPKRU", TYPE_UNIMPLEMENTED, FLAGS_NONE, 0}},
// REG /7
{((4 << 3) | 0), 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = ModRM_ArchSelect_LUT[ENTRY_7_0] }}},
{((4 << 3) | 1), 1, X86InstInfo{"RDTSCP", TYPE_INST, FLAGS_NONE, 0}},
{((4 << 3) | 2), 1, X86InstInfo{"MONITORX", TYPE_PRIV, FLAGS_NONE, 0}},
{((4 << 3) | 3), 1, X86InstInfo{"MWAITX", TYPE_PRIV, FLAGS_NONE, 0}},
{((4 << 3) | 4), 1, X86InstInfo{"CLZERO", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_DEBUG_MEM_ACCESS, 0}},
{((4 << 3) | 5), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0}},
{((4 << 3) | 6), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0}},
{((4 << 3) | 7), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0}},
{((3 << 3) | 0), 1, X86InstInfo{"SWAPGS", TYPE_INST, FLAGS_NONE, 0}},
{((3 << 3) | 1), 1, X86InstInfo{"RDTSCP", TYPE_INST, FLAGS_NONE, 0}},
{((3 << 3) | 2), 1, X86InstInfo{"MONITORX", TYPE_PRIV, FLAGS_NONE, 0}},
{((3 << 3) | 3), 1, X86InstInfo{"MWAITX", TYPE_PRIV, FLAGS_NONE, 0}},
{((3 << 3) | 4), 1, X86InstInfo{"CLZERO", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SF_SRC_RAX | FLAGS_DEBUG_MEM_ACCESS, 0}},
{((3 << 3) | 5), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0}},
{((3 << 3) | 6), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0}},
{((3 << 3) | 7), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0}},
};
GenerateTable(Table.data(), SecondaryModRMExtensionOpTable, std::size(SecondaryModRMExtensionOpTable));
@@ -204,17 +204,17 @@ constexpr std::array<X86InstInfo, MAX_SECOND_TABLE_SIZE> SecondBaseOps = []() co
{0xA0, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_DEBUG_MEM_ACCESS | FLAGS_NO_OVERLAY, 0, { .Indirect = Secondary_ArchSelect_LUT[ENTRY_A0] }}},
{0xA1, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_DEBUG_MEM_ACCESS | FLAGS_NO_OVERLAY, 0, { .Indirect = Secondary_ArchSelect_LUT[ENTRY_A1] }}},
{0xA2, 1, X86InstInfo{"CPUID", TYPE_INST, FLAGS_NO_OVERLAY | FLAGS_BLOCK_END, 0}},
{0xA2, 1, X86InstInfo{"CPUID", TYPE_INST, FLAGS_SF_SRC_RAX | FLAGS_NO_OVERLAY, 0}},
{0xA3, 1, X86InstInfo{"BT", TYPE_INST, FLAGS_DEBUG_MEM_ACCESS | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY | FLAGS_SUPPORTS_LOCK, 0}},
{0xA4, 1, X86InstInfo{"SHLD", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 1}},
{0xA5, 1, X86InstInfo{"SHLD", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 0}},
{0xA5, 1, X86InstInfo{"SHLD", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX | FLAGS_NO_OVERLAY, 0}},
{0xA6, 2, X86InstInfo{"", TYPE_INVALID, FLAGS_NO_OVERLAY, 0}},
{0xA8, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_DEBUG_MEM_ACCESS | FLAGS_NO_OVERLAY, 0, { .Indirect = Secondary_ArchSelect_LUT[ENTRY_A8] }}},
{0xA9, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_DEBUG_MEM_ACCESS | FLAGS_NO_OVERLAY, 0, { .Indirect = Secondary_ArchSelect_LUT[ENTRY_A9] }}},
{0xAA, 1, X86InstInfo{"RSM", TYPE_PRIV, FLAGS_NO_OVERLAY, 0}},
{0xAB, 1, X86InstInfo{"BTS", TYPE_INST, FLAGS_DEBUG_MEM_ACCESS | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY | FLAGS_SUPPORTS_LOCK, 0}},
{0xAC, 1, X86InstInfo{"SHRD", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 1}},
{0xAD, 1, X86InstInfo{"SHRD", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY, 0}},
{0xAD, 1, X86InstInfo{"SHRD", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX | FLAGS_NO_OVERLAY, 0}},
{0xAE, 1, X86InstInfo{"", TYPE_GROUP_15, FLAGS_NO_OVERLAY, 0}},
{0xAF, 1, X86InstInfo{"IMUL", TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},
@@ -226,12 +226,12 @@ constexpr std::array<X86InstInfo, MAX_SECOND_TABLE_SIZE> SecondBaseOps = []() co
{0xB5, 1, X86InstInfo{"LGS", TYPE_INVALID, FLAGS_NO_OVERLAY, 0}},
{0xB6, 1, X86InstInfo{"MOVZX", TYPE_INST, GenFlagsSrcSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},
{0xB7, 1, X86InstInfo{"MOVZX", TYPE_INST, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},
{0xB8, 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NO_PREFIX_ORDER, 0}},
{0xB8, 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0}},
{0xB9, 1, X86InstInfo{"", TYPE_GROUP_10, FLAGS_NO_OVERLAY, 0}},
{0xBA, 1, X86InstInfo{"", TYPE_GROUP_8, FLAGS_NO_OVERLAY, 0}},
{0xBB, 1, X86InstInfo{"BTC", TYPE_INST, FLAGS_DEBUG_MEM_ACCESS | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY | FLAGS_SUPPORTS_LOCK, 0}},
{0xBC, 1, X86InstInfo{"BSF", TYPE_INST, FLAGS_MODRM | FLAGS_NO_PREFIX_ORDER | FLAGS_NO_OVERLAY66, 0}},
{0xBD, 1, X86InstInfo{"BSR", TYPE_INST, FLAGS_MODRM | FLAGS_NO_PREFIX_ORDER | FLAGS_NO_OVERLAY66, 0}},
{0xBC, 1, X86InstInfo{"BSF", TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY66, 0}},
{0xBD, 1, X86InstInfo{"BSR", TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY66, 0}},
{0xBE, 1, X86InstInfo{"MOVSX", TYPE_INST, GenFlagsSrcSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},
{0xBF, 1, X86InstInfo{"MOVSX", TYPE_INST, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_MODRM | FLAGS_NO_OVERLAY, 0}},
@@ -312,16 +312,10 @@ namespace AVX128 {
{OPD(2, 0b01, 0x46), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VariableShiftImpl, IROps::OP_VSSHR>}, // VPSRAVD
{OPD(2, 0b01, 0x47), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VariableShiftImpl, IROps::OP_VUSHL>}, // VPSLLV
{OPD(2, 0b01, 0x50), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPDPBUSD, false>},
{OPD(2, 0b01, 0x51), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPDPBUSD, true>},
{OPD(2, 0b01, 0x52), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPDPWSSD, false>},
{OPD(2, 0b01, 0x53), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPDPWSSD, true>},
{OPD(2, 0b01, 0x58), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBROADCAST, OpSize::i32Bit>},
{OPD(2, 0b01, 0x59), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBROADCAST, OpSize::i64Bit>},
{OPD(2, 0b01, 0x5A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBROADCAST, OpSize::i128Bit>},
{OPD(2, 0b10, 0x72), 1, &OpDispatchBuilder::AVX128_VCVTNEPS2BF16},
{OPD(2, 0b01, 0x78), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBROADCAST, OpSize::i8Bit>},
{OPD(2, 0b01, 0x79), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBROADCAST, OpSize::i16Bit>},
@@ -366,13 +360,6 @@ namespace AVX128 {
{OPD(2, 0b01, 0xA6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAddSubImpl, true, 2, 1, 3>}, // VFMADDSUB
{OPD(2, 0b01, 0xA7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAddSubImpl, false, 2, 1, 3>}, // VFMSUBADD
{OPD(2, 0b00, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVX128_VCVTNEEPH2PS, false>},
{OPD(2, 0b01, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVX128_VCVTNEEPH2PS, true>},
{OPD(2, 0b10, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVX128_VCVTNEEBF162PS, true>},
{OPD(2, 0b11, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVX128_VCVTNEEBF162PS, false>},
{OPD(2, 0b01, 0xB1), 1, &IR::OpDispatchBuilder::AVX128_VBCSTNESH2PS},
{OPD(2, 0b10, 0xB1), 1, &IR::OpDispatchBuilder::AVX128_VBCSTNEBF162PS},
{OPD(2, 0b01, 0xB6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAddSubImpl, true, 2, 3, 1>}, // VFMADDSUB
{OPD(2, 0b01, 0xB7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAddSubImpl, false, 2, 3, 1>}, // VFMSUBADD
@@ -770,16 +757,10 @@ namespace AVX256 {
{OPD(2, 0b01, 0x46), 1, &OpDispatchBuilder::VPSRAVDOp},
{OPD(2, 0b01, 0x47), 1, &OpDispatchBuilder::VPSLLVOp},
{OPD(2, 0b01, 0x50), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPDPBUSDOp, false>},
{OPD(2, 0b01, 0x51), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPDPBUSDOp, true>},
{OPD(2, 0b01, 0x52), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPDPWSSDOp, false>},
{OPD(2, 0b01, 0x53), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPDPWSSDOp, true>},
{OPD(2, 0b01, 0x58), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i32Bit>},
{OPD(2, 0b01, 0x59), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i64Bit>},
{OPD(2, 0b01, 0x5A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i128Bit>},
{OPD(2, 0b10, 0x72), 1, &OpDispatchBuilder::AVXVCVTNEPS2BF16},
{OPD(2, 0b01, 0x78), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i8Bit>},
{OPD(2, 0b01, 0x79), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i16Bit>},
@@ -824,13 +805,6 @@ namespace AVX256 {
{OPD(2, 0b01, 0xA6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, true, 2, 1, 3>}, // VFMADDSUB
{OPD(2, 0b01, 0xA7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, false, 2, 1, 3>}, // VFMSUBADD
{OPD(2, 0b00, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVXVCVTNEEPH2PS, false>},
{OPD(2, 0b01, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVXVCVTNEEPH2PS, true>},
{OPD(2, 0b10, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVXVCVTNEEBF162PS, true>},
{OPD(2, 0b11, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVXVCVTNEEBF162PS, false>},
{OPD(2, 0b01, 0xB1), 1, &IR::OpDispatchBuilder::AVXVBCSTNESH2PS},
{OPD(2, 0b10, 0xB1), 1, &IR::OpDispatchBuilder::AVXVBCSTNEBF162PS},
{OPD(2, 0b01, 0xB6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, true, 2, 3, 1>}, // VFMADDSUB
{OPD(2, 0b01, 0xB7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, false, 2, 3, 1>}, // VFMSUBADD
@@ -1233,16 +1207,10 @@ auto BaseTableLambda = [](const auto RuntimeTable) consteval {
{OPD(2, 0b01, 0x46), 1, X86InstInfo{"VPSRAVD", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},
{OPD(2, 0b01, 0x47), 1, X86InstInfo{"VPSLLV", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},
{OPD(2, 0b01, 0x50), 1, X86InstInfo{"VPDPBUSD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},
{OPD(2, 0b01, 0x51), 1, X86InstInfo{"VPDPBUSDS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},
{OPD(2, 0b01, 0x52), 1, X86InstInfo{"VPDPWSSD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},
{OPD(2, 0b01, 0x53), 1, X86InstInfo{"VPDPWSSDS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},
{OPD(2, 0b01, 0x58), 1, X86InstInfo{"VPBROADCASTD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},
{OPD(2, 0b01, 0x59), 1, X86InstInfo{"VPBROADCASTQ", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},
{OPD(2, 0b01, 0x5A), 1, X86InstInfo{"VBROADCASTI128", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_L_1 | FLAGS_SF_MOD_MEM_ONLY | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},
{OPD(2, 0b10, 0x72), 1, X86InstInfo{"VCVTNEPS2BF16", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},
{OPD(2, 0b01, 0x78), 1, X86InstInfo{"VPBROADCASTB", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},
{OPD(2, 0b01, 0x79), 1, X86InstInfo{"VPBROADCASTW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},
@@ -1287,13 +1255,6 @@ auto BaseTableLambda = [](const auto RuntimeTable) consteval {
{OPD(2, 0b01, 0xA6), 1, X86InstInfo{"VFMADDSUB213", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},
{OPD(2, 0b01, 0xA7), 1, X86InstInfo{"VFMSUBADD213", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},
{OPD(2, 0b00, 0xB0), 1, X86InstInfo{"VCVTNEOPH2PS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},
{OPD(2, 0b01, 0xB0), 1, X86InstInfo{"VCVTNEEPH2PS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},
{OPD(2, 0b10, 0xB0), 1, X86InstInfo{"VCVTNEEBF162PS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},
{OPD(2, 0b11, 0xB0), 1, X86InstInfo{"VCVTNEOBF162PS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},
{OPD(2, 0b01, 0xB1), 1, X86InstInfo{"VBCSTNESH2PS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},
{OPD(2, 0b10, 0xB1), 1, X86InstInfo{"VBCSTNEBF162PS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},
{OPD(2, 0b01, 0xB6), 1, X86InstInfo{"VFMADDSUB231", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},
{OPD(2, 0b01, 0xB7), 1, X86InstInfo{"VFMSUBADD231", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},
@@ -185,14 +185,10 @@ struct DecodedOperand {
struct {
int64_t Displacement;
uint8_t GPR;
bool PatchableDisp;
uint8_t DispOffset;
} GPRIndirect; // Shared with GPRIndirectRelocation
struct {
int64_t Value;
bool PatchableDisp;
uint8_t DispOffset;
} RIPLiteral; // Shared with RIPLiteralRelocation
struct LiteralType {
@@ -215,9 +211,7 @@ struct DecodedOperand {
uint8_t Scale;
uint8_t Index; // ~0 invalid
uint8_t Base; // ~0 invalid
bool PatchableDisp;
uint8_t DispOffset;
} SIB; // Shared with SIBRelocation
} SIB; // Shared with SIBRelocation
};
TypeUnion Data;
@@ -272,7 +266,6 @@ enum InstType {
TYPE_INST,
TYPE_X87 = TYPE_INST,
TYPE_INVALID,
TYPE_UNIMPLEMENTED,
TYPE_COPY_OTHER,
// Changes `X86InstInfo::OpcodeDispatcher` member to use the `Indirect` version.
// Points to a 2 member array of X86InstInfo to choose instruction description based on executing bitness.
@@ -356,8 +349,11 @@ namespace InstFlags {
constexpr InstFlagType FLAGS_X87_FLAGS = (1ULL << 10);
// Non-XMM subflags
constexpr InstFlagType FLAGS_SF_REX_IN_BYTE = (1ULL << 11);
// subflag [15:12] unused
constexpr InstFlagType FLAGS_SF_DST_RAX = (1ULL << 11);
constexpr InstFlagType FLAGS_SF_DST_RDX = (1ULL << 12);
constexpr InstFlagType FLAGS_SF_SRC_RAX = (1ULL << 13);
constexpr InstFlagType FLAGS_SF_SRC_RCX = (1ULL << 14);
constexpr InstFlagType FLAGS_SF_REX_IN_BYTE = (1ULL << 15);
// XMM subflags
constexpr InstFlagType FLAGS_SF_UNUSED = (1ULL << 11); // No assigned behavior yet
@@ -407,10 +403,7 @@ namespace InstFlags {
constexpr InstFlagType FLAGS_CALL = (1ULL << 30);
constexpr InstFlagType FLAGS_SUPPORTS_LOCK = (1ULL << 31);
constexpr InstFlagType FLAGS_LITERAL_PATCHABLE = (1ULL << 32);
constexpr InstFlagType FLAGS_NO_PREFIX_ORDER = (1ULL << 33);
// Flags [57..34]: Undefined
// Flags [57..32]: Undefined
// Flags [60..58]: Dst size
constexpr InstFlagType FLAGS_SIZE_DST_OFF = 58;
// Flags [63..61]: Src size
@@ -522,7 +515,7 @@ constexpr size_t MAX_INST_GROUP_TABLE_SIZE = 512;
// 12 (groups) | 3(max indexes) | 8 ops = 0b1111'11'111 = 9 bits
constexpr size_t MAX_INST_SECOND_GROUP_TABLE_SIZE = 512;
constexpr size_t MAX_X87_TABLE_SIZE = 1 << 11;
constexpr size_t MAX_SECOND_MODRM_TABLE_SIZE = 40;
constexpr size_t MAX_SECOND_MODRM_TABLE_SIZE = 32;
// (3 bit prefixes) | 8 bit opcode
constexpr size_t MAX_0F_38_TABLE_SIZE = (1 << 11);
// 1 REX | 1 prefixes | 8 bit opcode
@@ -69,7 +69,7 @@ constexpr std::array<DispatchTableEntry, 140> X87F64OpTable = {{
{OPD(0xD9, 0xC0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDFromStack>},
{OPD(0xD9, 0xC8), 8, &OpDispatchBuilder::FXCH},
{OPD(0xD9, 0xD0), 1, &OpDispatchBuilder::X87NOP}, // FNOP
{OPD(0xD9, 0xD0), 1, &OpDispatchBuilder::NOPOp}, // FNOP
// D1 = Invalid
// D8 = Invalid
{OPD(0xD9, 0xE0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80STACKCHANGESIGN, false>},
@@ -216,7 +216,7 @@ constexpr std::array<DispatchTableEntry, 140> X87F64OpTable = {{
// 5 = Invalid
{OPDReg(0xDD, 6) | 0x00, 8, &OpDispatchBuilder::X87FNSAVE},
{OPDReg(0xDD, 7) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87FNSTSW, false>},
{OPDReg(0xDD, 7) | 0x00, 8, &OpDispatchBuilder::X87FNSTSW},
{OPD(0xDD, 0xC0), 8, &OpDispatchBuilder::X87FFREE},
{OPD(0xDD, 0xC8), 8, &OpDispatchBuilder::FXCH},
@@ -284,7 +284,7 @@ constexpr std::array<DispatchTableEntry, 140> X87F64OpTable = {{
{OPD(0xDF, 0xD0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSTToStack>},
{OPD(0xDF, 0xD8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSTToStack>},
{OPD(0xDF, 0xE0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87FNSTSW, true>},
{OPD(0xDF, 0xE0), 8, &OpDispatchBuilder::X87FNSTSW},
{OPD(0xDF, 0xE8), 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_RFLAGS, false>},
{OPD(0xDF, 0xF0), 8,
@@ -341,7 +341,7 @@ constexpr std::array<DispatchTableEntry, 140> X87F80OpTable = {{
{OPD(0xD9, 0xC0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDFromStack>},
{OPD(0xD9, 0xC8), 8, &OpDispatchBuilder::FXCH},
{OPD(0xD9, 0xD0), 1, &OpDispatchBuilder::X87NOP}, // FNOP
{OPD(0xD9, 0xD0), 1, &OpDispatchBuilder::NOPOp}, // FNOP
// D1 = Invalid
// D8 = Invalid
{OPD(0xD9, 0xE0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80STACKCHANGESIGN, false>},
@@ -483,7 +483,7 @@ constexpr std::array<DispatchTableEntry, 140> X87F80OpTable = {{
// 5 = Invalid
{OPDReg(0xDD, 6) | 0x00, 8, &OpDispatchBuilder::X87FNSAVE},
{OPDReg(0xDD, 7) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87FNSTSW, false>},
{OPDReg(0xDD, 7) | 0x00, 8, &OpDispatchBuilder::X87FNSTSW},
{OPD(0xDD, 0xC0), 8, &OpDispatchBuilder::X87FFREE},
{OPD(0xDD, 0xC8), 8, &OpDispatchBuilder::FXCH},
@@ -545,7 +545,7 @@ constexpr std::array<DispatchTableEntry, 140> X87F80OpTable = {{
{OPD(0xDF, 0xD0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSTToStack>},
{OPD(0xDF, 0xD8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSTToStack>},
{OPD(0xDF, 0xE0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87FNSTSW, true>},
{OPD(0xDF, 0xE0), 8, &OpDispatchBuilder::X87FNSTSW},
{OPD(0xDF, 0xE8), 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_RFLAGS, false>},
{OPD(0xDF, 0xF0), 8,
@@ -794,7 +794,7 @@ auto GenerateX87TableLambda = [](const auto DispatchTable) consteval {
// / 3
{OPD(0xDF, 0xD8), 8, X86InstInfo{"FSTP", TYPE_X87, FLAGS_SF_MOD_DST | FLAGS_POP, 0}},
// / 4
{OPD(0xDF, 0xE0), 1, X86InstInfo{"FNSTSW", TYPE_INST, GenFlagsSameSize(SIZE_16BIT), 0}},
{OPD(0xDF, 0xE0), 1, X86InstInfo{"FNSTSW", TYPE_INST, GenFlagsSameSize(SIZE_16BIT) | FLAGS_SF_DST_RAX, 0}},
{OPD(0xDF, 0xE1), 7, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0}},
// / 5
{OPD(0xDF, 0xE8), 8, X86InstInfo{"FUCOMIP", TYPE_INST, FLAGS_POP, 0}},
+12 -104
View File
@@ -201,7 +201,7 @@
"DestSize": "OpSize::i64Bit"
},
"ThreadRemoveCodeEntry GPR:$EntryToInvalidate, GPR:$NewRIP": {
"ThreadRemoveCodeEntry GPR:$Entry": {
"HasSideEffects": true
},
@@ -275,8 +275,7 @@
"The boolean argument asks if we should be reading the reseeded number or not",
"Reseeded RNG calculation is more expensive and will be heavier to use",
"Returns the 64-bit number",
"Falls back to a host RNG call when the hardware doesn't support it",
"Sets the Z flag if the number is invalid.",
"Sets the Z flag if the number is valid.",
"RNG hardware is allowed to fail early and return. Software must always check this"
],
"HasSideEffects": true,
@@ -321,7 +320,7 @@
"DestSize": "Size",
"RAOverride": "2"
},
"Break BreakDefinition:$Reason, GPR:$FaultAddress": {
"Break BreakDefinition:$Reason": {
"HasSideEffects": true
},
"CallbackReturn": {
@@ -337,8 +336,9 @@
"HasSideEffects": true
},
"CPUID i8:$InstSize": {
"GPR:$EAX, GPR:$EBX, GPR:$ECX, GPR:$EDX = CPUID GPR:$Function, GPR:$Leaf": {
"Desc": ["Calls in to the CPUID handler function to return emulated CPUID"],
"DestSize": "OpSize::i32Bit",
"HasSideEffects": true
},
"GPR:$EAX, GPR:$EDX = XGetBV GPR:$Function": {
@@ -529,7 +529,6 @@
},
"SSA = LoadMem RegisterClass:$Class, OpSize:#Size, GPR:$Addr, GPR:$Offset, OpSize:$Align, MemOffsetType:$OffsetType, u8:$OffsetScale": {
"HasSideEffects": true,
"Inline": ["", "Mem"],
"DestSize": "Size"
},
@@ -575,7 +574,6 @@
},
"SSA = LoadMemTSO RegisterClass:$Class, OpSize:#Size, GPR:$Addr, GPR:$Offset, OpSize:$Align, MemOffsetType:$OffsetType, u8:$OffsetScale": {
"HasSideEffects": true,
"Desc": ["Does a x86 TSO compatible load from memory. Offset must be Invalid()."
],
"Inline": ["", "Memtso"],
@@ -591,7 +589,6 @@
},
"FPR = VLoadVectorMasked OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Mask, GPR:$Addr, GPR:$Offset, MemOffsetType:$OffsetType, u8:$OffsetScale": {
"HasSideEffects": true,
"Desc": ["Does a masked load similar to VPMASKMOV/VMASKMOV where the upper bit of each element",
"determines whether or not that element will be loaded from memory"],
"ImplicitFlagClobber": true,
@@ -606,13 +603,11 @@
"DestSize": "RegisterSize",
"ElementSize": "ElementSize"
},
"FPR = VLoadVectorGatherMasked OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Incoming, FPR:$Mask, GPR:$AddrBase, FPR:$VectorIndexLow, FPR:$VectorIndexHigh, OpSize:$VectorIndexElementSize, u8:$OffsetScale, u8:$DataElementOffsetStart, u8:$IndexElementOffsetStart, OpSize:$AddrSize, i1:$MaskAllOnes{false}": {
"HasSideEffects": true,
"FPR = VLoadVectorGatherMasked OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Incoming, FPR:$Mask, GPR:$AddrBase, FPR:$VectorIndexLow, FPR:$VectorIndexHigh, OpSize:$VectorIndexElementSize, u8:$OffsetScale, u8:$DataElementOffsetStart, u8:$IndexElementOffsetStart, OpSize:$AddrSize": {
"Desc": [
"Does a masked load similar to VPGATHERD* where the upper bit of each element",
"determines whether or not that element will be loaded from memory.",
"Most of VSIB encoding is passed directly through to the IR operation.",
"MaskAllOnes is set when every mask element is provably all ones."
"Most of VSIB encoding is passed directly through to the IR operation."
],
"TiedSource": 0,
"ImplicitFlagClobber": true,
@@ -622,14 +617,12 @@
"$VectorIndexElementSize == OpSize::i32Bit || $VectorIndexElementSize == OpSize::i64Bit"
]
},
"FPR = VLoadVectorGatherMaskedQPS OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Incoming, FPR:$MaskReg, GPR:$AddrBase, FPR:$VectorIndexLow, FPR:$VectorIndexHigh, u8:$OffsetScale, OpSize:$AddrSize, i1:$MaskAllOnes{false}": {
"HasSideEffects": true,
"FPR = VLoadVectorGatherMaskedQPS OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Incoming, FPR:$MaskReg, GPR:$AddrBase, FPR:$VectorIndexLow, FPR:$VectorIndexHigh, u8:$OffsetScale, OpSize:$AddrSize": {
"Desc": [
"Does a masked load similar to VPGATHERQPS where the upper bit of each element",
"determines whether or not that element will be loaded from memory.",
"Most of VSIB encoding is passed directly through to the IR operation.",
"Only supports the case of 32-bit data element sizes from 64-bit addresses",
"MaskAllOnes is set when every mask element is provably all ones."
"Only supports the case of 32-bit data element sizes from 64-bit addresses"
],
"TiedSource": 0,
"ImplicitFlagClobber": true,
@@ -641,7 +634,6 @@
]
},
"FPR = VLoadVectorElement OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$DstSrc, u8:$Index, GPR:$Addr": {
"HasSideEffects": true,
"Desc": ["Does a memory load to a single element of a vector.",
"Leaves the rest of the vector's data intact.",
"Matches arm64 ld1 semantics"],
@@ -781,7 +773,6 @@
]
},
"FPR = VLoadNonTemporal OpSize:#RegisterSize, GPR:$Addr, i8:$Offset": {
"HasSideEffects": true,
"Desc": ["Does a non-temporal memory load of a vector.",
"Matches arm64 SVE ldnt1b semantics.",
"Specifically weak-memory model ordered to match x86 non-temporal stores."
@@ -969,20 +960,6 @@
"DestSize": "Size"
},
"GPR = PatchableGuestRIP OpSize:#Size, i64:$Value, i64:$SiteAddress, i64:$SiteSize": {
"Desc": ["Loads GuestRIP-relative Value in a patchable way",
"On disk cache load the value is patched from live guest PC and live displacement at SiteAddress"
],
"DestSize": "Size"
},
"GPR = PatchableGuestCRC OpSize:#Size, i64:$Value, i64:$GuestAddress, i64:$GuestSize": {
"Desc": ["Loads Guest CRC in a patchable way",
"On disk cache load the value is recomputed from live guest bytes and patched"
],
"DestSize": "Size"
},
"GPR = Constant i64:$Constant, ConstPad:$Pad{IR::ConstPad::NoPad}, i32:$MaxBytes{0}": {
"Desc": ["Generates a 64bit constant inside of a GPR",
"Unsupported to create a constant in FPR"
@@ -1869,7 +1846,7 @@
"DestSize": "RegisterSize"
},
"FPR = VectorImm OpSize:#RegisterSize, OpSize:#ElementSize, u8:$Immediate, u8:$ShiftAmount{0}, i1:$Do64BitExpansion{false}": {
"FPR = VectorImm OpSize:#RegisterSize, OpSize:#ElementSize, u8:$Immediate, u8:$ShiftAmount{0}": {
"Desc": ["Generates a vector with each element containg the immediate zexted"
],
"DestSize": "RegisterSize",
@@ -2386,43 +2363,6 @@
"DestSize": "RegisterSize",
"ElementSize": "ElementSize << 1"
},
"FPR = VUSDot OpSize:#RegisterSize, FPR:$Acc, FPR:$Vector1, FPR:$Vector2": {
"Desc": ["Unsigned by signed 8-bit dot product, accumulating four products in to each 32-bit element of Acc.",
"Requires FEAT_I8MM."
],
"DestSize": "RegisterSize",
"ElementSize": "OpSize::i32Bit",
"TiedSource": 0,
"EmitValidation": [
"RegisterSize == FEXCore::IR::OpSize::i128Bit || RegisterSize == FEXCore::IR::OpSize::i256Bit"
]
},
"FPR = VSDot OpSize:#RegisterSize, FPR:$Acc, FPR:$Vector1, FPR:$Vector2": {
"Desc": ["Signed 8-bit dot product, accumulating four products in to each 32-bit element of Acc.",
"Requires FEAT_DotProd."
],
"DestSize": "RegisterSize",
"ElementSize": "OpSize::i32Bit",
"TiedSource": 0,
"EmitValidation": [
"RegisterSize == FEXCore::IR::OpSize::i128Bit || RegisterSize == FEXCore::IR::OpSize::i256Bit"
]
},
"FPR = VSAddLP OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector": {
"Desc": ["Signed add long pairwise. Adds adjacent pairs of elements in to elements of twice the size.",
"ElementSize is the source size"
],
"DestSize": "RegisterSize",
"ElementSize": "ElementSize << 1"
},
"FPR = VSAdALP OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Acc, FPR:$Vector": {
"Desc": ["Signed add and accumulate long pairwise. Adds adjacent pairs of elements in to the elements of Acc, which are twice the size.",
"ElementSize is the source size"
],
"DestSize": "RegisterSize",
"ElementSize": "ElementSize << 1",
"TiedSource": 0
},
"FPR = VUMulH OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2": {
"Desc": ["Wide unsigned multiply returning the high results"],
"DestSize": "RegisterSize",
@@ -2526,14 +2466,6 @@
"DestSize": "RegisterSize",
"ElementSize": "ElementSize"
},
"FPR = VUCMPGT OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2": {
"Desc": ["Vector compare unsigned greater than",
"Each element is compared, if the result is true then the resulting element is ~0, else zero"
],
"DestSize": "RegisterSize",
"ElementSize": "ElementSize"
},
"FPR = VFCMPEQ OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2": {
"DestSize": "RegisterSize",
"ElementSize": "ElementSize"
@@ -2601,8 +2533,7 @@
},
"GPR = VPCMPESTRX FPR:$LHS, FPR:$RHS, GPR:$RAX, GPR:$RDX, u16:$Control": {
"Desc": ["NOTE: Currently unused. The OpcodeDispatcher implements the SSE4.2 string instructions inline.",
"Performs intermediate behavior analogous to the x86 PCMPESTRI/PCMPESTRM instruction",
"Desc": ["Performs intermediate behavior analogous to the x86 PCMPESTRI/PCMPESTRM instruction",
"This will return the intermediate result of a PCMPESTR-type operation, but NOT the final",
"result. This must be derived from the intermediate result",
@@ -2614,8 +2545,7 @@
"JITDispatch": false
},
"GPR = VPCMPISTRX FPR:$LHS, FPR:$RHS, u8:$Control": {
"Desc": ["NOTE: Currently unused. The OpcodeDispatcher implements the SSE4.2 string instructions inline.",
"Performs intermediate behavior analogous to the x86 PCMPISTRI/PCMPISTRM instruction",
"Desc": ["Performs intermediate behavior analogous to the x86 PCMPISTRI/PCMPISTRM instruction",
"This will return the intermediate result of a PCMPISTR-type operation, but NOT the final",
"result. This must be derived from the intermediate result",
@@ -2767,28 +2697,6 @@
"RegisterSize != FEXCore::IR::OpSize::i256Bit && \"What does 256-bit mean in this context?\""
]
},
"FPR = VBFCVTN OpSize:#RegisterSize, FPR:$Vector": {
"Desc": [
"Converts fp32 to bf16."
],
"DestSize": "RegisterSize",
"ElementSize": "FEXCore::IR::OpSize::i16Bit",
"EmitValidation": [
"RegisterSize != FEXCore::IR::OpSize::i256Bit && \"What does 256-bit mean in this context?\""
]
},
"FPR = VBFCVTN2 OpSize:#RegisterSize, FPR:$VectorLower, FPR:$VectorUpper": {
"TiedSource": 0,
"Desc": [
"Converts fp32 to bf16 and inserting in to the high bits.",
"Bottom half is untouched"
],
"DestSize": "RegisterSize",
"ElementSize": "FEXCore::IR::OpSize::i16Bit",
"EmitValidation": [
"RegisterSize != FEXCore::IR::OpSize::i256Bit && \"What does 256-bit mean in this context?\""
]
},
"FPR = Vector_FToI OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector, RoundType:$Round": {
"Desc": ["Vector op: Rounds float to integral",
"Rounding mode determined by argument"
-2
View File
@@ -172,8 +172,6 @@ static void PrintArg(fextl::ostringstream* out, const IRListView*, NamedVectorCo
return "u16_incremental_index";
case NamedVectorConstant::NAMED_VECTOR_INCREMENTAL_U16_INDEX_UPPER:
return "u16_incremental_index_upper";
case NamedVectorConstant::NAMED_VECTOR_INCREMENTAL_U8_INDEX:
return "u8_incremental_index";
case NamedVectorConstant::NAMED_VECTOR_PADDSUBPS_INVERT:
return "addsubps_invert";
case NamedVectorConstant::NAMED_VECTOR_PADDSUBPS_INVERT_UPPER:
@@ -19,9 +19,6 @@ namespace FEXCore::IR {
static bool IsFragmentExit(FEXCore::IR::IROps Op) {
switch (Op) {
case OP_THREADREMOVECODEENTRY:
case OP_SYSCALL:
case OP_CPUID:
case OP_EXITFUNCTION:
case OP_BREAK: return true;
default: return false;
-30
View File
@@ -341,27 +341,6 @@ public:
return false;
}
// Returns true if every bit of the vector value is set.
// Currently this only recognizes the self compare equal idiom.
// TODO: Should this recognize other patterns?
bool IsValueVectorAllOnes(OrderedNodeWrapper ssa) const {
auto RealNode = ssa.GetNode(DualListData.ListBegin());
const auto* IROp = RealNode->Op(DualListData.DataBegin());
return IROp->Op == OP_VCMPEQ && IROp->Args[0].ID() == IROp->Args[1].ID();
}
// Returns true if the value is a load of register Reg of class Class.
bool IsValueLoadRegister(OrderedNodeWrapper ssa, RegClass Class, uint32_t Reg) const {
auto RealNode = ssa.GetNode(DualListData.ListBegin());
const auto* IROp = RealNode->Op(DualListData.DataBegin());
if (IROp->Op != OP_LOADREGISTER) {
return false;
}
auto Op = IROp->C<IR::IROp_LoadRegister>();
return Op->Class == Class && Op->Reg == Reg;
}
FEXCore::IR::IROp_Header* GetOpHeader(OrderedNodeWrapper ssa) {
Ref RealNode = ssa.GetNode(DualListData.ListBegin());
return RealNode->Op(DualListData.DataBegin());
@@ -375,15 +354,6 @@ public:
return node->Wrapped(DualListData.ListBegin());
}
// Returns argument Index of Node if the op of Node is Op. Else returns nullptr.
Ref GetArgIfOp(Ref Node, IROps Op, unsigned Index) {
if (!Node || GetOpHeader(WrapNode(Node))->Op != Op) {
return nullptr;
}
return UnwrapNode(GetOpHeader(WrapNode(Node))->Args[Index]);
}
NodeIterator GetIterator(OrderedNodeWrapper wrapper) {
return NodeIterator(DualListData.ListBegin(), DualListData.DataBegin(), wrapper);
}
@@ -527,6 +527,27 @@ bool ConstrainedRAPass::TryPostRAMerge(Ref LastNode, Ref CodeNode, IROp_Header*
IREmit->RemovePostRA(CodeNode);
return false;
}
} else if (IROp->Op == OP_CPUID && PhysicalRegister(IROp->Args[0]) == PhysicalRegister(LastNode) && LastOp->Op == OP_CONSTANT) {
// Try to constant fold. As a limitation of merging only 2 instructions, we
// can only handle constant functions, not constant leafs. This could be
// lifted if we generalized at a (significant) complexity cost.
uint64_t ConstantFunction = LastOp->C<IROp_Constant>()->Constant;
auto Op = IROp->CW<IR::IROp_CPUID>();
const auto SupportsConstant = CPUID->DoesFunctionReportConstantData(ConstantFunction);
if (SupportsConstant.SupportsConstantFunction == CPUIDEmu::SupportsConstant::CONSTANT &&
SupportsConstant.NeedsLeaf != CPUIDEmu::NeedsLeafConstant::NEEDSLEAFCONSTANT) {
const auto Result = CPUID->RunFunction(ConstantFunction, 0 /* leaf */);
IREmit->SetWriteCursorBefore(CodeNode);
IREmit->_Fence(IR::FenceType::Inst);
IREmit->_Constant(Result.eax).Node->Reg = PhysicalRegister(Op->OutEAX).Raw;
IREmit->_Constant(Result.ebx).Node->Reg = PhysicalRegister(Op->OutEBX).Raw;
IREmit->_Constant(Result.ecx).Node->Reg = PhysicalRegister(Op->OutECX).Raw;
IREmit->_Constant(Result.edx).Node->Reg = PhysicalRegister(Op->OutEDX).Raw;
IREmit->RemovePostRA(CodeNode);
return false;
}
}
// Merge moves that are immediately consumed.
+5 -85
View File
@@ -23,9 +23,7 @@
#include <fcntl.h>
#ifndef _WIN32
#include <sys/mman.h>
#include <sys/resource.h>
#include <sys/user.h>
#include <unistd.h>
#endif
namespace fextl::pmr {
@@ -216,62 +214,11 @@ fextl::vector<MemoryRegion> CollectMemoryGaps(uintptr_t Begin, uintptr_t End, in
FEX_UNREACHABLE;
}
// Tracks the region below the main thread stack, which is reserved for stack growth,
// as well as the guard region directly below the stack window.
static uintptr_t MainThreadStackReserveBegin {};
static uintptr_t MainThreadStackWindowBegin {};
static uintptr_t MainThreadStackWindowEnd {};
static size_t MainThreadStackGuardSize {};
static uintptr_t MainThreadStackGuardBegin {};
// Currently, guest signal handlers leak space on the main stack
// if they exit via longjmp instead of sigreturn. Because of this memory leak
// we must allow the main thread stack to grow without bound. This technically
// violates RLIMIT_STACK, but until we fix the leak, we don't have much choice.
// TODO: Once the leak is fixed, remove this function entirely.
bool TryGrowMainThreadStack(uintptr_t FaultAddress) {
// Only grow the stack when the fault is in the guard region.
if (FaultAddress < MainThreadStackGuardBegin || FaultAddress >= MainThreadStackWindowBegin) {
return false;
}
// Compute the required size of the stack region.
// Double the current stack, up to 1GB, then just add 1GB at a time.
constexpr size_t OneGB = 1ULL << 30;
const size_t RequiredSize = MainThreadStackWindowEnd - FaultAddress;
size_t NewSize = MainThreadStackWindowEnd - MainThreadStackWindowBegin;
while (NewSize < RequiredSize) {
if (NewSize < OneGB) {
NewSize = std::min(NewSize * 2, OneGB);
} else {
NewSize += OneGB;
}
}
// Don't grow past the bottom of the reserved region,
// which is where the first dynamically loaded libraries live.
const uintptr_t NewWindowBegin = std::max(MainThreadStackWindowEnd - NewSize, MainThreadStackReserveBegin);
if (mprotect(reinterpret_cast<void*>(NewWindowBegin), MainThreadStackWindowBegin - NewWindowBegin, PROT_READ | PROT_WRITE) != 0) {
return false;
}
// Move the window and the guard region below it down.
MainThreadStackWindowBegin = NewWindowBegin;
MainThreadStackGuardBegin = std::max(NewWindowBegin - MainThreadStackGuardSize, MainThreadStackReserveBegin);
return true;
}
fextl::vector<MemoryRegion> StealMemoryRegion(uintptr_t Begin, uintptr_t End) {
const uintptr_t StackLocation_u64 = reinterpret_cast<uintptr_t>(alloca(0));
const int MapsFD = open("/proc/self/maps", O_RDONLY);
if (MapsFD == -1) {
// No procfs, as in BuildKit's emulator probe (an empty chroot). The assert
// above it compiles out in release, and CollectMemoryGaps then spins on
// read(-1) forever; reserve nothing instead.
return {};
}
LogMan::Throw::AFmt(MapsFD != -1, "Failed to open /proc/self/maps");
auto Regions = CollectMemoryGaps(Begin, End, MapsFD);
close(MapsFD);
@@ -279,7 +226,7 @@ fextl::vector<MemoryRegion> StealMemoryRegion(uintptr_t Begin, uintptr_t End) {
// If the memory bounds include the stack, blocking all memory regions will
// limit the stack size to the current value. To allow some stack growth,
// we don't block the memory gap directly below the stack memory but
// instead reserve it for the stack, see TryGrowMainThreadStack.
// instead map it as readable+writable.
{
auto StackRegionIt = std::find_if(Regions.begin(), Regions.end(), [StackLocation_u64](auto& Region) {
return reinterpret_cast<uintptr_t>(Region.Ptr) + Region.Size > StackLocation_u64;
@@ -291,40 +238,16 @@ fextl::vector<MemoryRegion> StealMemoryRegion(uintptr_t Begin, uintptr_t End) {
if (IsStackMapping && StackRegionIt != Regions.begin() &&
reinterpret_cast<uintptr_t>(std::prev(StackRegionIt)->Ptr) + std::prev(StackRegionIt)->Size <= End) {
// Allocate the region under the stack as READ | WRITE so the stack can still grow
--StackRegionIt;
// Get the page size, so we can make sure to align on page boundaries.
const auto PageSize = sysconf(_SC_PAGESIZE);
const size_t HostPageSize = PageSize > 0 ? PageSize : FEXCore::Utils::FEX_PAGE_SIZE;
// Allocate RLIMIT_STACK worth of memory for the main thread stack, or 64MB if no limit is set.
struct rlimit StackLimit {};
size_t Size = 1ULL << 26;
if (getrlimit(RLIMIT_STACK, &StackLimit) == 0 && StackLimit.rlim_cur != RLIM_INFINITY) {
Size = std::max<size_t>(FEXCore::AlignUp(StackLimit.rlim_cur, HostPageSize), HostPageSize);
}
Size = std::min(Size, StackRegionIt->Size);
// Reserve all of the memory between the top of the stack, and the start of the next region.
// We must map it with PROT_NONE, otherwise the MM will assume these pages are "in use"
// and charge this usage towards VmData. In practice this is multiple TBs of VA space.
auto Alloc = ::mmap(StackRegionIt->Ptr, StackRegionIt->Size, PROT_NONE, MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, -1, 0);
auto Alloc =
::mmap(StackRegionIt->Ptr, StackRegionIt->Size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, -1, 0);
LogMan::Throw::AFmt(Alloc != MAP_FAILED, "StealMemoryRegion:Stack: mmap({}, {:x}) failed: {}", fmt::ptr(StackRegionIt->Ptr),
StackRegionIt->Size, errno);
LogMan::Throw::AFmt(Alloc == StackRegionIt->Ptr, "mmap returned {} instead of {}", Alloc, fmt::ptr(StackRegionIt->Ptr));
// Now determine the bounds for the stack region, and its guard region, which is used by TryGrowMainThreadStack().
MainThreadStackReserveBegin = reinterpret_cast<uintptr_t>(StackRegionIt->Ptr);
MainThreadStackWindowEnd = MainThreadStackReserveBegin + StackRegionIt->Size;
MainThreadStackWindowBegin = MainThreadStackWindowEnd - Size;
MainThreadStackGuardSize = 256 * HostPageSize;
MainThreadStackGuardBegin = std::max(MainThreadStackWindowBegin - MainThreadStackGuardSize, MainThreadStackReserveBegin);
// Protect the stack window.
const auto Result = mprotect(reinterpret_cast<void*>(MainThreadStackWindowBegin), Size, PROT_READ | PROT_WRITE);
LogMan::Throw::AFmt(Result == 0, "StealMemoryRegion:Stack: mprotect({:x}, {:x}) failed: {}", MainThreadStackWindowBegin, Size, errno);
Regions.erase(StackRegionIt);
}
}
@@ -359,9 +282,6 @@ fextl::vector<MemoryRegion> Setup48BitAllocatorIfExists(size_t PageSize) {
uintptr_t Begin48BitVA = 0x0'8000'0000'0000ULL;
uintptr_t End48BitVA = 0x1'0000'0000'0000ULL;
auto Regions = StealMemoryRegion(Begin48BitVA, End48BitVA);
if (Regions.empty()) {
return {};
}
Alloc64 = Alloc::OSAllocator::Create64BitAllocatorWithRegions(Regions);
AssignHookOverrides(PageSize);
+2 -18
View File
@@ -27,13 +27,9 @@ using vma_name_hook_type = void (*)(const char* name, const void* address, size_
#ifdef ENABLE_FEX_ALLOCATOR
typedef void* (*rp_mmap_hook_type)(size_t size, size_t alignment, size_t* offset, size_t* mapped_size);
typedef void (*rp_munmap_hook_type)(void* address, size_t offset, size_t mapped_size);
typedef void (*rp_mcommit_hook_type)(void* address, size_t size);
typedef void (*rp_mdecommit_hook_type)(void* address, size_t size);
typedef void (*vma_name_hook_type)(const char* name, const void* address, size_t size);
extern "C" rp_mmap_hook_type rp_mmap_hook;
extern "C" rp_munmap_hook_type rp_munmap_hook;
extern "C" rp_mcommit_hook_type rp_mcommit_hook;
extern "C" rp_mdecommit_hook_type rp_mdecommit_hook;
extern "C" vma_name_hook_type rp_name_hook;
#ifndef _WIN32
@@ -116,11 +112,8 @@ static void* FEX_rp_mmap(size_t size, size_t alignment, size_t* offset, size_t*
alignment = 0;
}
// Protect with PROT_NONE so that the whole heap is not charged against VmData.
// In the memory commit callback, FEX_rp_memory_commit, we reprotect with
// PROT_READ | PROT_WRITE to commit memory on demand.
size_t map_size = AlignUp(size + alignment, global_config.page_size);
auto ptr = fex_mmap_hook(0, map_size, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
auto ptr = fex_mmap_hook(0, map_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
if (ptr == MAP_FAILED) {
ptr = nullptr;
@@ -156,9 +149,7 @@ static void* FEX_rp_mmap(size_t size, size_t alignment, size_t* offset, size_t*
}
static void FEX_rp_memory_commit(void* address, size_t size) {
if (mprotect(address, size, PROT_READ | PROT_WRITE)) {
fprintf(stderr, "Failed to commit VMA region.");
}
// NOP-implementation.
}
static void FEX_rp_memory_decommit(void* address, size_t size) {
@@ -169,11 +160,6 @@ static void FEX_rp_memory_decommit(void* address, size_t size) {
if (madvise(address, size, MADV_DONTNEED)) {
fprintf(stderr, "Failed to decommit VMA region.");
}
// Drop the protections back to PROT_NONE so the pages are no longer charged towards VmData.
if (mprotect(address, size, PROT_NONE)) {
fprintf(stderr, "Failed to decommit VMA region.");
}
}
static void FEX_rp_memory_unmap(void* address, size_t offset, size_t mapped_size) {
@@ -204,8 +190,6 @@ void InitializeAllocator(size_t PageSize) {
rpmalloc_initialize_config(&global_interface, &global_config);
rp_mmap_hook = FEX_rp_mmap;
rp_munmap_hook = FEX_rp_memory_unmap;
rp_mcommit_hook = FEX_rp_memory_commit;
rp_mdecommit_hook = FEX_rp_memory_decommit;
rp_name_hook = LocalVirtualName;
}
#else
-366
View File
@@ -1,366 +0,0 @@
// SPDX-License-Identifier: MIT
#include <FEXCore/fextl/fmt.h>
#include <FEXCore/fextl/functional.h>
#include <FEXCore/fextl/string.h>
#include <FEXCore/Utils/CompilerDefs.h>
#include <FEXCore/Utils/LogManager.h>
#ifndef _WIN32
#include <dirent.h>
#include <fcntl.h>
#include <unistd.h>
#else
#include <windows.h>
#include <winnt.h>
#include <winternl.h>
#include <ntstatus.h>
// FEX today doesn't have Windows include in FEXCore.
extern "C" NTSTATUS WINAPI NtQueryDirectoryFile(HANDLE, HANDLE, PIO_APC_ROUTINE, PVOID, PIO_STATUS_BLOCK, PVOID, ULONG,
FILE_INFORMATION_CLASS, BOOLEAN, PUNICODE_STRING, BOOLEAN);
extern "C" NTSTATUS RtlUnicodeToUTF8N(OUT PCHAR UTF8StringDestination, IN ULONG UTF8StringMaxByteCount,
OUT PULONG UTF8StringActualByteCount, IN PCWCH UnicodeStringSource, IN ULONG UnicodeStringByteCount);
#endif
namespace FEXCore::FileUtils {
#ifndef _WIN32
static inline bool unlinkat(int fd, const char* path, bool dir) {
if (::unlinkat(fd, path, dir ? AT_REMOVEDIR : 0) == -1) {
return errno == ENOENT;
}
return true;
}
static bool RecursiveRemoveDirectory(int parent_fd, const char* Directory) {
// Don't follow symlinks and ensure it closes on exec.
constexpr int DIR_FLAGS = O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC;
int dir_fd = ::openat(parent_fd, Directory, DIR_FLAGS);
if (dir_fd == -1) {
if (errno == ENOENT) {
// Probably raced something. Non-error.
return true;
}
}
// Walk the directory listing.
bool Result = true;
// Four pages arbitrary chosen to be a balance between NFS wanting to return data in page-size granules and
// local filesystems returning arbitrary sizes.
size_t dirent_size = 4096 * 4;
uint8_t* dirent_buffer = reinterpret_cast<uint8_t*>(FEXCore::Allocator::malloc(dirent_size));
if (!dirent_buffer) {
// Ran out of memory?
Result = false;
goto end;
}
while (true) {
ssize_t read = getdents64(dir_fd, dirent_buffer, dirent_size);
if (read == -1) {
if (errno == EINVAL) {
// Buffer too small? Scale and try again.
dirent_size *= 2;
FEXCore::Allocator::free(dirent_buffer);
dirent_buffer = reinterpret_cast<uint8_t*>(FEXCore::Allocator::malloc(dirent_size));
if (!dirent_buffer) {
// Ran out of memory?
Result = false;
goto end;
}
continue;
}
// Anything else just exit.
Result = false;
goto end;
}
if (read == 0) {
// Done.
break;
}
for (size_t dirent_offset = 0; dirent_offset < read;) {
auto path_dirent = reinterpret_cast<const struct dirent*>(dirent_buffer + dirent_offset);
std::string_view path_name_view = path_dirent->d_name;
if (path_name_view == "." || path_name_view == "..") {
// Skip these two special files.
dirent_offset += path_dirent->d_reclen;
continue;
}
if (path_dirent->d_type == DT_DIR) {
// Recurse directories as we find them and remove them.
if (!RecursiveRemoveDirectory(dir_fd, path_dirent->d_name)) {
Result = false;
goto end;
}
}
// Remove anything possible.
if (!unlinkat(dir_fd, path_dirent->d_name, path_dirent->d_type == DT_DIR)) {
// Couldn't unlink the file for some reason.
LogMan::Msg::IFmt("Failed to remove file: {}", path_name_view);
Result = false;
goto end;
}
// dirent is a VLA so we need to increment by reported size.
dirent_offset += path_dirent->d_reclen;
}
}
end:
if (dirent_buffer) {
FEXCore::Allocator::free(dirent_buffer);
}
close(dir_fd);
return Result;
}
FEX_DEFAULT_VISIBILITY bool RecursiveRemoveDirectory(const fextl::string& Directory) {
return RecursiveRemoveDirectory(AT_FDCWD, Directory.c_str()) && unlinkat(AT_FDCWD, Directory.c_str(), true);
}
FEX_DEFAULT_VISIBILITY void WalkDirectory(std::string_view Directory,
fextl::move_only_function<void(std::string_view name, bool is_dir, const void* user_data)> Callback,
const void* user_data) {
constexpr int DIR_FLAGS = O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC;
int dir_fd = ::openat(AT_FDCWD, fextl::string(Directory).c_str(), DIR_FLAGS);
if (dir_fd == -1) {
if (errno == ENOENT) {
// Probably raced something. Non-error.
return;
}
}
// Four pages arbitrary chosen to be a balance between NFS wanting to return data in page-size granules and
// local filesystems returning arbitrary sizes.
size_t dirent_size = 4096 * 4;
uint8_t* dirent_buffer = reinterpret_cast<uint8_t*>(FEXCore::Allocator::malloc(dirent_size));
if (!dirent_buffer) {
// Ran out of memory?
goto end;
}
while (true) {
ssize_t read = getdents64(dir_fd, dirent_buffer, dirent_size);
if (read == -1) {
if (errno == EINVAL) {
// Buffer too small? Scale and try again.
dirent_size *= 2;
FEXCore::Allocator::free(dirent_buffer);
dirent_buffer = reinterpret_cast<uint8_t*>(FEXCore::Allocator::malloc(dirent_size));
if (!dirent_buffer) {
// Ran out of memory?
goto end;
}
continue;
}
// Anything else just exit.
goto end;
}
if (read == 0) {
// Done.
break;
}
for (size_t dirent_offset = 0; dirent_offset < read;) {
auto path_dirent = reinterpret_cast<const struct dirent*>(dirent_buffer + dirent_offset);
std::string_view path_name_view = path_dirent->d_name;
if (path_name_view == "." || path_name_view == "..") {
// Skip these two special files.
dirent_offset += path_dirent->d_reclen;
continue;
}
Callback(path_name_view, path_dirent->d_type == DT_DIR, user_data);
// dirent is a VLA so we need to increment by reported size.
dirent_offset += path_dirent->d_reclen;
}
}
end:
if (dirent_buffer) {
FEXCore::Allocator::free(dirent_buffer);
}
close(dir_fd);
}
#else
static inline std::optional<UNICODE_STRING> PathToNTPath(std::string_view Path) {
UNICODE_STRING PathW;
if (!RtlCreateUnicodeStringFromAsciiz(&PathW, fextl::string(Path).c_str())) {
return std::nullopt;
}
UNICODE_STRING NTPath;
bool Success = RtlDosPathNameToNtPathName_U(PathW.Buffer, &NTPath, nullptr, nullptr);
RtlFreeUnicodeString(&PathW);
if (!Success) {
return std::nullopt;
}
return NTPath;
}
static inline void FreeNTPath(UNICODE_STRING Path) {
RtlFreeUnicodeString(&Path);
}
FEX_DEFAULT_VISIBILITY void WalkDirectory(std::string_view Directory,
fextl::move_only_function<void(std::string_view name, bool is_dir, const void* user_data)> Callback,
const void* user_data) {
auto NTPath = PathToNTPath(Directory);
if (!NTPath) {
return;
}
NTSTATUS Status {};
HANDLE dir_fd {};
OBJECT_ATTRIBUTES attr {};
IO_STATUS_BLOCK io {};
InitializeObjectAttributes(&attr, &*NTPath, OBJ_CASE_INSENSITIVE, nullptr, nullptr);
Status = NtOpenFile(&dir_fd, FILE_LIST_DIRECTORY | SYNCHRONIZE, &attr, &io, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
FILE_DIRECTORY_FILE | FILE_SYNCHRONOUS_IO_NONALERT);
FreeNTPath(*NTPath);
if (!NT_SUCCESS(Status)) {
return;
}
BOOLEAN FirstQuery = TRUE;
size_t dirent_size = 4096 * 4;
uint8_t* dirent_buffer = reinterpret_cast<uint8_t*>(FEXCore::Allocator::malloc(dirent_size));
if (!dirent_buffer) {
// Ran out of memory?
goto end;
}
while (true) {
Status = NtQueryDirectoryFile(dir_fd,
nullptr, // Event
nullptr, // ApcRoutine
nullptr, // ApcContext
&io, dirent_buffer, dirent_size, FileDirectoryInformation,
FALSE, // ReturnSingleEntry
nullptr, // FileName
FirstQuery);
FirstQuery = FALSE;
if (Status == STATUS_NO_MORE_FILES) {
// No more files
break;
}
if (!NT_SUCCESS(Status)) {
if (Status == STATUS_BUFFER_TOO_SMALL || Status == STATUS_INFO_LENGTH_MISMATCH) {
// Buffer too small? Scale and try again.
dirent_size *= 2;
FEXCore::Allocator::free(dirent_buffer);
uint8_t* dirent_buffer = reinterpret_cast<uint8_t*>(FEXCore::Allocator::malloc(dirent_size));
if (!dirent_buffer) {
// Ran out of memory?
goto end;
}
continue;
}
// Any other failure, exit loop.
break;
}
// Iterate the entries returned.
for (size_t dirent_offset = 0;;) {
auto Info = reinterpret_cast<FILE_DIRECTORY_INFORMATION*>(dirent_buffer + dirent_offset);
std::wstring_view EntryName(Info->FileName, Info->FileNameLength / sizeof(wchar_t));
if (EntryName != L"." && EntryName != L"..") {
ULONG utf8_bytes_needed = 0;
ULONG actual_utf8_bytes = 0;
RtlUnicodeToUTF8N(nullptr, 0, &utf8_bytes_needed, Info->FileName, Info->FileNameLength);
char* dynamic_buf = reinterpret_cast<char*>(FEXCore::Allocator::malloc(utf8_bytes_needed + 1));
if (dynamic_buf) {
RtlUnicodeToUTF8N(dynamic_buf, utf8_bytes_needed + 1, &actual_utf8_bytes, Info->FileName, Info->FileNameLength);
std::string_view name_view(dynamic_buf, actual_utf8_bytes);
bool is_dir = (Info->FileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0;
Callback(name_view, is_dir, user_data);
FEXCore::Allocator::free(dynamic_buf);
}
}
if (Info->NextEntryOffset == 0) {
break;
}
dirent_offset += Info->NextEntryOffset;
}
}
end:
if (dirent_buffer) {
FEXCore::Allocator::free(dirent_buffer);
}
NtClose(dir_fd);
}
FEX_DEFAULT_VISIBILITY bool RecursiveRemoveDirectory(const fextl::string& Directory) {
using CallbackType = void (*)(std::string_view name, bool is_dir, const void* user_data);
struct UserData {
CallbackType RemoveFile {};
std::string_view base_path;
};
CallbackType RemoveFile = [](std::string_view name, bool is_dir, const void* user_data) {
auto Data = reinterpret_cast<const UserData*>(user_data);
auto full_path = std::format("{}/{}", Data->base_path, name);
if (is_dir) {
UserData NewData {
.RemoveFile = Data->RemoveFile,
.base_path = full_path,
};
WalkDirectory(full_path, Data->RemoveFile, &NewData);
}
if (is_dir) {
RemoveDirectoryA(full_path.c_str());
} else {
DeleteFileA(full_path.c_str());
}
};
UserData Data {
.RemoveFile = RemoveFile,
.base_path = Directory,
};
WalkDirectory(Directory, RemoveFile, &Data);
return RemoveDirectoryA(Directory.c_str()) != 0;
}
#endif
} // namespace FEXCore::FileUtils
+3 -5
View File
@@ -7,8 +7,7 @@
#include <unistd.h>
namespace FEXCore::Threads {
static fextl::unique_ptr<FEXCore::Threads::Thread>
CreateThread_Default(ThreadFunc Func, void* Arg, FEXCore::Threads::Flags Flags, const char* ThreadName) {
static fextl::unique_ptr<FEXCore::Threads::Thread> CreateThread_Default(ThreadFunc Func, void* Arg, FEXCore::Threads::Flags Flags) {
ERROR_AND_DIE_FMT("Frontend didn't setup thread creation!");
}
@@ -21,9 +20,8 @@ static FEXCore::Threads::Pointers Ptrs = {
.CleanupAfterFork = CleanupAfterFork_Default,
};
fextl::unique_ptr<FEXCore::Threads::Thread>
FEXCore::Threads::Thread::Create(ThreadFunc Func, void* Arg, FEXCore::Threads::Flags Flags, const char* ThreadName) {
return Ptrs.CreateThread(Func, Arg, Flags, ThreadName);
fextl::unique_ptr<FEXCore::Threads::Thread> FEXCore::Threads::Thread::Create(ThreadFunc Func, void* Arg, FEXCore::Threads::Flags Flags) {
return Ptrs.CreateThread(Func, Arg, Flags);
}
void FEXCore::Threads::Thread::CleanupAfterFork() {
+2 -2
View File
@@ -4,8 +4,8 @@
namespace FEXCore {
WorkQueueThread::WorkQueueThread(FEXCore::Threads::Flags ThreadFlags, const char* ThreadName) {
Thread = FEXCore::Threads::Thread::Create(ThreadEntry, this, ThreadFlags, ThreadName);
WorkQueueThread::WorkQueueThread(FEXCore::Threads::Flags ThreadFlags) {
Thread = FEXCore::Threads::Thread::Create(ThreadEntry, this, ThreadFlags);
}
WorkQueueThread::~WorkQueueThread() {
-59
View File
@@ -1,59 +0,0 @@
// SPDX-License-Identifier: MIT
#pragma once
#include <cstddef>
#include <cstdint>
#if defined(ARCHITECTURE_arm64)
#include <arm_acle.h>
#endif
namespace FEXCore::Utils {
template<typename T>
static inline uint32_t crc32(const T* Ptr, size_t Size) {
uint32_t Result {};
#if defined(ARCHITECTURE_arm64)
#define do_crc(type, suffix) \
while (Size >= sizeof(type)) { \
Result = __crc32##suffix(Result, *reinterpret_cast<const type*>(Ptr)); \
Ptr += sizeof(type); \
Size -= sizeof(type); \
}
do_crc(uint64_t, d);
do_crc(uint32_t, w);
do_crc(uint16_t, h);
do_crc(uint8_t, b);
#undef do_crc
#else
// Emulate arm64 crc32.
// This is basically just the pseudo-code for crc32b.
// Doesn't need to be fast, just needs to match.
auto reverse_bits = [](auto bits) {
decltype(bits) Result {};
for (size_t i = 0; i < (sizeof(decltype(bits)) * 8); ++i) {
Result = (Result << 1) | ((bits >> i) & 1);
}
return Result;
};
auto Poly32Mod2 = [](uint64_t data) -> uint32_t {
constexpr static size_t bits = 40;
constexpr static uint64_t poly = 0x04C11DB7U;
for (size_t i = (bits - 1); i >= 32; --i) {
if (((data >> i) & 1) != 0) {
const uint64_t poly_shift = poly << (i - 32);
const uint64_t data_mask = (1ULL << i) - 1;
data = (data & data_mask) ^ poly_shift;
}
}
return data;
};
for (size_t i = 0; i < Size; ++i) {
uint64_t TempAcc = static_cast<uint64_t>(reverse_bits(Result)) << 8;
uint64_t TempVal = static_cast<uint64_t>(reverse_bits(reinterpret_cast<const uint8_t*>(Ptr)[i])) << 32;
Result = reverse_bits(Poly32Mod2(TempAcc ^ TempVal));
}
#endif
return Result;
}
} // namespace FEXCore::Utils
-10
View File
@@ -297,16 +297,6 @@ struct FEX_DEFAULT_VISIBILITY Getter : public Value<typename detail::ConfigOptio
using OptionInfo = detail::ConfigOptionInfo<Option>;
Getter()
: Value<typename OptionInfo::Type> {Option, OptionInfo::Default()} {}
// Check if the config option is actually the default.
bool IsDefault() const {
auto Value = FEXCore::Config::GetConv<typename detail::ConfigOptionInfo<Option>::Type>(Option);
if (!Value) {
return true;
}
return *Value == OptionInfo::Default();
}
};
/**
+8 -1
View File
@@ -51,7 +51,6 @@ struct ExecutableFileInfo {
uint64_t FileId = 0;
fextl::string Filename;
fextl::robin_map<uint32_t, GuestRelocationType> Relocations;
uint64_t MappedSize = 0;
};
// Information associated with a specific section of an executable file
@@ -235,6 +234,14 @@ class AbstractCodeCache {
public:
virtual ~AbstractCodeCache() = default;
/**
* Computes a unique identifier for the referenced binary file to be used for
* generating the code map.
* This identifier is independent of FEX build/runtime configuration and
* stable across FEX updates.
*/
virtual uint64_t ComputeCodeMapId(std::string_view Filename, int FD) = 0;
/**
* Bundles the current Core state (CodeBuffer, GuestToHostMapping, ...) to a code cache and writes it to the given file descriptor.
* Returns true on success.
+11 -58
View File
@@ -71,12 +71,6 @@ static_assert(std::is_standard_layout_v<NonAtomicRefCounter<uint64_t>>, "Needs t
static_assert(std::is_trivially_copyable_v<NonAtomicRefCounter<uint64_t>>, "needs to be trivially copyable");
static_assert(sizeof(NonAtomicRefCounter<uint64_t>) == sizeof(uint64_t), "Needs to be correct size");
// Flags about the core state that the JIT can query.
enum class CoreFlagsEnum : uint64_t {
CPUID_ENABLED = (1ULL << 0),
};
FEX_DEF_NUM_OPS(CoreFlagsEnum)
struct alignas(64) CPUState {
// Allows more efficient handling of the register
// file in the event AVX is not supported.
@@ -113,9 +107,7 @@ struct alignas(64) CPUState {
uint64_t L1Pointer {};
uint64_t L1Mask {};
uint64_t callret_sp {};
// CPUID is default enabled.
CoreFlagsEnum CoreFlags {CoreFlagsEnum::CPUID_ENABLED};
uint64_t _pad1 {};
// Cacheline: 1,2,3,4
// The high 128-bits of AVX registers when not being emulated by SVE256.
@@ -196,41 +188,10 @@ struct alignas(64) CPUState {
GDT->Limit1 = Limit >> 16;
}
static bool IsInstructionBPEnabled(unsigned BPIndex, uint64_t Dr7) {
LOGMAN_THROW_A_FMT(BPIndex < NUM_DEBUG_ADDRESSES, "Too large of an index");
// If the GE/LE flags and Transaction type is instruction then it is enabled.
uint64_t GELEEnabledMask = 0b11ULL << (2 * BPIndex);
uint64_t TransactionTypeMask = 0b11ULL << (16 + (4 * BPIndex));
return (Dr7 & GELEEnabledMask) && (Dr7 & TransactionTypeMask) == 0;
};
static size_t BreakpointSize(unsigned BPIndex, uint64_t Dr7) {
LOGMAN_THROW_A_FMT(BPIndex < NUM_DEBUG_ADDRESSES, "Too large of an index");
switch ((Dr7 >> (18 + (4 * BPIndex))) & 0b11) {
case 0: return 1;
case 1: return 2;
case 2: return 8;
case 3: return 4;
default: FEX_UNREACHABLE;
}
}
uint16_t FCW {0x37F};
uint8_t AbridgedFTW {};
uint8_t _pad2[5];
uint64_t FIP {};
// Debug registers 0-3 are addresses.
static constexpr size_t NUM_DEBUG_ADDRESSES = 4;
uint64_t Dr[NUM_DEBUG_ADDRESSES] {};
// Debug register 6 is a status return register.
uint64_t Dr6 {};
// Debug register 7 is the control register.
uint64_t Dr7 {};
// PF/AF are statically mapped as-if they were r16/r17 (which do not exist in
// x86 otherwise). This allows a straightforward mapping for SRA.
static constexpr uint8_t PF_AS_GREG = 16;
@@ -286,6 +247,7 @@ struct alignas(64) CPUState {
// Follows encoding of the TI bit in segment selector encoding.
constexpr static uint32_t SEGMENT_ARRAY_INDEX_GDT = 0;
constexpr static uint32_t SEGMENT_ARRAY_INDEX_LDT = 1;
Core::CPUState::gdt_segment private_gdt[32] {};
};
static_assert(std::is_trivially_copyable_v<CPUState>, "Needs to be trivial");
static_assert(std::is_standard_layout_v<CPUState>, "This needs to be standard layout");
@@ -386,17 +348,12 @@ struct JITPointers {
uint64_t SyscallHandlerFunc {};
uint64_t ExitFunctionLink {};
uint64_t MonoBackpatcherWrite {};
uint64_t RDRANDFallback {};
uint64_t LUDIV {};
uint64_t LDIV {};
uint64_t ThunkCallbackRet {};
/**
* @name Frontend handler pointers.
* @{ */
// Handles returning/calling ARM64EC code from the JIT, expects the target PC in TMP3
uint64_t ExitFunctionEC {};
// Handles a synchronous exception without faulting to jump in to the handler. Frontend specific.
uint64_t SynchronousExceptionHandler {};
/** @} */
FallbackABIInfo FallbackHandlerPointers[FallbackHandlerIndex::OPINDEX_MAX];
uint64_t NamedVectorConstantPointers[FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_CONST_POOL_MAX];
@@ -413,16 +370,14 @@ struct JITPointers {
uint64_t ExitFunctionLinker {};
uint64_t ThreadStopHandlerSpillSRA {};
uint64_t ThreadPauseHandlerSpillSRA {};
uint64_t ThreadDispatchSyscallHandler {};
uint64_t ThreadDispatchCPUIDHandler {};
uint64_t ThreadDispatchRemoveCodeEntry {};
uint64_t GuestSignal_SIGILL {};
uint64_t GuestSignal_SIGTRAP {};
uint64_t GuestSignal_SIGSEGV {};
uint64_t GuestSignal_SynchronousFault {};
uint64_t SignalReturnHandler {};
uint64_t SignalReturnHandlerRT {};
uint64_t L2Pointer {};
uint64_t LUDIVHandler {};
uint64_t LDIVHandler {};
uint64_t F64SinHandler {};
uint64_t F64CosHandler {};
uint64_t F64TanHandler {};
@@ -459,22 +414,20 @@ struct CpuStateFrame {
*/
uint64_t InSyscallInfo {};
struct SynchronousFaultDataStruct {
uint32_t SignalHandlerRefCounter {};
struct alignas(8) SynchronousFaultDataStruct {
bool FaultToTopAndGeneratedException {};
uint8_t Signal;
uint8_t TrapNo;
uint8_t si_code;
uint16_t err_code;
uint16_t _pad : 16;
// Fault address can be different from the FaultRIP so need to distinguish.
uint64_t FaultAddress;
} SynchronousFaultData;
uint32_t SignalHandlerRefCounter {};
InternalThreadState* Thread;
#ifdef _WIN32
#ifdef ARCHITECTURE_arm64ec
// Set by the kernel on ARM64EC whenever the JIT should cooperatively suspend running guest code.
uint32_t SuspendDoorbell {};
#endif
@@ -488,7 +441,7 @@ static_assert(offsetof(CpuStateFrame, Pointers) + sizeof(CpuStateFrame::Pointers
"than architecture maximum 32768");
static_assert(std::is_standard_layout<CpuStateFrame>::value, "This needs to be standard layout");
static_assert(sizeof(CpuStateFrame::SynchronousFaultData) == 16, "This needs to be 16 bytes");
static_assert(sizeof(CpuStateFrame::SynchronousFaultData) == 8, "This needs to be 8 bytes");
static_assert(alignof(CpuStateFrame::SynchronousFaultDataStruct) == 8, "This needs to be 8 bytes");
static_assert(offsetof(CpuStateFrame, SynchronousFaultData) % 8 == 0, "This needs to be aligned");
} // namespace FEXCore::Core
+226 -5
View File
@@ -1,10 +1,231 @@
// SPDX-License-Identifier: MIT
#pragma once
#include "FEXCore/Core/CodeCache.h"
#include "FEXCore/Core/Context.h"
#include "Interface/Core/JIT/Relocations.h"
#include "Interface/Core/Frontend.h"
#include "Interface/Core/CPUBackend.h"
#include "FEXCore/Config/Config.h"
#include "FEXCore/Utils/File.h"
#include "FEXCore/Utils/WorkQueueThread.h"
#include "FEXCore/fextl/memory.h"
#include <FEXCore/fextl/string.h>
#include <FEXCore/fextl/unordered_set.h>
#include <FEXCore/fextl/robin_map.h>
#include <FEXCore/fextl/map.h>
#include <FEXCore/fextl/vector.h>
#include <stdint.h>
#include <mutex>
#include <optional>
#include <span>
#include <xxhash.h>
#include "FEXCore/Utils/CompilerDefs.h"
namespace FEXCore {
#include <cstdint>
namespace Context {
class ContextImpl;
}
namespace FEXCore::DiskCache {
FEX_DEFAULT_VISIBILITY uint16_t GetFormatVersion();
} // namespace FEXCore::DiskCache
namespace DiskCache {
namespace MesaFOZ {
#define FOSSILIZE_BLOB_HASH_LENGTH 40 /* SHA1 hexadecimal string length */
struct __attribute__((packed)) foz_payload_key {
uint8_t bytes[FOSSILIZE_BLOB_HASH_LENGTH];
};
struct __attribute__((packed)) foz_payload_header {
uint32_t payload_size;
uint32_t format;
uint32_t crc;
uint32_t uncompressed_size;
};
} // namespace MesaFOZ
class IndexedDB;
struct IndexEntry {
IndexedDB* DB;
uint64_t Offset;
uint32_t Size;
uint32_t GuestSize;
XXH128_hash_t GuestHash;
fextl::vector<uint32_t> GuestExtents;
};
struct IndexCacheHead {
struct IndexEntry MainEntry;
uint64_t MainEntryFootprint;
fextl::unique_ptr<fextl::multimap<uint64_t, IndexEntry>> MoreEntries; // sorted by guest footprint
};
struct __attribute__((packed)) BlobFixedHeader {
uint32_t GuestSize;
uint32_t HostSize;
uint32_t EntryPointCount;
uint32_t SmallRelocCount;
uint32_t ThunkRelocCount;
XXH128_hash_t GuestHash;
};
// packed struct for types 0, 2 and 3. type 1 is bigger and separate below
struct __attribute__((packed)) BlobSmallRelocation {
uint32_t Offset;
uint8_t Type;
union {
struct __attribute__((packed)) {
uint32_t Symbol;
} Named;
struct __attribute__((packed)) {
uint64_t GuestRIP;
} RIPLiteral;
struct __attribute__((packed)) {
uint8_t RegisterIndex;
uint64_t GuestRIP;
} RIPMove;
struct __attribute__((packed)) {
uint8_t RegisterIndex;
uint8_t ValueSize;
uint32_t SiteOffset;
} PatchableData;
};
};
// type 1, implicit
struct __attribute__((packed)) BlobThunkRelocation {
uint32_t Offset;
uint8_t RegisterIndex;
uint8_t SymbolHash[32]; // sha256sum in the real RelocNamedThunkMove
};
struct CodeHitData {
fextl::vector<uint8_t> Blob;
std::span<uint8_t> HostCode;
std::span<const uint64_t> GuestPages;
std::span<uint64_t> EntryPointRIPs;
std::span<const uint32_t> EntryPointHostOffsets;
// the spans above point to memory owned by the Blob vec, so it's important this can't be copied
CodeHitData() = default;
CodeHitData(CodeHitData&&) = default;
CodeHitData& operator=(CodeHitData&&) = default;
CodeHitData(const CodeHitData&) = delete;
CodeHitData& operator=(const CodeHitData&) = delete;
};
using Index = fextl::robin_map<uint64_t, IndexCacheHead>;
class FOZFile {
public:
bool Open(const fextl::string& CacheFileName, bool ReadOnly);
bool Lock(uint32_t TimeoutMS) {
if (!FD) {
return false;
}
return FD->Lock(TimeoutMS);
}
bool Unlock() {
if (!FD) {
return false;
}
return FD->Unlock();
}
File::File::FileHandleType GetHandle() {
return FD ? FD->GetHandle() : (File::File::FileHandleType)-1;
}
ssize_t Size();
bool ReadAll(fextl::vector<uint8_t>& Out); // from first blob
bool ReadBlob(uint64_t Offset, std::span<uint8_t> OutBlob);
bool WriteBlob(const MesaFOZ::foz_payload_key& Key, std::span<const std::span<const uint8_t>> BlobChunks, uint64_t& OutBlobOffset);
private:
static constexpr uint32_t OPEN_LOCK_TIMEOUT_MS = 100;
fextl::string FileName;
fextl::unique_ptr<File::File> FD;
bool ReadOnly = false;
};
class IndexedDB {
public:
bool Open(const fextl::string& CacheDBName, bool ReadOnly);
void PopulateIndex(Index& CacheIndex, bool& FoundMetadata);
bool ReadCacheBlob(uint64_t Offset, std::span<uint8_t> OutBlob);
bool StoreCacheBlob(const MesaFOZ::foz_payload_key& UniqueKey, uint64_t LookupKey, std::span<const uint8_t> Blob, Index& CacheIndex,
std::mutex& IndexMutex, std::span<const uint8_t> IndexBlob);
private:
// stores run on the Writer, so returning quick isn't as important
static constexpr uint32_t STORE_LOCK_TIMEOUT_MS = 1000;
static constexpr uint64_t BIG_MAPPING_SIZE = 1ULL << 33;
static constexpr uint32_t LOOKUP_KEY_MAX_BUCKET_DEPTH = 20;
FOZFile CacheFOZ;
uint8_t* CacheFileMapping = nullptr;
std::atomic<uint64_t> CacheFileSize;
FOZFile IndexFOZ;
bool ReadOnly = false;
};
class DiskCache {
public:
void Init(FEXCore::Context::ContextImpl* CTX);
std::optional<CodeHitData> Lookup(Core::InternalThreadState* Thread, std::optional<ExecutableFileSectionInfo> Region, uint64_t GuestRIP,
std::optional<uint64_t>& GuestCodeKey);
void Validate(uint64_t GuestCodeKey, const CodeHitData& Hit, const CPU::CPUBackend::CompiledCode& CompiledCode,
std::optional<ExecutableFileSectionInfo> Region);
bool Store(Core::InternalThreadState* Thread, std::optional<ExecutableFileSectionInfo> Region, uint64_t GuestRIP, uint64_t GuestCodeKey,
std::span<const uint8_t> GuestCode, const CPU::CPUBackend::CompiledCode& CompiledCode,
std::span<const FEXCore::CPU::Relocation> Relocations, const Frontend::Decoder::DecodedBlockInformation* DecodedBlockInfo);
bool IsWritingDiskCache() const {
return WritingDiskCache;
}
bool IsReadingDiskCache() const {
return ReadingDiskCache;
}
bool IsValidating() const {
return Validation;
}
private:
bool OpenCacheDB(const fextl::string& CacheDBName, bool ReadOnly);
uint64_t MakeLookupKey(Core::InternalThreadState* Thread, const uint64_t ModuleOffset, bool Writable, bool MonoBackpatcher);
bool ReadingDiskCache {};
bool WritingDiskCache {};
FEXCore::Context::ContextImpl* CTX;
XXH128_hash_t BucketHash;
fextl::vector<fextl::unique_ptr<IndexedDB>> ROCacheDBs;
fextl::unique_ptr<IndexedDB> RWCacheDB;
Index Index;
std::mutex IndexLock;
bool FoundMetadata = false;
struct CacheStoreWorkItem;
// the Writer holds references to all this stuff above and needs to be last
fextl::unique_ptr<WorkQueueThread> Writer;
FEX_CONFIG_OPT(EnableDiskCache, DISKCACHE);
FEX_CONFIG_OPT(Validation, DISKCACHEVALIDATION);
FEX_CONFIG_OPT(MapDiskCacheFiles, DISKCACHEFILEMAPPING);
FEX_CONFIG_OPT(RelocationFilter, DISKCACHERELOCATIONFILTER);
FEX_CONFIG_OPT(AnonCaching, DISKCACHEANONCACHING);
FEX_CONFIG_OPT(BasePathOverride, DISKCACHEPATH);
FEX_CONFIG_OPT(RODBNames, DISKCACHERODBNAMES);
};
static constexpr uint16_t AnonPrefixGuestBytes = 64;
// TODO: This header is in global installed header path, but uses internal headers.
// Migrate this once that is fixed.
static constexpr uint16_t FormatVersion = 20;
FEX_DEFAULT_VISIBILITY uint16_t GetFormatVersion();
} // namespace DiskCache
} // namespace FEXCore
+43 -95
View File
@@ -14,7 +14,7 @@ namespace FEXCore {
*/
struct HostFeatures {
// Changes code generation slightly.
enum class HostTypeEnum : uint8_t {
enum class HostTypeEnum : uint32_t {
Unknown,
Linux,
Wow64,
@@ -24,22 +24,7 @@ struct HostFeatures {
// Whether or not the host supports any kind of SVE implementation.
[[nodiscard]]
bool SupportsSVE() const {
return ProcessFeatures.SupportsSVE128 || ProcessFeatures.SupportsSVE256;
}
[[nodiscard]]
bool SupportsSVE128() const {
return ProcessFeatures.SupportsSVE128;
}
[[nodiscard]]
bool SupportsSVE256() const {
return ProcessFeatures.SupportsSVE256;
}
[[nodiscard]]
bool SupportsAVX() const {
return ProcessFeatures.SupportsAVX;
return SupportsSVE128 || SupportsSVE256;
}
[[nodiscard]]
@@ -47,92 +32,55 @@ struct HostFeatures {
return 4 << DCacheLineLog2;
}
/**
* @brief This structure contains all the data required to determine hashing at the "process" level.
* Unlike the `HostFeaturesHash` below, this data changes codegen depending on if the process is Linux, Arm64EC, or Wow64.
*/
struct ProcessFeaturesHash {
HostTypeEnum HostType : 2 {};
uint8_t SupportsFaultFreeSynchronousFaults : 1 {};
// This only determines the starting mode a process started in.
// The process can technically long jump to-from the other modes.
uint8_t Is64BitMode : 1 {};
// These are technically HostFeatures, but they differ between Linux, wow64, arm64.
uint8_t SupportsAVX : 1 {};
uint8_t SupportsSVE128 : 1 {};
uint8_t SupportsSVE256 : 1 {};
uint8_t pad : 1 {};
};
static_assert(sizeof(ProcessFeaturesHash) == 1);
struct CacheHash {
// This hash is used to determine the "bucket" a machine goes in to.
// This should match between Linux, Arm64EC, and Wow64.
uint64_t HostFeaturesHash;
// This hash is used to determine features that are different between process types.
ProcessFeaturesHash ProcessFeaturesHash;
};
[[nodiscard]]
CacheHash HashForCaching() const {
static_assert(offsetof(HostFeatures, ProcessFeatures) == 8);
CacheHash Result {};
memcpy(&Result.HostFeaturesHash, this, sizeof(uint64_t));
Result.ProcessFeaturesHash = ProcessFeatures;
uint64_t HashForCaching() const {
// As long as the number of options is 64-bit or below, we can just return it.
// Skip CPUMIDRs as it doesn't affect codegen.
static_assert(offsetof(HostFeatures, CPUMIDRs) == 8);
uint64_t Result {};
memcpy(&Result, this, sizeof(Result));
return Result;
}
// Affects codegen and is basically machine description.
uint32_t DCacheLineLog2 : 4 {}; // [3:0]
uint32_t SupportsCacheMaintenanceOps : 1 {}; // [4]
uint32_t SupportsAES : 1 {}; // [5]
uint32_t SupportsCRC : 1 {}; // [6]
uint32_t SupportsCLZERO : 1 {}; // [7]
uint32_t SupportsAtomics : 1 {}; // [8]
uint32_t SupportsRCPC : 1 {}; // [9]
uint32_t SupportsTSOImm9 : 1 {}; // [10]
uint32_t SupportsRAND : 1 {}; // [11]
uint32_t SupportsSHA : 1 {}; // [12]
uint32_t SupportsPMULL_128Bit : 1 {}; // [13]
uint32_t SupportsCSSC : 1 {}; // [14]
uint32_t SupportsFCMA : 1 {}; // [15]
uint32_t SupportsFlagM : 1 {}; // [16]
uint32_t SupportsFlagM2 : 1 {}; // [17]
uint32_t SupportsRPRES : 1 {}; // [18]
uint32_t SupportsPreserveAllABI : 1 {}; // [19]
uint32_t SupportsAES256 : 1 {}; // [20]
uint32_t SupportsSVEBitPerm : 1 {}; // [21]
uint32_t SupportsCPUIndexInTPIDRRO : 1 {}; // [22]
uint32_t SupportsFRINTTS : 1 {}; // [23]
uint32_t SupportsECV : 1 {}; // [24]
uint32_t SupportsWFXT : 1 {}; // [25]
uint32_t Supports3DNow : 1 {}; // [26]
uint32_t SupportsSSE4a : 1 {}; // [27]
uint32_t SupportsMOPS : 1 {}; // [28]
uint32_t SupportsI8MM : 1 {}; // [29]
uint32_t SupportsDotProd : 1 {}; // [30]
uint32_t PreferZVAForVZero : 1 {}; // [31]
uint32_t SupportsAFP : 1 {}; // [32]
uint32_t SupportsFloatExceptions : 1 {}; // [33]
uint32_t SupportsBF16 : 1 {}; // [34]
uint32_t DCacheLineLog2 : 4 {};
uint32_t SupportsCacheMaintenanceOps : 1 {};
uint32_t SupportsAES : 1 {};
uint32_t SupportsCRC : 1 {};
uint32_t SupportsCLZERO : 1 {};
uint32_t SupportsAtomics : 1 {};
uint32_t SupportsRCPC : 1 {};
uint32_t SupportsTSOImm9 : 1 {};
uint32_t SupportsRAND : 1 {};
uint32_t SupportsAVX : 1 {};
uint32_t SupportsSVE128 : 1 {};
uint32_t SupportsSVE256 : 1 {};
uint32_t SupportsSHA : 1 {};
uint32_t SupportsPMULL_128Bit : 1 {};
uint32_t SupportsCSSC : 1 {};
uint32_t SupportsFCMA : 1 {};
uint32_t SupportsFlagM : 1 {};
uint32_t SupportsFlagM2 : 1 {};
uint32_t SupportsRPRES : 1 {};
uint32_t SupportsPreserveAllABI : 1 {};
uint32_t SupportsAES256 : 1 {};
uint32_t SupportsSVEBitPerm : 1 {};
uint32_t SupportsCPUIndexInTPIDRRO : 1 {};
uint32_t SupportsFRINTTS : 1 {};
uint32_t SupportsECV : 1 {};
uint32_t SupportsWFXT : 1 {};
uint32_t Supports3DNow : 1 {};
uint32_t SupportsSSE4a : 1 {};
uint32_t SupportsMOPS : 1 {};
uint32_t PreferZVAForVZero : 1 {};
uint32_t SupportsAFP : 1 {};
uint32_t SupportsFloatExceptions : 1 {};
// Flag if this is InstCountCI
uint32_t IsInstCountCI : 1 {}; // [38]
uint32_t pad : 28 {}; // [63:35]
// This affects codegen, but it isn't machine state
ProcessFeaturesHash ProcessFeatures {};
uint32_t IsInstCountCI : 1 {};
HostTypeEnum HostType : 2 {};
uint32_t pad : 26 {};
// MIDR information
// Also used for determining number of CPU cores for CPUID
fextl::vector<uint32_t> CPUMIDRs;
// The Linux PID of this process. Useful for punching through perf-top information.
uint32_t ProcessPID {};
uint32_t pad2 {};
};
} // namespace FEXCore
-1
View File
@@ -15,7 +15,6 @@ namespace FEXCore::IR {
enum NamedVectorConstant : uint8_t {
NAMED_VECTOR_INCREMENTAL_U16_INDEX = 0,
NAMED_VECTOR_INCREMENTAL_U16_INDEX_UPPER,
NAMED_VECTOR_INCREMENTAL_U8_INDEX,
NAMED_VECTOR_PADDSUBPS_INVERT,
NAMED_VECTOR_PADDSUBPS_INVERT_UPPER,
NAMED_VECTOR_PADDSUBPD_INVERT,
@@ -79,6 +79,5 @@ FEX_DEFAULT_VISIBILITY fextl::vector<MemoryRegion> Setup48BitAllocatorIfExists(s
#ifndef _WIN32
FEX_DEFAULT_VISIBILITY void RegisterTLSData(FEXCore::Core::InternalThreadState* Thread);
FEX_DEFAULT_VISIBILITY void UninstallTLSData(FEXCore::Core::InternalThreadState* Thread);
FEX_DEFAULT_VISIBILITY bool TryGrowMainThreadStack(uintptr_t FaultAddress);
#endif
} // namespace FEXCore::Allocator
-19
View File
@@ -1,19 +0,0 @@
// SPDX-License-Identifier: MIT
#pragma once
#include <FEXCore/fextl/functional.h>
#include <FEXCore/fextl/string.h>
#include <FEXCore/Utils/CompilerDefs.h>
namespace FEXCore::FileUtils {
/**
* @brief Removes an entirely directory passed in. Including the path itself.
*
* @return True if the directory didn't exist or was deleted.
*/
FEX_DEFAULT_VISIBILITY bool RecursiveRemoveDirectory(const fextl::string& Directory);
FEX_DEFAULT_VISIBILITY void WalkDirectory(std::string_view Directory,
fextl::move_only_function<void(std::string_view name, bool is_dir, const void* user_data)> Callback,
const void* user_data);
} // namespace FEXCore::FileUtils
@@ -53,8 +53,4 @@
#define PR_GET_MDWE 66
#endif
#ifndef PR_GET_TPIDRRO_CPU
#define PR_GET_TPIDRRO_CPU 0x7F1DBB0
#endif
#endif // ifndef _WIN32
+2 -2
View File
@@ -13,7 +13,7 @@ struct Flags {
using ThreadFunc = void* (*)(void* user_ptr);
class Thread;
using CreateThreadFunc = fextl::unique_ptr<Thread> (*)(ThreadFunc Func, void* Arg, Flags Flags, const char* ThreadName);
using CreateThreadFunc = fextl::unique_ptr<Thread> (*)(ThreadFunc Func, void* Arg, Flags Flags);
using CleanupAfterForkFunc = void (*)();
struct Pointers {
@@ -34,7 +34,7 @@ public:
* @name Calls provided API functions
* @{ */
static fextl::unique_ptr<Thread> Create(ThreadFunc Func, void* Arg, Flags Flags = {}, const char* ThreadName = nullptr);
static fextl::unique_ptr<Thread> Create(ThreadFunc Func, void* Arg, Flags Flags = {});
static void CleanupAfterFork();
@@ -18,7 +18,7 @@ public:
virtual void Run() = 0;
};
WorkQueueThread(FEXCore::Threads::Flags Flags = {}, const char* ThreadName = nullptr);
WorkQueueThread(FEXCore::Threads::Flags Flags = {});
~WorkQueueThread();
void QueueWork(fextl::unique_ptr<WorkItem> Work);
Loaded 100 of 366 files, more files were not shown because too many files have changed in this diff. Show more