Initial Commit

.gitignore

@@ -1,3 +1,6 @@
-
 compile_commands.json
 vim_rc
+Config.json
+
+[Bb]uild*/
+.vscode/

.gitmodules

@@ -1,6 +1,15 @@
-[submodule "External/SonicUtils"]
-	path = External/SonicUtils
-	url = https://github.com/Sonicadvance1/SonicUtils.git
 [submodule "External/vixl"]
 	path = External/vixl
 	url = https://github.com/Sonicadvance1/vixl.git
+[submodule "External/cpp-optparse"]
+	path = External/cpp-optparse
+	url = https://github.com/Sonicadvance1/cpp-optparse
+[submodule "External/imgui"]
+	path = External/imgui
+	url = https://github.com/Sonicadvance1/imgui.git
+[submodule "External/json-maker"]
+	path = External/json-maker
+	url = https://github.com/Sonicadvance1/json-maker.git
+[submodule "External/tiny-json"]
+	path = External/tiny-json
+	url = https://github.com/Sonicadvance1/tiny-json.git

CMakeLists.txt

@@ -0,0 +1,101 @@
+cmake_minimum_required(VERSION 3.10)
+project(FEX)
+
+option(BUILD_TESTS "Build unit tests to ensure sanity" TRUE)
+option(ENABLE_CLANG_FORMAT "Run clang format over the source" FALSE)
+option(ENABLE_LTO "Enable LTO with compilation" TRUE)
+option(ENABLE_XRAY "Enable building with LLVM X-Ray" FALSE)
+option(ENABLE_LLD "Enable linking with LLD" FALSE)
+option(ENABLE_ASAN "Enables Clang ASAN" FALSE)
+option(ENABLE_TSAN "Enables Clang TSAN" FALSE)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/Bin)
+set(CMAKE_INCLUDE_CURRENT_DIR ON)
+
+if (ENABLE_LTO)
+  set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)
+endif()
+
+if (ENABLE_XRAY)
+  add_compile_options(-fxray-instrument)
+  link_libraries(-fxray-instrument)
+endif()
+
+if (ENABLE_LLD)
+  link_libraries(-fuse-ld=lld)
+endif()
+
+if (ENABLE_ASAN)
+  add_compile_options(-fno-omit-frame-pointer -fsanitize=address)
+  link_libraries(-fno-omit-frame-pointer -fsanitize=address)
+endif()
+
+if (ENABLE_TSAN)
+  add_compile_options(-fno-omit-frame-pointer -fsanitize=thread)
+  link_libraries(-fno-omit-frame-pointer -fsanitize=thread)
+endif()
+
+set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-omit-frame-pointer")
+set (CMAKE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_LINKER_FLAGS_RELWITHDEBINFO} -fno-omit-frame-pointer")
+
+set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fomit-frame-pointer")
+set (CMAKE_LINKER_FLAGS_RELEASE "${CMAKE_LINKER_FLAGS_RELEASE} -fomit-frame-pointer")
+
+add_definitions(-Wno-trigraphs)
+
+# add_subdirectory(External/SonicUtils/)
+include_directories(External/SonicUtils/)
+
+add_subdirectory(External/cpp-optparse/)
+include_directories(External/cpp-optparse/)
+
+add_subdirectory(External/imgui/)
+include_directories(External/imgui/)
+
+add_subdirectory(External/json-maker/)
+include_directories(External/json-maker/)
+
+add_subdirectory(External/tiny-json/)
+include_directories(External/tiny-json/)
+
+include_directories(Source/)
+include_directories("${CMAKE_BINARY_DIR}/Source/")
+
+add_subdirectory(External/FEXCore)
+
+find_package(LLVM CONFIG QUIET)
+if(LLVM_FOUND AND TARGET LLVM)
+  message(STATUS "LLVM found!")
+  include_directories(${LLVM_INCLUDE_DIRS})
+endif()
+
+include(CheckCXXCompilerFlag)
+
+# Add in diagnostic colours if the option is available.
+# Ninja code generator will kill colours if this isn't here
+check_cxx_compiler_flag(-fdiagnostics-color=always GCC_COLOR)
+check_cxx_compiler_flag(-fcolor-diagnostics CLANG_COLOR)
+
+if (GCC_COLOR)
+  add_compile_options(-fdiagnostics-color=always)
+endif()
+if (CLANG_COLOR)
+  add_compile_options(-fcolor-diagnostics)
+endif()
+
+check_cxx_compiler_flag("-march=native" COMPILER_SUPPORTS_MARCH_NATIVE)
+if(COMPILER_SUPPORTS_MARCH_NATIVE)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
+endif()
+
+add_compile_options(-Wall)
+
+add_subdirectory(Source/)
+
+if (BUILD_TESTS)
+  enable_testing()
+  message(STATUS "Unit tests are enabled")
+  add_subdirectory(unittests/)
+endif()

External/cpp-optparse

@@ -0,0 +1 @@
+Subproject commit 5d46ee5bb5c7b5e8988c719aa2b45119df6c5092

External/imgui

@@ -0,0 +1 @@
+Subproject commit 6ad37300cc85c8b195d87055875244585b531cc6

External/json-maker

@@ -0,0 +1 @@
+Subproject commit 8ecb8ecc348bf88c592fac808c03efb342f69e0a

External/tiny-json

@@ -0,0 +1 @@
+Subproject commit 9d09127f87ea6a128fb17d1ffd0b444517343f1c

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Ryan Houdek <Sonicadvance1@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Readme.md

@@ -0,0 +1,16 @@
+# FEX - Fast x86 emulation frontend
+This is the frontend application and tooling used for development and debugging of the FEXCore library.
+
+### Dependencies
+* [SonicUtils](https://github.com/Sonicadvance1/SonicUtils)
+* FEXCore
+* cpp-optparse
+* imgui
+* json-maker
+* tiny-json
+* boost interprocess (sadly)
+* A C++17 compliant compiler (There are assumptions made about using Clang and LTO)
+* clang-tidy if you want the code cleaned up
+* cmake
+
+![FEX diagram](docs/Diagram.svg)

Scripts/Threaded_Lockstep_Runner.py

@@ -0,0 +1,98 @@
+#!/usr/bin/python3
+from enum import Flag
+import json
+import os
+import struct
+import sys
+import glob
+from threading import Thread
+import subprocess
+import time
+import multiprocessing
+
+if sys.version_info[0] < 3:
+        raise Exception("Python 3 or a more recent version is required.")
+
+if (len(sys.argv) < 3):
+    sys.exit("We need two arguments. Location of LockStepRunner and folder containing the tests")
+
+# Remove our SHM regions if they still exist
+SHM_Files = glob.glob("/dev/shm/*_Lockstep")
+for file in SHM_Files:
+    os.remove(file)
+
+UnitTests = sorted(glob.glob(sys.argv[2] + "*"))
+UnitTestsSize = len(UnitTests)
+Threads = [None] * UnitTestsSize
+Results = [None] * UnitTestsSize
+ThreadResults = [[None] * 2] * UnitTestsSize
+MaxFileNameStringLen = 0
+
+def Threaded_Runner(Args, ID, Client):
+    Log = open("Log_" + str(ID) + "_" + str(Client), "w")
+    Log.write("Args: %s\n" % " ".join(Args))
+    Log.flush()
+    Process = subprocess.Popen(Args, stdout=Log, stderr=Log)
+    Process.wait()
+    Log.flush()
+    ThreadResults[ID][Client] = Process.returncode
+
+def Threaded_Manager(Runner, ID, File):
+    ServerArgs = ["catchsegv", Runner, "-c", "vm", "-n", "1", "-I", "R" + str(ID), File]
+    ClientArgs = ["catchsegv", Runner, "-c", "vm", "-n", "1", "-I", "R" + str(ID), "-C"]
+
+    ServerThread = Thread(target = Threaded_Runner, args = (ServerArgs, ID, 0))
+    ClientThread = Thread(target = Threaded_Runner, args = (ClientArgs, ID, 1))
+
+    ServerThread.start()
+    ClientThread.start()
+
+    ClientThread.join()
+    ServerThread.join()
+
+    # The server is the one we should listen to for results
+    if (ThreadResults[ID][1] != 0 and ThreadResults[ID][0] == 0):
+        # If the client died for some reason but server thought we were fine then take client data
+        Results[ID] = ThreadResults[ID][1]
+    else:
+        # Else just take the server data
+        Results[ID] = ThreadResults[ID][0]
+
+    DupLen = MaxFileNameStringLen - len(UnitTests[ID])
+
+    if (Results[ID] == 0):
+        print("\t'%s'%s - PASSED ID: %d - 0" % (UnitTests[ID], " "*DupLen, ID))
+    else:
+        print("\t'%s'%s - FAILED ID: %d - %s" % (UnitTests[ID], " "*DupLen, ID, hex(Results[ID])))
+
+RunnerSlot = 0
+MaxRunnerSlots = min(32, multiprocessing.cpu_count() / 2)
+RunnerSlots = [None] * MaxRunnerSlots
+for RunnerID in range(UnitTestsSize):
+    File = UnitTests[RunnerID]
+    print("'%s' Running Test" % File)
+    MaxFileNameStringLen = max(MaxFileNameStringLen, len(File))
+    Threads[RunnerID] = Thread(target = Threaded_Manager, args = (sys.argv[1], RunnerID, File))
+    Threads[RunnerID].start()
+    if (MaxRunnerSlots != 0):
+        RunnerSlots[RunnerSlot] = Threads[RunnerID]
+        RunnerSlot += 1
+        if (RunnerSlot == MaxRunnerSlots):
+            for i in range(MaxRunnerSlots):
+                RunnerSlots[i].join()
+            RunnerSlot = 0
+
+for i in range(UnitTestsSize):
+    Threads[i].join()
+
+print("====== PASSED RESULTS ======")
+for i in range(UnitTestsSize):
+    DupLen = MaxFileNameStringLen - len(UnitTests[i])
+    if (Results[i] == 0):
+        print("\t'%s'%s - PASSED ID: %d - 0" % (UnitTests[i], " "*DupLen, i))
+
+print("====== FAILED RESULTS ======")
+for i in range(UnitTestsSize):
+    DupLen = MaxFileNameStringLen - len(UnitTests[i])
+    if (Results[i] != 0):
+        print("\t'%s'%s - FAILED ID: %d - %s" % (UnitTests[i], " "*DupLen, i, hex(Results[i])))

Scripts/json_asm_parse.py

@@ -0,0 +1,229 @@
+from enum import Flag
+import json
+import struct
+import sys
+
+class Regs(Flag):
+    REG_NONE  = 0
+    REG_RIP   = (1 << 0)
+    REG_RAX   = (1 << 1)
+    REG_RBX   = (1 << 2)
+    REG_RCX   = (1 << 3)
+    REG_RDX   = (1 << 4)
+    REG_RSI   = (1 << 5)
+    REG_RDI   = (1 << 6)
+    REG_RBP   = (1 << 7)
+    REG_RSP   = (1 << 8)
+    REG_R8    = (1 << 9)
+    REG_R9    = (1 << 10)
+    REG_R10   = (1 << 11)
+    REG_R11   = (1 << 12)
+    REG_R12   = (1 << 13)
+    REG_R13   = (1 << 14)
+    REG_R14   = (1 << 15)
+    REG_R15   = (1 << 16)
+    REG_XMM0  = (1 << 17)
+    REG_XMM1  = (1 << 18)
+    REG_XMM2  = (1 << 19)
+    REG_XMM3  = (1 << 20)
+    REG_XMM4  = (1 << 21)
+    REG_XMM5  = (1 << 22)
+    REG_XMM6  = (1 << 23)
+    REG_XMM7  = (1 << 24)
+    REG_XMM8  = (1 << 25)
+    REG_XMM9  = (1 << 26)
+    REG_XMM10 = (1 << 27)
+    REG_XMM11 = (1 << 28)
+    REG_XMM12 = (1 << 29)
+    REG_XMM13 = (1 << 30)
+    REG_XMM14 = (1 << 31)
+    REG_XMM15 = (1 << 32)
+    REG_GS    = (1 << 33)
+    REG_FS    = (1 << 34)
+    REG_FLAGS = (1 << 35)
+    REG_ALL   = (1 << 36) - 1
+    REG_INVALID = (1 << 36)
+
+class ABI(Flag) :
+    ABI_SYSTEMV = 0
+    ABI_WIN64   = 1
+    ABI_NONE    = 2
+
+RegStringLookup = {
+    "NONE":  Regs.REG_NONE,
+    "RAX":   Regs.REG_RAX,
+    "RIP":   Regs.REG_RIP,
+    "RBX":   Regs.REG_RBX,
+    "RCX":   Regs.REG_RCX,
+    "RDX":   Regs.REG_RDX,
+    "RSI":   Regs.REG_RSI,
+    "RDI":   Regs.REG_RDI,
+    "RBP":   Regs.REG_RBP,
+    "RSP":   Regs.REG_RSP,
+    "R8":    Regs.REG_R8,
+    "R9":    Regs.REG_R9,
+    "R10":   Regs.REG_R10,
+    "R11":   Regs.REG_R11,
+    "R12":   Regs.REG_R12,
+    "R13":   Regs.REG_R13,
+    "R14":   Regs.REG_R14,
+    "R15":   Regs.REG_R15,
+    "XMM0":  Regs.REG_XMM0,
+    "XMM1":  Regs.REG_XMM1,
+    "XMM2":  Regs.REG_XMM2,
+    "XMM3":  Regs.REG_XMM3,
+    "XMM4":  Regs.REG_XMM4,
+    "XMM5":  Regs.REG_XMM5,
+    "XMM6":  Regs.REG_XMM6,
+    "XMM7":  Regs.REG_XMM7,
+    "XMM8":  Regs.REG_XMM8,
+    "XMM9":  Regs.REG_XMM9,
+    "XMM10": Regs.REG_XMM10,
+    "XMM11": Regs.REG_XMM11,
+    "XMM12": Regs.REG_XMM12,
+    "XMM13": Regs.REG_XMM13,
+    "XMM14": Regs.REG_XMM14,
+    "XMM15": Regs.REG_XMM15,
+    "GS":    Regs.REG_GS,
+    "FS":    Regs.REG_FS,
+    "FLAGS": Regs.REG_FLAGS,
+    "ALL":   Regs.REG_ALL,
+}
+
+ABIStringLookup = {
+    "SYSTEMV": ABI.ABI_SYSTEMV,
+    "WIN64": ABI.ABI_WIN64,
+    "NONE": ABI.ABI_NONE,
+}
+
+if (len(sys.argv) < 3):
+    sys.exit()
+
+output_file = sys.argv[2]
+asm_file = open(sys.argv[1], "r")
+asm_text = asm_file.read()
+asm_file.close()
+
+# Default options
+OptionMatch = Regs.REG_INVALID
+OptionIgnore = Regs.REG_NONE
+OptionABI = ABI.ABI_SYSTEMV
+OptionStackSize = 4096
+OptionEntryPoint = 1
+OptionRegData = {}
+OptionMemoryRegions = {}
+
+json_text = asm_text.split("%ifdef CONFIG")
+if (len(json_text) > 1):
+        json_text = json_text[1].split("%endif")
+        if (len(json_text) > 1):
+            json_text = json_text[0].strip()
+
+            json_object = json.loads(json_text)
+            json_object = {k.upper(): v for k, v in json_object.items()}
+
+            # Begin parsing the JSON
+            if ("MATCH" in json_object):
+                data = json_object["MATCH"]
+                if (type(data) is str):
+                    data = [data]
+
+                for data_val in data:
+                    data_val = data_val.upper()
+                    if not (data_val in RegStringLookup):
+                        sys.exit("Invalid Match register option")
+                    if (OptionMatch == Regs.REG_INVALID):
+                        OptionMatch = Regs.REG_NONE
+                    RegOption = RegStringLookup[data_val]
+                    OptionMatch = OptionMatch | RegOption
+
+            if ("IGNORE" in json_object):
+                data = json_object["IGNORE"]
+                if (type(data) is str):
+                    data = [data]
+
+                for data_val in data:
+                    data_val = data_val.upper()
+                    if not (data_val in RegStringLookup):
+                        sys.exit("Invalid Ignore register option")
+                    if (OptionMatch == Regs.REG_INVALID):
+                        OptionMatch = Regs.REG_NONE
+                    RegOption = RegStringLookup[data_val]
+                    OptionIgnore = OptionIgnore | RegOption
+
+            if ("ABI" in json_object):
+                data = json_object["ABI"]
+                data = data.upper()
+                if not (data in ABIStringLookup):
+                    sys.exit("Invalid ABI")
+                OptionABI = ABIStringLookup[data]
+
+            if ("STACKSIZE" in json_object):
+                data = json_object["STACKSIZE"]
+                OptionStackSize = int(data, 0)
+
+            if ("ENTRYPOINT" in json_object):
+                data = json_object["ENTRYPOINT"]
+                data = int(data, 0)
+                if (data == 0):
+                    sys.exit("Invalid entrypoint of 0")
+                OptionEntryPoint = data
+
+            if ("MEMORYREGIONS" in json_object):
+                data = json_object["MEMORYREGIONS"]
+                if not (type(data) is dict):
+                    sys.exit("RegData value must be list of key:value pairs")
+                for data_key, data_val in data.items():
+                    OptionMemoryRegions[int(data_key, 0)] = int(data_val, 0);
+
+            if ("REGDATA" in json_object):
+                data = json_object["REGDATA"]
+                if not (type(data) is dict):
+                    sys.exit("RegData value must be list of key:value pairs")
+                for data_key, data_val in data.items():
+                    data_key = data_key.upper()
+                    if not (data_key in RegStringLookup):
+                        sys.exit("Invalid RegData register option")
+
+                    data_key_index = RegStringLookup[data_key]
+                    data_key_values = []
+
+                    # Create a list of values for this register as an integer
+                    if (type(data_val) is list):
+                        for data_key_value in data_val:
+                            data_key_values.append(int(data_key_value, 0))
+                    else:
+                        data_key_values.append(int(data_val, 0))
+                    OptionRegData[data_key_index] = data_key_values
+
+            # If Match option wasn't touched then set it to the default
+            if (OptionMatch == Regs.REG_INVALID):
+                OptionMatch = Regs.REG_NONE
+
+            config_file = open(output_file, "wb")
+            config_file.write(struct.pack('Q', OptionMatch.value))
+            config_file.write(struct.pack('Q', OptionIgnore.value))
+            config_file.write(struct.pack('Q', OptionStackSize))
+            config_file.write(struct.pack('Q', OptionEntryPoint))
+            config_file.write(struct.pack('I', OptionABI.value))
+
+            # Number of memory regions
+            config_file.write(struct.pack('I', len(OptionMemoryRegions)))
+
+            # Number of register values
+            config_file.write(struct.pack('I', len(OptionRegData)))
+
+            # Print number of memory regions
+            for reg_key, reg_val in OptionMemoryRegions.items():
+                config_file.write(struct.pack('Q', reg_key))
+                config_file.write(struct.pack('Q', reg_val))
+
+           # Print Register values
+            for reg_key, reg_val in OptionRegData.items():
+                config_file.write(struct.pack('I', len(reg_val)))
+                config_file.write(struct.pack('Q', reg_key.value))
+                for reg_vals in reg_val:
+                    config_file.write(struct.pack('Q', reg_vals))
+
+            config_file.close()
+

Source/CMakeLists.txt

@@ -0,0 +1,52 @@
+if (ENABLE_CLANG_FORMAT)
+  find_program(CLANG_TIDY_EXE "clang-tidy")
+  set(CLANG_TIDY_FLAGS
+    "-checks=*"
+    "-fuchsia*"
+    "-bugprone-macro-parentheses"
+    "-clang-analyzer-core.*"
+    "-cppcoreguidelines-pro-type-*"
+    "-cppcoreguidelines-pro-bounds-array-to-pointer-decay"
+    "-cppcoreguidelines-pro-bounds-pointer-arithmetic"
+    "-cppcoreguidelines-avoid-c-arrays"
+    "-cppcoreguidelines-avoid-magic-numbers"
+    "-cppcoreguidelines-pro-bounds-constant-array-index"
+    "-cppcoreguidelines-no-malloc"
+    "-cppcoreguidelines-special-member-functions"
+    "-cppcoreguidelines-owning-memory"
+    "-cppcoreguidelines-macro-usage"
+    "-cppcoreguidelines-avoid-goto"
+    "-google-readability-function-size"
+    "-google-readability-namespace-comments"
+    "-google-readability-braces-around-statements"
+    "-google-build-using-namespace"
+    "-hicpp-*"
+    "-llvm-namespace-comment"
+    "-llvm-include-order" # Messes up with case sensitivity
+    "-misc-unused-parameters"
+    "-modernize-loop-convert"
+    "-modernize-use-auto"
+    "-modernize-avoid-c-arrays"
+    "-modernize-use-nodiscard"
+    "readability-*"
+    "-readability-function-size"
+    "-readability-implicit-bool-conversion"
+    "-readability-braces-around-statements"
+    "-readability-else-after-return"
+    "-readability-magic-numbers"
+    "-readability-named-parameter"
+    "-readability-uppercase-literal-suffix"
+    "-cert-err34-c"
+    "-cert-err58-cpp"
+    "-bugprone-exception-escape"
+    )
+  string(REPLACE ";" "," CLANG_TIDY_FLAGS "${CLANG_TIDY_FLAGS}")
+
+  set(CMAKE_CXX_CLANG_TIDY ${CLANG_TIDY_EXE} "${CLANG_TIDY_FLAGS}")
+endif()
+
+
+add_subdirectory(Common/)
+add_subdirectory(CommonCore/)
+add_subdirectory(Tests/)
+add_subdirectory(Tools/)

Source/Common/ArgumentLoader.cpp

@@ -0,0 +1,150 @@
+#include "Common/Config.h"
+
+#include "OptionParser.h"
+
+namespace FEX::ArgLoader {
+  std::vector<std::string> RemainingArgs;
+
+  void Load(int argc, char **argv) {
+
+    optparse::OptionParser Parser{};
+    optparse::OptionGroup CPUGroup(Parser, "CPU Core options");
+    optparse::OptionGroup TestGroup(Parser, "Test Harness options");
+
+    {
+      CPUGroup.add_option("-c", "--core")
+        .dest("Core")
+        .help("Which CPU core to use")
+        .choices({"irint", "irjit", "llvm", "host", "vm"})
+        .set_default("irint");
+
+      Parser.set_defaults("Break", "0");
+      Parser.set_defaults("Multiblock", "0");
+
+
+      CPUGroup.add_option("-b", "--break")
+        .dest("Break")
+        .action("store_true")
+        .help("Break when op dispatcher doesn't understand instruction");
+      CPUGroup.add_option("--no-break")
+        .dest("Break")
+        .action("store_false")
+        .help("Break when op dispatcher doesn't understand instruction");
+
+      CPUGroup.add_option("-s", "--single-step")
+        .dest("SingleStep")
+        .action("store_true")
+        .help("Single Step config");
+
+      CPUGroup.add_option("-n", "--max-inst")
+        .dest("MaxInst")
+        .help("Maximum number of instructions to stick in a block")
+        .set_default(~0U);
+     CPUGroup.add_option("-m", "--multiblock")
+        .dest("Multiblock")
+        .action("store_true")
+        .help("Enable Multiblock code compilation");
+     CPUGroup.add_option("--no-multiblock")
+        .dest("Multiblock")
+        .action("store_false")
+        .help("Enable Multiblock code compilation");
+
+      Parser.add_option_group(CPUGroup);
+    }
+    {
+      TestGroup.add_option("-g", "--dump-gprs")
+        .dest("DumpGPRs")
+        .action("store_true")
+        .help("When Test Harness ends, print GPR state")
+        .set_default(false);
+
+      TestGroup.add_option("-C", "--ipc-client")
+        .dest("IPCClient")
+        .action("store_true")
+        .help("If the lockstep runner is a client or server")
+        .set_default(false);
+
+      TestGroup.add_option("-I", "--ID")
+        .dest("IPCID")
+        .help("Sets an ID that is prepended to IPC names. For multiple runners")
+        .set_default("0");
+
+      TestGroup.add_option("-e", "--elf")
+        .dest("ELFType")
+        .action("store_true")
+        .help("Lockstep runner should load argument as ELF")
+        .set_default(false);
+
+      Parser.add_option_group(TestGroup);
+    }
+    optparse::Values Options = Parser.parse_args(argc, argv);
+
+    {
+      if (Options.is_set_by_user("Core")) {
+        auto Core = Options["Core"];
+        if (Core == "irint")
+          Config::Add("Core", "0");
+        else if (Core == "irjit")
+          Config::Add("Core", "1");
+        else if (Core == "llvm")
+          Config::Add("Core", "2");
+        else if (Core == "host")
+          Config::Add("Core", "3");
+        else if (Core == "vm")
+          Config::Add("Core", "4");
+      }
+
+      if (Options.is_set_by_user("Break")) {
+        bool Break = Options.get("Break");
+        Config::Add("Break", std::to_string(Break));
+      }
+
+      if (Options.is_set_by_user("SingleStep")) {
+        bool SingleStep = Options.get("SingleStep");
+        Config::Add("SingleStep", std::to_string(SingleStep));
+
+        // Single stepping also enforces single instruction size blocks
+        Config::Add("MaxInst", std::to_string(1u));
+      }
+      else {
+        if (Options.is_set_by_user("MaxInst")) {
+          uint32_t MaxInst = Options.get("MaxInst");
+          Config::Add("MaxInst", std::to_string(MaxInst));
+        }
+      }
+
+      if (Options.is_set_by_user("Multiblock")) {
+        bool Multiblock = Options.get("Multiblock");
+        Config::Add("Multiblock", std::to_string(Multiblock));
+      }
+
+    }
+
+    {
+      if (Options.is_set_by_user("DumpGPRs")) {
+        bool DumpGPRs = Options.get("DumpGPRs");
+        Config::Add("DumpGPRs", std::to_string(DumpGPRs));
+      }
+
+      if (Options.is_set_by_user("IPCClient")) {
+        bool IPCClient = Options.get("IPCClient");
+        Config::Add("IPCClient", std::to_string(IPCClient));
+      }
+
+      if (Options.is_set_by_user("ELFType")) {
+        bool ELFType = Options.get("ELFType");
+        Config::Add("ELFType", std::to_string(ELFType));
+      }
+      if (Options.is_set_by_user("IPCID")) {
+        const char* Value = Options.get("IPCID");
+        Config::Add("IPCID", Value);
+      }
+
+    }
+    RemainingArgs = Parser.args();
+  }
+
+  std::vector<std::string> Get() {
+    return RemainingArgs;
+  }
+}

Source/Common/ArgumentLoader.h

@@ -0,0 +1,10 @@
+#pragma once
+
+#include <string>
+#include <vector>
+
+namespace FEX::ArgLoader {
+  void Load(int argc, char **argv);
+
+  std::vector<std::string> Get();
+}

Source/Common/CMakeLists.txt

@@ -0,0 +1,9 @@
+set(NAME Common)
+set(SRCS
+  ArgumentLoader.cpp
+  Config.cpp
+  StringUtil.cpp)
+
+add_library(${NAME} STATIC ${SRCS})
+target_link_libraries(${NAME} cpp-optparse tiny-json json-maker)
+target_include_directories(${NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/External/cpp-optparse/)

Source/Common/Config.cpp

@@ -0,0 +1,115 @@
+#include "Common/Config.h"
+#include <cassert>
+#include <cstring>
+#include <fstream>
+#include <memory>
+#include <list>
+#include <unordered_map>
+#include <vector>
+#include <tiny-json.h>
+#include <json-maker.h>
+
+namespace FEX::Config {
+  std::unordered_map<std::string, std::string> ConfigMap;
+
+  struct JsonAllocator {
+    jsonPool_t PoolObject;
+    std::unique_ptr<std::list<json_t>> json_objects;
+  };
+
+  json_t* PoolInit(jsonPool_t* Pool) {
+    JsonAllocator* alloc = json_containerOf(Pool, JsonAllocator, PoolObject);
+    alloc->json_objects = std::make_unique<std::list<json_t>>();
+    return &*alloc->json_objects->emplace(alloc->json_objects->end());
+  }
+
+  json_t* PoolAlloc(jsonPool_t* Pool) {
+    JsonAllocator* alloc = json_containerOf(Pool, JsonAllocator, PoolObject);
+    return &*alloc->json_objects->emplace(alloc->json_objects->end());
+  }
+
+  void Init() {
+    JsonAllocator Pool {
+      .PoolObject = {
+        .init = PoolInit,
+        .alloc = PoolAlloc,
+      },
+    };
+
+    std::fstream ConfigFile;
+    std::vector<char> Data;
+    ConfigFile.open("Config.json", std::fstream::in);
+
+    if (ConfigFile.is_open()) {
+      ConfigFile.seekg(0, std::fstream::end);
+      size_t FileSize = ConfigFile.tellg();
+      ConfigFile.seekg(0, std::fstream::beg);
+
+      if (FileSize > 0) {
+        Data.resize(FileSize);
+        ConfigFile.read(&Data.at(0), FileSize);
+        ConfigFile.close();
+
+        json_t const *json = json_createWithPool(&Data.at(0), &Pool.PoolObject);
+        json_t const* ConfigList = json_getProperty(json, "Config");
+
+        for (json_t const* ConfigItem = json_getChild(ConfigList);
+          ConfigItem != nullptr;
+          ConfigItem = json_getSibling(ConfigItem)) {
+          const char* ConfigName = json_getName(ConfigItem);
+          const char* ConfigString = json_getValue(ConfigItem);
+
+          FEX::Config::Add(ConfigName, ConfigString);
+        }
+      }
+    }
+  }
+
+  void Shutdown() {
+    char Buffer[4096];
+    char *Dest;
+    Dest = json_objOpen(Buffer, nullptr);
+    Dest = json_objOpen(Dest, "Config");
+    for (auto &it : ConfigMap) {
+      Dest = json_str(Dest, it.first.data(), it.second.c_str());
+    }
+    Dest = json_objClose(Dest);
+    Dest = json_objClose(Dest);
+    json_end(Dest);
+
+    std::fstream ConfigFile;
+    ConfigFile.open("Config.json", std::fstream::out);
+
+    if (ConfigFile.is_open()) {
+      ConfigFile.write(Buffer, strlen(Buffer));
+      ConfigFile.close();
+    }
+
+    ConfigMap.clear();
+  }
+
+  void Add(std::string const &Key, std::string_view const Value) {
+    ConfigMap[Key] = Value;
+  }
+
+  bool Exists(std::string const &Key) {
+    return ConfigMap.find(Key) != ConfigMap.end();
+  }
+
+  std::string_view Get(std::string const &Key) {
+    auto Value = ConfigMap.find(Key);
+    if (Value == ConfigMap.end())
+      assert(0 && "Not a real config value");
+    std::string_view ValueView = Value->second;
+    return ValueView;
+  }
+
+  std::string_view GetIfExists(std::string const &Key, std::string_view const Default) {
+    auto Value = ConfigMap.find(Key);
+    if (Value == ConfigMap.end())
+      return Default;
+
+    std::string_view ValueView = Value->second;
+    return ValueView;
+  }
+}

Source/Common/Config.h

@@ -0,0 +1,64 @@
+#pragma once
+
+#include "Common/StringConv.h"
+
+#include <cassert>
+#include <string>
+#include <string_view>
+
+/**
+ * @brief This is a singleton for storing global configuration state
+ */
+namespace FEX::Config {
+  void Init();
+  void Shutdown();
+
+  void Add(std::string const &Key, std::string_view const Value);
+  bool Exists(std::string const &Key);
+  std::string_view Get(std::string const &Key);
+  std::string_view GetIfExists(std::string const &Key, std::string_view const Default = "");
+
+  template<typename T>
+  T Get(std::string const &Key) {
+    T Value;
+    if (!FEX::StrConv::Conv(Get(Key), &Value)) {
+      assert(0 && "Attempted to convert invalid value");
+    }
+    return Value;
+  }
+
+  template<typename T>
+  T GetIfExists(std::string const &Key, T Default) {
+    T Value;
+    if (Exists(Key) && FEX::StrConv::Conv(FEX::Config::Get(Key), &Value)) {
+      return Value;
+    }
+    else {
+      return Default;
+    }
+  }
+
+  template<typename T>
+  class Value {
+  public:
+    Value(std::string const &key, T Default)
+      : Key {key}
+    {
+      ValueData = GetFromConfig(Key, Default);
+    }
+    void Set(T NewValue) {
+      ValueData = NewValue;
+      FEX::Config::Add(Key, std::to_string(NewValue));
+    }
+
+    T operator()() { return ValueData; }
+
+  private:
+    std::string const Key;
+    T ValueData;
+    T GetFromConfig(std::string const &Key, T Default) {
+      return FEX::Config::GetIfExists(Key, Default);
+    }
+  };
+
+}

Source/Common/MathUtils.h

@@ -0,0 +1,13 @@
+#pragma once
+
+#include <stdint.h>
+
+[[maybe_unused]] inline uint64_t AlignUp(uint64_t value, uint64_t size) {
+  return value + (size - value % size) % size;
+};
+
+[[maybe_unused]] inline uint64_t AlignDown(uint64_t value, uint64_t size) {
+  return value - value % size;
+};
+
+

Source/Common/StringConv.h

@@ -0,0 +1,42 @@
+#pragma once
+#include <cstdint>
+#include <string>
+#include <string_view>
+#include <optional>
+
+namespace FEX::StrConv {
+  [[maybe_unused]] static bool Conv(std::string_view Value, bool *Result) {
+    *Result = std::stoi(std::string(Value), nullptr, 0);
+    return true;
+  }
+
+  [[maybe_unused]] static bool Conv(std::string_view Value, uint8_t *Result) {
+    *Result = std::stoi(std::string(Value), nullptr, 0);
+    return true;
+  }
+
+  [[maybe_unused]] static bool Conv(std::string_view Value, uint16_t *Result) {
+    *Result = std::stoi(std::string(Value), nullptr, 0);
+    return true;
+  }
+
+  [[maybe_unused]] static bool Conv(std::string_view Value, uint32_t *Result) {
+    *Result = std::stoi(std::string(Value), nullptr, 0);
+    return true;
+  }
+
+  [[maybe_unused]] static bool Conv(std::string_view Value, int32_t *Result) {
+    *Result = std::stoi(std::string(Value), nullptr, 0);
+    return true;
+  }
+
+  [[maybe_unused]] static bool Conv(std::string_view Value, uint64_t *Result) {
+    *Result = std::stoull(std::string(Value), nullptr, 0);
+    return true;
+  }
+  [[maybe_unused]] static bool Conv(std::string_view Value, std::string *Result) {
+    *Result = Value;
+    return true;
+  }
+
+}

Source/Common/StringUtil.cpp

@@ -0,0 +1,19 @@
+#include "Common/StringUtil.h"
+
+namespace FEX::StringUtil {
+void ltrim(std::string &s) {
+  s.erase(std::find_if(s.begin(), s.end(), [](int ch) {
+      return !std::isspace(ch);
+    }));
+}
+
+void rtrim(std::string &s) {
+  s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) {
+      return !std::isspace(ch);
+    }).base(), s.end());
+}
+void trim(std::string &s) {
+  ltrim(s);
+  rtrim(s);
+}
+}

Source/Common/StringUtil.h

@@ -0,0 +1,9 @@
+#pragma once
+#include <algorithm>
+#include <string>
+
+namespace FEX::StringUtil {
+void ltrim(std::string &s);
+void rtrim(std::string &s);
+void trim(std::string &s);
+}

Source/CommonCore/CMakeLists.txt

@@ -0,0 +1,6 @@
+set(NAME CommonCore)
+set(SRCS
+  VMFactory.cpp)
+
+add_library(${NAME} STATIC ${SRCS})
+target_link_libraries(${NAME} FEXCore)

Source/CommonCore/HostFactory.cpp

@@ -0,0 +1,296 @@
+#include <FEXCore/Config/Config.h>
+#include <FEXCore/Core/CodeLoader.h>
+#include <FEXCore/Core/Context.h>
+#include <FEXCore/Core/CoreState.h>
+#include <FEXCore/Core/CPUBackend.h>
+#include <FEXCore/Core/X86Enums.h>
+#include <FEXCore/HLE/SyscallHandler.h>
+#include <FEXCore/Debug/InternalThreadState.h>
+#include <FEXCore/Memory/SharedMem.h>
+
+#include <ucontext.h>
+#include <cassert>
+#include <iostream>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <sys/ptrace.h>
+#include <errno.h>
+#include <sys/user.h>
+#include <stdint.h>
+
+namespace HostFactory {
+
+class HostCore final : public FEXCore::CPU::CPUBackend {
+public:
+  explicit HostCore(FEXCore::Core::ThreadState *Thread, bool Fallback);
+  ~HostCore() override = default;
+  std::string GetName() override { return "Host Stepper"; }
+  void* CompileCode(FEXCore::IR::IntrusiveIRList const *ir, FEXCore::Core::DebugData *DebugData) override;
+
+  void *MapRegion(void *HostPtr, uint64_t GuestPtr, uint64_t Size) override {
+    // Map locally to unprotected
+    printf("Mapping Guest Ptr: 0x%lx\n", GuestPtr);
+    MemoryRegions.emplace_back(MemoryRegion{HostPtr, GuestPtr, Size});
+    return HostPtr;
+  }
+
+  void ExecuteCode(FEXCore::Core::ThreadState *Thread);
+
+  void SignalHandler(int sig, siginfo_t *info, void *RawContext);
+
+  bool NeedsOpDispatch() override { return false; }
+private:
+  void HandleSyscall();
+  void ExecutionThreadFunction();
+
+  void InstallSignalHandler();
+
+  template<typename T>
+  T GetPointerToGuest(uint64_t Addr) {
+    for (auto const& Region : MemoryRegions) {
+      if (Addr >= Region.VirtualGuestPtr && Addr < (Region.VirtualGuestPtr + Region.Size)) {
+        return reinterpret_cast<T>(reinterpret_cast<uintptr_t>(Region.HostPtr) + (Addr - Region.VirtualGuestPtr));
+      }
+    }
+
+    return nullptr;
+  }
+
+  FEXCore::Core::ThreadState *ThreadState;
+  bool IsFallback;
+
+  std::thread ExecutionThread;
+  struct MemoryRegion {
+    void *HostPtr;
+    uint64_t VirtualGuestPtr;
+    uint64_t Size;
+  };
+  std::vector<MemoryRegion> MemoryRegions;
+
+  pid_t ChildPid;
+  struct sigaction OldSigAction_SEGV;
+  struct sigaction OldSigAction_TRAP;
+
+  int PipeFDs[2];
+  std::atomic_bool ShouldStart{false};
+};
+
+static HostCore *GlobalCore;
+static void SigAction_SEGV(int sig, siginfo_t* info, void* RawContext) {
+  GlobalCore->SignalHandler(sig, info, RawContext);
+}
+
+HostCore::HostCore(FEXCore::Core::ThreadState *Thread, bool Fallback)
+  : ThreadState {Thread}
+  , IsFallback {Fallback} {
+
+  GlobalCore = this;
+
+  ExecutionThread = std::thread(&HostCore::ExecutionThreadFunction, this);
+}
+
+static void HostExecution(FEXCore::Core::ThreadState *Thread) {
+  auto InternalThread = reinterpret_cast<FEXCore::Core::InternalThreadState*>(Thread);
+  HostCore *Core = reinterpret_cast<HostCore*>(InternalThread->CPUBackend.get());
+  Core->ExecuteCode(Thread);
+}
+
+static void HostExecutionFallback(FEXCore::Core::ThreadState *Thread) {
+  auto InternalThread = reinterpret_cast<FEXCore::Core::InternalThreadState*>(Thread);
+  HostCore *Core = reinterpret_cast<HostCore*>(InternalThread->FallbackBackend.get());
+  Core->ExecuteCode(Thread);
+}
+
+void HostCore::SignalHandler(int sig, siginfo_t *info, void *RawContext) {
+  ucontext_t* Context = (ucontext_t*)RawContext;
+  static uint64_t LastEMURip = ~0ULL;
+  static uint64_t LastInstSize = 0;
+  if (sig == SIGSEGV) {
+    // RIP == Base instruction
+  }
+  else if (sig == SIGTRAP) {
+    HostToEmuRIP -= 1; // 0xCC moves us ahead by one
+  }
+
+  uint8_t *LocalData = GetPointerToGuest<uint8_t*>(Context->uc_mcontext.gregs[REG_RIP]);
+  uint8_t *ActualData = ThreadState->CPUCore->MemoryMapper->GetPointer<uint8_t*>(HostToEmuRIP);
+  uint64_t TotalInstructionsLength {0};
+
+  ThreadState->CPUCore->FrontendDecoder.DecodeInstructionsInBlock(ActualData, HostToEmuRIP);
+  auto DecodedOps = ThreadState->CPUCore->FrontendDecoder.GetDecodedInsts();
+  if (sig == SIGSEGV) {
+    for (size_t i = 0; i < DecodedOps.second; ++i) {
+      FEXCore::X86Tables::DecodedInst const* DecodedInfo {nullptr};
+      DecodedInfo = &DecodedOps.first->at(i);
+      auto CheckOp = [&](char const* Name, FEXCore::X86Tables::DecodedOperand const &Operand) {
+        if (Operand.TypeNone.Type != FEXCore::X86Tables::DecodedOperand::TYPE_NONE &&
+            Operand.TypeNone.Type != FEXCore::X86Tables::DecodedOperand::TYPE_LITERAL &&
+            Operand.TypeNone.Type != FEXCore::X86Tables::DecodedOperand::TYPE_GPR) {
+          printf("Operand type is %d\n", Operand.TypeNone.Type);
+          const std::vector<unsigned> EmulatorToSystemContext = {
+            offsetof(mcontext_t, gregs[REG_RAX]),
+            offsetof(mcontext_t, gregs[REG_RBX]),
+            offsetof(mcontext_t, gregs[REG_RCX]),
+            offsetof(mcontext_t, gregs[REG_RDX]),
+            offsetof(mcontext_t, gregs[REG_RSI]),
+            offsetof(mcontext_t, gregs[REG_RDI]),
+            offsetof(mcontext_t, gregs[REG_RBP]),
+            offsetof(mcontext_t, gregs[REG_RSP]),
+            offsetof(mcontext_t, gregs[REG_R8]),
+            offsetof(mcontext_t, gregs[REG_R9]),
+            offsetof(mcontext_t, gregs[REG_R10]),
+            offsetof(mcontext_t, gregs[REG_R11]),
+            offsetof(mcontext_t, gregs[REG_R12]),
+            offsetof(mcontext_t, gregs[REG_R13]),
+            offsetof(mcontext_t, gregs[REG_R14]),
+            offsetof(mcontext_t, gregs[REG_R15]),
+          };
+
+          // Modify the registers to match the memory operands necessary.
+          // This will propagate some addresses
+          if (Operand.TypeNone.Type == FEXCore::X86Tables::DecodedOperand::TYPE_GPR_DIRECT) {
+            uint64_t *GPR = (uint64_t*)((uintptr_t)&Context->uc_mcontext + EmulatorToSystemContext[Operand.TypeGPR.GPR]);
+            uint64_t HostPointer = LocalMemoryMapper.GetPointer<uint64_t>(*GPR);
+            printf("Changing host pointer from 0x%lx to %lx\n", *GPR, HostPointer);
+            *GPR = HostPointer;
+          }
+          else {
+            OldSigAction_SEGV.sa_sigaction(sig, info, RawContext);
+            return;
+          }
+        }
+      };
+      printf("Insts: %ld\n", DecodedOps.second);
+      CheckOp("Dest", DecodedInfo->Dest);
+      CheckOp("Src1", DecodedInfo->Src1);
+      CheckOp("Src2", DecodedInfo->Src2);
+
+      // Reset RIP to the start of the instruction
+      Context->uc_mcontext.gregs[REG_RIP] = (uint64_t)LocalData;
+      return;
+    }
+
+    OldSigAction_SEGV.sa_sigaction(sig, info, RawContext);
+    return;
+  }
+  else if (sig == SIGTRAP) {
+    for (size_t i = 0; i < DecodedOps.second; ++i) {
+      FEXCore::X86Tables::DecodedInst const* DecodedInfo {nullptr};
+      DecodedInfo = &DecodedOps.first->at(i);
+      TotalInstructionsLength += DecodedInfo->InstSize;
+    }
+
+    if (LastEMURip != ~0ULL) {
+      uint8_t *PreviousLocalData = LocalMemoryMapper.GetPointer<uint8_t*>(LastEMURip);
+      memset(PreviousLocalData, 0xCC, LastInstSize);
+    }
+    LastInstSize = TotalInstructionsLength;
+    printf("\tHit an instruction of length %ld 0x%lx 0x%lx 0x%lx\n", TotalInstructionsLength, (uint64_t)Context->uc_mcontext.gregs[REG_RIP], (uint64_t)LocalData, (uint64_t)ActualData);
+    memcpy(LocalData, ActualData, TotalInstructionsLength);
+    printf("\tData Source:");
+    for (uint64_t i = 0; i < TotalInstructionsLength; ++i) {
+      printf("%02x ", ActualData[i]);
+    }
+    printf("\n");
+
+    // Reset RIP to the start of the instruction
+    Context->uc_mcontext.gregs[REG_RIP] = (uint64_t)LocalData;
+  }
+}
+
+void HostCore::InstallSignalHandler() {
+  struct sigaction sa;
+  sa.sa_handler = nullptr;
+  sa.sa_sigaction = &SigAction_SEGV;
+  sa.sa_flags = SA_SIGINFO;
+  sigemptyset(&sa.sa_mask);
+  // We use sigsegv to capture invalid memory accesses
+  // We then patch the GPR state of that instruction to point to the correct location
+  sigaction(SIGSEGV, &sa, &OldSigAction_SEGV);
+  // We use trapping to determine when we've stepped to a new instruction
+  sigaction(SIGTRAP, &sa, &OldSigAction_TRAP);
+}
+
+void HostCore::ExecutionThreadFunction() {
+  printf("Host Core created\n");
+
+  if (pipe(PipeFDs) == -1) {
+    LogMan::Msg::A("Couldn't pipe");
+    return;
+  }
+  ChildPid = fork();
+  if (ChildPid == 0) {
+    // Child
+    // Set that we want to be traced
+    if (ptrace(PTRACE_TRACEME, 0, 0, 0)) {
+      LogMan::Msg::A("Couldn't start trace");
+    }
+    raise(SIGSTOP);
+    InstallSignalHandler();
+
+    close(PipeFDs[1]);
+    using Ptr = void (*)();
+    read(PipeFDs[0], &ThreadState->CPUState, sizeof(FEXCore::X86State::State));
+
+    printf("Child is running! 0x%lx\n", ThreadState->CPUState.rip);
+    Ptr Loc = LocalMemoryMapper.GetPointer<Ptr>(ThreadState->CPUState.rip);
+    Loc();
+    printf("Oh Snap. We returned in the child\n");
+  }
+  else {
+    // Parent
+    // Parent will be the ptrace control thread
+    int status;
+    close(PipeFDs[0]);
+
+    waitpid(ChildPid, &status, 0);
+
+    if (WIFSTOPPED(status) && WSTOPSIG(status) == 19) {
+      // Attach to child
+      ptrace(PTRACE_ATTACH, ChildPid, 0, 0);
+      ptrace(PTRACE_CONT, ChildPid, 0, 0);
+    }
+
+    while (!ShouldStart.load());
+    printf("Telling child to go!\n");
+    write(PipeFDs[1], &ThreadState->CPUState, sizeof(FEXCore::X86State::State));
+
+    while (1) {
+
+      ShouldStart.store(false);
+
+      waitpid(ChildPid, &status, 0);
+      if (WIFEXITED(status)) {
+        return;
+      }
+      if (WIFSTOPPED(status) && WSTOPSIG(status) == 5) {
+        ptrace(PTRACE_CONT, ChildPid, 0, 5);
+      }
+      if (WIFSTOPPED(status) && WSTOPSIG(status) == 11) {
+        ptrace(PTRACE_DETACH, ChildPid, 0, 11);
+        break;
+      }
+    }
+  }
+}
+
+void* HostCore::CompileCode([[maybe_unused]] FEXCore::IR::IntrusiveIRList const* ir, FEXCore::Core::DebugData *DebugData) {
+  printf("Attempting to compile: 0x%lx\n", ThreadState->State.rip);
+  if (IsFallback)
+    return reinterpret_cast<void*>(HostExecutionFallback);
+  else
+    return reinterpret_cast<void*>(HostExecution);
+}
+
+void HostCore::ExecuteCode(FEXCore::Core::ThreadState *Thread) {
+  ShouldStart = true;
+  while(1);
+}
+
+FEXCore::CPU::CPUBackend *CreateHostCore(FEXCore::Core::ThreadState *Thread) {
+  return new HostCore(Thread);
+}
+
+}

Source/CommonCore/HostFactory.h

@@ -0,0 +1,14 @@
+namespace FEXCore::CPU {
+  class CPUBackend;
+}
+namespace FEXCore::Context{
+  struct Context;
+}
+namespace FEXCore::Core {
+  struct ThreadState;
+}
+
+namespace HostFactory {
+  FEXCore::CPU::CPUBackend *CPUCreationFactory(FEXCore::Context::Context* CTX, FEXCore::Core::ThreadState *Thread);
+  FEXCore::CPU::CPUBackend *CPUCreationFactoryFallback(FEXCore::Context::Context* CTX, FEXCore::Core::ThreadState *Thread);
+}

Source/CommonCore/VMFactory.cpp

@@ -0,0 +1,293 @@
+#include "VM.h"
+
+#include <FEXCore/Config/Config.h>
+#include <FEXCore/Core/CodeLoader.h>
+#include <FEXCore/Core/Context.h>
+#include <FEXCore/Core/CoreState.h>
+#include <FEXCore/Core/CPUBackend.h>
+#include <FEXCore/Core/X86Enums.h>
+#include <FEXCore/HLE/SyscallHandler.h>
+#include <FEXCore/Debug/InternalThreadState.h>
+#include <FEXCore/Memory/SharedMem.h>
+
+namespace VMFactory {
+  class VMCore final : public FEXCore::CPU::CPUBackend {
+  public:
+    explicit VMCore(FEXCore::Context::Context* CTX, FEXCore::Core::ThreadState *Thread, bool Fallback);
+    ~VMCore() override;
+    std::string GetName() override { return "VM Core"; }
+    void* CompileCode(FEXCore::IR::IRListView<true> const *IR, FEXCore::Core::DebugData *DebugData) override;
+
+    void *MapRegion(void *HostPtr, uint64_t VirtualGuestPtr, uint64_t Size) override {
+      MemoryRegions.emplace_back(MemoryRegion{HostPtr, VirtualGuestPtr, Size});
+      LogMan::Throw::A(!IsInitialized, "Tried mapping a new VM region post initialization");
+      return HostPtr;
+    }
+
+    void Initialize() override;
+    void ExecuteCode(FEXCore::Core::ThreadState *Thread);
+    bool NeedsOpDispatch() override { return false; }
+
+  private:
+    FEXCore::Context::Context* CTX;
+    FEXCore::Core::ThreadState *ThreadState;
+    bool IsFallback;
+
+    void CopyHostStateToGuest();
+    void CopyGuestCPUToHost();
+    void CopyHostMemoryToGuest();
+    void CopyGuestMemoryToHost();
+
+    void RecalculatePML4();
+
+    struct MemoryRegion {
+      void *HostPtr;
+      uint64_t VirtualGuestPtr;
+      uint64_t Size;
+    };
+    std::vector<MemoryRegion> MemoryRegions;
+
+    struct PhysicalToVirtual {
+      uint64_t PhysicalPtr;
+      uint64_t GuestVirtualPtr;
+      void *HostPtr;
+      uint64_t Size;
+    };
+    std::vector<PhysicalToVirtual> PhysToVirt;
+
+    SU::VM::VMInstance *VM;
+    bool IsInitialized{false};
+  };
+
+  VMCore::~VMCore() {
+    delete VM;
+  }
+
+  VMCore::VMCore(FEXCore::Context::Context* CTX, FEXCore::Core::ThreadState *Thread, bool Fallback)
+    : CTX {CTX}
+    , ThreadState {Thread}
+    , IsFallback {Fallback} {
+    VM = SU::VM::VMInstance::Create();
+  }
+
+  void VMCore::Initialize() {
+    // Scan the mapped memory regions and see how much memory backing we need
+    uint64_t PhysicalMemoryNeeded {};
+
+    PhysToVirt.reserve(MemoryRegions.size());
+
+    for (auto &Region : MemoryRegions) {
+      PhysToVirt.emplace_back(PhysicalToVirtual{PhysicalMemoryNeeded, Region.VirtualGuestPtr, Region.HostPtr, Region.Size});
+      PhysicalMemoryNeeded += Region.Size;
+    }
+    LogMan::Msg::D("We need 0x%lx physical memory", PhysicalMemoryNeeded);
+    VM->SetPhysicalMemorySize(PhysicalMemoryNeeded);
+
+    // Map these regions in the VM
+    for (auto &Region : PhysToVirt) {
+      if (!Region.Size) continue;
+      VM->AddMemoryMapping(Region.GuestVirtualPtr, Region.PhysicalPtr, Region.Size);
+    }
+
+    // Initialize the VM with the memory mapping
+    VM->Initialize();
+
+    CopyHostMemoryToGuest();
+
+    // Set initial register states
+    CopyHostStateToGuest();
+
+    IsInitialized = true;
+  }
+
+  void VMCore::CopyHostMemoryToGuest() {
+    void *PhysBase = VM->GetPhysicalMemoryPointer();
+    for (auto &Region : PhysToVirt) {
+      void *GuestPtr = reinterpret_cast<void*>((reinterpret_cast<uintptr_t>(PhysBase) + Region.PhysicalPtr));
+      memcpy(GuestPtr, Region.HostPtr, Region.Size);
+    }
+  }
+
+  void VMCore::CopyGuestMemoryToHost() {
+    void *PhysBase = VM->GetPhysicalMemoryPointer();
+    for (auto &Region : PhysToVirt) {
+      void *GuestPtr = reinterpret_cast<void*>((reinterpret_cast<uintptr_t>(PhysBase) + Region.PhysicalPtr));
+      memcpy(Region.HostPtr, GuestPtr, Region.Size);
+    }
+  }
+
+  void VMCore::CopyHostStateToGuest() {
+    auto CompactRFlags = [](auto Arg) -> uint32_t {
+      uint32_t Res = 2;
+      for (int i = 0; i < 32; ++i) {
+        Res |= Arg->flags[i] << i;
+      }
+      return Res;
+    };
+
+    SU::VM::VMInstance::RegisterState State;
+    SU::VM::VMInstance::SpecialRegisterState SpecialState;
+
+    State.rax    = ThreadState->State.gregs[FEXCore::X86State::REG_RAX];
+    State.rbx    = ThreadState->State.gregs[FEXCore::X86State::REG_RBX];
+    State.rcx    = ThreadState->State.gregs[FEXCore::X86State::REG_RCX];
+    State.rdx    = ThreadState->State.gregs[FEXCore::X86State::REG_RDX];
+    State.rsi    = ThreadState->State.gregs[FEXCore::X86State::REG_RSI];
+    State.rdi    = ThreadState->State.gregs[FEXCore::X86State::REG_RDI];
+    State.rsp    = ThreadState->State.gregs[FEXCore::X86State::REG_RSP];
+    State.rbp    = ThreadState->State.gregs[FEXCore::X86State::REG_RBP];
+    State.r8     = ThreadState->State.gregs[FEXCore::X86State::REG_R8];
+    State.r9     = ThreadState->State.gregs[FEXCore::X86State::REG_R9];
+    State.r10    = ThreadState->State.gregs[FEXCore::X86State::REG_R10];
+    State.r11    = ThreadState->State.gregs[FEXCore::X86State::REG_R11];
+    State.r12    = ThreadState->State.gregs[FEXCore::X86State::REG_R12];
+    State.r13    = ThreadState->State.gregs[FEXCore::X86State::REG_R13];
+    State.r14    = ThreadState->State.gregs[FEXCore::X86State::REG_R14];
+    State.r15    = ThreadState->State.gregs[FEXCore::X86State::REG_R15];
+    State.rip    = ThreadState->State.rip;
+    State.rflags = CompactRFlags(&ThreadState->State);
+    VM->SetRegisterState(State);
+
+    SpecialState.fs.base = ThreadState->State.fs;
+    SpecialState.gs.base = ThreadState->State.gs;
+
+    VM->SetSpecialRegisterState(SpecialState);
+  }
+
+  void VMCore::CopyGuestCPUToHost() {
+    auto UnpackRFLAGS = [](auto ThreadState, uint64_t GuestFlags) {
+      for (int i = 0; i < 32; ++i) {
+        ThreadState->flags[i] = (GuestFlags >> i) & 1;
+      }
+    };
+
+    SU::VM::VMInstance::RegisterState State;
+    SU::VM::VMInstance::SpecialRegisterState SpecialState;
+
+    State = VM->GetRegisterState();
+    SpecialState = VM->GetSpecialRegisterState();
+
+    // Copy the VM's register state to our host context
+    ThreadState->State.gregs[FEXCore::X86State::REG_RAX] = State.rax;
+    ThreadState->State.gregs[FEXCore::X86State::REG_RBX] = State.rbx;
+    ThreadState->State.gregs[FEXCore::X86State::REG_RCX] = State.rcx;
+    ThreadState->State.gregs[FEXCore::X86State::REG_RDX] = State.rdx;
+    ThreadState->State.gregs[FEXCore::X86State::REG_RSI] = State.rsi;
+    ThreadState->State.gregs[FEXCore::X86State::REG_RDI] = State.rdi;
+    ThreadState->State.gregs[FEXCore::X86State::REG_RSP] = State.rsp;
+    ThreadState->State.gregs[FEXCore::X86State::REG_RBP] = State.rbp;
+    ThreadState->State.gregs[FEXCore::X86State::REG_R8]  = State.r8;
+    ThreadState->State.gregs[FEXCore::X86State::REG_R9]  = State.r9;
+    ThreadState->State.gregs[FEXCore::X86State::REG_R10] = State.r10;
+    ThreadState->State.gregs[FEXCore::X86State::REG_R11] = State.r11;
+    ThreadState->State.gregs[FEXCore::X86State::REG_R12] = State.r12;
+    ThreadState->State.gregs[FEXCore::X86State::REG_R13] = State.r13;
+    ThreadState->State.gregs[FEXCore::X86State::REG_R14] = State.r14;
+    ThreadState->State.gregs[FEXCore::X86State::REG_R15] = State.r15;
+    ThreadState->State.rip = State.rip;
+    UnpackRFLAGS(&ThreadState->State, State.rflags);
+
+    ThreadState->State.fs = SpecialState.fs.base;
+    ThreadState->State.gs = SpecialState.gs.base;
+  }
+
+
+  static void VMExecution(FEXCore::Core::ThreadState *Thread) {
+    auto InternalThread = reinterpret_cast<FEXCore::Core::InternalThreadState*>(Thread);
+    VMCore *Core = reinterpret_cast<VMCore*>(InternalThread->CPUBackend.get());
+    Core->ExecuteCode(Thread);
+  }
+
+  static void VMExecutionFallback(FEXCore::Core::ThreadState *Thread) {
+    auto InternalThread = reinterpret_cast<FEXCore::Core::InternalThreadState*>(Thread);
+    VMCore *Core = reinterpret_cast<VMCore*>(InternalThread->FallbackBackend.get());
+    Core->ExecuteCode(Thread);
+  }
+
+  void* VMCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const *IR, FEXCore::Core::DebugData *DebugData) {
+    if (IsFallback)
+      return reinterpret_cast<void*>(VMExecutionFallback);
+    else
+      return reinterpret_cast<void*>(VMExecution);
+  }
+
+  void VMCore::ExecuteCode(FEXCore::Core::ThreadState *Thread) {
+
+#if 0
+    auto DumpState = [this]() {
+      LogMan::Msg::D("RIP: 0x%016lx", ThreadState->State.rip);
+      LogMan::Msg::D("RAX              RBX             RCX            RDX");
+      LogMan::Msg::D("0x%016lx 0x%016lx 0x%016lx 0x%016lx",
+        ThreadState->State.gregs[FEXCore::X86State::REG_RAX],
+        ThreadState->State.gregs[FEXCore::X86State::REG_RBX],
+        ThreadState->State.gregs[FEXCore::X86State::REG_RCX],
+        ThreadState->State.gregs[FEXCore::X86State::REG_RDX]);
+      LogMan::Msg::D("RSI              RDI             RSP            RBP");
+      LogMan::Msg::D("0x%016lx 0x%016lx 0x%016lx 0x%016lx",
+        ThreadState->State.gregs[FEXCore::X86State::REG_RSI],
+        ThreadState->State.gregs[FEXCore::X86State::REG_RDI],
+        ThreadState->State.gregs[FEXCore::X86State::REG_RSP],
+        ThreadState->State.gregs[FEXCore::X86State::REG_RBP]);
+      LogMan::Msg::D("R8               R9              R10            R11");
+      LogMan::Msg::D("0x%016lx 0x%016lx 0x%016lx 0x%016lx",
+        ThreadState->State.gregs[FEXCore::X86State::REG_R8],
+        ThreadState->State.gregs[FEXCore::X86State::REG_R9],
+        ThreadState->State.gregs[FEXCore::X86State::REG_R10],
+        ThreadState->State.gregs[FEXCore::X86State::REG_R11]);
+      LogMan::Msg::D("R12              R13             R14            R15");
+      LogMan::Msg::D("0x%016lx 0x%016lx 0x%016lx 0x%016lx",
+        ThreadState->State.gregs[FEXCore::X86State::REG_R12],
+        ThreadState->State.gregs[FEXCore::X86State::REG_R13],
+        ThreadState->State.gregs[FEXCore::X86State::REG_R14],
+        ThreadState->State.gregs[FEXCore::X86State::REG_R15]);
+    };
+#endif
+
+    CopyHostMemoryToGuest();
+    CopyHostStateToGuest();
+    VM->SetStepping(true);
+
+    if (VM->Run()) {
+      int ExitReason = VM->ExitReason();
+
+      // 4 = DEBUG
+      if (ExitReason == 4 || ExitReason == 5) {
+        CopyGuestCPUToHost();
+        CopyGuestMemoryToHost();
+      }
+
+      // 5 = HLT
+      if (ExitReason == 5) {
+        ThreadState->RunningEvents.ShouldStop = true;
+      }
+
+      // 8 = Shutdown. Due to an unhandled error
+      if (ExitReason == 8) {
+        LogMan::Msg::E("Unhandled VM Fault");
+        VM->Debug();
+        CopyGuestCPUToHost();
+        ThreadState->RunningEvents.ShouldStop = true;
+      }
+
+      // 9 = Failed Entry
+      if (ExitReason == 9) {
+        LogMan::Msg::E("Failed to enter VM due to: 0x%lx", VM->GetFailEntryReason());
+        ThreadState->RunningEvents.ShouldStop = true;
+      }
+    }
+    else {
+      LogMan::Msg::E("VM failed to run");
+      ThreadState->RunningEvents.ShouldStop = true;
+    }
+  }
+
+  FEXCore::CPU::CPUBackend *CPUCreationFactory(FEXCore::Context::Context* CTX, FEXCore::Core::ThreadState *Thread) {
+    return new VMCore(CTX, Thread, false);
+  }
+
+  FEXCore::CPU::CPUBackend *CPUCreationFactoryFallback(FEXCore::Context::Context* CTX, FEXCore::Core::ThreadState *Thread) {
+    return new VMCore(CTX, Thread, true);
+  }
+
+}
+

Source/CommonCore/VMFactory.h

@@ -0,0 +1,14 @@
+namespace FEXCore::CPU {
+  class CPUBackend;
+}
+namespace FEXCore::Context{
+  struct Context;
+}
+namespace FEXCore::Core {
+  struct ThreadState;
+}
+
+namespace VMFactory {
+  FEXCore::CPU::CPUBackend *CPUCreationFactory(FEXCore::Context::Context* CTX, FEXCore::Core::ThreadState *Thread);
+  FEXCore::CPU::CPUBackend *CPUCreationFactoryFallback(FEXCore::Context::Context* CTX, FEXCore::Core::ThreadState *Thread);
+}

Source/Core/CPU/HostCore/HostCore.cpp

@@ -0,0 +1,268 @@
+#include "Common/MathUtils.h"
+#include "Core/CPU/CPUBackend.h"
+#include "Core/CPU/CPUCore.h"
+#include "Core/CPU/DebugData.h"
+#include "Core/CPU/IR.h"
+#include "Core/CPU/IntrusiveIRList.h"
+#include "Core/CPU/HostCore/HostCore.h"
+#include "LogManager.h"
+#include <ucontext.h>
+
+#include <cassert>
+#include <iostream>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <sys/ptrace.h>
+#include <errno.h>
+#include <sys/user.h>
+#include <stdint.h>
+
+namespace FEX::CPU {
+
+class HostCore final : public CPUBackend {
+public:
+  explicit HostCore(FEX::ThreadState *Thread);
+  ~HostCore() override = default;
+  std::string GetName() override { return "Host Stepper"; }
+  void* CompileCode(FEX::IR::IntrusiveIRList const *ir, FEX::CPU::DebugData *DebugData) override;
+
+  void *MapRegion(void *HostPtr, uint64_t GuestPtr, uint64_t Size) override {
+    // Map locally to unprotected
+    printf("Mapping Guest Ptr: 0x%lx\n", GuestPtr);
+    void *Mem = LocalMemoryMapper.MapRegionFlags(GuestPtr, Size, PROT_READ | PROT_WRITE | PROT_EXEC);
+    memset(Mem, 0xCC, Size);
+    return HostPtr;
+  }
+
+  void ExecuteCode(FEX::ThreadState *Thread);
+
+  void SignalHandler(int sig, siginfo_t *info, void *RawContext);
+
+  bool NeedsOpDispatch() override { return false; }
+private:
+  void HandleSyscall();
+  void ExecutionThreadFunction();
+
+  void InstallSignalHandler();
+
+  FEX::ThreadState *ThreadState;
+  FEX::Config::Value<bool> ConfigMultiblock{"Multiblock", false};
+  std::thread ExecutionThread;
+  Memmap LocalMemoryMapper{};
+  pid_t ChildPid;
+  struct sigaction OldSigAction_SEGV;
+  struct sigaction OldSigAction_TRAP;
+
+  int PipeFDs[2];
+  std::atomic_bool ShouldStart{false};
+};
+
+static HostCore *GlobalCore;
+static void SigAction_SEGV(int sig, siginfo_t* info, void* RawContext) {
+  GlobalCore->SignalHandler(sig, info, RawContext);
+}
+
+HostCore::HostCore(FEX::ThreadState *Thread)
+  : ThreadState {Thread} {
+
+  GlobalCore = this;
+  LocalMemoryMapper.AllocateSHMRegion(1ULL << 36);
+
+  ExecutionThread = std::thread(&HostCore::ExecutionThreadFunction, this);
+}
+static void HostExecution(FEX::ThreadState *Thread) {
+  HostCore *Core = reinterpret_cast<HostCore*>(Thread->CPUBackend.get());
+  Core->ExecuteCode(Thread);
+}
+
+void HostCore::SignalHandler(int sig, siginfo_t *info, void *RawContext) {
+  ucontext_t* Context = (ucontext_t*)RawContext;
+  uint64_t HostToEmuRIP = (uint64_t)Context->uc_mcontext.gregs[REG_RIP] - LocalMemoryMapper.GetBaseOffset<uint64_t>(0);
+  printf("We hecked up and faulted! %d Emu:0x%lx Host:0x%lx\n", sig, HostToEmuRIP, (uint64_t)Context->uc_mcontext.gregs[REG_RIP]);
+  static uint64_t LastEMURip = ~0ULL;
+  static uint64_t LastInstSize = 0;
+  if (sig == SIGSEGV) {
+    // RIP == Base instruction
+  }
+  else if (sig == SIGTRAP) {
+    HostToEmuRIP -= 1; // 0xCC moves us ahead by one
+  }
+
+  uint8_t *LocalData = LocalMemoryMapper.GetPointer<uint8_t*>(HostToEmuRIP);
+  uint8_t *ActualData = ThreadState->CPUCore->MemoryMapper->GetPointer<uint8_t*>(HostToEmuRIP);
+  uint64_t TotalInstructionsLength {0};
+
+  ThreadState->CPUCore->FrontendDecoder.DecodeInstructionsInBlock(ActualData, HostToEmuRIP);
+  auto DecodedOps = ThreadState->CPUCore->FrontendDecoder.GetDecodedInsts();
+  if (sig == SIGSEGV) {
+    for (size_t i = 0; i < DecodedOps.second; ++i) {
+      FEX::X86Tables::DecodedInst const* DecodedInfo {nullptr};
+      DecodedInfo = &DecodedOps.first->at(i);
+      auto CheckOp = [&](char const* Name, FEX::X86Tables::DecodedOperand const &Operand) {
+        if (Operand.TypeNone.Type != FEX::X86Tables::DecodedOperand::TYPE_NONE &&
+            Operand.TypeNone.Type != FEX::X86Tables::DecodedOperand::TYPE_LITERAL &&
+            Operand.TypeNone.Type != FEX::X86Tables::DecodedOperand::TYPE_GPR) {
+          printf("Operand type is %d\n", Operand.TypeNone.Type);
+          const std::vector<unsigned> EmulatorToSystemContext = {
+            offsetof(mcontext_t, gregs[REG_RAX]),
+            offsetof(mcontext_t, gregs[REG_RBX]),
+            offsetof(mcontext_t, gregs[REG_RCX]),
+            offsetof(mcontext_t, gregs[REG_RDX]),
+            offsetof(mcontext_t, gregs[REG_RSI]),
+            offsetof(mcontext_t, gregs[REG_RDI]),
+            offsetof(mcontext_t, gregs[REG_RBP]),
+            offsetof(mcontext_t, gregs[REG_RSP]),
+            offsetof(mcontext_t, gregs[REG_R8]),
+            offsetof(mcontext_t, gregs[REG_R9]),
+            offsetof(mcontext_t, gregs[REG_R10]),
+            offsetof(mcontext_t, gregs[REG_R11]),
+            offsetof(mcontext_t, gregs[REG_R12]),
+            offsetof(mcontext_t, gregs[REG_R13]),
+            offsetof(mcontext_t, gregs[REG_R14]),
+            offsetof(mcontext_t, gregs[REG_R15]),
+          };
+
+          // Modify the registers to match the memory operands necessary.
+          // This will propagate some addresses
+          if (Operand.TypeNone.Type == FEX::X86Tables::DecodedOperand::TYPE_GPR_DIRECT) {
+            uint64_t *GPR = (uint64_t*)((uintptr_t)&Context->uc_mcontext + EmulatorToSystemContext[Operand.TypeGPR.GPR]);
+            uint64_t HostPointer = LocalMemoryMapper.GetPointer<uint64_t>(*GPR);
+            printf("Changing host pointer from 0x%lx to %lx\n", *GPR, HostPointer);
+            *GPR = HostPointer;
+          }
+          else {
+            OldSigAction_SEGV.sa_sigaction(sig, info, RawContext);
+            return;
+          }
+        }
+      };
+      printf("Insts: %ld\n", DecodedOps.second);
+      CheckOp("Dest", DecodedInfo->Dest);
+      CheckOp("Src1", DecodedInfo->Src1);
+      CheckOp("Src2", DecodedInfo->Src2);
+
+      // Reset RIP to the start of the instruction
+      Context->uc_mcontext.gregs[REG_RIP] = (uint64_t)LocalData;
+      return;
+    }
+
+    OldSigAction_SEGV.sa_sigaction(sig, info, RawContext);
+    return;
+  }
+  else if (sig == SIGTRAP) {
+    for (size_t i = 0; i < DecodedOps.second; ++i) {
+      FEX::X86Tables::DecodedInst const* DecodedInfo {nullptr};
+      DecodedInfo = &DecodedOps.first->at(i);
+      TotalInstructionsLength += DecodedInfo->InstSize;
+    }
+
+    if (LastEMURip != ~0ULL) {
+      uint8_t *PreviousLocalData = LocalMemoryMapper.GetPointer<uint8_t*>(LastEMURip);
+      memset(PreviousLocalData, 0xCC, LastInstSize);
+    }
+    LastInstSize = TotalInstructionsLength;
+    printf("\tHit an instruction of length %ld 0x%lx 0x%lx 0x%lx\n", TotalInstructionsLength, (uint64_t)Context->uc_mcontext.gregs[REG_RIP], (uint64_t)LocalData, (uint64_t)ActualData);
+    memcpy(LocalData, ActualData, TotalInstructionsLength);
+    printf("\tData Source:");
+    for (uint64_t i = 0; i < TotalInstructionsLength; ++i) {
+      printf("%02x ", ActualData[i]);
+    }
+    printf("\n");
+
+    // Reset RIP to the start of the instruction
+    Context->uc_mcontext.gregs[REG_RIP] = (uint64_t)LocalData;
+  }
+}
+
+void HostCore::InstallSignalHandler() {
+  struct sigaction sa;
+  sa.sa_handler = nullptr;
+  sa.sa_sigaction = &SigAction_SEGV;
+  sa.sa_flags = SA_SIGINFO;
+  sigemptyset(&sa.sa_mask);
+  // We use sigsegv to capture invalid memory accesses
+  // We then patch the GPR state of that instruction to point to the correct location
+  sigaction(SIGSEGV, &sa, &OldSigAction_SEGV);
+  // We use trapping to determine when we've stepped to a new instruction
+  sigaction(SIGTRAP, &sa, &OldSigAction_TRAP);
+}
+
+void HostCore::ExecutionThreadFunction() {
+  printf("Host Core created\n");
+
+  if (pipe(PipeFDs) == -1) {
+    LogMan::Msg::A("Couldn't pipe");
+    return;
+  }
+  ChildPid = fork();
+  if (ChildPid == 0) {
+    // Child
+    // Set that we want to be traced
+    if (ptrace(PTRACE_TRACEME, 0, 0, 0)) {
+      LogMan::Msg::A("Couldn't start trace");
+    }
+    raise(SIGSTOP);
+    InstallSignalHandler();
+
+    close(PipeFDs[1]);
+    using Ptr = void (*)();
+    read(PipeFDs[0], &ThreadState->CPUState, sizeof(FEX::X86State::State));
+
+    printf("Child is running! 0x%lx\n", ThreadState->CPUState.rip);
+    Ptr Loc = LocalMemoryMapper.GetPointer<Ptr>(ThreadState->CPUState.rip);
+    Loc();
+    printf("Oh Snap. We returned in the child\n");
+  }
+  else {
+    // Parent
+    // Parent will be the ptrace control thread
+    int status;
+    close(PipeFDs[0]);
+
+    waitpid(ChildPid, &status, 0);
+
+    if (WIFSTOPPED(status) && WSTOPSIG(status) == 19) {
+      // Attach to child
+      ptrace(PTRACE_ATTACH, ChildPid, 0, 0);
+      ptrace(PTRACE_CONT, ChildPid, 0, 0);
+    }
+
+    while (!ShouldStart.load());
+    printf("Telling child to go!\n");
+    write(PipeFDs[1], &ThreadState->CPUState, sizeof(FEX::X86State::State));
+
+    while (1) {
+
+      ShouldStart.store(false);
+
+      waitpid(ChildPid, &status, 0);
+      if (WIFEXITED(status)) {
+        return;
+      }
+      if (WIFSTOPPED(status) && WSTOPSIG(status) == 5) {
+        ptrace(PTRACE_CONT, ChildPid, 0, 5);
+      }
+      if (WIFSTOPPED(status) && WSTOPSIG(status) == 11) {
+        ptrace(PTRACE_DETACH, ChildPid, 0, 11);
+        break;
+      }
+    }
+  }
+}
+void* HostCore::CompileCode([[maybe_unused]] FEX::IR::IntrusiveIRList const* ir, FEX::CPU::DebugData *DebugData) {
+  printf("Attempting to compile: 0x%lx\n", ThreadState->CPUState.rip);
+  return reinterpret_cast<void*>(HostExecution);
+}
+
+void HostCore::ExecuteCode(FEX::ThreadState *Thread) {
+  ShouldStart = true;
+  while(1);
+}
+
+FEX::CPU::CPUBackend *CreateHostCore(FEX::ThreadState *Thread) {
+  return new HostCore(Thread);
+}
+
+}

Source/Core/CPU/HostCore/HostCore.h

@@ -0,0 +1,10 @@
+#pragma once
+
+namespace FEX {
+struct ThreadState;
+}
+
+namespace FEX::CPU {
+class CPUBackend;
+FEX::CPU::CPUBackend *CreateHostCore(FEX::ThreadState *Thread);
+}

Source/Tests/CMakeLists.txt

@@ -0,0 +1,55 @@
+set(LIBS FEXCore Common CommonCore SonicUtils pthread LLVM)
+set(NAME ELFLoader)
+set(SRCS ELFLoader.cpp)
+
+add_executable(${NAME} ${SRCS})
+target_include_directories(${NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/Source/)
+target_include_directories(${NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/External/SonicUtils/)
+
+target_link_libraries(${NAME} ${LIBS})
+
+set(NAME TestHarness)
+set(SRCS TestHarness.cpp)
+
+add_executable(${NAME} ${SRCS})
+target_include_directories(${NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/Source/)
+target_include_directories(${NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/External/SonicUtils/)
+
+target_link_libraries(${NAME} ${LIBS})
+
+set(NAME TestHarnessRunner)
+set(SRCS TestHarnessRunner.cpp)
+
+add_executable(${NAME} ${SRCS})
+target_include_directories(${NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/Source/)
+target_include_directories(${NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/External/SonicUtils/)
+
+target_link_libraries(${NAME} ${LIBS})
+
+set(NAME LockstepRunner)
+set(SRCS LockstepRunner.cpp)
+
+add_executable(${NAME} ${SRCS})
+target_include_directories(${NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/Source/)
+target_include_directories(${NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/External/SonicUtils/)
+
+target_link_libraries(${NAME} ${LIBS})
+
+set(NAME UnitTestGenerator)
+set(SRCS UnitTestGenerator.cpp)
+
+add_executable(${NAME} ${SRCS})
+target_include_directories(${NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/Source/)
+target_include_directories(${NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/External/SonicUtils/)
+
+target_link_libraries(${NAME} ${LIBS})
+
+set(NAME PTrace)
+set(SRCS TestSingleStepHardware.cpp)
+
+add_executable(${NAME} ${SRCS})
+target_include_directories(${NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/Source/)
+target_include_directories(${NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/External/SonicUtils/)
+
+target_link_libraries(${NAME} ${LIBS})
+

Source/Tests/ELFLoader.cpp

@@ -0,0 +1,102 @@
+#include "Common/ArgumentLoader.h"
+#include "CommonCore/VMFactory.h"
+#include "Common/Config.h"
+#include "ELFLoader.h"
+#include "HarnessHelpers.h"
+#include "LogManager.h"
+
+#include <FEXCore/Config/Config.h>
+#include <FEXCore/Core/CodeLoader.h>
+#include <FEXCore/Core/Context.h>
+#include <FEXCore/Memory/SharedMem.h>
+
+#include <cstdint>
+#include <filesystem>
+#include <string>
+#include <vector>
+
+void MsgHandler(LogMan::DebugLevels Level, char const *Message) {
+  const char *CharLevel{nullptr};
+
+  switch (Level) {
+  case LogMan::NONE:
+    CharLevel = "NONE";
+    break;
+  case LogMan::ASSERT:
+    CharLevel = "ASSERT";
+    break;
+  case LogMan::ERROR:
+    CharLevel = "ERROR";
+    break;
+  case LogMan::DEBUG:
+    CharLevel = "DEBUG";
+    break;
+  case LogMan::INFO:
+    CharLevel = "Info";
+    break;
+  case LogMan::STDOUT:
+    CharLevel = "STDOUT";
+    break;
+  case LogMan::STDERR:
+    CharLevel = "STDERR";
+    break;
+  default:
+    CharLevel = "???";
+    break;
+  }
+
+  printf("[%s] %s\n", CharLevel, Message);
+  fflush(nullptr);
+}
+
+void AssertHandler(char const *Message) {
+  printf("[ASSERT] %s\n", Message);
+  fflush(nullptr);
+}
+
+int main(int argc, char **argv, char **const envp) {
+  LogMan::Throw::InstallHandler(AssertHandler);
+  LogMan::Msg::InstallHandler(MsgHandler);
+  FEX::Config::Init();
+  FEX::ArgLoader::Load(argc, argv);
+
+  FEX::Config::Value<uint8_t> CoreConfig{"Core", 0};
+  FEX::Config::Value<uint64_t> BlockSizeConfig{"MaxInst", 1};
+  FEX::Config::Value<bool> SingleStepConfig{"SingleStep", false};
+  FEX::Config::Value<bool> MultiblockConfig{"Multiblock", false};
+
+  auto Args = FEX::ArgLoader::Get();
+
+  LogMan::Throw::A(!Args.empty(), "Not enough arguments");
+
+  FEX::HarnessHelper::ELFCodeLoader Loader{Args[0], Args, envp};
+
+  FEXCore::Context::InitializeStaticTables();
+  auto SHM = FEXCore::SHM::AllocateSHMRegion(1ULL << 36);
+  auto CTX = FEXCore::Context::CreateNewContext();
+  FEXCore::Context::InitializeContext(CTX);
+  FEXCore::Context::SetApplicationFile(CTX, std::filesystem::canonical(Args[0]));
+
+  FEXCore::Config::SetConfig(CTX, FEXCore::Config::CONFIG_DEFAULTCORE, CoreConfig() > 3 ? FEXCore::Config::CONFIG_CUSTOM : CoreConfig());
+  FEXCore::Config::SetConfig(CTX, FEXCore::Config::CONFIG_MULTIBLOCK, MultiblockConfig());
+  FEXCore::Config::SetConfig(CTX, FEXCore::Config::CONFIG_SINGLESTEP, SingleStepConfig());
+  FEXCore::Config::SetConfig(CTX, FEXCore::Config::CONFIG_MAXBLOCKINST, BlockSizeConfig());
+  FEXCore::Context::SetCustomCPUBackendFactory(CTX, VMFactory::CPUCreationFactory);
+  // FEXCore::Context::SetFallbackCPUBackendFactory(CTX, VMFactory::CPUCreationFactoryFallback);
+
+  FEXCore::Context::AddGuestMemoryRegion(CTX, SHM);
+  FEXCore::Context::InitCore(CTX, &Loader);
+
+  while (FEXCore::Context::RunLoop(CTX, true) == FEXCore::Context::ExitReason::EXIT_DEBUG);
+  auto ShutdownReason = FEXCore::Context::GetExitReason(CTX);
+  LogMan::Msg::D("Reason we left VM: %d", ShutdownReason);
+  bool Result = ShutdownReason == 0;
+
+  FEXCore::Context::DestroyContext(CTX);
+  FEXCore::SHM::DestroyRegion(SHM);
+
+  printf("Managed to load? %s\n", Result ? "Yes" : "No");
+
+  FEX::Config::Shutdown();
+  return 0;
+}

Source/Tests/HarnessHelpers.h

@@ -0,0 +1,598 @@
+#pragma once
+#include "Common/Config.h"
+#include "Common/MathUtils.h"
+#include "ELFLoader.h"
+#include "ELFSymbolDatabase.h"
+#include "LogManager.h"
+
+#include <FEXCore/Core/CodeLoader.h>
+#include <bitset>
+#include <cassert>
+#include <cstring>
+#include <fstream>
+#include <vector>
+#include <FEXCore/Core/CodeLoader.h>
+#include <FEXCore/Core/CoreState.h>
+#include <FEXCore/Core/X86Enums.h>
+
+namespace FEX::HarnessHelper {
+   bool CompareStates(FEXCore::Core::CPUState const& State1,
+       FEXCore::Core::CPUState const& State2,
+       uint64_t MatchMask,
+       bool OutputGPRs) {
+    bool Matches = true;
+
+    auto DumpGPRs = [OutputGPRs](auto Name, uint64_t A, uint64_t B) {
+      if (!OutputGPRs) return;
+      if (A == B) return;
+
+      printf("%s: 0x%016lx %s 0x%016lx\n", Name.c_str(), A, A==B ? "==" : "!=", B);
+    };
+
+    auto DumpFLAGs = [OutputGPRs](auto Name, uint64_t A, uint64_t B) {
+      if (!OutputGPRs) return;
+      if (A == B) return;
+
+      constexpr std::array<unsigned, 17> Flags = {
+        FEXCore::X86State::RFLAG_CF_LOC,
+        FEXCore::X86State::RFLAG_PF_LOC,
+        FEXCore::X86State::RFLAG_AF_LOC,
+        FEXCore::X86State::RFLAG_ZF_LOC,
+        FEXCore::X86State::RFLAG_SF_LOC,
+        FEXCore::X86State::RFLAG_TF_LOC,
+        FEXCore::X86State::RFLAG_IF_LOC,
+        FEXCore::X86State::RFLAG_DF_LOC,
+        FEXCore::X86State::RFLAG_OF_LOC,
+        FEXCore::X86State::RFLAG_IOPL_LOC,
+        FEXCore::X86State::RFLAG_NT_LOC,
+        FEXCore::X86State::RFLAG_RF_LOC,
+        FEXCore::X86State::RFLAG_VM_LOC,
+        FEXCore::X86State::RFLAG_AC_LOC,
+        FEXCore::X86State::RFLAG_VIF_LOC,
+        FEXCore::X86State::RFLAG_VIP_LOC,
+        FEXCore::X86State::RFLAG_ID_LOC,
+      };
+
+      printf("%s: 0x%016lx %s 0x%016lx\n", Name.c_str(), A, A==B ? "==" : "!=", B);
+      for (auto &Flag : Flags) {
+        uint64_t FlagMask = 1 << Flag;
+        if ((A & FlagMask) != (B & FlagMask)) {
+          printf("\t%s: %ld != %ld\n", FEXCore::Core::GetFlagName(Flag).data(), (A >> Flag) & 1, (B >> Flag) & 1);
+        }
+      }
+    };
+
+    auto CheckGPRs = [&Matches, DumpGPRs](std::string Name, uint64_t A, uint64_t B){
+      DumpGPRs(std::move(Name), A, B);
+      Matches &= A == B;
+    };
+
+    auto CheckFLAGS = [&Matches, DumpFLAGs](std::string Name, uint64_t A, uint64_t B){
+      DumpFLAGs(std::move(Name), A, B);
+      Matches &= A == B;
+    };
+
+
+    // RIP
+    if (MatchMask & 1) {
+      CheckGPRs("RIP", State1.rip, State2.rip);
+    }
+
+    MatchMask >>= 1;
+
+    // GPRS
+    for (unsigned i = 0; i < 16; ++i, MatchMask >>= 1) {
+      if (MatchMask & 1) {
+        CheckGPRs("GPR" + std::to_string(i), State1.gregs[i], State2.gregs[i]);
+      }
+    }
+
+    // XMM
+    for (unsigned i = 0; i < 16; ++i, MatchMask >>= 1) {
+      if (MatchMask & 1) {
+        CheckGPRs("XMM0_" + std::to_string(i), State1.xmm[i][0], State2.xmm[i][0]);
+        CheckGPRs("XMM1_" + std::to_string(i), State1.xmm[i][1], State2.xmm[i][1]);
+      }
+    }
+
+    // GS
+    if (MatchMask & 1) {
+      CheckGPRs("GS", State1.gs, State2.gs);
+    }
+    MatchMask >>= 1;
+
+    // FS
+    if (MatchMask & 1) {
+      CheckGPRs("FS", State1.fs, State2.fs);
+    }
+    MatchMask >>= 1;
+
+    auto CompactRFlags = [](auto Arg) -> uint32_t {
+      uint32_t Res = 2;
+      for (int i = 0; i < 32; ++i) {
+        Res |= Arg->flags[i] << i;
+      }
+      return Res;
+    };
+
+    // FLAGS
+    if (MatchMask & 1) {
+      uint32_t rflags1 = CompactRFlags(&State1);
+      uint32_t rflags2 = CompactRFlags(&State2);
+
+      CheckFLAGS("FLAGS", rflags1, rflags2);
+    }
+    MatchMask >>= 1;
+    return Matches;
+  }
+
+  void ReadFile(std::string const &Filename, std::vector<char> *Data) {
+    std::fstream TestFile;
+    TestFile.open(Filename, std::fstream::in | std::fstream::binary);
+    LogMan::Throw::A(TestFile.is_open(), "Failed to open file");
+
+    TestFile.seekg(0, std::fstream::end);
+    size_t FileSize = TestFile.tellg();
+    TestFile.seekg(0, std::fstream::beg);
+
+    Data->resize(FileSize);
+
+    TestFile.read(&Data->at(0), FileSize);
+
+    TestFile.close();
+  }
+
+  class ConfigLoader final {
+  public:
+    void Init(std::string const &ConfigFilename) {
+      ReadFile(ConfigFilename, &RawConfigFile);
+      memcpy(&BaseConfig, RawConfigFile.data(), sizeof(ConfigStructBase));
+    }
+
+    bool CompareStates(FEXCore::Core::CPUState const& State1, FEXCore::Core::CPUState const& State2) {
+      bool Matches = true;
+      uint64_t MatchMask = BaseConfig.OptionMatch & ~BaseConfig.OptionIgnore;
+      Matches &= FEX::HarnessHelper::CompareStates(State1, State2, MatchMask, ConfigDumpGPRs());
+
+      if (BaseConfig.OptionRegDataCount > 0) {
+        uintptr_t DataOffset = sizeof(ConfigStructBase);
+        constexpr std::array<std::pair<uint64_t, unsigned>, 36> OffsetArray = {{
+          {offsetof(FEXCore::Core::CPUState, rip), 1},
+          {offsetof(FEXCore::Core::CPUState, gregs[0]), 1},
+          {offsetof(FEXCore::Core::CPUState, gregs[1]), 1},
+          {offsetof(FEXCore::Core::CPUState, gregs[2]), 1},
+          {offsetof(FEXCore::Core::CPUState, gregs[3]), 1},
+          {offsetof(FEXCore::Core::CPUState, gregs[4]), 1},
+          {offsetof(FEXCore::Core::CPUState, gregs[5]), 1},
+          {offsetof(FEXCore::Core::CPUState, gregs[6]), 1},
+          {offsetof(FEXCore::Core::CPUState, gregs[7]), 1},
+          {offsetof(FEXCore::Core::CPUState, gregs[8]), 1},
+          {offsetof(FEXCore::Core::CPUState, gregs[9]), 1},
+          {offsetof(FEXCore::Core::CPUState, gregs[10]), 1},
+          {offsetof(FEXCore::Core::CPUState, gregs[11]), 1},
+          {offsetof(FEXCore::Core::CPUState, gregs[12]), 1},
+          {offsetof(FEXCore::Core::CPUState, gregs[13]), 1},
+          {offsetof(FEXCore::Core::CPUState, gregs[14]), 1},
+          {offsetof(FEXCore::Core::CPUState, gregs[15]), 1},
+          {offsetof(FEXCore::Core::CPUState, xmm[0][0]), 2},
+          {offsetof(FEXCore::Core::CPUState, xmm[1][0]), 2},
+          {offsetof(FEXCore::Core::CPUState, xmm[2][0]), 2},
+          {offsetof(FEXCore::Core::CPUState, xmm[3][0]), 2},
+          {offsetof(FEXCore::Core::CPUState, xmm[4][0]), 2},
+          {offsetof(FEXCore::Core::CPUState, xmm[5][0]), 2},
+          {offsetof(FEXCore::Core::CPUState, xmm[6][0]), 2},
+          {offsetof(FEXCore::Core::CPUState, xmm[7][0]), 2},
+          {offsetof(FEXCore::Core::CPUState, xmm[8][0]), 2},
+          {offsetof(FEXCore::Core::CPUState, xmm[9][0]), 2},
+          {offsetof(FEXCore::Core::CPUState, xmm[10][0]), 2},
+          {offsetof(FEXCore::Core::CPUState, xmm[11][0]), 2},
+          {offsetof(FEXCore::Core::CPUState, xmm[12][0]), 2},
+          {offsetof(FEXCore::Core::CPUState, xmm[13][0]), 2},
+          {offsetof(FEXCore::Core::CPUState, xmm[14][0]), 2},
+          {offsetof(FEXCore::Core::CPUState, xmm[15][0]), 2},
+          {offsetof(FEXCore::Core::CPUState, gs), 1},
+          {offsetof(FEXCore::Core::CPUState, fs), 1},
+          {offsetof(FEXCore::Core::CPUState, flags), 8},
+        }};
+
+        // Offset past the Memory regions if there are any
+        DataOffset += sizeof(MemoryRegionBase) * BaseConfig.OptionMemoryRegionCount;
+        for (unsigned i = 0; i < BaseConfig.OptionRegDataCount; ++i) {
+          RegDataStructBase *RegData = reinterpret_cast<RegDataStructBase*>(RawConfigFile.data() + DataOffset);
+          [[maybe_unused]] std::bitset<64> RegFlags = RegData->RegKey;
+          assert(RegFlags.count() == 1  && "Must set reg data explicitly per register");
+          assert(RegData->RegKey != (1UL << 36) && "Invalid register selection");
+
+          size_t NameIndex = __builtin_ffsl(RegData->RegKey)- 1;
+          auto Offset = OffsetArray[NameIndex];
+          uint64_t *State1Data = reinterpret_cast<uint64_t*>(reinterpret_cast<uint64_t>(&State1) + Offset.first);
+          uint64_t *State2Data = reinterpret_cast<uint64_t*>(reinterpret_cast<uint64_t>(&State2) + Offset.first);
+
+          auto DumpGPRs = [this](auto Name, uint64_t A, uint64_t B) {
+            if (!ConfigDumpGPRs())
+              return;
+
+            printf("%s: 0x%016lx %s 0x%016lx\n", Name.c_str(), A, A==B ? "==" : "!=", B);
+          };
+
+          auto CheckGPRs = [&Matches, DumpGPRs](std::string Name, uint64_t A, uint64_t B) {
+            DumpGPRs(std::move(Name), A, B);
+            Matches &= A == B;
+          };
+
+          for (unsigned j = 0; j < Offset.second; ++j) {
+            std::string Name;
+            if (NameIndex == 0) // RIP
+              Name = "RIP";
+            else if (NameIndex >= 1 && NameIndex < 17)
+              Name = "GPR" + std::to_string(NameIndex - 1);
+            else if (NameIndex >= 17 && NameIndex < 33)
+              Name = "XMM[" + std::to_string(NameIndex - 17) + "][" + std::to_string(j) + "]";
+            else if (NameIndex == 33)
+              Name = "gs";
+            else if (NameIndex == 34)
+              Name ="fs";
+            else if (NameIndex == 35)
+              Name = "rflags";
+
+            CheckGPRs("Core1: " + Name + ": ", State1Data[j], RegData->RegValues[j]);
+            CheckGPRs("Core2: " + Name + ": ", State2Data[j], RegData->RegValues[j]);
+          }
+
+          // Get the correct data offset
+          DataOffset += sizeof(RegDataStructBase) + Offset.second * 8;
+        }
+      }
+      return Matches;
+    }
+
+  private:
+    FEX::Config::Value<bool> ConfigDumpGPRs{"DumpGPRs", false};
+
+    struct ConfigStructBase {
+      uint64_t OptionMatch;
+      uint64_t OptionIgnore;
+      uint64_t OptionStackSize;
+      uint64_t OptionEntryPoint;
+      uint32_t OptionABI;
+      uint32_t OptionMemoryRegionCount;
+      uint32_t OptionRegDataCount;
+      uint8_t  AdditionalData[];
+    }__attribute__((packed));
+
+    struct MemoryRegionBase {
+      uint64_t Region;
+      uint64_t Size;
+    } __attribute__((packed));
+
+    struct RegDataStructBase {
+      uint32_t RegDataCount;
+      uint64_t RegKey;
+      uint64_t RegValues[];
+    } __attribute__((packed));
+
+    std::vector<char> RawConfigFile;
+    ConfigStructBase BaseConfig;
+  };
+
+  class HarnessCodeLoader final : public FEXCore::CodeLoader {
+
+    static constexpr uint32_t PAGE_SIZE = 4096;
+  public:
+
+    HarnessCodeLoader(std::string const &Filename, const char *ConfigFilename) {
+      ReadFile(Filename, &RawFile);
+      if (ConfigFilename) {
+        Config.Init(ConfigFilename);
+      }
+    }
+
+    uint64_t StackSize() const override {
+      return STACK_SIZE;
+    }
+
+    uint64_t SetupStack([[maybe_unused]] void *HostPtr, uint64_t GuestPtr) const override {
+      return GuestPtr + STACK_SIZE - 16;
+    }
+
+    uint64_t DefaultRIP() const override {
+      return RIP;
+    }
+
+    MemoryLayout GetLayout() const override {
+      uint64_t CodeSize = RawFile.size();
+      CodeSize = AlignUp(CodeSize, PAGE_SIZE);
+      return std::make_tuple(CODE_START_RANGE, CODE_START_RANGE + CodeSize, CodeSize);
+    }
+
+    void MapMemoryRegion(std::function<void*(uint64_t, uint64_t)> Mapper) override {
+      bool LimitedSize = true;
+      if (LimitedSize) {
+        Mapper(0xe000'0000, PAGE_SIZE * 10);
+
+        // SIB8
+        // We test [-128, -126] (Bottom)
+        // We test [-8, 8] (Middle)
+        // We test [120, 127] (Top)
+        // Can fit in two pages
+        Mapper(0xe800'0000 - PAGE_SIZE, PAGE_SIZE * 2);
+
+        // SIB32 Bottom
+        // We test INT_MIN, INT_MIN + 8
+        Mapper(0x2'0000'0000, PAGE_SIZE);
+        // SIB32 Middle
+        // We test -8 + 8
+        Mapper(0x2'8000'0000 - PAGE_SIZE, PAGE_SIZE * 2);
+
+        // SIB32 Top
+        // We Test INT_MAX - 8, INT_MAX
+        Mapper(0x3'0000'0000 - PAGE_SIZE, PAGE_SIZE * 2);
+      }
+      else {
+        // This is scratch memory location and SIB8 location
+        Mapper(0xe000'0000, 0x1000'0000);
+        // This is for large SIB 32bit displacement testing
+        Mapper(0x2'0000'0000, 0x1'0000'1000);
+      }
+
+      // Map in the memory region for the test file
+      Mapper(CODE_START_PAGE, AlignUp(RawFile.size(), PAGE_SIZE));
+    }
+
+    void LoadMemory(MemoryWriter Writer) override {
+      // Memory base here starts at the start location we passed back with GetLayout()
+      // This will write at [CODE_START_RANGE + 0, RawFile.size() )
+      Writer(&RawFile.at(0), CODE_START_RANGE, RawFile.size());
+    }
+
+    uint64_t GetFinalRIP() override { return CODE_START_RANGE + RawFile.size(); }
+
+    bool CompareStates(FEXCore::Core::CPUState const& State1, FEXCore::Core::CPUState const& State2) {
+      return Config.CompareStates(State1, State2);
+    }
+
+  private:
+    constexpr static uint64_t STACK_SIZE = PAGE_SIZE;
+    // Zero is special case to know when we are done
+    constexpr static uint64_t CODE_START_PAGE = 0x0'1000;
+    constexpr static uint64_t CODE_START_RANGE = CODE_START_PAGE + 0x1;
+    constexpr static uint64_t RIP = CODE_START_RANGE;
+
+    std::vector<char> RawFile;
+    ConfigLoader Config;
+  };
+
+class ELFCodeLoader final : public FEXCore::CodeLoader {
+public:
+  ELFCodeLoader(std::string const &Filename, [[maybe_unused]] std::vector<std::string> const &args, char **const envp = nullptr)
+    : File {Filename, false}
+    , DB {&File}
+    , Args {args} {
+
+    if (File.WasDynamic()) {
+      // If the file isn't static then we need to add the filename of interpreter
+      // to the front of the argument list
+      Args.emplace(Args.begin(), File.InterpreterLocation());
+    }
+
+    // Just block pretty much everything
+    const std::vector<const char*> NotAllowed = {
+      "XDG_CONFIG_DIRS",
+      "XDG_SEAT",
+      "XDG_SESSION_DESKTOP",
+      "XDG_SESSION_TYPE",
+      "XDG_CURRENT_DESKTOP",
+      "XDG_SESSION_CLASS",
+      "XDG_VTNR",
+      "XDG_SESSION_ID",
+      "XDG_RUNTIME_DIR",
+      "XDG_DATA_DIRS",
+
+      "QT_ACCESSIBILITY",
+      "COLORTERM",
+      "DERBY_HOME",
+      "TERMCAP",
+      "JAVA_HOME",
+      "SSH_AUTH_SOCK",
+      "DESKTOP_SESSION",
+      "SSH_AGENT_PID",
+      "GTK_MODULES",
+      "LOGNAME",
+      "GPG_AGENT_INFO",
+      "XAUTHORITY",
+      "J2REDIR",
+      "WINDOWPATH",
+      "HOME",
+      "LANG",
+      "LS_COLORS",
+      "PM_PACKAGES_ROOT",
+      "TERM",
+      "DEFAULT_PATH",
+      "J2SDKDIR",
+      "LESSCLOSE",
+      "LESSOPEN",
+      "LIBVIRT_DEFAULT_URI",
+      "USER",
+      "DISPLAY",
+      "SHLVL",
+      "PATH",
+      "STY",
+      "GDMSESSION",
+      "DBUS_SESSION_BUS_ADDRESS",
+      "OLDPWD",
+      "_",
+
+      "WINIT_HIDPI_FACTOR",
+      "SHELL",
+      "WINDOWID",
+      "MANDATORY_PATH",
+      "WINDOW",
+      "PWD",
+      "DEFAULTS_PATH",
+
+      "ALACRITTY_LOG",
+      "USERNAME",
+    };
+
+    if (!!envp) {
+      // If we had envp passed in then make sure to set it up on the guest
+      for (unsigned i = 0;; ++i) {
+        if (envp[i] == nullptr)
+          break;
+        bool Invalid = false;
+
+        for (auto Str : NotAllowed) {
+          if (strncmp(Str, envp[i], strlen(Str)) == 0) {
+            Invalid = true;
+            break;
+          }
+        }
+
+        if (Invalid) continue;
+
+        printf("Allowing '%s'\n", envp[i]);
+        EnvironmentVariables.emplace_back(envp[i]);
+      }
+    }
+  }
+
+  uint64_t StackSize() const override {
+    return STACK_SIZE;
+  }
+
+  uint64_t SetupStack(void *HostPtr, uint64_t GuestPtr) const override {
+    uintptr_t StackPointer = reinterpret_cast<uintptr_t>(HostPtr) + StackSize();
+    // Set up our initial CPU state
+    uint64_t rsp = GuestPtr + StackSize();
+
+    uint64_t TotalArgumentMemSize{};
+    uint64_t ArgumentBackingSize{};
+
+    TotalArgumentMemSize += 8; // Argument counter size
+    TotalArgumentMemSize += 8 * Args.size(); // Pointers to strings
+    TotalArgumentMemSize += 8; // Padding for something
+    TotalArgumentMemSize += 8 * EnvironmentVariables.size(); // Argument location for envp
+
+    for (unsigned i = 0; i < Args.size(); ++i) {
+      TotalArgumentMemSize += Args[i].size() + 1;
+      ArgumentBackingSize += Args[i].size() + 1;
+    }
+
+    for (unsigned i = 0; i < EnvironmentVariables.size(); ++i) {
+      TotalArgumentMemSize += EnvironmentVariables[i].size() + 1;
+    }
+
+    TotalArgumentMemSize += 8;
+
+    // Burn some SP space for a redzone
+    rsp -= TotalArgumentMemSize + 0x1000;
+    StackPointer -= TotalArgumentMemSize + 0x1000;
+
+    // Stack setup
+    // [0, 8):   Argument Count
+    // [8, 16):  Argument Pointer 0
+    // [16, 24): Argument Pointer 1
+    // ....
+    // [Pad1, +8): Some Pointer
+    // [envp, +8): envp pointer
+    // [Pad2End, +8): Argument String 0
+    // [+8, +8): String 1
+    // ...
+    // [argvend, +8): envp[0]
+    // ...
+    // [envpend, +8): nullptr
+
+    uint64_t *ArgumentPointers = reinterpret_cast<uint64_t*>(StackPointer + 8);
+    uint64_t *PadPointers = reinterpret_cast<uint64_t*>(StackPointer + 8 + Args.size() * 8);
+    uint64_t *EnvpPointers = reinterpret_cast<uint64_t*>(StackPointer + 8 + Args.size() * 8 + 8);
+
+    uint64_t ArgumentBackingOffset = 8 * Args.size() + 8 * 4;
+    uint8_t *ArgumentBackingBase = reinterpret_cast<uint8_t*>(StackPointer + ArgumentBackingOffset);
+    uint64_t ArgumentBackingBaseGuest = rsp + ArgumentBackingOffset;
+
+    uint64_t EnvpBackingOffset = ArgumentBackingOffset + ArgumentBackingSize + EnvironmentVariables.size() * 8;
+    uint8_t *EnvpBackingBase = reinterpret_cast<uint8_t*>(StackPointer + EnvpBackingOffset);
+    uint64_t EnvpBackingBaseGuest = rsp + EnvpBackingOffset;
+
+    *reinterpret_cast<uint64_t*>(StackPointer + 0) = Args.size();
+    PadPointers[0] = 0;
+
+    // If we don't have any, just make sure the first is nullptr
+    EnvpPointers[0] = 0;
+
+    uint64_t CurrentOffset = 0;
+    for (size_t i = 0; i < Args.size(); ++i) {
+      size_t ArgSize = Args[i].size();
+      // Set the pointer to this argument
+      ArgumentPointers[i] = ArgumentBackingBaseGuest + CurrentOffset;
+      // Copy the string in to the final location
+      memcpy(reinterpret_cast<void*>(ArgumentBackingBase + CurrentOffset), &Args[i].at(0), ArgSize);
+
+      // Set the null terminator for the string
+      *reinterpret_cast<uint8_t*>(ArgumentBackingBase + CurrentOffset + ArgSize + 1) = 0;
+
+      CurrentOffset += ArgSize + 1;
+    }
+
+    CurrentOffset = 0;
+    for (size_t i = 0; i < EnvironmentVariables.size(); ++i) {
+      size_t EnvpSize = EnvironmentVariables[i].size();
+      // Set the pointer to this argument
+      EnvpPointers[i] = EnvpBackingBaseGuest + CurrentOffset;
+
+      // Copy the string in to the final location
+      memcpy(reinterpret_cast<void*>(EnvpBackingBase + CurrentOffset), &EnvironmentVariables[i].at(0), EnvpSize);
+
+      // Set the null terminator for the string
+      *reinterpret_cast<uint8_t*>(EnvpBackingBase + CurrentOffset + EnvpSize + 1) = 0;
+
+      CurrentOffset += EnvpSize + 1;
+    }
+    // Last envp needs to be nullptr
+    EnvpPointers[EnvironmentVariables.size()] = 0;
+
+    return rsp;
+  }
+
+  uint64_t DefaultRIP() const override {
+    return DB.DefaultRIP();
+  }
+
+  void MapMemoryRegion(std::function<void*(uint64_t, uint64_t)> Mapper) override {
+    DB.MapMemoryRegions(Mapper);
+  }
+
+  MemoryLayout GetLayout() const override {
+    return DB.GetFileLayout();
+  }
+
+  void LoadMemory(MemoryWriter Writer) override {
+    DB.WriteLoadableSections(Writer);
+  }
+
+  char const *FindSymbolNameInRange(uint64_t Address) override {
+    ELFLoader::ELFSymbol const *Sym;
+    Sym = DB.GetSymbolInRange(std::make_pair(Address, 1));
+    if (Sym) {
+      return Sym->Name;
+    }
+    return nullptr;
+  }
+
+  void GetInitLocations(std::vector<uint64_t> *Locations) override {
+    DB.GetInitLocations(Locations);
+  }
+
+  uint64_t InitializeThreadSlot(std::function<void(void const*, uint64_t)> Writer) const override {
+    return DB.InitializeThreadSlot(Writer);
+  };
+
+private:
+  ::ELFLoader::ELFContainer File;
+  ::ELFLoader::ELFSymbolDatabase DB;
+  std::vector<std::string> Args;
+  std::vector<std::string> EnvironmentVariables;
+  constexpr static uint64_t STACK_SIZE = 8 * 1024 * 1024;
+};
+
+}

Source/Tests/LockstepRunner.cpp

@@ -0,0 +1,738 @@
+#include "Common/ArgumentLoader.h"
+#include "Common/Config.h"
+#include "CommonCore/VMFactory.h"
+#include "HarnessHelpers.h"
+#include "LogManager.h"
+
+#include <FEXCore/Config/Config.h>
+#include <FEXCore/Core/CodeLoader.h>
+#include <FEXCore/Core/Context.h>
+#include <FEXCore/Core/CoreState.h>
+#include <FEXCore/Memory/SharedMem.h>
+
+#include <cstdint>
+#include <fcntl.h>
+#include <unistd.h>
+#include <string>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <thread>
+#include <vector>
+#include <boost/interprocess/detail/config_begin.hpp>
+#include <boost/interprocess/mapped_region.hpp>
+#include <boost/interprocess/shared_memory_object.hpp>
+#include <boost/interprocess/ipc/message_queue.hpp>
+#include <boost/thread/thread_time.hpp>
+
+void MsgHandler(LogMan::DebugLevels Level, char const *Message) {
+  const char *CharLevel{nullptr};
+
+  switch (Level) {
+  case LogMan::NONE:
+    CharLevel = "NONE";
+    break;
+  case LogMan::ASSERT:
+    CharLevel = "ASSERT";
+    break;
+  case LogMan::ERROR:
+    CharLevel = "ERROR";
+    break;
+  case LogMan::DEBUG:
+    CharLevel = "DEBUG";
+    break;
+  case LogMan::INFO:
+    CharLevel = "Info";
+    break;
+  default:
+    CharLevel = "???";
+    break;
+  }
+  printf("[%s] %s\n", CharLevel, Message);
+}
+
+void AssertHandler(char const *Message) {
+  printf("[ASSERT] %s\n", Message);
+}
+
+class Flag final {
+public:
+  bool TestAndSet(bool SetValue = true) {
+    bool Expected = !SetValue;
+    return Value.compare_exchange_strong(Expected, SetValue);
+  }
+
+  bool TestAndClear() {
+    return TestAndSet(false);
+  }
+
+  void Set() {
+    Value.store(true);
+  }
+
+  bool Load() {
+    return Value.load();
+  }
+
+private:
+  std::atomic_bool Value {false};
+};
+
+
+class IPCEvent final {
+private:
+/**
+ * @brief Literally just an atomic bool that we are using for this class
+ */
+public:
+  IPCEvent(std::string_view Name, bool Client)
+    : EventName {Name} {
+
+    using namespace boost::interprocess;
+    if (!Client) {
+      shared_memory_object::remove(EventName.c_str());
+      SHM = std::make_unique<shared_memory_object>(
+        create_only,
+        EventName.c_str(),
+        read_write
+      );
+      SHMSize = sizeof(SHMObject);
+      SHM->truncate(SHMSize);
+      SHMRegion = std::make_unique<mapped_region>(*SHM, read_write);
+
+      Obj = new (SHMRegion->get_address()) SHMObject{};
+    }
+    else {
+      SHM = std::make_unique<shared_memory_object>(
+        open_only,
+        EventName.c_str(),
+        read_write
+      );
+      SHMSize = sizeof(SHMObject);
+      SHMRegion = std::make_unique<mapped_region>(*SHM, read_write);
+
+      // Load object from shared memory, don't construct it
+      Obj = reinterpret_cast<SHMObject*>(SHMRegion->get_address());
+    }
+  }
+
+  ~IPCEvent() {
+    using namespace boost::interprocess;
+    shared_memory_object::remove(EventName.c_str());
+  }
+
+  void NotifyOne() {
+    using namespace boost::interprocess;
+    if (Obj->FlagObject.TestAndSet()) {
+      scoped_lock<interprocess_mutex> lk(Obj->MutexObject);
+      Obj->CondObject.notify_one();
+    }
+  }
+
+  void NotifyAll() {
+    using namespace boost::interprocess;
+    if (Obj->FlagObject.TestAndSet()) {
+      scoped_lock<interprocess_mutex> lk(Obj->MutexObject);
+      Obj->CondObject.notify_all();
+    }
+  }
+
+  void Wait() {
+    using namespace boost::interprocess;
+
+    // Have we signaled before we started waiting?
+    if (Obj->FlagObject.TestAndClear())
+      return;
+
+    scoped_lock<interprocess_mutex> lk(Obj->MutexObject);
+    Obj->CondObject.wait(lk, [this] { return Obj->FlagObject.TestAndClear(); });
+  }
+
+  bool WaitFor(boost::posix_time::ptime const& time) {
+    using namespace boost::interprocess;
+
+    // Have we signaled before we started waiting?
+    if (Obj->FlagObject.TestAndClear())
+      return true;
+
+    scoped_lock<interprocess_mutex> lk(Obj->MutexObject);
+    bool DidSignal = Obj->CondObject.timed_wait(lk, time, [this] { return Obj->FlagObject.TestAndClear(); });
+    return DidSignal;
+  }
+
+private:
+  std::string EventName;
+  std::unique_ptr<boost::interprocess::shared_memory_object> SHM;
+  std::unique_ptr<boost::interprocess::mapped_region> SHMRegion;
+  struct SHMObject {
+    Flag FlagObject;
+    boost::interprocess::interprocess_condition CondObject;
+    boost::interprocess::interprocess_mutex MutexObject;
+  };
+  SHMObject *Obj;
+  size_t SHMSize;
+};
+
+class IPCFlag {
+public:
+
+  IPCFlag(std::string_view Name, bool Client)
+    : EventName {Name} {
+
+    using namespace boost::interprocess;
+    if (!Client) {
+      shared_memory_object::remove(EventName.c_str());
+      SHM = std::make_unique<shared_memory_object>(
+        create_only,
+        EventName.c_str(),
+        read_write
+      );
+      SHMSize = sizeof(FlagObj);
+      SHM->truncate(SHMSize);
+      SHMRegion = std::make_unique<mapped_region>(*SHM, read_write);
+
+      FlagObj = new (SHMRegion->get_address()) Flag{};
+    }
+    else {
+      SHM = std::make_unique<shared_memory_object>(
+        open_only,
+        EventName.c_str(),
+        read_write
+      );
+      SHMSize = sizeof(Flag);
+      SHMRegion = std::make_unique<mapped_region>(*SHM, read_write);
+
+      // Load object from shared memory, don't construct it
+      FlagObj = reinterpret_cast<Flag*>(SHMRegion->get_address());
+    }
+  }
+
+  ~IPCFlag() {
+    using namespace boost::interprocess;
+    shared_memory_object::remove(EventName.c_str());
+  }
+
+  Flag *GetFlag() { return FlagObj; }
+
+private:
+  std::string EventName;
+  std::unique_ptr<boost::interprocess::shared_memory_object> SHM;
+  std::unique_ptr<boost::interprocess::mapped_region> SHMRegion;
+  size_t SHMSize;
+
+  Flag *FlagObj;
+};
+
+class IPCMessage {
+public:
+  IPCMessage(std::string_view Name, bool Client, size_t QueueSize, size_t QueueDepth)
+    : QueueName {Name} {
+    using namespace boost::interprocess;
+    if (!Client) {
+      message_queue::remove(QueueName.c_str());
+      MessageQueue = std::make_unique<message_queue>(
+        create_only,
+        QueueName.c_str(),
+        QueueDepth,
+        QueueSize
+      );
+    }
+    else {
+      MessageQueue = std::make_unique<message_queue>(
+        open_only,
+        QueueName.c_str()
+      );
+    }
+  }
+
+  void Send(void *Buf, size_t Size) {
+    MessageQueue->send(Buf, Size, 0);
+  }
+
+  size_t Recv(void *Buf, size_t Size) {
+    uint32_t Priority {};
+    size_t Recv_Size{};
+    if (!MessageQueue->timed_receive(Buf, Size, Recv_Size, Priority, boost::get_system_time() + boost::posix_time::seconds(10))) {
+      return 0;
+    }
+    return Recv_Size;
+  }
+
+  void ThrowAwayRecv() {
+    uint32_t Priority {};
+    size_t Recv_Size{};
+    MessageQueue->try_receive(nullptr, 0, Recv_Size, Priority);
+  }
+
+  ~IPCMessage() {
+    using namespace boost::interprocess;
+    message_queue::remove(QueueName.c_str());
+  }
+
+private:
+  std::string QueueName;
+  std::unique_ptr<boost::interprocess::message_queue> MessageQueue;
+};
+
+/**
+ * @brief Heartbeat to know if processes are alive
+ */
+class IPCHeartbeat {
+public:
+  IPCHeartbeat(std::string_view Name, bool Client)
+    : EventName {Name} {
+    using namespace boost::interprocess;
+
+    if (Client) {
+      bool Trying = true;
+      while (Trying) {
+        try {
+          SHM = std::make_unique<shared_memory_object>(
+            open_only,
+            EventName.c_str(),
+            read_write
+          );
+
+          boost::interprocess::offset_t size;
+          while (!SHM->get_size(size)) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); }
+
+          // Map the SHM region
+          // This can fault for...some reason?
+          SHMRegion = std::make_unique<mapped_region>(*SHM, read_write);
+
+          HeartBeatEvents = reinterpret_cast<SHMHeartbeatEvent*>(SHMRegion->get_address());
+          while (HeartBeatEvents->Allocated.load() == 0)  { std::this_thread::sleep_for(std::chrono::milliseconds(100)); }
+          Trying = false;
+        }
+        catch(...) {
+          std::this_thread::sleep_for(std::chrono::milliseconds(100));
+        }
+      }
+
+    }
+    else {
+      SHM = std::make_unique<shared_memory_object>(
+        open_or_create,
+        EventName.c_str(),
+        read_write
+      );
+
+      SHM->truncate(sizeof(SHMHeartbeatEvent) + sizeof(HeartbeatClient) * 32);
+
+      // Map the SHM region
+      SHMRegion = std::make_unique<mapped_region>(*SHM, read_write);
+
+      HeartBeatEvents = new (SHMRegion->get_address()) SHMHeartbeatEvent{};
+      HeartBeatEvents->Allocated = 1;
+    }
+
+    ThreadID = AllocateThreadID();
+    if (ThreadID == ~0U) {
+      Running = false;
+      return;
+    }
+    HeartBeatThread = std::thread(&IPCHeartbeat::Thread, this);
+  }
+
+  ~IPCHeartbeat() {
+    using namespace boost::interprocess;
+    Running = false;
+    if (HeartBeatThread.joinable()) {
+      HeartBeatThread.join();
+    }
+
+    // If we are the last client then we should just destroy this region
+    if (DeallocateThreadID(ThreadID)) {
+      shared_memory_object::remove(EventName.c_str());
+    }
+  }
+
+  bool WaitForServer() {
+    auto Start = std::chrono::system_clock::now();
+    while (!IsServerAlive()) {
+      auto Now = std::chrono::system_clock::now();
+      using namespace std::chrono_literals;
+      if ((Now - Start) >= 10s) {
+        // We waited 10 seconds and the server never came up
+        return false;
+      }
+      std::this_thread::sleep_for(std::chrono::milliseconds(100));
+    }
+    return true;
+  }
+
+  bool WaitForClient() {
+    auto Start = std::chrono::system_clock::now();
+    while (!IsClientAlive()) {
+      auto Now = std::chrono::system_clock::now();
+      using namespace std::chrono_literals;
+      if ((Now - Start) >= 10s) {
+        // We waited 10 seconds and the server never came up
+        return false;
+      }
+      std::this_thread::sleep_for(std::chrono::milliseconds(100));
+    }
+    return true;
+  }
+
+
+  void Thread() {
+    HeartbeatClient *ThisThread = &HeartBeatEvents->Clients[ThreadID];
+    while (Running) {
+      std::this_thread::sleep_for(HeartBeatRate);
+      ThisThread->HeartbeatCounter++;
+    }
+  }
+
+  /**
+   * @brief Once the heartbeat is active we can enable a server flag
+   */
+  void EnableServer() {
+    LogMan::Msg::E("[SERVER %s] Enabling server on thread ID %d", EventName.c_str(), ThreadID);
+    HeartbeatClient *ThisThread = &HeartBeatEvents->Clients[ThreadID];
+    ThisThread->Type = 1;
+  }
+
+  void EnableClient() {
+    LogMan::Msg::E("[CLIENT %s] Enabling client on thread ID %d", EventName.c_str(), ThreadID);
+    HeartbeatClient *ThisThread = &HeartBeatEvents->Clients[ThreadID];
+    ThisThread->Type = 2;
+  }
+
+private:
+
+  bool IsServerAlive() {
+    return IsTypeAlive(1);
+  }
+
+  bool IsClientAlive() {
+    return IsTypeAlive(2);
+  }
+
+  bool IsTypeAlive(uint32_t Type) {
+    uint32_t OriginalThreadMask = HeartBeatEvents->NumClients.load();
+    for (uint32_t Index = 0; Index < 32; ++Index) {
+      if ((OriginalThreadMask & (1 << Index)) == 0)
+        continue;
+
+      HeartbeatClient *ThisThread = &HeartBeatEvents->Clients[Index];
+      if (ThisThread->Type == Type) {
+        // Found a Server, check if it is live
+
+        uint32_t Counter = ThisThread->HeartbeatCounter.load();
+        std::this_thread::sleep_for(HeartBeatRate * 2);
+        if (Counter != ThisThread->HeartbeatCounter.load()) {
+          return true;
+        }
+      }
+    }
+
+    return false;
+  }
+
+
+  uint32_t AllocateThreadID() {
+    while (true) {
+      uint32_t Index = 0;
+      uint32_t OriginalIDMask = HeartBeatEvents->NumClients.load();
+      if (OriginalIDMask == ~0U) {
+        // All slots taken
+        return ~0U;
+      }
+
+      // Scan every index and try to find a free slot
+      while (Index != 33) {
+        uint32_t Slot = 1 << Index;
+        if ((OriginalIDMask & Slot) == 0) {
+          // Free slot, let's try and take it
+          uint32_t NewIDMask = OriginalIDMask | Slot;
+
+          if (HeartBeatEvents->NumClients.compare_exchange_strong(OriginalIDMask, NewIDMask)) {
+            // We successfully got a slow
+            return Index;
+          }
+          else {
+            // We failed to get a slot, try again
+            break;
+          }
+        }
+
+        // Try next index
+        ++Index;
+      }
+    }
+  }
+
+  /**
+   * @brief Deallocates the thread ID from the clients
+   *
+   * @param LocalThreadID
+   *
+   * @return true is returned if we are the last thread to deallocate
+   */
+  bool DeallocateThreadID(uint32_t LocalThreadID) {
+    uint32_t ThreadIDMask = (1 << LocalThreadID);
+    while (true) {
+      uint32_t OriginalIDMask = HeartBeatEvents->NumClients.load();
+      uint32_t NewIDMask = OriginalIDMask & ~ThreadIDMask;
+      if (HeartBeatEvents->NumClients.compare_exchange_strong(OriginalIDMask, NewIDMask)) {
+        // We set the atomic. If the new mask is zero then we were the last to deallocate
+        return NewIDMask == 0;
+      }
+      else {
+        // Failed to deallocate, try again
+      }
+    }
+  }
+
+  struct HeartbeatClient {
+    std::atomic<uint32_t> Type; // 1 = Server, 2 = Client
+    std::atomic<uint32_t> HeartbeatCounter;
+  };
+
+  struct SHMHeartbeatEvent {
+    std::atomic<uint32_t> Allocated;
+    std::atomic<uint32_t> NumClients;
+    HeartbeatClient Clients[0];
+  };
+  std::string EventName;
+  std::unique_ptr<boost::interprocess::shared_memory_object> SHM;
+  std::unique_ptr<boost::interprocess::mapped_region> SHMRegion;
+  SHMHeartbeatEvent *HeartBeatEvents;
+
+  bool Running{true};
+  uint32_t ThreadID;
+  std::thread HeartBeatThread;
+
+  const std::chrono::milliseconds HeartBeatRate = std::chrono::seconds(1);
+};
+
+int main(int argc, char **argv) {
+  LogMan::Throw::InstallHandler(AssertHandler);
+  LogMan::Msg::InstallHandler(MsgHandler);
+  FEX::Config::Init();
+  FEX::ArgLoader::Load(argc, argv);
+
+  auto Args = FEX::ArgLoader::Get();
+
+  FEX::Config::Value<uint8_t> CoreConfig{"Core", 0};
+  FEX::Config::Value<bool> ConfigIPCClient{"IPCClient", false};
+  FEX::Config::Value<bool> ConfigELFType{"ELFType", false};
+  FEX::Config::Value<std::string> ConfigIPCID{"IPCID", "0"};
+
+  char File[256]{};
+
+  std::unique_ptr<IPCMessage> StateMessage;
+  std::unique_ptr<IPCMessage> StateFile;
+  std::unique_ptr<IPCEvent> HostEvent;
+  std::unique_ptr<IPCEvent> ClientEvent;
+  std::unique_ptr<IPCFlag> QuitFlag;
+
+  // Heartbeat is the only thing robust enough to support client or server starting in any order
+  printf("Initializing Heartbeat\n"); fflush(stdout);
+  IPCHeartbeat HeartBeat(ConfigIPCID() + "IPCHeart_Lockstep", ConfigIPCClient());
+
+  printf("Client? %s\n", ConfigIPCClient() ? "Yes" : "No"); fflush(stdout);
+
+  if (ConfigIPCClient()) {
+    if (!HeartBeat.WaitForServer()) {
+      // Server managed to time out
+      LogMan::Msg::E("[CLIENT %s] Timed out waiting for server", ConfigIPCID().c_str());
+      return -1;
+    }
+
+    // Now that we know the server is online, create our objects
+    StateMessage = std::make_unique<IPCMessage>(ConfigIPCID() + "IPCState_Lockstep", ConfigIPCClient(), sizeof(FEXCore::Core::CPUState), 1);
+    StateFile = std::make_unique<IPCMessage>(ConfigIPCID() + "IPCFile_Lockstep", ConfigIPCClient(), 256, 1);
+    HostEvent = std::make_unique<IPCEvent>(ConfigIPCID() + "IPCHost_Lockstep", ConfigIPCClient());
+    ClientEvent = std::make_unique<IPCEvent>(ConfigIPCID() + "IPCClient_Lockstep", ConfigIPCClient());
+    QuitFlag = std::make_unique<IPCFlag>(ConfigIPCID() + "IPCFlag_Lockstep", ConfigIPCClient());
+
+    HeartBeat.EnableClient();
+    HostEvent->Wait();
+    StateFile->Recv(File, 256);
+  }
+  else {
+    LogMan::Throw::A(!Args.empty(), "[SERVER %s] Not enough arguments", ConfigIPCID().c_str());
+    strncpy(File, Args[0].c_str(), 256);
+
+    // Create all of our objects now
+    StateMessage = std::make_unique<IPCMessage>(ConfigIPCID() + "IPCState_Lockstep", ConfigIPCClient(), sizeof(FEXCore::Core::CPUState), 1);
+    StateFile = std::make_unique<IPCMessage>(ConfigIPCID() + "IPCFile_Lockstep", ConfigIPCClient(), 256, 1);
+    HostEvent = std::make_unique<IPCEvent>(ConfigIPCID() + "IPCHost_Lockstep", ConfigIPCClient());
+    ClientEvent = std::make_unique<IPCEvent>(ConfigIPCID() + "IPCClient_Lockstep", ConfigIPCClient());
+    QuitFlag = std::make_unique<IPCFlag>(ConfigIPCID() + "IPCFlag_Lockstep", ConfigIPCClient());
+
+    HeartBeat.EnableServer();
+    HeartBeat.WaitForClient();
+    StateFile->Send(File, strlen(File));
+    HostEvent->NotifyAll();
+  }
+
+  bool ShowProgress = false;
+  uint64_t PCEnd;
+  uint64_t PCStart = 1;
+  auto LastTime = std::chrono::high_resolution_clock::now();
+
+  FEXCore::Context::InitializeStaticTables();
+  auto SHM = FEXCore::SHM::AllocateSHMRegion(1ULL << 36);
+  auto CTX = FEXCore::Context::CreateNewContext();
+  FEXCore::Context::InitializeContext(CTX);
+
+  FEXCore::Config::SetConfig(CTX, FEXCore::Config::CONFIG_DEFAULTCORE, CoreConfig() > 3 ? FEXCore::Config::CONFIG_CUSTOM : CoreConfig());
+  FEXCore::Config::SetConfig(CTX, FEXCore::Config::CONFIG_SINGLESTEP, 1);
+  FEXCore::Config::SetConfig(CTX, FEXCore::Config::CONFIG_MAXBLOCKINST, 1);
+
+  if (CoreConfig() == 4) {
+    FEXCore::Context::SetCustomCPUBackendFactory(CTX, VMFactory::CPUCreationFactory);
+  }
+
+  FEXCore::Context::AddGuestMemoryRegion(CTX, SHM);
+
+  if (ConfigELFType()) {
+    FEX::HarnessHelper::ELFCodeLoader Loader{File, {}};
+    bool Result = FEXCore::Context::InitCore(CTX, &Loader);
+    printf("Did we Load? %s\n", Result ? "Yes" : "No");
+  }
+  else {
+    FEX::HarnessHelper::HarnessCodeLoader Loader{File, nullptr};
+    bool Result = FEXCore::Context::InitCore(CTX, &Loader);
+    printf("Did we Load? %s\n", Result ? "Yes" : "No");
+    ShowProgress = true;
+    PCEnd = Loader.GetFinalRIP();
+    PCStart = Loader.DefaultRIP();
+  }
+
+  FEXCore::Core::CPUState State1;
+  bool MaskFlags = true;
+  bool MaskGPRs = false;
+  uint64_t LastRIP = 0;
+
+  auto PrintProgress = [&](bool PrintFinal = false) {
+    if (ShowProgress) {
+      auto CurrentTime = std::chrono::high_resolution_clock::now();
+      auto Diff = CurrentTime - LastTime;
+      if (Diff >= std::chrono::seconds(1) || PrintFinal) {
+        LastTime = CurrentTime;
+        uint64_t CurrentRIP = State1.rip - PCStart;
+        uint64_t EndRIP = PCEnd - PCStart;
+        double Progress = static_cast<double>(CurrentRIP) / static_cast<double>(EndRIP) * 25.0;
+        printf("Progress: [");
+        for (uint32_t i = 0; i < 25; ++i) {
+          printf("%c", (Progress > static_cast<double>(i)) ? '#' : '|');
+        }
+        printf("] RIP: 0x%lx\n", State1.rip);
+      }
+    }
+  };
+
+  uint32_t ErrorLocation = 0;
+  bool Done = false;
+  bool ServerTimedOut = false;
+  while (!Done)
+  {
+    if (MaskFlags) {
+      // We need to reset the CPU flags to somethign standard so we don't need to handle flags in this case
+      FEXCore::Core::CPUState CPUState;
+      FEXCore::Context::GetCPUState(CTX, &CPUState);
+      for (int i = 0; i < 32; ++i) {
+        CPUState.flags[i] = 0;
+      }
+      CPUState.flags[1] = 1; // Default state
+      FEXCore::Context::SetCPUState(CTX, &CPUState);
+    }
+
+    FEXCore::Context::ExitReason ExitReason;
+    ExitReason = FEXCore::Context::RunLoop(CTX, true);
+    FEXCore::Context::GetCPUState(CTX, &State1);
+
+    if (ExitReason == FEXCore::Context::ExitReason::EXIT_SHUTDOWN) {
+      Done = true;
+    }
+    else if (ExitReason == FEXCore::Context::ExitReason::EXIT_UNKNOWNERROR) {
+      QuitFlag->GetFlag()->Set();
+      ErrorLocation = -2;
+      break;
+    }
+
+    if (!ConfigIPCClient()) {
+      FEXCore::Core::CPUState State2;
+      if (StateMessage->Recv(&State2, sizeof(FEXCore::Core::CPUState)) == 0) {
+        // We had a time out
+        // This can happen when the client managed to early exit
+        LogMan::Msg::E("Client Timed out");
+        ErrorLocation = -2;
+        QuitFlag->GetFlag()->Set();
+        ClientEvent->NotifyAll();
+        break;
+      }
+
+      PrintProgress();
+
+      uint64_t MatchMask = (1ULL << 36) - 1;
+      if (MaskFlags) {
+        MatchMask &= ~(1ULL << 35); // Remove FLAGS for now
+      }
+
+      if (MaskGPRs) {
+        MatchMask &= ~((1ULL << 35) - 1);
+      }
+
+      bool Matches = FEX::HarnessHelper::CompareStates(State1, State2, MatchMask, true);
+
+      if (!Matches) {
+        LogMan::Msg::E("[SERVER %s] Stated ended up different at RIPS 0x%lx - 0x%lx - LastRIP: 0x%lx\n", ConfigIPCID().c_str(), State1.rip, State2.rip, LastRIP);
+        ErrorLocation = -3;
+        QuitFlag->GetFlag()->Set();
+        break;
+      }
+      LastRIP = State1.rip;
+      ClientEvent->NotifyAll();
+    }
+    else {
+      StateMessage->Send(&State1, sizeof(FEXCore::Core::CPUState));
+      if (!ClientEvent->WaitFor(boost::get_system_time() + boost::posix_time::seconds(10))) {
+        // Timed out
+        LogMan::Msg::E("Server timed out");
+        QuitFlag->GetFlag()->Set();
+        ErrorLocation = -1;
+        ServerTimedOut = true;
+      }
+    }
+
+    if (QuitFlag->GetFlag()->Load()) {
+      break;
+    }
+  }
+
+  // Some cleanup that needs to be done in case some side failed
+  if (!ConfigIPCClient()) {
+    ClientEvent->NotifyAll();
+  }
+  else {
+    if (!ServerTimedOut) {
+      // Send the latest state to the server so it can die
+      StateMessage->Send(&State1, sizeof(FEXCore::Core::CPUState));
+    }
+  }
+
+  FEXCore::Context::GetCPUState(CTX, &State1);
+  PrintProgress(true);
+
+  if (ErrorLocation == 0) {
+    // One final check to make sure the RIP ended up in the correct location
+    if (State1.rip != PCEnd) {
+      ErrorLocation = State1.rip;
+    }
+  }
+
+  if (ErrorLocation != 0) {
+    LogMan::Msg::E("Error Location: 0x%lx", ErrorLocation);
+  }
+
+  FEXCore::Context::DestroyContext(CTX);
+  FEXCore::SHM::DestroyRegion(SHM);
+  FEX::Config::Shutdown();
+  return ErrorLocation != 0;
+}
+
+#include <boost/interprocess/detail/config_end.hpp>

Source/Tests/TestHarness.cpp

@@ -0,0 +1,112 @@
+#include "Common/ArgumentLoader.h"
+#include "CommonCore/VMFactory.h"
+#include "HarnessHelpers.h"
+#include "LogManager.h"
+
+#include <FEXCore/Config/Config.h>
+#include <FEXCore/Core/CodeLoader.h>
+#include <FEXCore/Core/Context.h>
+#include <FEXCore/Core/CoreState.h>
+#include <FEXCore/Core/CPUBackend.h>
+#include <FEXCore/Core/X86Enums.h>
+#include <FEXCore/HLE/SyscallHandler.h>
+#include <FEXCore/Debug/InternalThreadState.h>
+#include <FEXCore/Memory/SharedMem.h>
+
+#include <cassert>
+#include <cstdint>
+#include <string>
+#include <vector>
+
+void MsgHandler(LogMan::DebugLevels Level, char const *Message) {
+  const char *CharLevel{nullptr};
+
+  switch (Level) {
+  case LogMan::NONE:
+    CharLevel = "NONE";
+    break;
+  case LogMan::ASSERT:
+    CharLevel = "ASSERT";
+    break;
+  case LogMan::ERROR:
+    CharLevel = "ERROR";
+    break;
+  case LogMan::DEBUG:
+    CharLevel = "DEBUG";
+    break;
+  case LogMan::INFO:
+    CharLevel = "Info";
+    break;
+  default:
+    CharLevel = "???";
+    break;
+  }
+  printf("[%s] %s\n", CharLevel, Message);
+}
+
+void AssertHandler(char const *Message) {
+  printf("[ASSERT] %s\n", Message);
+}
+
+int main(int argc, char **argv) {
+  LogMan::Throw::InstallHandler(AssertHandler);
+  LogMan::Msg::InstallHandler(MsgHandler);
+
+  FEX::ArgLoader::Load(argc, argv);
+
+  auto Args = FEX::ArgLoader::Get();
+
+  LogMan::Throw::A(Args.size() > 1, "Not enough arguments");
+
+  FEXCore::Context::InitializeStaticTables();
+  auto SHM1 = FEXCore::SHM::AllocateSHMRegion(1ULL << 36);
+  auto CTX1 = FEXCore::Context::CreateNewContext();
+
+  auto SHM2 = FEXCore::SHM::AllocateSHMRegion(1ULL << 36);
+  auto CTX2 = FEXCore::Context::CreateNewContext();
+
+  FEXCore::Context::SetCustomCPUBackendFactory(CTX1, VMFactory::CPUCreationFactory);
+  FEXCore::Config::SetConfig(CTX1, FEXCore::Config::CONFIG_DEFAULTCORE, FEXCore::Config::CONFIG_CUSTOM);
+  FEXCore::Config::SetConfig(CTX1, FEXCore::Config::CONFIG_SINGLESTEP, 1);
+  FEXCore::Config::SetConfig(CTX1, FEXCore::Config::CONFIG_MAXBLOCKINST, 1);
+
+  FEXCore::Context::AddGuestMemoryRegion(CTX1, SHM1);
+
+  FEXCore::Config::SetConfig(CTX2, FEXCore::Config::CONFIG_DEFAULTCORE, FEXCore::Config::CONFIG_INTERPRETER);
+  FEXCore::Config::SetConfig(CTX2, FEXCore::Config::CONFIG_SINGLESTEP, 1);
+  FEXCore::Config::SetConfig(CTX2, FEXCore::Config::CONFIG_MAXBLOCKINST, 1);
+
+  FEXCore::Context::AddGuestMemoryRegion(CTX2, SHM2);
+
+  FEXCore::Context::InitializeContext(CTX1);
+  FEXCore::Context::InitializeContext(CTX2);
+
+  FEX::HarnessHelper::HarnessCodeLoader Loader{Args[0], Args[1].c_str()};
+
+  bool Result1 = FEXCore::Context::InitCore(CTX1, &Loader);
+  bool Result2 = FEXCore::Context::InitCore(CTX2, &Loader);
+
+ if (!Result1 || !Result2)
+   return 2;
+
+ while (FEXCore::Context::RunLoop(CTX1, true) == FEXCore::Context::ExitReason::EXIT_DEBUG);
+ LogMan::Msg::I("Running Core2");
+  while (FEXCore::Context::RunLoop(CTX2, true) == FEXCore::Context::ExitReason::EXIT_DEBUG);
+
+  FEXCore::Core::CPUState State1;
+  FEXCore::Core::CPUState State2;
+
+  FEXCore::Context::GetCPUState(CTX1, &State1);
+  FEXCore::Context::GetCPUState(CTX2, &State2);
+
+  bool Passed = Loader.CompareStates(State1, State2);
+  LogMan::Msg::I("Passed? %s\n", Passed ? "Yes" : "No");
+
+  FEXCore::SHM::DestroyRegion(SHM1);
+  FEXCore::Context::DestroyContext(CTX1);
+  FEXCore::SHM::DestroyRegion(SHM2);
+  FEXCore::Context::DestroyContext(CTX2);
+
+  return Passed ? 0 : 1;
+}
+

Source/Tests/TestHarnessRunner.cpp

@@ -0,0 +1,96 @@
+#include "Common/ArgumentLoader.h"
+#include "CommonCore/VMFactory.h"
+#include "HarnessHelpers.h"
+#include "LogManager.h"
+
+#include <FEXCore/Config/Config.h>
+#include <FEXCore/Core/CodeLoader.h>
+#include <FEXCore/Core/Context.h>
+#include <FEXCore/Core/CoreState.h>
+#include <FEXCore/Core/CPUBackend.h>
+#include <FEXCore/Core/X86Enums.h>
+#include <FEXCore/HLE/SyscallHandler.h>
+#include <FEXCore/Debug/InternalThreadState.h>
+#include <FEXCore/Memory/SharedMem.h>
+
+#include <cassert>
+#include <cstdint>
+#include <string>
+#include <vector>
+
+void MsgHandler(LogMan::DebugLevels Level, char const *Message) {
+  const char *CharLevel{nullptr};
+
+  switch (Level) {
+  case LogMan::NONE:
+    CharLevel = "NONE";
+    break;
+  case LogMan::ASSERT:
+    CharLevel = "ASSERT";
+    break;
+  case LogMan::ERROR:
+    CharLevel = "ERROR";
+    break;
+  case LogMan::DEBUG:
+    CharLevel = "DEBUG";
+    break;
+  case LogMan::INFO:
+    CharLevel = "Info";
+    break;
+  default:
+    CharLevel = "???";
+    break;
+  }
+  printf("[%s] %s\n", CharLevel, Message);
+}
+
+void AssertHandler(char const *Message) {
+  printf("[ASSERT] %s\n", Message);
+}
+
+int main(int argc, char **argv) {
+  LogMan::Throw::InstallHandler(AssertHandler);
+  LogMan::Msg::InstallHandler(MsgHandler);
+  FEX::Config::Init();
+  FEX::ArgLoader::Load(argc, argv);
+
+  FEX::Config::Value<uint8_t> CoreConfig{"Core", 0};
+  FEX::Config::Value<uint64_t> BlockSizeConfig{"MaxInst", 1};
+  FEX::Config::Value<bool> SingleStepConfig{"SingleStep", false};
+  FEX::Config::Value<bool> MultiblockConfig{"Multiblock", false};
+
+  auto Args = FEX::ArgLoader::Get();
+
+  LogMan::Throw::A(Args.size() > 1, "Not enough arguments");
+
+  FEXCore::Context::InitializeStaticTables();
+  auto SHM = FEXCore::SHM::AllocateSHMRegion(1ULL << 36);
+  auto CTX = FEXCore::Context::CreateNewContext();
+
+  FEXCore::Context::SetCustomCPUBackendFactory(CTX, VMFactory::CPUCreationFactory);
+  FEXCore::Config::SetConfig(CTX, FEXCore::Config::CONFIG_DEFAULTCORE, CoreConfig() > 3 ? FEXCore::Config::CONFIG_CUSTOM : CoreConfig());
+  FEXCore::Config::SetConfig(CTX, FEXCore::Config::CONFIG_MULTIBLOCK, MultiblockConfig());
+  FEXCore::Config::SetConfig(CTX, FEXCore::Config::CONFIG_SINGLESTEP, SingleStepConfig());
+  FEXCore::Config::SetConfig(CTX, FEXCore::Config::CONFIG_MAXBLOCKINST, BlockSizeConfig());
+  FEXCore::Context::SetCustomCPUBackendFactory(CTX, VMFactory::CPUCreationFactory);
+
+  FEXCore::Context::AddGuestMemoryRegion(CTX, SHM);
+
+  FEXCore::Context::InitializeContext(CTX);
+
+  FEX::HarnessHelper::HarnessCodeLoader Loader{Args[0], Args[1].c_str()};
+
+  bool Result1 = FEXCore::Context::InitCore(CTX, &Loader);
+
+  if (!Result1)
+    return 1;
+
+  while (FEXCore::Context::RunLoop(CTX, true) == FEXCore::Context::ExitReason::EXIT_DEBUG)
+    ;
+
+  FEXCore::SHM::DestroyRegion(SHM);
+  FEXCore::Context::DestroyContext(CTX);
+
+  return 0;
+}
+

Source/Tests/TestSingleStepHardware.cpp

@@ -0,0 +1,113 @@
+#include "Common/ArgumentLoader.h"
+#include "Common/Config.h"
+#include "ELFLoader.h"
+#include "HarnessHelpers.h"
+#include "LogManager.h"
+
+#include <FEXCore/Core/CodeLoader.h>
+#include <cstdint>
+#include <string>
+#include <vector>
+#include <sys/mman.h>
+
+
+namespace FEX {
+uint8_t data[] = {
+  0xb8, 0x00, 0x00, 0x80, 0x3f, // mov eax, 0x3f800000
+  0x0f, 0xc7, 0xf8, // rdseed eax
+  0xba, 0x00, 0x00, 0x00, 0xe0, // mov edx, 0xe0000000
+  0x89, 0x02, // mov dword [edx], eax
+	0xC3, // RET
+};
+
+  class TestCodeLoader final : public FEXCore::CodeLoader {
+    static constexpr uint32_t PAGE_SIZE = 4096;
+  public:
+
+    TestCodeLoader() = default;
+
+    uint64_t StackSize() const override {
+      return STACK_SIZE;
+    }
+
+    uint64_t SetupStack([[maybe_unused]] void *HostPtr, uint64_t GuestPtr) const override {
+      return GuestPtr + STACK_SIZE - 8;
+    }
+
+    uint64_t DefaultRIP() const override {
+      return RIP;
+    }
+
+    MemoryLayout GetLayout() const override {
+      uint64_t CodeSize = 0x500;
+      CodeSize = AlignUp(CodeSize, PAGE_SIZE);
+      return std::make_tuple(0, CodeSize, CodeSize);
+    }
+
+    void MapMemoryRegion(std::function<void*(uint64_t, uint64_t)> Mapper) override {
+      // XXX: Pull this from the config
+      Mapper(0xe000'0000, 0x1000'0000);
+      Mapper(0x2'0000'0000, 0x1'0000'1000);
+    }
+
+    void LoadMemory(MemoryWriter Writer) override {
+      Writer(data, DefaultRIP(), sizeof(data));
+    }
+
+  private:
+    constexpr static uint64_t STACK_SIZE = PAGE_SIZE;
+    // Zero is special case to know when we are done
+    constexpr static uint64_t RIP = 0x1;
+  };
+}
+
+void MsgHandler(LogMan::DebugLevels Level, char const *Message) {
+  const char *CharLevel{nullptr};
+
+  switch (Level) {
+  case LogMan::NONE:
+    CharLevel = "NONE";
+    break;
+  case LogMan::ASSERT:
+    CharLevel = "ASSERT";
+    break;
+  case LogMan::ERROR:
+    CharLevel = "ERROR";
+    break;
+  case LogMan::DEBUG:
+    CharLevel = "DEBUG";
+    break;
+  case LogMan::INFO:
+    CharLevel = "Info";
+    break;
+  default:
+    CharLevel = "???";
+    break;
+  }
+  printf("[%s] %s\n", CharLevel, Message);
+}
+
+void AssertHandler(char const *Message) {
+  printf("[ASSERT] %s\n", Message);
+}
+
+int main(int argc, char **argv) {
+  LogMan::Throw::InstallHandler(AssertHandler);
+  LogMan::Msg::InstallHandler(MsgHandler);
+  FEX::Config::Init();
+  FEX::ArgLoader::Load(argc, argv);
+
+  FEX::Config::Value<uint8_t> CoreConfig{"Core", 0};
+  auto Args = FEX::ArgLoader::Get();
+
+//  FEX::Core localCore {static_cast<FEX::CPUCore::CPUCoreType>(CoreConfig())};
+//  FEX::TestCodeLoader Loader{};
+//  bool Result = localCore.Load(&Loader);
+//
+//  auto Base = localCore.GetCPU()->MemoryMapper->GetPointer<void*>(0);
+//  localCore.RunLoop(true);
+//  printf("Managed to load? %s\n", Result ? "Yes" : "No");
+
+  FEX::Config::Shutdown();
+  return 0;
+}

Source/Tools/CMakeLists.txt

@@ -0,0 +1,10 @@
+add_subdirectory(Debugger/)
+
+set(NAME Opt)
+set(SRCS Opt.cpp)
+
+add_executable(${NAME} ${SRCS})
+target_include_directories(${NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/Source/)
+target_include_directories(${NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/External/SonicUtils/)
+
+target_link_libraries(${NAME} FEXCore Common CommonCore SonicUtils pthread LLVM)

Source/Tools/Debugger/CMakeLists.txt

@@ -0,0 +1,29 @@
+set(NAME Debugger)
+set(SRCS Main.cpp
+  DebuggerState.cpp
+  Context.cpp
+  FEXImGui.cpp
+  IMGui.cpp
+  IRLexer.cpp
+  GLUtils.cpp
+  MainWindow.cpp
+  Disassembler.cpp
+  ${CMAKE_SOURCE_DIR}/External/imgui/examples/imgui_impl_glfw.cpp
+  ${CMAKE_SOURCE_DIR}/External/imgui/examples/imgui_impl_opengl3.cpp
+  )
+
+find_library(EPOXY_LIBRARY epoxy)
+find_library(GLFW_LIBRARY glfw3)
+find_package(LLVM CONFIG QUIET)
+
+add_definitions(-DIMGUI_IMPL_OPENGL_LOADER_CUSTOM=<epoxy/gl.h>)
+add_executable(${NAME} ${SRCS})
+
+target_link_libraries(${NAME} PRIVATE LLVM)
+target_include_directories(${NAME} PRIVATE ${LLVM_INCLUDE_DIRS})
+
+target_include_directories(${NAME} PRIVATE ${CMAKE_SOURCE_DIR}/Source/)
+target_include_directories(${NAME} PRIVATE ${CMAKE_SOURCE_DIR}/External/SonicUtils/)
+target_include_directories(${NAME} PRIVATE ${CMAKE_SOURCE_DIR}/External/imgui/examples/)
+
+target_link_libraries(${NAME} PRIVATE FEXCore Common CommonCore SonicUtils pthread LLVM epoxy glfw X11 EGL imgui tiny-json json-maker)

Source/Tools/Debugger/Context.cpp

@@ -0,0 +1,90 @@
+#include "Context.h"
+
+#include <cassert>
+#include <GLFW/glfw3.h>
+#include <vector>
+
+namespace GLContext {
+void glfw_error_callback(int error, const char* description)
+{
+    fprintf(stderr, "Glfw Error %d: %s\n", error, description);
+}
+
+class GLFWContext final : public GLContext::Context {
+public:
+  void Create(const char *Title) override {
+    glfwSetErrorCallback(glfw_error_callback);
+    if (!glfwInit()) {
+      assert(0 && "Couldn't init glfw");
+    }
+    glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 4);
+    glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 1);
+    glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE);
+    glfwWindowHint(GLFW_OPENGL_FORWARD_COMPAT, GL_TRUE);
+    glfwWindowHint(GLFW_RED_BITS, 8);
+    glfwWindowHint(GLFW_GREEN_BITS, 8);
+    glfwWindowHint(GLFW_BLUE_BITS, 8);
+    glfwWindowHint(GLFW_ALPHA_BITS, 8);
+    glfwWindowHint(GLFW_RESIZABLE, 1);
+    glfwWindowHint(GLFW_DOUBLEBUFFER, 1);
+
+    Window = glfwCreateWindow(640, 640, Title, nullptr, nullptr);
+    if (!Window) {
+      assert(0 && "Couldn't create window");
+    }
+    glfwMakeContextCurrent(Window);
+    glfwSwapInterval(1);
+  }
+
+  void Shutdown() override {
+    glfwMakeContextCurrent(nullptr);
+    glfwDestroyWindow(Window);
+    glfwTerminate();
+  }
+
+  void Swap() override {
+    glfwSwapBuffers(Window);
+    CheckWindowDimensions();
+  }
+
+  void RegisterResizeEvent(ResizeEvent Event) override {
+    ResizeEvents.emplace_back(Event);
+  }
+
+  void GetDim(uint32_t *Dim) override {
+    Dim[0] = Width;
+    Dim[1] = Height;
+  }
+
+private:
+
+  void CheckWindowDimensions() {
+    int LocalWidth;
+    int LocalHeight;
+    glfwGetWindowSize(Window, &LocalWidth, &LocalHeight);
+
+    if (LocalHeight != Height || LocalWidth != Width) {
+      Width = LocalWidth;
+      Height = LocalHeight;
+
+      for (auto const &Event : ResizeEvents) {
+        Event(Width, Height);
+      }
+    }
+  }
+  void* GetWindow() override {
+    return Window;
+  }
+
+  std::vector<ResizeEvent> ResizeEvents;
+
+  GLFWwindow *Window;
+  uint32_t Width{};
+  uint32_t Height{};
+};
+
+std::unique_ptr<Context> CreateContext() {
+  return std::make_unique<GLFWContext>();
+}
+
+}

Source/Tools/Debugger/Context.h

@@ -0,0 +1,20 @@
+#pragma once
+#include <functional>
+#include <memory>
+
+namespace GLContext {
+class Context {
+public:
+  virtual ~Context() {}
+  virtual void Create(const char *Title) = 0;
+  virtual void Shutdown() = 0;
+  virtual void Swap() = 0;
+  virtual void GetDim(uint32_t *Dim) = 0;
+
+  using ResizeEvent = std::function<void(uint32_t, uint32_t)>;
+  virtual void RegisterResizeEvent(ResizeEvent Event) = 0;
+  virtual void* GetWindow() = 0;
+};
+
+std::unique_ptr<Context> CreateContext();
+}

Source/Tools/Debugger/DebuggerState.cpp

@@ -0,0 +1,149 @@
+#include "DebuggerState.h"
+
+#include <FEXCore/Core/Context.h>
+#include <FEXCore/Debug/ContextDebug.h>
+
+namespace FEX::DebuggerState {
+
+FEXCore::Context::Context *s_CTX{};
+FEXCore::Config::ConfigCore s_CoreType = FEXCore::Config::ConfigCore::CONFIG_INTERPRETER;
+int s_IsStepping = 0;
+bool NewState = false;
+
+std::function<void()> StepCallback;
+std::function<void()> PauseCallback;
+std::function<void()> ContinueCallback;
+std::function<void()> NewStateCallback;
+std::function<void()> CloseCallback;
+std::function<void(char const *, bool)> CreateCallback;
+std::function<void(uint64_t)> CompileRIPCallback;
+std::function<void(std::stringstream *out, uint64_t PC)> GetIRCallback;
+
+bool ActiveCore() {
+  return s_CTX != nullptr;
+}
+
+void SetContext(FEXCore::Context::Context *ctx) {
+  s_CTX = ctx;
+}
+
+FEXCore::Context::Context *GetContext() {
+  return s_CTX;
+}
+
+FEXCore::Config::ConfigCore GetCoreType() {
+  return s_CoreType;
+}
+
+void SetCoreType(FEXCore::Config::ConfigCore CoreType) {
+  s_CoreType = CoreType;
+}
+
+int GetRunningMode() {
+  return s_IsStepping;
+}
+
+int GetCoreCurrentRunningMode() {
+  if (ActiveCore()) {
+    return FEXCore::Config::GetConfig(s_CTX, FEXCore::Config::CONFIG_SINGLESTEP);
+  }
+  return s_IsStepping;
+}
+
+void SetRunningMode(int RunningMode) {
+  s_IsStepping = RunningMode;
+
+  if (ActiveCore()) {
+    FEXCore::Config::SetConfig(s_CTX, FEXCore::Config::CONFIG_SINGLESTEP, RunningMode);
+  }
+}
+
+FEXCore::Core::CPUState GetCPUState() {
+  if (!ActiveCore()) {
+    return FEXCore::Core::CPUState{};
+  }
+  return FEXCore::Context::Debug::GetCPUState(s_CTX);
+}
+
+bool IsCoreRunning() {
+  if (!ActiveCore()) {
+    return false;
+  }
+
+  return !FEXCore::Context::IsDone(s_CTX);
+}
+
+// Client interface
+void RegisterStepCallback(std::function<void()> Callback) {
+  StepCallback = std::move(Callback);
+}
+
+void RegisterPauseCallback(std::function<void()> Callback) {
+  PauseCallback = std::move(Callback);
+}
+
+void RegisterContinueCallback(std::function<void()> Callback) {
+  ContinueCallback = std::move(Callback);
+}
+
+void Step() {
+  StepCallback();
+}
+
+void Pause() {
+  PauseCallback();
+}
+
+void Continue() {
+  ContinueCallback();
+}
+
+void RegisterCreateCallback(std::function<void(char const *Filename, bool)> Callback)
+{
+  CreateCallback = std::move(Callback);
+}
+
+void Create(char const *Filename, bool ELF) {
+  CreateCallback(Filename, ELF);
+}
+
+void RegisterCompileRIPCallback(std::function<void(uint64_t)> Callback) {
+  CompileRIPCallback = std::move(Callback);
+}
+void CompileRIP(uint64_t RIP) {
+  CompileRIPCallback(RIP);
+}
+
+void RegisterCloseCallback(std::function<void()> Callback) {
+  CloseCallback = std::move(Callback);
+}
+
+void Close() {
+  CloseCallback();
+}
+
+void RegisterNewStateCallback(std::function<void()> Callback) {
+  NewStateCallback = std::move(Callback);
+}
+
+void RegisterGetIRCallback(std::function<void(std::stringstream *out, uint64_t PC)> Callback) {
+  GetIRCallback = std::move(Callback);
+}
+
+void GetIR(std::stringstream *out, uint64_t PC) {
+  GetIRCallback(out, PC);
+}
+
+void CallNewState() {
+  NewStateCallback();
+}
+
+bool HasNewState() {
+  return NewState;
+}
+
+void SetHasNewState(bool State) {
+  NewState = State;
+}
+
+}

Source/Tools/Debugger/DebuggerState.h

@@ -0,0 +1,49 @@
+#pragma once
+#include <FEXCore/Config/Config.h>
+#include <FEXCore/Core/Context.h>
+#include <FEXCore/Core/CoreState.h>
+#include <sstream>
+
+namespace FEX::DebuggerState {
+bool ActiveCore();
+
+void SetContext(FEXCore::Context::Context *ctx);
+FEXCore::Context::Context *GetContext();
+
+FEXCore::Config::ConfigCore GetCoreType();
+void SetCoreType(FEXCore::Config::ConfigCore CoreType);
+
+int GetRunningMode(); ///< This is the running mode we've set
+int GetCoreCurrentRunningMode(); ///< This typically matches `GetRunningMode()` but can differ when we are in the middle of stepping
+void SetRunningMode(int RunningMode);
+
+FEXCore::Core::CPUState GetCPUState();
+
+bool IsCoreRunning();
+
+// Client interface
+void RegisterStepCallback(std::function<void()> Callback);
+void RegisterPauseCallback(std::function<void()> Callback);
+void RegisterContinueCallback(std::function<void()> Callback);
+void Step();
+void Pause();
+void Continue();
+
+void RegisterCreateCallback(std::function<void(char const *Filename, bool ELF)> Callback);
+void Create(char const *Filename, bool ELF);
+
+void RegisterCompileRIPCallback(std::function<void(uint64_t)> Callback);
+void CompileRIP(uint64_t RIP);
+
+void RegisterCloseCallback(std::function<void()> Callback);
+void Close();
+
+void RegisterNewStateCallback(std::function<void()> Callback);
+void CallNewState();
+
+void RegisterGetIRCallback(std::function<void(std::stringstream *out, uint64_t PC)>);
+void GetIR(std::stringstream *out, uint64_t PC);
+
+bool HasNewState();
+void SetHasNewState(bool State = true);
+}

Source/Tools/Debugger/Disassembler.cpp

@@ -0,0 +1,94 @@
+#include "Disassembler.h"
+#include <llvm-c/Disassembler.h>
+#include <llvm-c/Target.h>
+#include <memory>
+#include <sstream>
+#include <iomanip>
+
+namespace FEX::Debugger {
+
+class LLVMDisassembler final : public Disassembler {
+public:
+  ~LLVMDisassembler() override {
+    LLVMDisasmDispose(LLVMContext);
+  }
+  explicit LLVMDisassembler(const char *Arch);
+  std::string Disassemble(uint8_t *Code, uint32_t CodeSize, uint32_t MaxInst, uint64_t StartingPC, uint32_t *InstructionCount) override;
+
+private:
+  LLVMDisasmContextRef LLVMContext;
+
+};
+
+LLVMDisassembler::LLVMDisassembler(const char *Arch) {
+  LLVMInitializeAllTargetInfos();
+  LLVMInitializeAllTargetMCs();
+  LLVMInitializeAllDisassemblers();
+
+  LLVMContext = LLVMCreateDisasmCPU(Arch, "", nullptr, 0, nullptr, nullptr);
+
+  if (!LLVMContext)
+    return;
+
+  LLVMSetDisasmOptions(LLVMContext, LLVMDisassembler_Option_AsmPrinterVariant |
+                                           LLVMDisassembler_Option_PrintLatency);
+}
+
+std::string LLVMDisassembler::Disassemble(uint8_t *Code, uint32_t CodeSize, uint32_t MaxInst, uint64_t StartingPC, uint32_t *InstructionCount) {
+  std::ostringstream Output;
+
+  uint8_t *CurrentRIPAddr = Code;
+  uint8_t *EndRIPAddr = Code + CodeSize;
+  uint64_t CurrentRIP = StartingPC;
+  uint32_t NumberOfInstructions = 0;
+
+  while (CurrentRIPAddr <= EndRIPAddr) {
+    char OutputText[128];
+    size_t InstSize = LLVMDisasmInstruction(LLVMContext,
+        CurrentRIPAddr,
+        static_cast<uint64_t>(EndRIPAddr - CurrentRIPAddr),
+        CurrentRIP,
+        OutputText,
+        128);
+
+    Output << "0x" << std::hex << CurrentRIP << ": ";
+    if (!InstSize) {
+      Output <<  "<Invalid Inst>" << std::endl;
+      break;
+    }
+    else {
+      // Print instruction hex encodings if we want
+      if (!true) {
+        for (size_t i = 0; i < InstSize; ++i) {
+          Output << std::setw(2) << std::setfill('0') << static_cast<uint32_t>(CurrentRIPAddr[i]);
+          if ((i + 1) == InstSize) {
+            Output << ": ";
+          }
+          else {
+            Output << " ";
+          }
+        }
+      }
+      Output << OutputText << std::endl;
+    }
+
+    CurrentRIP += InstSize;
+    CurrentRIPAddr += InstSize;
+    NumberOfInstructions++;
+    if (NumberOfInstructions >= MaxInst) {
+      break;
+    }
+  }
+
+  *InstructionCount = NumberOfInstructions;
+  return Output.str();
+}
+
+std::unique_ptr<Disassembler> CreateHostDisassembler() {
+  return std::make_unique<LLVMDisassembler>("x86_64-none-unknown");
+}
+std::unique_ptr<Disassembler> CreateGuestDisassembler() {
+  return std::make_unique<LLVMDisassembler>("x86_64-none-unknown");
+}
+
+}

Source/Tools/Debugger/Disassembler.h

@@ -0,0 +1,15 @@
+#pragma once
+#include <memory>
+
+namespace FEX::Debugger {
+
+class Disassembler {
+public:
+  virtual ~Disassembler() {}
+  virtual std::string Disassemble(uint8_t *Code, uint32_t CodeSize, uint32_t MaxInst, uint64_t StartingPC, uint32_t *InstructionCount) = 0;
+};
+
+std::unique_ptr<Disassembler> CreateHostDisassembler();
+std::unique_ptr<Disassembler> CreateGuestDisassembler();
+
+}

Source/Tools/Debugger/FEXImGui.cpp

@@ -0,0 +1,153 @@
+#include "FEXImGui.h"
+#include <imgui.h>
+#ifndef IMGUI_DEFINE_MATH_OPERATORS
+#define IMGUI_DEFINE_MATH_OPERATORS
+#endif
+#include <imgui_internal.h>
+#include <vector>
+
+namespace FEXImGui {
+using namespace ImGui;
+
+// FIXME: In principle this function should be called EndListBox(). We should rename it after re-evaluating if we want to keep the same signature.
+bool ListBoxHeader(const char* label, int items_count)
+{
+    // Size default to hold ~7.25 items.
+    // We add +25% worth of item height to allow the user to see at a glance if there are more items up/down, without looking at the scrollbar.
+    // We don't add this extra bit if items_count <= height_in_items. It is slightly dodgy, because it means a dynamic list of items will make the widget resize occasionally when it crosses that size.
+    // I am expecting that someone will come and complain about this behavior in a remote future, then we can advise on a better solution.
+    const ImGuiStyle& style = GetStyle();
+
+    // We include ItemSpacing.y so that a list sized for the exact number of items doesn't make a scrollbar appears. We could also enforce that by passing a flag to BeginChild().
+    ImVec2 size;
+    size.x = 0.0f;
+
+    auto Window = GetCurrentWindowRead();
+    ImVec2 Base = Window->Rect().Min;
+    ImVec2 Size = Window->Rect().Max;
+    ImVec2 Height = Size - Base;
+    size.y = Height.y - style.FramePadding.y * 2.0f;
+
+    return ImGui::ListBoxHeader(label, size);
+}
+
+bool ListBox(const char* label, int* current_item, bool (*items_getter)(void*, int, const char**), void* data, int items_count)
+{
+    if (!ListBoxHeader(label, items_count))
+        return false;
+
+    // Assume all items have even height (= 1 line of text). If you need items of different or variable sizes you can create a custom version of ListBox() in your code without using the clipper.
+    ImGuiContext& g = *GImGui;
+    bool value_changed = false;
+    ImGuiListClipper clipper(items_count, GetTextLineHeightWithSpacing()); // We know exactly our line height here so we pass it as a minor optimization, but generally you don't need to.
+    while (clipper.Step())
+        for (int i = clipper.DisplayStart; i < clipper.DisplayEnd; i++)
+        {
+            const bool item_selected = (i == *current_item);
+            const char* item_text;
+            if (!items_getter(data, i, &item_text))
+                item_text = "*Unknown item*";
+
+            PushID(i);
+            if (Selectable(item_text, item_selected))
+            {
+                *current_item = i;
+                value_changed = true;
+            }
+            if (item_selected)
+                SetItemDefaultFocus();
+            PopID();
+        }
+    ListBoxFooter();
+    if (value_changed)
+        MarkItemEdited(g.CurrentWindow->DC.LastItemId);
+
+    return value_changed;
+}
+
+bool CustomIRViewer(char const *buf, size_t buf_size, std::vector<IRLines> *lines) {
+  ImGuiContext& g = *GImGui;
+  const float inner_spacing = g.Style.ItemInnerSpacing.x;
+  ImGui::Columns(2);
+  ImGui::SetColumnWidth(0, 100.0f + inner_spacing * 2.0f);
+  static float TextOffset = 0.0f;
+  if (ImGui::BeginChildFrame(GetID(""), ImVec2(100, -1))) {
+    const std::vector<ImVec4> LineColors = {
+      ImVec4(1, 0, 0, 1),
+      ImVec4(1, 1, 0, 1),
+      ImVec4(0, 0, 0, 1),
+      ImVec4(0, 0, 1, 1),
+      ImVec4(1, 0, 1, 1),
+      ImVec4(0, 1, 1, 1),
+    };
+
+    auto Window = GetCurrentWindowRead();
+    auto DrawList = GetWindowDrawList();
+
+    auto DrawIRLine = [&](size_t Index, float From, float To, ImVec4 Color) {
+      ImVec2 Size = Window->Rect().Max;
+
+      float LineWidth = 3.0;
+      float LineSpacing = LineWidth * 5;
+
+      // Zero indexing, add one
+      Index = Index % LineColors.size();
+      Index++;
+      From -= TextOffset;
+      To -= TextOffset;
+
+      ImVec2 LeftFrom = ImVec2(Size.x - LineSpacing * static_cast<float>(Index), From);
+      ImVec2 RightFrom = ImVec2(Size.x, From);
+      ImVec2 LeftTo = ImVec2(Size.x - LineSpacing * static_cast<float>(Index), To);
+      ImVec2 RightTo = ImVec2(Size.x, To);
+
+      // We want to draw a three lines
+      // One horizontal one from the "FROM" offset
+      // One vertical one down the middle. Offset by Index
+      // Another horizontol to the "TO" offset
+      //
+      // Additionally we want a triangle pointing to the TO location
+
+      // Draw the from line
+      DrawList->AddLine(LeftFrom, RightFrom, GetColorU32(Color), LineWidth);
+
+      // Draw the to line
+      DrawList->AddLine(LeftTo, RightTo, GetColorU32(Color), LineWidth);
+
+      // Draw a small filled rectangle at the target
+      DrawList->AddTriangleFilled(ImVec2(RightTo.x - LineWidth * 2, RightTo.y - LineWidth), ImVec2(RightTo.x - LineWidth * 2, RightTo.y + LineWidth), RightTo, GetColorU32(Color));
+
+      // Draw the vertical line between the two
+      DrawList->AddLine(LeftFrom, LeftTo, GetColorU32(Color), LineWidth);
+    };
+
+    ImVec2 Base = Window->Rect().Min;
+    ImVec2 Size = Window->Rect().Max;
+    DrawList->AddRectFilled(Base, Size, GetColorU32(ImVec4(0.5, 0.5, 0.5, 1.0)));
+
+    float FontSize = Window->CalcFontSize();
+
+    for (size_t i = 0; i < lines->size(); ++i) {
+      float BaseOffset = Base.y + inner_spacing + FontSize / 2.0f;
+      DrawIRLine(i, BaseOffset + static_cast<float>(lines->at(i).From) * FontSize, BaseOffset + static_cast<float>(lines->at(i).To) * FontSize, LineColors[i % LineColors.size()]);
+    }
+  }
+  ImGui::EndChildFrame();
+
+  ImGui::NextColumn();
+  ImGui::InputTextMultiline("##IR", const_cast<char*>(buf), buf_size, ImVec2(-1, -1), ImGuiInputTextFlags_ReadOnly);
+
+  // This is a bit filthy
+  // Pulls the window and if the child is available then we can pull its child and read the scroll amount
+  // This lets the IR CFG lines match up with textbox, albiet a frame behind so it gets a bit of a wiggle
+  auto win = GetCurrentWindow();
+  if (win->DC.ChildWindows.size() == 2) {
+    win = win->DC.ChildWindows[win->DC.ChildWindows.size() - 1];
+    TextOffset = win->Scroll.y;
+  }
+
+  return false;
+}
+
+
+}

Source/Tools/Debugger/FEXImGui.h

@@ -0,0 +1,18 @@
+#pragma once
+#include <imgui.h>
+#include <cstdint>
+#include <vector>
+
+namespace FEXImGui {
+
+// Listbox that fills the child window
+bool ListBox(const char* label, int* current_item, bool (*items_getter)(void*, int, const char**), void* data, int items_count);
+
+struct IRLines {
+  size_t From;
+  size_t To;
+};
+
+bool CustomIRViewer(char const *buf, size_t buf_size, std::vector<IRLines> *lines);
+
+}

Source/Tools/Debugger/GLUtils.cpp

@@ -0,0 +1,5 @@
+#include "GLUtils.h"
+#include <cstdio>
+
+namespace GLUtils {
+}

Source/Tools/Debugger/GLUtils.h

@@ -0,0 +1,7 @@
+#pragma once
+
+#include <epoxy/gl.h>
+#include <imgui.h>
+
+namespace GLUtils {
+}

Source/Tools/Debugger/IMGui.h

@@ -0,0 +1,8 @@
+#include "Context.h"
+
+namespace FEX::Debugger {
+void Init();
+void Shutdown();
+void DrawDebugUI(GLContext::Context *Context);
+
+}

Source/Tools/Debugger/IRLexer.cpp

@@ -0,0 +1,43 @@
+#include <cstring>
+#include <sstream>
+#include "IRLexer.h"
+#include "LogManager.h"
+#include "Common/StringUtil.h"
+
+#include <FEXCore/IR/IR.h>
+
+namespace FEX::Debugger::IR {
+bool Lexer::Lex(char const *IR) {
+//  std::istringstream iss {std::string(IR)};
+//  HadError = false;
+//
+//  [[maybe_unused]] int CurrentLine {};
+//  [[maybe_unused]] int CurrentColumn {};
+//  while (true) {
+//    char Line[256];
+//    char *Str = Line;
+//    std::string LineStr;
+//    std::getline(iss, LineStr);
+//    FEX::StringUtil::trim(LineStr);
+//    strncpy(Line, LineStr.c_str(), 256);
+//
+//    struct {
+//      bool HadDest;
+//      FEXCore::IR::AlignmentType DestLoc;
+//    } IRData;
+//
+//    if (strstr(Str, "%ssa") == nullptr) {
+//      IRData.HadDest = true;
+//      sscanf(Str, "%%ssa%d =", reinterpret_cast<int*>(&IRData.DestLoc));
+//      strtok(Str, " = ");
+//      strtok(nullptr, " = ");
+//    }
+//
+//    CurrentColumn = false;
+//    ++CurrentLine;
+//  }
+//  // Our IR is fairly simple to lex, just spin through it
+//  return HadError;
+  return false;
+}
+}

Source/Tools/Debugger/IRLexer.h

@@ -0,0 +1,11 @@
+#pragma once
+
+namespace FEX::Debugger::IR {
+class Lexer {
+public:
+  bool Lex(char const *IR);
+
+private:
+};
+
+}

Source/Tools/Debugger/Main.cpp

@@ -0,0 +1,207 @@
+#include "Common/ArgumentLoader.h"
+#include "Common/Config.h"
+#include "CommonCore/VMFactory.h"
+#include "Tests/HarnessHelpers.h"
+#include "MainWindow.h"
+#include "Context.h"
+#include "GLUtils.h"
+#include "IMGui.h"
+#include "DebuggerState.h"
+#include "LogManager.h"
+
+#include "Event.h"
+#include <FEXCore/Config/Config.h>
+#include <FEXCore/Core/Context.h>
+#include <FEXCore/Core/CodeLoader.h>
+#include <FEXCore/Debug/ContextDebug.h>
+#include <FEXCore/Memory/SharedMem.h>
+
+#include <imgui.h>
+#include <epoxy/gl.h>
+#include <memory>
+#include <GLFW/glfw3.h>
+#include <thread>
+
+#include "imgui_impl_glfw.h"
+#include "imgui_impl_opengl3.h"
+
+Event SteppingEvent;
+FEXCore::Context::Context *CTX{};
+FEXCore::SHM::SHMObject *SHM{};
+std::atomic_bool ShouldClose = false;
+std::thread CoreThread;
+
+void CoreWorker() {
+  while (!ShouldClose) {
+    SteppingEvent.Wait();
+    if (ShouldClose) {
+      break;
+    }
+    FEXCore::Context::RunLoop(CTX, false);
+    FEX::DebuggerState::SetHasNewState();
+    FEX::DebuggerState::CallNewState();
+  }
+}
+
+void StepCallback() {
+  FEXCore::Config::SetConfig(CTX, FEXCore::Config::CONFIG_SINGLESTEP, 1);
+  SteppingEvent.NotifyAll();
+}
+
+void CreateCoreCallback(char const *Filename, bool ELF) {
+  SHM = FEXCore::SHM::AllocateSHMRegion(1ULL << 36);
+  CTX = FEXCore::Context::CreateNewContext();
+
+  FEXCore::Context::AddGuestMemoryRegion(CTX, SHM);
+
+  FEXCore::Config::SetConfig(CTX, FEXCore::Config::CONFIG_SINGLESTEP, 1);
+  FEXCore::Config::SetConfig(CTX, FEXCore::Config::CONFIG_DEFAULTCORE, FEX::DebuggerState::GetCoreType());
+  FEXCore::Context::SetFallbackCPUBackendFactory(CTX, VMFactory::CPUCreationFactoryFallback);
+
+  FEXCore::Context::InitializeContext(CTX);
+
+  bool Result{};
+  if (ELF) {
+    FEX::HarnessHelper::ELFCodeLoader Loader{Filename, {}};
+    Result = FEXCore::Context::InitCore(CTX, &Loader);
+  }
+  else {
+    std::string ConfigName = Filename;
+    ConfigName.erase(ConfigName.end() - 3, ConfigName.end());
+    ConfigName += "config.bin";
+    LogMan::Msg::I("Opening '%s'", Filename);
+    LogMan::Msg::I("Opening '%s'", ConfigName.c_str());
+    FEX::HarnessHelper::HarnessCodeLoader Loader{Filename, ConfigName.c_str()};
+    Result = FEXCore::Context::InitCore(CTX, &Loader);
+  }
+
+  LogMan::Throw::A(Result, "Couldn't initialize CTX");
+
+  FEX::DebuggerState::SetContext(CTX);
+  FEX::DebuggerState::SetHasNewState();
+  FEX::DebuggerState::CallNewState();
+  CoreThread = std::thread(CoreWorker);
+}
+
+void CompileRIPCallback(uint64_t RIP) {
+  FEXCore::Context::Debug::CompileRIP(CTX, RIP);
+  FEX::DebuggerState::SetHasNewState();
+  FEX::DebuggerState::CallNewState();
+}
+
+void CloseCallback() {
+  ShouldClose = true;
+  SteppingEvent.NotifyAll();
+  if (CoreThread.joinable()) {
+    CoreThread.join();
+  }
+
+  if (SHM) {
+    FEXCore::SHM::DestroyRegion(SHM);
+  }
+
+  if (CTX) {
+    FEXCore::Context::DestroyContext(CTX);
+  }
+
+  SHM = nullptr;
+  CTX = nullptr;
+
+  ShouldClose = false;
+  FEX::DebuggerState::SetContext(CTX);
+  FEX::DebuggerState::SetHasNewState();
+  FEX::DebuggerState::CallNewState();
+}
+
+void PauseCallback() {
+  FEXCore::Context::Pause(CTX);
+  FEX::DebuggerState::SetHasNewState();
+  FEX::DebuggerState::CallNewState();
+}
+
+void ContinueCallback() {
+  FEXCore::Config::SetConfig(CTX, FEXCore::Config::CONFIG_SINGLESTEP, FEX::DebuggerState::GetRunningMode());
+  SteppingEvent.NotifyAll();
+}
+
+void GetIRCallback(std::stringstream *out, uint64_t PC) {
+  if (!CTX) {
+    *out << "<No Core>";
+    return;
+  }
+  // FEXCore::IR::IntrusiveIRList *ir;
+  // bool HadIR = FEXCore::Context::Debug::FindIRForRIP(FEX::DebuggerState::GetContext(), PC, &ir);
+  // if (HadIR) {
+  //   FEXCore::IR::Dump(out, ir);
+  // }
+  // else {
+  //   *out << "<No IR Found>";
+  // }
+}
+
+int main(int argc, char **argv) {
+  FEX::Config::Init();
+  FEX::ArgLoader::Load(argc, argv);
+
+  FEXCore::Context::InitializeStaticTables();
+
+  auto Context = GLContext::CreateContext();
+  Context->Create("FEX Debugger");
+  FEX::Debugger::Init();
+  FEX::DebuggerState::RegisterStepCallback(StepCallback);
+  FEX::DebuggerState::RegisterPauseCallback(PauseCallback);
+  FEX::DebuggerState::RegisterContinueCallback(ContinueCallback);
+  FEX::DebuggerState::RegisterCreateCallback(CreateCoreCallback);
+  FEX::DebuggerState::RegisterCompileRIPCallback(CompileRIPCallback);
+
+  FEX::DebuggerState::RegisterCloseCallback(CloseCallback);
+  FEX::DebuggerState::RegisterGetIRCallback(GetIRCallback);
+
+  // Setup Dear ImGui context
+  IMGUI_CHECKVERSION();
+  ImGui::CreateContext();
+  ImGuiIO& io = ImGui::GetIO();
+  io.ConfigFlags |= ImGuiConfigFlags_NavEnableKeyboard;       // Enable Keyboard Controls
+  io.ConfigFlags |= ImGuiConfigFlags_DockingEnable;           // Enable Docking
+  io.ConfigFlags |= ImGuiConfigFlags_ViewportsEnable;         // Enable Multi-Viewport / Platform Windows
+
+  GLFWwindow *Window = static_cast<GLFWwindow*>(Context->GetWindow());
+  // Setup Platform/Renderer bindings
+  ImGui_ImplGlfw_InitForOpenGL(Window, true);
+  const char* glsl_version = "#version 410";
+  ImGui_ImplOpenGL3_Init(glsl_version);
+
+  while (!glfwWindowShouldClose(Window)) {
+    glfwPollEvents();
+
+    // Start the Dear ImGui frame
+    ImGui_ImplOpenGL3_NewFrame();
+    ImGui_ImplGlfw_NewFrame();
+    FEX::Debugger::DrawDebugUI(Context.get());
+
+    glClear(GL_COLOR_BUFFER_BIT);
+    ImGui_ImplOpenGL3_RenderDrawData(ImGui::GetDrawData());
+
+    if (io.ConfigFlags & ImGuiConfigFlags_ViewportsEnable)
+    {
+        GLFWwindow* backup_current_context = glfwGetCurrentContext();
+        ImGui::UpdatePlatformWindows();
+        ImGui::RenderPlatformWindowsDefault();
+        glfwMakeContextCurrent(backup_current_context);
+    }
+
+    Context->Swap();
+  }
+
+  FEX::Debugger::Shutdown();
+
+  // Cleanup
+  ImGui_ImplOpenGL3_Shutdown();
+  ImGui_ImplGlfw_Shutdown();
+  ImGui::DestroyContext();
+  Context->Shutdown();
+
+  FEX::Config::Shutdown();
+
+  return 0;
+}

Source/Tools/Debugger/MainWindow.cpp

@@ -0,0 +1 @@
+#include "MainWindow.h"

Source/Tools/Debugger/MainWindow.h

@@ -0,0 +1,3 @@
+#pragma once
+#include <memory>
+

Source/Tools/Debugger/Util/DataRingBuffer.h

@@ -0,0 +1,55 @@
+#pragma once
+#include <cstddef>
+#include <sys/types.h>
+#include <vector>
+
+namespace FEX::Debugger::Util {
+  template<typename T>
+  class DataRingBuffer {
+    public:
+    DataRingBuffer(size_t Size)
+      : RingSize {Size}
+      , BufferSize {RingSize * SIZE_MULTIPLE} {
+      Data.resize(BufferSize);
+    }
+
+    T const* operator()() const {
+      return &Data.at(ReadOffset);
+    }
+
+    T back() const {
+      return Data.at(WriteOffset - 1);
+    }
+
+    bool empty() const {
+      return WriteOffset == 0;
+    }
+
+    size_t size() const {
+      return WriteOffset - ReadOffset;
+    }
+
+    void push_back(T Val) {
+      if (WriteOffset + 1 >= BufferSize) {
+        // We reached the end of the buffer size. Time to wrap around
+        // Copy the ring buffer expected size to the start
+        memcpy(&Data.at(0), &Data.at(ReadOffset), sizeof(T) * RingSize);
+        WriteOffset = RingSize;
+        ReadOffset = 0;
+      }
+      Data.at(WriteOffset) = Val;
+      ++WriteOffset;
+      if (WriteOffset - ReadOffset > RingSize) {
+        ++ReadOffset;
+      }
+    }
+
+  private:
+    constexpr static ssize_t SIZE_MULTIPLE = 3;
+    size_t RingSize;
+    size_t WriteOffset{};
+    size_t ReadOffset{};
+    size_t BufferSize;
+    std::vector<T> Data;
+  };
+}

Source/Tools/Opt.cpp

@@ -0,0 +1,11 @@
+#include "Common/ArgumentLoader.h"
+#include "Common/Config.h"
+
+#include <FEXCore/IR/IntrusiveIRList.h>
+
+int main(int argc, char **argv) {
+  FEX::Config::Init();
+  FEX::ArgLoader::Load(argc, argv);
+
+  FEX::Config::Shutdown();
+}

docs/Debugger.md

@@ -0,0 +1,30 @@
+# FEX - Debugger GUI
+---
+FEX has a debugger GUI for debugging the translation of x86 code.
+This UI is written in [Dear ImGui](https://github.com/ocornut/imgui) since it is purely for debugging and not user facing.
+
+## Features
+* Stopping code execution at any point
+* Single stepping code
+* ELF code loading
+* Test Harness code loading
+* CPU State inspection
+* IR inspection
+* CF inspection of IR
+* Application stderr, stdout viewing
+* Disassembler for viewing both guest and host code
+
+## Nice to have
+* IR writing and direct compiling
+  * Requires writing an IR lexer
+* Profiling of compiled blocks directly in debugger
+  * Save CPU state prior to running, time compiled block for microprofiling
+* Save all state to file to allow offline inspection and profiling
+  * Save original code
+  * Save CPU state on entry
+  * Save IR
+  * Save compiled host code
+  * Profiling data
+  * Allow comparison of all state before(What was saved to file) and after (New version compiled in this version of FEX for iterative debugging)
+* Single stepping of IR state and inspection of SSA values
+* Complete disconnection of debugger from FEXCore to ensure robustness if core crashes.

docs/VMCore.md

@@ -0,0 +1,17 @@
+# FEX - Virtual Machine Custom CPUBackend
+---
+This is a custom CPU backend that is used by the FEX frontend for its Lockstep Runner.
+This is used for hardware validation of CPU emulation of the FEXCore's emulation.
+
+## Implementation details
+This is implemented using the VM helper library that is living inside of [SonicUtils](https://github.com/Sonicadvance1/SonicUtils).
+The helper library sets up a VM using KVM under Linux. 
+
+## Limitations
+* Only works under KVM
+  * I don't care about running under Windows, MacOS, or other hypervisors
+* Only works under AMD
+  * Haven't spent time figuring out why it breaks, Intel throws a VMX invalid state entry error.
+* Can't launch a bunch of instances due to memory limitations
+  * Each VM allocates the full VM's memory region upon initialization
+  * If you launch a VM with 64GB of memory then that full amount is allocated right away

unittests/ASM/CALL.asm

@@ -0,0 +1,36 @@
+%ifdef CONFIG
+{
+  "Ignore": [],
+  "RegData": {
+    "RAX": "1",
+    "RDX": "2"
+  },
+  "MemoryRegions": {
+    "0x100000000": "4096"
+  }
+}
+%endif
+
+jmp label
+label:
+
+mov rsp, 0xe8000000
+
+; Test direct literal call
+call function
+
+; Move the absolute address of function2 in to rbx and call it
+mov rbx, function2+1
+call rbx
+
+hlt
+
+function:
+mov rax, 1
+ret
+
+function2:
+mov rbx, 2
+ret
+
+hlt

unittests/ASM/CMakeLists.txt

@@ -0,0 +1,45 @@
+
+# Careful. Globbing can't see changes to the contents of files
+# Need to do a fresh clean to see changes
+file(GLOB ASM_SOURCES CONFIGURE_DEPENDS *.asm)
+
+set(ASM_DEPENDS "")
+foreach(ASM_SRC ${ASM_SOURCES})
+  get_filename_component(ASM_NAME ${ASM_SRC} NAME)
+
+  # Generate a temporary file
+ set(ASM_TMP "${ASM_NAME}_TMP.asm")
+ set(TMP_FILE "${CMAKE_CURRENT_BINARY_DIR}/${ASM_TMP}")
+  add_custom_command(OUTPUT ${TMP_FILE}
+    DEPENDS "${ASM_SRC}"
+    COMMAND "cp" ARGS "${ASM_SRC}" "${TMP_FILE}"
+    COMMAND "sed" ARGS "-i" "-e" "\'1s;^;BITS 64\\n;\'" "-e" "\'\$\$a\\ret\\n\'" "${TMP_FILE}"
+    )
+
+  set(OUTPUT_NAME "${ASM_NAME}.bin")
+  set(OUTPUT_CONFIG_NAME "${ASM_NAME}.config.bin")
+
+  add_custom_command(OUTPUT ${OUTPUT_NAME}
+    DEPENDS "${TMP_FILE}"
+    COMMAND "nasm" ARGS "${TMP_FILE}" "-o" "${CMAKE_CURRENT_BINARY_DIR}/${OUTPUT_NAME}")
+
+  add_custom_command(OUTPUT ${OUTPUT_CONFIG_NAME}
+    DEPENDS "${ASM_SRC}"
+    COMMAND "python3" ARGS "${CMAKE_SOURCE_DIR}/Scripts/json_asm_parse.py" "${ASM_SRC}" "${CMAKE_CURRENT_BINARY_DIR}/${OUTPUT_CONFIG_NAME}")
+
+  list(APPEND ASM_DEPENDS "${OUTPUT_NAME};${OUTPUT_CONFIG_NAME}")
+
+  set(TEST_NAME "Test_${ASM_NAME}")
+  add_test(NAME ${TEST_NAME}
+    COMMAND "${CMAKE_BINARY_DIR}/Bin/TestHarness" "${OUTPUT_NAME}" "${OUTPUT_CONFIG_NAME}")
+  # This will cause the ASM tests to fail if it can't find the TestHarness or ASMN files
+  # Prety crap way to work around the fact that tests can't have a build dependency in a different directory
+  # Just make sure to independently run `make all` then `make test`
+  set_property(TEST ${TEST_NAME} APPEND PROPERTY DEPENDS "${CMAKE_BINARY_DIR}/Bin/TestHarness")
+  set_property(TEST ${TEST_NAME} APPEND PROPERTY DEPENDS "${OUTPUT_NAME}")
+  set_property(TEST ${TEST_NAME} APPEND PROPERTY DEPENDS "${OUTPUT_CONFIG_NAME}")
+
+endforeach()
+
+add_custom_target(asm_files ALL
+  DEPENDS "${ASM_DEPENDS}")

unittests/ASM/JMP.asm

@@ -0,0 +1,55 @@
+%ifdef CONFIG
+{
+  "Ignore": [],
+  "RegData": {
+    "RAX": "1",
+    "RBX": "2",
+    "RCX": "3",
+    "RDX": "4"
+  },
+  "MemoryRegions": {
+    "0x100000000": "4096"
+  }
+}
+%endif
+
+jmp label
+label:
+
+mov rsp, 0xe8000000
+
+jmp function
+func_return:
+
+mov rbx, function2+1
+jmp rbx
+func2_return:
+
+cmp rcx, rcx
+je function3
+func3_return:
+
+mov rdx, 4
+jne function4
+func4_return:
+
+hlt
+
+function:
+mov rax, 1
+jmp func_return
+
+function2:
+mov rbx, 2
+jmp func2_return
+
+function3:
+mov rcx, 3
+jmp func3_return
+
+function4:
+mov rdx, 0xDEADBEEF
+jmp func4_return
+
+hlt
+

unittests/ASM/STOS.asm

@@ -0,0 +1,61 @@
+%ifdef CONFIG
+{
+  "Match": "All",
+  "MemoryRegions": {
+    "0x100000000": "4096"
+  }
+}
+%endif
+
+; Data we want to store
+mov rax, 0xDEADBEEFBAD0DAD1
+
+; Starting address to store to
+mov rdi, 0xe8000000
+
+; How many elements we want to store
+mov rcx, 0x0
+
+; Direction to increment (Increment when cleared)
+cld
+
+; Store bytes
+rep stosw
+
+mov r11, 0
+mov r10, 0xe8000000
+
+movzx r12, word [r10 + 0]
+add r11, r12
+movzx r12, word [r10 + 1]
+add r11, r12
+movzx r12, word [r10 + 2]
+add r11, r12
+movzx r12, word [r10 + 3]
+add r11, r12
+movzx r12, word [r10 + 4]
+add r11, r12
+movzx r12, word [r10 + 5]
+add r11, r12
+movzx r12, word [r10 + 6]
+add r11, r12
+movzx r12, word [r10 + 7]
+add r11, r12
+movzx r12, word [r10 + 8]
+add r11, r12
+movzx r12, word [r10 + 9]
+add r11, r12
+movzx r12, word [r10 + 10]
+add r11, r12
+movzx r12, word [r10 + 11]
+add r11, r12
+movzx r12, word [r10 + 12]
+add r11, r12
+movzx r12, word [r10 + 13]
+add r11, r12
+movzx r12, word [r10 + 14]
+add r11, r12
+movzx r12, word [r10 + 15]
+add r11, r12
+
+hlt

unittests/ASM/jump.asm

@@ -0,0 +1,35 @@
+%ifdef CONFIG
+{
+  "Match": "All",
+  "RegData": {
+    "RAX": "0x1"
+  }
+}
+%endif
+
+mov esi, 50
+
+.jump_start:
+mov edi, 1
+test edi, edi
+nop
+nop
+nop
+nop
+nop
+nop
+nop
+nop
+
+jz .local
+mov eax, 1
+jmp .end
+
+.local:
+mov eax, 0
+
+.end:
+sub esi, 1
+test esi, esi
+jz .jump_start
+hlt

unittests/ASM/lea.asm

@@ -0,0 +1,16 @@
+%ifdef CONFIG
+{
+  "Match": "All",
+  "RegData": {
+    "RAX": "0x1BD5B7DDE",
+    "RBX": "0x0DEADBF18"
+  }
+}
+%endif
+
+mov r15, 0xDEADBEEF
+mov r14, 0x5
+
+lea rax, [r15*2]
+lea rbx, [r15+r14*8 + 1]
+

unittests/ASM/mov.asm

@@ -0,0 +1,36 @@
+%ifdef CONFIG
+{
+  "Match": "All",
+  "RegData": {
+    "RAX": "0xFFFFFFFFFFFFFFD1",
+    "RBX": "0xFFFFFFFFFFFFDAD1",
+    "RCX": "0xBAD0DAD1",
+    "RDX": "0xDEADBEEFBAD0DAD1",
+    "R15": "0xDEADBEEFBAD0DAD1"
+  }
+}
+%endif
+
+mov rax, -1
+mov rbx, -1
+mov rcx, -1
+
+mov r15, qword 0xDEADBEEFBAD0DAD1
+mov rdx, qword 0xDEADBEEFBAD0DAD1
+
+;mov al, dl
+;mov bx, dx
+;mov ecx, edx
+;mov al, -1
+;mov ax, -1
+;mov eax, -1
+;mov rax, qword -1
+;mov rax, 0
+;mov al, al
+;mov rbx, -1
+;mov bx, ax
+;mov ax, ax
+;mov ax, ax
+;mov eax, eax
+;mov rax, rax
+hlt

unittests/ASM/movups.asm

@@ -0,0 +1,66 @@
+%ifdef CONFIG
+{
+  "Ignore": ["RAX", "RDX"],
+  "RegData": {
+    "RAX" : "0x0000FFFF",
+    "XMM0": ["0x3f800000", "0x40000000"],
+    "XMM1": ["0x3f800000", "0x40000000"],
+    "XMM2": ["0x3f800000", "0x40000000"],
+    "XMM3": ["0x3f800000", "0x8100000080000000"],
+    "XMM4": ["0xDEADBEEFBFD0DAD1", "0x4141414142424242"],
+    "XMM5": ["0xDEADBEEFBAD0DAD1", "0"],
+    "XMM6": ["0xDEADBEEFBFD0DAD1", "0"]
+  },
+  "MemoryRegions": {
+    "0x100000000": "4096"
+  }
+}
+%endif
+
+jmp label
+label:
+mov rax, 0x3f800000
+;mov rsi, 0xdeadbeefbaddad1
+;rdseed eax
+;vpermd ymm0, ymm1, ymm2
+mov rdx, 0xe0000000
+mov [rdx], eax
+mov eax, 0x40000000
+mov [rdx + 8], eax
+
+movups xmm0, [rdx]
+movups xmm1, xmm0
+
+movups [rdx + 16], xmm1
+movups xmm2, [rdx + 16]
+
+; Upper moves
+mov eax, 0x80000000
+
+mov [rdx + 32], eax
+mov eax, 0x81000000
+mov [rdx + 36], eax
+
+movups xmm3, xmm0
+movhps xmm3, [rdx + 32]
+
+mov rax, 0xDEADBEEFBAD0DAD1
+mov [rdx + 32], rax
+mov rax, 0x4141414142424242
+mov [rdx + 40], rax
+movups xmm4, [rdx + 32]
+movq xmm5, [rdx + 32]
+
+por xmm4, xmm0
+movq xmm6, xmm4
+paddq xmm7, xmm6
+mov rax, 0xFFFFFFFFFFFFFFFF
+mov [rdx + 32], rax
+mov [rdx + 40], rax
+movdqu xmm8, [rdx + 32]
+pmovmskb eax, xmm8
+
+;fcomp dword [0]
+;fldl2t
+;fld1
+hlt

unittests/ASM/movzx.asm

@@ -0,0 +1,23 @@
+%ifdef CONFIG
+{
+  "Match": "All",
+  "RegData": {
+    "RBX": "0xFFFFFFFFFFFF00D1",
+    "RCX": "0x00000000000000D1",
+    "RDX": "0xDAD1",
+    "RDI": "0xDAD1"
+  }
+}
+%endif
+
+mov rax, qword 0xDEADBEEFBAD0DAD1
+
+mov rbx, -1
+mov rcx, -1
+mov rdx, -1
+mov rdi, -1
+
+movzx bx,  al ; 8bit-> 16bit
+movzx ecx, al ; 8bit-> 32bit
+movzx edx, ax ; 16bit-> 32bit
+movzx rdi, ax ; 16bit -> 64bit

unittests/ASM/mul.asm

@@ -0,0 +1,17 @@
+%ifdef CONFIG
+{
+  "Match": "All",
+  "RegData": {
+    "RAX": "0xFFFFFFFFFFFFFFFE",
+    "RDX": "1",
+    "RBX": "1"
+  }
+}
+%endif
+mov rax, -1
+mov rdx, 0
+mov rbx, 1
+
+mul rbx
+
+hlt

unittests/ASM/simple_loop.asm

@@ -0,0 +1,42 @@
+%ifdef CONFIG
+{
+  "Ignore": [],
+  "RegData": {
+    "RAX": "1",
+    "RDX": "2"
+  },
+  "MemoryRegions": {
+    "0x100000000": "4096"
+  }
+}
+%endif
+
+mov rsp, 0xe8000000
+mov rdi, 0xe0000000
+call .simple_loop
+
+
+hlt
+
+.simple_loop:
+pxor xmm0, xmm0
+pxor xmm1, xmm1
+mov rax, 0xffffffffffffd8f0
+
+.loop_top:
+movdqu xmm2, [rdi + rax * 4 + 0x9c40]
+paddd xmm0, xmm2
+movdqu xmm2, [rdi + rax * 4 + 0x9c50]
+paddd xmm1, xmm2
+
+add rax, 8
+jne .loop_top
+
+paddd xmm1, xmm0
+pshufd xmm0, xmm1, 0x4e
+paddd xmm0, xmm1
+pshufd xmm1, xmm0, 0xe5
+paddd xmm1, xmm0
+movd eax, xmm1
+
+retq

unittests/CMakeLists.txt

@@ -0,0 +1 @@
+add_subdirectory(ASM/)

unittests/Example.asm

@@ -0,0 +1,64 @@
+; If you want a specific configuration at the top of asm file then make sure to wrap it in ifdef and endif.
+; This allows the python script to extract the json and nasm to ignore the section
+;
+; X86 State option that can be compared
+; - All: Makes sure all options are compared
+; - None: No options
+; ===== Specific options ====
+; -- GPRs --
+; RAX, RBX, RCX, RDX
+; RSI, RDI, RBP, RSP
+; R8-R15
+; -- XMM --
+; XMM0-XX15
+; -- Misc --
+; RIP
+; FS, GS
+; Flags
+; ===========================
+; Match: Forces full matching of types
+;   - Type: String or List of strings
+;   - Default: All
+; Ignore: Forces types to be ignored when matching. Overwrites Matches
+;   - Default: None
+;   - Type: String or List of strings
+; RegData: Makes sure that a register contains specific data
+;   - Default: Any data
+;   - Type: Dict of key:value pairs
+;   - >64bit registers should contain a list of values for each 64bit value
+;
+; Additional config options
+; ABI : {SystemV-64, Win64, None}
+;   - Default: SystemV-64
+; StackSize : Stack size that the test needs
+;   - Default : 4096
+;   - Stack address starts at: [0xc000'0000, 0xc000'0000 + StackSize)
+; EntryPoint : Entrypoint for the assembly
+;   - Default: 1
+;   - 0 is invalid since that is special cased
+; MemoryRegions: Memory Regions for the tests to use
+;   - Default: No memory regions generated
+;   - Dict of key:value pairs
+;   - Key indicates the memory base
+;   - Value indicates the memory region size
+;   - WARNING: Emulator sets up some default regions that you don't want to intersect with
+;   - Additionally the VM only has 64GB of virtual memory. If you go past this sizer, expect failure
+;   - 0xb000'0000 - FS Memory base
+;   - 0xc000'0000 - Stack pointer base
+;   - 0xd000'0000 - Linux BRK memory base
+
+%ifdef CONFIG
+{
+  "Match": "All",
+  "Ignore": ["XMM0", "Flags"],
+  "RegData": {
+    "RAX": "1"
+  },
+  "MemoryRegions": {
+    "0x100000000": "4096"
+  }
+}
+%endif
+
+mov eax, 1
+ret