Capture a 64-bit process trying to jump to 32-bit syscall handler

Fixes #591

Adds a simple unittest

FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp

@@ -5218,6 +5218,9 @@ void OpDispatchBuilder::INTOp(OpcodeArgs) {
     constexpr uint8_t SYSCALL_LITERAL = 0x2E;
 #endif
     if (Literal == SYSCALL_LITERAL) {
+      if (CTX->Config.Is64BitMode()) [[unlikely]] {
+        ERROR_AND_DIE_FMT("[Unsupported] Trying to execute 32-bit syscall from a 64-bit process.");
+      }
       // Syscall on linux
       SyscallOp<false>(Op);
       return;

unittests/ASM/FEX_bugs/32bit_syscall.asm

@@ -0,0 +1,32 @@
+%ifdef CONFIG
+{
+  "RegData": {
+    "RAX": "0"
+  }
+}
+%endif
+
+; 32-bit:
+; 265 = clock_gettime
+; 64-bit
+; 265 = linkat
+
+; rax = syscall on both 32-bit and 64-bit
+mov rax, 265
+
+; rdi/rbx = first argument on 64-bit and 32-bit respectively
+mov rdi, 0
+mov rbx, 0
+
+; rsi/rcx = second argument on 64-bit and 32-bit respectively
+lea rsi, [rel .data]
+lea rcx, [rel .data]
+
+; Do a 32-bit syscall
+; On a real linux kernel this will execute clock_gettime
+; Under FEX without 32-bit syscall support this might try to execute linkat and return -ENOENT.
+int 0x80
+hlt
+
+.data:
+dq 0, 0, 0, 0

unittests/ASM/Known_Failures_jit

@@ -1,3 +1,5 @@
 # FPREM is incorrect
 Test_X87/D9_F5_2.asm
 Test_X87/D9_F5_3.asm
+
+Test_FEX_bugs/32bit_syscall.asm