From 64358ac8eab95fab6390db1aeee32b47e33d48e1 Mon Sep 17 00:00:00 2001 From: JordanTheToaster Date: Mon, 7 Sep 2026 21:18:52 +0100 Subject: [PATCH] 3rdparty: Update xbyak to v7.41.1 --- 3rdparty/xbyak/xbyak/xbyak.h | 1519 +++++++++++++++---------- 3rdparty/xbyak/xbyak/xbyak_mnemonic.h | 428 +++---- 3rdparty/xbyak/xbyak/xbyak_util.h | 189 ++- 3 files changed, 1346 insertions(+), 790 deletions(-) diff --git a/3rdparty/xbyak/xbyak/xbyak.h b/3rdparty/xbyak/xbyak/xbyak.h index 127fd41588..f9de8ac23a 100644 --- a/3rdparty/xbyak/xbyak/xbyak.h +++ b/3rdparty/xbyak/xbyak/xbyak.h @@ -151,6 +151,10 @@ #define XBYAK_CONSTEXPR #endif +#ifdef __cpp_inline_variables + #define XBYAK_USE_CONSTEXPR_REGISTERS 1 +#endif + #ifdef _MSC_VER #pragma warning(push) #pragma warning(disable : 4514) /* remove inline function */ @@ -166,17 +170,525 @@ #pragma GCC diagnostic ignored "-Warray-bounds" #endif -// Define this macro as 0 to disable strict checking of memory operand and register size matching. +// Heuristically check general-purpose register and memory operand size matching such as add(dword[eax], al); +// Not exact but useful to find mistakes. Define as 0 to disable. // This macro may be removed in future versions. #ifndef XBYAK_STRICT_CHECK_MEM_REG_SIZE #define XBYAK_STRICT_CHECK_MEM_REG_SIZE 1 #endif +// F is called: F(Type, name, init...) +#define XBYAK_FOR_EACH_REG_REG8(F) \ + F(Reg8, al, Operand::AL) \ + F(Reg8, cl, Operand::CL) \ + F(Reg8, dl, Operand::DL) \ + F(Reg8, bl, Operand::BL) \ + F(Reg8, ah, Operand::AH) \ + F(Reg8, ch, Operand::CH) \ + F(Reg8, dh, Operand::DH) \ + F(Reg8, bh, Operand::BH) \ + /**/ +#define XBYAK_FOR_EACH_REG_REG16(F) \ + F(Reg16, ax, Operand::AX) \ + F(Reg16, cx, Operand::CX) \ + F(Reg16, dx, Operand::DX) \ + F(Reg16, bx, Operand::BX) \ + F(Reg16, sp, Operand::SP) \ + F(Reg16, bp, Operand::BP) \ + F(Reg16, si, Operand::SI) \ + F(Reg16, di, Operand::DI) \ + /**/ +#define XBYAK_FOR_EACH_REG_REG32(F) \ + F(Reg32, eax, Operand::EAX) \ + F(Reg32, ecx, Operand::ECX) \ + F(Reg32, edx, Operand::EDX) \ + F(Reg32, ebx, Operand::EBX) \ + F(Reg32, esp, Operand::ESP) \ + F(Reg32, ebp, Operand::EBP) \ + F(Reg32, esi, Operand::ESI) \ + F(Reg32, edi, Operand::EDI) \ + /**/ +#define XBYAK_FOR_EACH_REG_MMX(F) \ + F(Mmx, mm0, 0) \ + F(Mmx, mm1, 1) \ + F(Mmx, mm2, 2) \ + F(Mmx, mm3, 3) \ + F(Mmx, mm4, 4) \ + F(Mmx, mm5, 5) \ + F(Mmx, mm6, 6) \ + F(Mmx, mm7, 7) \ + /**/ +#define XBYAK_FOR_EACH_REG_XMM(F) \ + F(Xmm, xmm0, 0) \ + F(Xmm, xmm1, 1) \ + F(Xmm, xmm2, 2) \ + F(Xmm, xmm3, 3) \ + F(Xmm, xmm4, 4) \ + F(Xmm, xmm5, 5) \ + F(Xmm, xmm6, 6) \ + F(Xmm, xmm7, 7) \ + /**/ +#define XBYAK_FOR_EACH_REG_YMM(F) \ + F(Ymm, ymm0, 0) \ + F(Ymm, ymm1, 1) \ + F(Ymm, ymm2, 2) \ + F(Ymm, ymm3, 3) \ + F(Ymm, ymm4, 4) \ + F(Ymm, ymm5, 5) \ + F(Ymm, ymm6, 6) \ + F(Ymm, ymm7, 7) \ + /**/ +#define XBYAK_FOR_EACH_REG_ZMM(F) \ + F(Zmm, zmm0, 0) \ + F(Zmm, zmm1, 1) \ + F(Zmm, zmm2, 2) \ + F(Zmm, zmm3, 3) \ + F(Zmm, zmm4, 4) \ + F(Zmm, zmm5, 5) \ + F(Zmm, zmm6, 6) \ + F(Zmm, zmm7, 7) \ + /**/ +// xword is same as oword of NASM +// ptr_b/xword_b/yword_b/zword_b are broadcast such as {1to2}, {1to4}, {1to8}, {1to16}, {b} +#define XBYAK_FOR_EACH_REG_ADDRESS(F) \ + F(AddressFrame, ptr, 0) \ + F(AddressFrame, byte, 8) \ + F(AddressFrame, word, 16) \ + F(AddressFrame, dword, 32) \ + F(AddressFrame, qword, 64) \ + F(AddressFrame, xword, 128) \ + F(AddressFrame, yword, 256) \ + F(AddressFrame, zword, 512) \ + F(AddressFrame, ptr_b, 0, true) \ + F(AddressFrame, xword_b, 128, true) \ + F(AddressFrame, yword_b, 256, true) \ + F(AddressFrame, zword_b, 512, true) \ + /**/ +#define XBYAK_FOR_EACH_REG_FPU(F) \ + F(Fpu, st0, 0) \ + F(Fpu, st1, 1) \ + F(Fpu, st2, 2) \ + F(Fpu, st3, 3) \ + F(Fpu, st4, 4) \ + F(Fpu, st5, 5) \ + F(Fpu, st6, 6) \ + F(Fpu, st7, 7) \ + /**/ +#define XBYAK_FOR_EACH_REG_OPMASK(F) \ + F(Opmask, k0, 0) \ + F(Opmask, k1, 1) \ + F(Opmask, k2, 2) \ + F(Opmask, k3, 3) \ + F(Opmask, k4, 4) \ + F(Opmask, k5, 5) \ + F(Opmask, k6, 6) \ + F(Opmask, k7, 7) \ + /**/ +#define XBYAK_FOR_EACH_REG_BOUNDS(F) \ + F(BoundsReg, bnd0, 0) \ + F(BoundsReg, bnd1, 1) \ + F(BoundsReg, bnd2, 2) \ + F(BoundsReg, bnd3, 3) \ + /**/ +// {sae}, {rn-sae}, {rd-sae}, {ru-sae}, {rz-sae} and {z} +#define XBYAK_FOR_EACH_EVEX_MODIFIER(F) \ + F(EvexModifierRounding, T_sae, EvexModifierRounding::T_SAE) \ + F(EvexModifierRounding, T_rn_sae, EvexModifierRounding::T_RN_SAE) \ + F(EvexModifierRounding, T_rd_sae, EvexModifierRounding::T_RD_SAE) \ + F(EvexModifierRounding, T_ru_sae, EvexModifierRounding::T_RU_SAE) \ + F(EvexModifierRounding, T_rz_sae, EvexModifierRounding::T_RZ_SAE) \ + F(EvexModifierZero, T_z, 0) \ + /**/ + +#ifdef XBYAK64 +#define XBYAK_FOR_EACH_REG_REG8_REX(F) \ + F(Reg8, spl, Operand::SPL, true) \ + F(Reg8, bpl, Operand::BPL, true) \ + F(Reg8, sil, Operand::SIL, true) \ + F(Reg8, dil, Operand::DIL, true) \ + /**/ +#define XBYAK_FOR_EACH_REG_REG8_EXT(F) \ + F(Reg8, r8b, 8) \ + F(Reg8, r9b, 9) \ + F(Reg8, r10b, 10) \ + F(Reg8, r11b, 11) \ + F(Reg8, r12b, 12) \ + F(Reg8, r13b, 13) \ + F(Reg8, r14b, 14) \ + F(Reg8, r15b, 15) \ + F(Reg8, r16b, 16) \ + F(Reg8, r17b, 17) \ + F(Reg8, r18b, 18) \ + F(Reg8, r19b, 19) \ + F(Reg8, r20b, 20) \ + F(Reg8, r21b, 21) \ + F(Reg8, r22b, 22) \ + F(Reg8, r23b, 23) \ + F(Reg8, r24b, 24) \ + F(Reg8, r25b, 25) \ + F(Reg8, r26b, 26) \ + F(Reg8, r27b, 27) \ + F(Reg8, r28b, 28) \ + F(Reg8, r29b, 29) \ + F(Reg8, r30b, 30) \ + F(Reg8, r31b, 31) \ + /**/ +#define XBYAK_FOR_EACH_REG_REG16_EXT(F) \ + F(Reg16, r8w, 8) \ + F(Reg16, r9w, 9) \ + F(Reg16, r10w, 10) \ + F(Reg16, r11w, 11) \ + F(Reg16, r12w, 12) \ + F(Reg16, r13w, 13) \ + F(Reg16, r14w, 14) \ + F(Reg16, r15w, 15) \ + F(Reg16, r16w, 16) \ + F(Reg16, r17w, 17) \ + F(Reg16, r18w, 18) \ + F(Reg16, r19w, 19) \ + F(Reg16, r20w, 20) \ + F(Reg16, r21w, 21) \ + F(Reg16, r22w, 22) \ + F(Reg16, r23w, 23) \ + F(Reg16, r24w, 24) \ + F(Reg16, r25w, 25) \ + F(Reg16, r26w, 26) \ + F(Reg16, r27w, 27) \ + F(Reg16, r28w, 28) \ + F(Reg16, r29w, 29) \ + F(Reg16, r30w, 30) \ + F(Reg16, r31w, 31) \ + /**/ +#define XBYAK_FOR_EACH_REG_REG32_EXT(F) \ + F(Reg32, r8d, 8) \ + F(Reg32, r9d, 9) \ + F(Reg32, r10d, 10) \ + F(Reg32, r11d, 11) \ + F(Reg32, r12d, 12) \ + F(Reg32, r13d, 13) \ + F(Reg32, r14d, 14) \ + F(Reg32, r15d, 15) \ + F(Reg32, r16d, 16) \ + F(Reg32, r17d, 17) \ + F(Reg32, r18d, 18) \ + F(Reg32, r19d, 19) \ + F(Reg32, r20d, 20) \ + F(Reg32, r21d, 21) \ + F(Reg32, r22d, 22) \ + F(Reg32, r23d, 23) \ + F(Reg32, r24d, 24) \ + F(Reg32, r25d, 25) \ + F(Reg32, r26d, 26) \ + F(Reg32, r27d, 27) \ + F(Reg32, r28d, 28) \ + F(Reg32, r29d, 29) \ + F(Reg32, r30d, 30) \ + F(Reg32, r31d, 31) \ + /**/ +#define XBYAK_FOR_EACH_REG_REG64(F) \ + F(Reg64, rax, Operand::RAX) \ + F(Reg64, rcx, Operand::RCX) \ + F(Reg64, rdx, Operand::RDX) \ + F(Reg64, rbx, Operand::RBX) \ + F(Reg64, rsp, Operand::RSP) \ + F(Reg64, rbp, Operand::RBP) \ + F(Reg64, rsi, Operand::RSI) \ + F(Reg64, rdi, Operand::RDI) \ + F(Reg64, r8, Operand::R8) \ + F(Reg64, r9, Operand::R9) \ + F(Reg64, r10, Operand::R10) \ + F(Reg64, r11, Operand::R11) \ + F(Reg64, r12, Operand::R12) \ + F(Reg64, r13, Operand::R13) \ + F(Reg64, r14, Operand::R14) \ + F(Reg64, r15, Operand::R15) \ + F(Reg64, r16, 16) \ + F(Reg64, r17, 17) \ + F(Reg64, r18, 18) \ + F(Reg64, r19, 19) \ + F(Reg64, r20, 20) \ + F(Reg64, r21, 21) \ + F(Reg64, r22, 22) \ + F(Reg64, r23, 23) \ + F(Reg64, r24, 24) \ + F(Reg64, r25, 25) \ + F(Reg64, r26, 26) \ + F(Reg64, r27, 27) \ + F(Reg64, r28, 28) \ + F(Reg64, r29, 29) \ + F(Reg64, r30, 30) \ + F(Reg64, r31, 31) \ + /**/ +#define XBYAK_FOR_EACH_REG_XMM_EXT(F) \ + F(Xmm, xmm8, 8) \ + F(Xmm, xmm9, 9) \ + F(Xmm, xmm10, 10) \ + F(Xmm, xmm11, 11) \ + F(Xmm, xmm12, 12) \ + F(Xmm, xmm13, 13) \ + F(Xmm, xmm14, 14) \ + F(Xmm, xmm15, 15) \ + F(Xmm, xmm16, 16) \ + F(Xmm, xmm17, 17) \ + F(Xmm, xmm18, 18) \ + F(Xmm, xmm19, 19) \ + F(Xmm, xmm20, 20) \ + F(Xmm, xmm21, 21) \ + F(Xmm, xmm22, 22) \ + F(Xmm, xmm23, 23) \ + F(Xmm, xmm24, 24) \ + F(Xmm, xmm25, 25) \ + F(Xmm, xmm26, 26) \ + F(Xmm, xmm27, 27) \ + F(Xmm, xmm28, 28) \ + F(Xmm, xmm29, 29) \ + F(Xmm, xmm30, 30) \ + F(Xmm, xmm31, 31) \ + /**/ +#define XBYAK_FOR_EACH_REG_YMM_EXT(F) \ + F(Ymm, ymm8, 8) \ + F(Ymm, ymm9, 9) \ + F(Ymm, ymm10, 10) \ + F(Ymm, ymm11, 11) \ + F(Ymm, ymm12, 12) \ + F(Ymm, ymm13, 13) \ + F(Ymm, ymm14, 14) \ + F(Ymm, ymm15, 15) \ + F(Ymm, ymm16, 16) \ + F(Ymm, ymm17, 17) \ + F(Ymm, ymm18, 18) \ + F(Ymm, ymm19, 19) \ + F(Ymm, ymm20, 20) \ + F(Ymm, ymm21, 21) \ + F(Ymm, ymm22, 22) \ + F(Ymm, ymm23, 23) \ + F(Ymm, ymm24, 24) \ + F(Ymm, ymm25, 25) \ + F(Ymm, ymm26, 26) \ + F(Ymm, ymm27, 27) \ + F(Ymm, ymm28, 28) \ + F(Ymm, ymm29, 29) \ + F(Ymm, ymm30, 30) \ + F(Ymm, ymm31, 31) \ + /**/ +#define XBYAK_FOR_EACH_REG_ZMM_EXT(F) \ + F(Zmm, zmm8, 8) \ + F(Zmm, zmm9, 9) \ + F(Zmm, zmm10, 10) \ + F(Zmm, zmm11, 11) \ + F(Zmm, zmm12, 12) \ + F(Zmm, zmm13, 13) \ + F(Zmm, zmm14, 14) \ + F(Zmm, zmm15, 15) \ + F(Zmm, zmm16, 16) \ + F(Zmm, zmm17, 17) \ + F(Zmm, zmm18, 18) \ + F(Zmm, zmm19, 19) \ + F(Zmm, zmm20, 20) \ + F(Zmm, zmm21, 21) \ + F(Zmm, zmm22, 22) \ + F(Zmm, zmm23, 23) \ + F(Zmm, zmm24, 24) \ + F(Zmm, zmm25, 25) \ + F(Zmm, zmm26, 26) \ + F(Zmm, zmm27, 27) \ + F(Zmm, zmm28, 28) \ + F(Zmm, zmm29, 29) \ + F(Zmm, zmm30, 30) \ + F(Zmm, zmm31, 31) \ + /**/ +#define XBYAK_FOR_EACH_REG_TMM(F) \ + F(Tmm, tmm0, 0) \ + F(Tmm, tmm1, 1) \ + F(Tmm, tmm2, 2) \ + F(Tmm, tmm3, 3) \ + F(Tmm, tmm4, 4) \ + F(Tmm, tmm5, 5) \ + F(Tmm, tmm6, 6) \ + F(Tmm, tmm7, 7) \ + /**/ +#define XBYAK_FOR_EACH_REG_BSR(F) \ + F(Bsr, bsr0, 0) \ + /**/ +#define XBYAK_FOR_EACH_REG_RIP(F) \ + F(RegRip, rip, 0) \ + /**/ +#else +#define XBYAK_FOR_EACH_REG_REG8_REX(F) +#define XBYAK_FOR_EACH_REG_REG8_EXT(F) +#define XBYAK_FOR_EACH_REG_REG16_EXT(F) +#define XBYAK_FOR_EACH_REG_REG32_EXT(F) +#define XBYAK_FOR_EACH_REG_REG64(F) +#define XBYAK_FOR_EACH_REG_XMM_EXT(F) +#define XBYAK_FOR_EACH_REG_YMM_EXT(F) +#define XBYAK_FOR_EACH_REG_ZMM_EXT(F) +#define XBYAK_FOR_EACH_REG_TMM(F) +#define XBYAK_FOR_EACH_REG_BSR(F) +#define XBYAK_FOR_EACH_REG_RIP(F) +#endif + +// T_nf/T_zu are used by cfcmov etc. even on 32-bit, so they are defined regardless of XBYAK64 +#define XBYAK_FOR_EACH_APX_MODIFIER(F) \ + F(ApxFlagNF, T_nf, 0) \ + F(ApxFlagZU, T_zu, 0) \ + /**/ + +#ifdef XBYAK_DISABLE_SEGMENT +#define XBYAK_FOR_EACH_REG_SEGMENT(F) +#else +#define XBYAK_FOR_EACH_REG_SEGMENT(F) \ + F(Segment, es, Segment::es) \ + F(Segment, cs, Segment::cs) \ + F(Segment, ss, Segment::ss) \ + F(Segment, ds, Segment::ds) \ + F(Segment, fs, Segment::fs) \ + F(Segment, gs, Segment::gs) \ + /**/ +#endif + +#define XBYAK_FOR_EACH_REGISTER(F) \ + XBYAK_FOR_EACH_REG_REG8(F) \ + XBYAK_FOR_EACH_REG_REG16(F) \ + XBYAK_FOR_EACH_REG_REG32(F) \ + XBYAK_FOR_EACH_REG_MMX(F) \ + XBYAK_FOR_EACH_REG_XMM(F) \ + XBYAK_FOR_EACH_REG_YMM(F) \ + XBYAK_FOR_EACH_REG_ZMM(F) \ + XBYAK_FOR_EACH_REG_ADDRESS(F) \ + XBYAK_FOR_EACH_REG_FPU(F) \ + XBYAK_FOR_EACH_REG_OPMASK(F) \ + XBYAK_FOR_EACH_REG_BOUNDS(F) \ + XBYAK_FOR_EACH_EVEX_MODIFIER(F) \ + XBYAK_FOR_EACH_APX_MODIFIER(F) \ + XBYAK_FOR_EACH_REG_REG8_REX(F) \ + XBYAK_FOR_EACH_REG_REG8_EXT(F) \ + XBYAK_FOR_EACH_REG_REG16_EXT(F) \ + XBYAK_FOR_EACH_REG_REG32_EXT(F) \ + XBYAK_FOR_EACH_REG_REG64(F) \ + XBYAK_FOR_EACH_REG_XMM_EXT(F) \ + XBYAK_FOR_EACH_REG_YMM_EXT(F) \ + XBYAK_FOR_EACH_REG_ZMM_EXT(F) \ + XBYAK_FOR_EACH_REG_TMM(F) \ + XBYAK_FOR_EACH_REG_BSR(F) \ + XBYAK_FOR_EACH_REG_RIP(F) \ + XBYAK_FOR_EACH_REG_SEGMENT(F) \ + /**/ + +// xmN/ymN/zmN are aliases of xmmN/ymmN/zmmN (for my convenience) +#define XBYAK_FOR_EACH_REG_CONVENIENCE(F) \ + F(Xmm, xm0, xmm0) \ + F(Xmm, xm1, xmm1) \ + F(Xmm, xm2, xmm2) \ + F(Xmm, xm3, xmm3) \ + F(Xmm, xm4, xmm4) \ + F(Xmm, xm5, xmm5) \ + F(Xmm, xm6, xmm6) \ + F(Xmm, xm7, xmm7) \ + F(Ymm, ym0, ymm0) \ + F(Ymm, ym1, ymm1) \ + F(Ymm, ym2, ymm2) \ + F(Ymm, ym3, ymm3) \ + F(Ymm, ym4, ymm4) \ + F(Ymm, ym5, ymm5) \ + F(Ymm, ym6, ymm6) \ + F(Ymm, ym7, ymm7) \ + F(Zmm, zm0, zmm0) \ + F(Zmm, zm1, zmm1) \ + F(Zmm, zm2, zmm2) \ + F(Zmm, zm3, zmm3) \ + F(Zmm, zm4, zmm4) \ + F(Zmm, zm5, zmm5) \ + F(Zmm, zm6, zmm6) \ + F(Zmm, zm7, zmm7) \ + /**/ +#ifdef XBYAK64 +#define XBYAK_FOR_EACH_REG_CONVENIENCE_EXT(F) \ + F(Xmm, xm8, xmm8) \ + F(Xmm, xm9, xmm9) \ + F(Xmm, xm10, xmm10) \ + F(Xmm, xm11, xmm11) \ + F(Xmm, xm12, xmm12) \ + F(Xmm, xm13, xmm13) \ + F(Xmm, xm14, xmm14) \ + F(Xmm, xm15, xmm15) \ + F(Xmm, xm16, xmm16) \ + F(Xmm, xm17, xmm17) \ + F(Xmm, xm18, xmm18) \ + F(Xmm, xm19, xmm19) \ + F(Xmm, xm20, xmm20) \ + F(Xmm, xm21, xmm21) \ + F(Xmm, xm22, xmm22) \ + F(Xmm, xm23, xmm23) \ + F(Xmm, xm24, xmm24) \ + F(Xmm, xm25, xmm25) \ + F(Xmm, xm26, xmm26) \ + F(Xmm, xm27, xmm27) \ + F(Xmm, xm28, xmm28) \ + F(Xmm, xm29, xmm29) \ + F(Xmm, xm30, xmm30) \ + F(Xmm, xm31, xmm31) \ + F(Ymm, ym8, ymm8) \ + F(Ymm, ym9, ymm9) \ + F(Ymm, ym10, ymm10) \ + F(Ymm, ym11, ymm11) \ + F(Ymm, ym12, ymm12) \ + F(Ymm, ym13, ymm13) \ + F(Ymm, ym14, ymm14) \ + F(Ymm, ym15, ymm15) \ + F(Ymm, ym16, ymm16) \ + F(Ymm, ym17, ymm17) \ + F(Ymm, ym18, ymm18) \ + F(Ymm, ym19, ymm19) \ + F(Ymm, ym20, ymm20) \ + F(Ymm, ym21, ymm21) \ + F(Ymm, ym22, ymm22) \ + F(Ymm, ym23, ymm23) \ + F(Ymm, ym24, ymm24) \ + F(Ymm, ym25, ymm25) \ + F(Ymm, ym26, ymm26) \ + F(Ymm, ym27, ymm27) \ + F(Ymm, ym28, ymm28) \ + F(Ymm, ym29, ymm29) \ + F(Ymm, ym30, ymm30) \ + F(Ymm, ym31, ymm31) \ + F(Zmm, zm8, zmm8) \ + F(Zmm, zm9, zmm9) \ + F(Zmm, zm10, zmm10) \ + F(Zmm, zm11, zmm11) \ + F(Zmm, zm12, zmm12) \ + F(Zmm, zm13, zmm13) \ + F(Zmm, zm14, zmm14) \ + F(Zmm, zm15, zmm15) \ + F(Zmm, zm16, zmm16) \ + F(Zmm, zm17, zmm17) \ + F(Zmm, zm18, zmm18) \ + F(Zmm, zm19, zmm19) \ + F(Zmm, zm20, zmm20) \ + F(Zmm, zm21, zmm21) \ + F(Zmm, zm22, zmm22) \ + F(Zmm, zm23, zmm23) \ + F(Zmm, zm24, zmm24) \ + F(Zmm, zm25, zmm25) \ + F(Zmm, zm26, zmm26) \ + F(Zmm, zm27, zmm27) \ + F(Zmm, zm28, zmm28) \ + F(Zmm, zm29, zmm29) \ + F(Zmm, zm30, zmm30) \ + F(Zmm, zm31, zmm31) \ + /**/ +#else +#define XBYAK_FOR_EACH_REG_CONVENIENCE_EXT(F) +#endif +#define XBYAK_FOR_EACH_CONVENIENCE_ALL(F) \ + XBYAK_FOR_EACH_REG_CONVENIENCE(F) \ + XBYAK_FOR_EACH_REG_CONVENIENCE_EXT(F) \ + /**/ + namespace Xbyak { enum { DEFAULT_MAX_CODE_SIZE = 4096, - VERSION = 0x7370 /* 0xABCD = A.BC(.D) */ + VERSION = 0x7411 /* 0xABCD = A.BC(.D) */ }; #ifndef MIE_INTEGER_TYPE_DEFINED @@ -200,130 +712,83 @@ typedef uint8_t uint8; #define MIE_PACK(x, y, z, w) ((x) * 64 + (y) * 16 + (z) * 4 + (w)) #endif +// (error code, error message) pairs. The enum and the string table of ConvertErrorToString() are generated from this list. +#define XBYAK_ERR_LIST(f) \ + f(ERR_NONE, "none") \ + f(ERR_BAD_ADDRESSING, "bad addressing") \ + f(ERR_CODE_IS_TOO_BIG, "code is too big") \ + f(ERR_BAD_SCALE, "bad scale") \ + f(ERR_ESP_CANT_BE_INDEX, "esp can't be index") \ + f(ERR_BAD_COMBINATION, "bad combination") \ + f(ERR_BAD_SIZE_OF_REGISTER, "bad size of register") \ + f(ERR_IMM_IS_TOO_BIG, "imm is too big") \ + f(ERR_BAD_ALIGN, "bad align") \ + f(ERR_LABEL_IS_REDEFINED, "label is redefined") \ + f(ERR_LABEL_IS_TOO_FAR, "label is too far") \ + f(ERR_LABEL_IS_NOT_FOUND, "label is not found") \ + f(ERR_CODE_ISNOT_COPYABLE, "code is not copyable") \ + f(ERR_BAD_PARAMETER, "bad parameter") \ + f(ERR_CANT_PROTECT, "can't protect") \ + f(ERR_CANT_USE_64BIT_DISP, "can't use 64bit disp(use (void*))") \ + f(ERR_OFFSET_IS_TOO_BIG, "offset is too big") \ + f(ERR_MEM_SIZE_IS_NOT_SPECIFIED, "MEM size is not specified") \ + f(ERR_BAD_MEM_SIZE, "bad mem size") \ + f(ERR_BAD_ST_COMBINATION, "bad st combination") \ + f(ERR_OVER_LOCAL_LABEL, "over local label") /* not used */ \ + f(ERR_UNDER_LOCAL_LABEL, "under local label") \ + f(ERR_CANT_ALLOC, "can't alloc") \ + f(ERR_ONLY_T_NEAR_IS_SUPPORTED_IN_AUTO_GROW, "T_SHORT is not supported in AutoGrow") \ + f(ERR_BAD_PROTECT_MODE, "bad protect mode") \ + f(ERR_BAD_PNUM, "bad pNum") \ + f(ERR_BAD_TNUM, "bad tNum") \ + f(ERR_BAD_VSIB_ADDRESSING, "bad vsib addressing") \ + f(ERR_CANT_CONVERT, "can't convert") \ + f(ERR_LABEL_ISNOT_SET_BY_L, "label is not set by L()") \ + f(ERR_LABEL_IS_ALREADY_SET_BY_L, "label is already set by L()") \ + f(ERR_BAD_LABEL_STR, "bad label string") \ + f(ERR_MUNMAP, "err munmap") \ + f(ERR_OPMASK_IS_ALREADY_SET, "opmask is already set") \ + f(ERR_ROUNDING_IS_ALREADY_SET, "rounding is already set") \ + f(ERR_K0_IS_INVALID, "k0 is invalid") \ + f(ERR_EVEX_IS_INVALID, "evex is invalid") \ + f(ERR_SAE_IS_INVALID, "sae(suppress all exceptions) is invalid") \ + f(ERR_ER_IS_INVALID, "er(embedded rounding) is invalid") \ + f(ERR_INVALID_BROADCAST, "invalid broadcast") \ + f(ERR_INVALID_OPMASK_WITH_MEMORY, "invalid opmask with memory") \ + f(ERR_INVALID_ZERO, "invalid zero") \ + f(ERR_INVALID_RIP_IN_AUTO_GROW, "invalid rip in AutoGrow") \ + f(ERR_INVALID_MIB_ADDRESS, "invalid mib address") \ + f(ERR_X2APIC_IS_NOT_SUPPORTED, "x2APIC is not supported") \ + f(ERR_NOT_SUPPORTED, "not supported") \ + f(ERR_SAME_REGS_ARE_INVALID, "same regs are invalid") \ + f(ERR_INVALID_NF, "invalid NF") \ + f(ERR_INVALID_ZU, "invalid ZU") \ + f(ERR_CANT_USE_REX2, "can't use rex2") \ + f(ERR_INVALID_DFV, "invalid dfv") \ + f(ERR_INVALID_REG_IDX, "invalid reg index") \ + f(ERR_BAD_ENCODING_MODE, "bad encoding mode") \ + f(ERR_CANT_USE_ABCDH, "can't use [abcd]h with rex") \ + f(ERR_CANT_INIT_CPUTOPOLOGY, "can't init CpuTopology") \ + f(ERR_INVALID_CPUMASK_INDEX, "invalid cpumask index") \ + f(ERR_INTERNAL, "internal error") /* Put it at last. */ + enum { - ERR_NONE = 0, - ERR_BAD_ADDRESSING, - ERR_CODE_IS_TOO_BIG, - ERR_BAD_SCALE, - ERR_ESP_CANT_BE_INDEX, - ERR_BAD_COMBINATION, - ERR_BAD_SIZE_OF_REGISTER, - ERR_IMM_IS_TOO_BIG, - ERR_BAD_ALIGN, - ERR_LABEL_IS_REDEFINED, - ERR_LABEL_IS_TOO_FAR, - ERR_LABEL_IS_NOT_FOUND, - ERR_CODE_ISNOT_COPYABLE, - ERR_BAD_PARAMETER, - ERR_CANT_PROTECT, - ERR_CANT_USE_64BIT_DISP, - ERR_OFFSET_IS_TOO_BIG, - ERR_MEM_SIZE_IS_NOT_SPECIFIED, - ERR_BAD_MEM_SIZE, - ERR_BAD_ST_COMBINATION, - ERR_OVER_LOCAL_LABEL, // not used - ERR_UNDER_LOCAL_LABEL, - ERR_CANT_ALLOC, - ERR_ONLY_T_NEAR_IS_SUPPORTED_IN_AUTO_GROW, - ERR_BAD_PROTECT_MODE, - ERR_BAD_PNUM, - ERR_BAD_TNUM, - ERR_BAD_VSIB_ADDRESSING, - ERR_CANT_CONVERT, - ERR_LABEL_ISNOT_SET_BY_L, - ERR_LABEL_IS_ALREADY_SET_BY_L, - ERR_BAD_LABEL_STR, - ERR_MUNMAP, - ERR_OPMASK_IS_ALREADY_SET, - ERR_ROUNDING_IS_ALREADY_SET, - ERR_K0_IS_INVALID, - ERR_EVEX_IS_INVALID, - ERR_SAE_IS_INVALID, - ERR_ER_IS_INVALID, - ERR_INVALID_BROADCAST, - ERR_INVALID_OPMASK_WITH_MEMORY, - ERR_INVALID_ZERO, - ERR_INVALID_RIP_IN_AUTO_GROW, - ERR_INVALID_MIB_ADDRESS, - ERR_X2APIC_IS_NOT_SUPPORTED, - ERR_NOT_SUPPORTED, - ERR_SAME_REGS_ARE_INVALID, - ERR_INVALID_NF, - ERR_INVALID_ZU, - ERR_CANT_USE_REX2, - ERR_INVALID_DFV, - ERR_INVALID_REG_IDX, - ERR_BAD_ENCODING_MODE, - ERR_CANT_USE_ABCDH, - ERR_CANT_INIT_CPUTOPOLOGY, - ERR_INVALID_CPUMASK_INDEX, - ERR_INTERNAL // Put it at last. +#define XBYAK_ERR_DEF(name, msg) name, + XBYAK_ERR_LIST(XBYAK_ERR_DEF) +#undef XBYAK_ERR_DEF + ERR_MAX_ // sentinel to avoid a trailing comma in C++03 (= ERR_INTERNAL + 1) }; inline const char *ConvertErrorToString(int err) { static const char *errTbl[] = { - "none", - "bad addressing", - "code is too big", - "bad scale", - "esp can't be index", - "bad combination", - "bad size of register", - "imm is too big", - "bad align", - "label is redefined", - "label is too far", - "label is not found", - "code is not copyable", - "bad parameter", - "can't protect", - "can't use 64bit disp(use (void*))", - "offset is too big", - "MEM size is not specified", - "bad mem size", - "bad st combination", - "over local label", - "under local label", - "can't alloc", - "T_SHORT is not supported in AutoGrow", - "bad protect mode", - "bad pNum", - "bad tNum", - "bad vsib addressing", - "can't convert", - "label is not set by L()", - "label is already set by L()", - "bad label string", - "err munmap", - "opmask is already set", - "rounding is already set", - "k0 is invalid", - "evex is invalid", - "sae(suppress all exceptions) is invalid", - "er(embedded rounding) is invalid", - "invalid broadcast", - "invalid opmask with memory", - "invalid zero", - "invalid rip in AutoGrow", - "invalid mib address", - "x2APIC is not supported", - "not supported", - "same regs are invalid", - "invalid NF", - "invalid ZU", - "can't use rex2", - "invalid dfv", - "invalid reg index", - "bad encoding mode", - "can't use [abcd]h with rex", - "can't init CpuTopology", - "invalid cpumask index", - "internal error" +#define XBYAK_ERR_DEF(name, msg) msg, + XBYAK_ERR_LIST(XBYAK_ERR_DEF) +#undef XBYAK_ERR_DEF }; - assert(ERR_INTERNAL + 1 == sizeof(errTbl) / sizeof(*errTbl)); return err <= ERR_INTERNAL ? errTbl[err] : "unknown err"; } +#undef XBYAK_ERR_LIST #ifdef XBYAK_NO_EXCEPTION namespace local { @@ -444,8 +909,9 @@ inline uint32_t VerifyInInt32(uint64_t x) enum LabelMode { LasIs, // as is - Labs, // absolute - LaddTop // (addr + top) for mov(reg, label) with AutoGrow + Labs, // absolute address (not used with AutoGrow) + LaddTop, // (addr + top) for mov(reg, label) with AutoGrow + LsubTop // (addr - top) for jmp/call to an absolute address with AutoGrow }; enum AddressMode { @@ -460,10 +926,17 @@ enum AddressMode { /* custom allocator + alloc() must allocate the buffer with size rounded up to a multiple of + the page size because protect() works at page granularity. */ struct Allocator { explicit Allocator(const std::string& = "") {} // same interface with MmapAllocator - virtual uint8_t *alloc(size_t size) { return reinterpret_cast(AlignedMalloc(size, inner::getPageSize())); } + virtual uint8_t *alloc(size_t size) + { + const size_t alignedSizeM1 = inner::getPageSize() - 1; + size = (size + alignedSizeM1) & ~alignedSizeM1; + return reinterpret_cast(AlignedMalloc(size, inner::getPageSize())); + } virtual void free(uint8_t *p) { AlignedFree(p); } virtual ~Allocator() {} /* override to return false if you call protect() manually */ @@ -496,6 +969,7 @@ inline int getMacOsVersion() #endif class MmapAllocator : public Allocator { struct Allocation { + uintptr_t addr; size_t size; #if defined(XBYAK_USE_MEMFD) // fd_ is only used with XBYAK_USE_MEMFD. We keep the file open @@ -505,7 +979,7 @@ class MmapAllocator : public Allocator { #endif }; const std::string name_; // only used with XBYAK_USE_MEMFD - typedef XBYAK_STD_UNORDERED_MAP AllocationList; + typedef std::vector AllocationList; AllocationList allocList_; public: explicit MmapAllocator(const std::string& name = "xbyak") : name_(name) {} @@ -547,23 +1021,30 @@ public: XBYAK_THROW_RET(ERR_CANT_ALLOC, 0) } assert(p); - Allocation &alloc = allocList_[(uintptr_t)p]; + Allocation alloc; + alloc.addr = (uintptr_t)p; alloc.size = size; #if defined(XBYAK_USE_MEMFD) alloc.fd = fd; #endif + allocList_.push_back(alloc); return (uint8_t*)p; } void free(uint8_t *p) XBYAK_OVERRIDE { if (p == 0) return; - AllocationList::iterator i = allocList_.find((uintptr_t)p); - if (i == allocList_.end()) XBYAK_THROW(ERR_BAD_PARAMETER) - if (munmap((void*)i->first, i->second.size) < 0) XBYAK_THROW(ERR_MUNMAP) + for (size_t idx = 0; idx < allocList_.size(); idx++) { + Allocation& a = allocList_[idx]; + if (a.addr != (uintptr_t)p) continue; + if (munmap((void*)a.addr, a.size) < 0) XBYAK_THROW(ERR_MUNMAP) #if defined(XBYAK_USE_MEMFD) - if (i->second.fd != -1) close(i->second.fd); + if (a.fd != -1) close(a.fd); #endif - allocList_.erase(i); + a = allocList_.back(); + allocList_.pop_back(); + return; + } + XBYAK_THROW(ERR_BAD_PARAMETER) } }; #else @@ -573,8 +1054,8 @@ typedef Allocator MmapAllocator; class Address; class Reg; -struct ApxFlagNF {}; -struct ApxFlagZU {}; +struct ApxFlagNF { explicit XBYAK_CONSTEXPR ApxFlagNF(int = 0) {} }; +struct ApxFlagZU { explicit XBYAK_CONSTEXPR ApxFlagZU(int = 0) {} }; // dfv (default flags value) is or operation of these flags static const int T_of = 8; @@ -585,7 +1066,7 @@ static const int T_cf = 1; class Operand { static const uint8_t EXT8BIT = 0x20; unsigned int idx_:6; // 0..31 + EXT8BIT = 1 if spl/bpl/sil/dil - unsigned int kind_:10; + unsigned int kind_:11; unsigned int bit_:14; protected: unsigned int zero_:1; @@ -606,7 +1087,8 @@ public: ZMM = 1 << 6, OPMASK = 1 << 7, BNDREG = 1 << 8, - TMM = 1 << 9 + TMM = 1 << 9, + BSR = 1 << 10 }; enum Code { #ifdef XBYAK64 @@ -631,6 +1113,9 @@ public: , bit_(bit) , zero_(0), mask_(0), rounding_(0), NF_(0), ZU_(0) { +#ifdef XBYAK32 + if (idx >= 8) XBYAK_THROW(ERR_INVALID_REG_IDX) +#endif assert((bit_ & (bit_ - 1)) == 0); // bit must be power of two } XBYAK_CONSTEXPR Kind getKind() const { return static_cast(kind_); } @@ -643,6 +1128,7 @@ public: XBYAK_CONSTEXPR bool isZMM() const { return is(ZMM); } XBYAK_CONSTEXPR bool isSIMD() const { return is(XMM|YMM|ZMM); } XBYAK_CONSTEXPR bool isTMM() const { return is(TMM); } + XBYAK_CONSTEXPR bool isBSR() const { return is(BSR); } XBYAK_CONSTEXPR bool isXMEM() const { return is(XMM | MEM); } XBYAK_CONSTEXPR bool isYMEM() const { return is(YMM | MEM); } XBYAK_CONSTEXPR bool isZMEM() const { return is(ZMM | MEM); } @@ -702,6 +1188,7 @@ public: XBYAK_CONSTEXPR uint32_t getBit() const { return bit_; } const char *toString() const { + if (isBSR()) return "bsr0"; const int idx = getIdx(); if (kind_ == REG) { if (isExt8bit()) { @@ -759,7 +1246,7 @@ public: static const char *tbl[4] = { "bnd0", "bnd1", "bnd2", "bnd3" }; return tbl[idx]; } - XBYAK_THROW_RET(ERR_INTERNAL, 0); + XBYAK_THROW_RET(ERR_INTERNAL, ""); } bool isEqualIfNotInherited(const Operand& rhs) const { return idx_ == rhs.idx_ && kind_ == rhs.kind_ && bit_ == rhs.bit_ && zero_ == rhs.zero_ && mask_ == rhs.mask_ && rounding_ == rhs.rounding_; } bool operator==(const Operand& rhs) const; @@ -876,7 +1363,7 @@ struct EvexModifierRounding { explicit XBYAK_CONSTEXPR EvexModifierRounding(int rounding) : rounding(rounding) {} int rounding; }; -struct EvexModifierZero{ XBYAK_CONSTEXPR EvexModifierZero() {}}; +struct EvexModifierZero{ explicit XBYAK_CONSTEXPR EvexModifierZero(int = 0) {}}; struct Xmm : public Mmx { explicit XBYAK_CONSTEXPR Xmm(int idx = 0, Kind kind = Operand::XMM, int bit = 128) : Mmx(idx, kind, bit) { } @@ -900,6 +1387,12 @@ struct Zmm : public Ymm { struct Tmm : public Reg { explicit XBYAK_CONSTEXPR Tmm(int idx = 0, Kind kind = Operand::TMM, int bit = 8192) : Reg(idx, kind, bit) { } }; +// Singleton register (idx always 0): 1024-bit width is its true size, but every memory-capable +// mnemonic that uses it sets T_N1 (not T_N_VL), so evex()'s VL==512 disp8N multiplier check is +// never reached with this width. +struct Bsr : public Reg { + explicit XBYAK_CONSTEXPR Bsr(int idx = 0) : Reg(idx, Operand::BSR, 1024) { } +}; #endif struct Opmask : public Reg { @@ -931,6 +1424,7 @@ struct Reg64 : public Reg32e { explicit XBYAK_CONSTEXPR Reg64(int idx = 0) : Reg32e(idx, 64) {} }; struct RegRip { + explicit XBYAK_CONSTEXPR RegRip(int = 0) {} }; #endif @@ -1047,16 +1541,6 @@ public: } #endif bool isVsib(int bit = 128 | 256 | 512) const { return index_.isBit(bit); } - RegExp optimize() const - { - RegExp exp = *this; - // [reg * 2] => [reg + reg] - if (index_.isBit(i32e) && !base_.getBit() && scale_ == 2) { - exp.base_ = index_; - exp.scale_ = 1; - } - return exp; - } bool operator==(const RegExp& rhs) const { return base_ == rhs.base_ && index_ == rhs.index_ && disp_ == rhs.disp_ && scale_ == rhs.scale_; @@ -1163,7 +1647,6 @@ class CodeArray { }; CodeArray(const CodeArray& rhs); void operator=(const CodeArray&); - bool isAllocType() const { return type_ == ALLOC_BUF || type_ == AUTO_GROW; } struct AddrInfo { size_t codeOffset; // position to write size_t jmpAddr; // value to write @@ -1173,7 +1656,7 @@ class CodeArray { : codeOffset(_codeOffset), jmpAddr(_jmpAddr), jmpSize(_jmpSize), mode(_mode) {} uint64_t getVal(const uint8_t *top) const { - uint64_t disp = (mode == inner::LaddTop) ? jmpAddr + size_t(top) : (mode == inner::LasIs) ? jmpAddr : jmpAddr - size_t(top); + uint64_t disp = (mode == inner::LaddTop) ? jmpAddr + size_t(top) : (mode == inner::LsubTop) ? jmpAddr - size_t(top) : jmpAddr; if (jmpSize == 4) disp = inner::VerifyInInt32(disp); return disp; } @@ -1207,6 +1690,11 @@ protected: top_ = newTop; maxSize_ = newSize; } + // grow memory in advance so that the code of a jmp is not split by growMemory() in AutoGrow mode + void growMemoryForJmp() + { + if (isAutoGrow() && size_ + 16 >= maxSize_) growMemory(); + } /* calc jmp address for AutoGrow mode */ @@ -1225,6 +1713,9 @@ public: PROTECT_RWE = 1, // read/write/exec PROTECT_RE = 2 // read/exec }; +protected: + ProtectMode curMode_; +public: explicit CodeArray(size_t maxSize, void *userPtr = 0, Allocator *allocator = 0) : type_(userPtr == AutoGrow ? AUTO_GROW : (userPtr == 0 || userPtr == DontSetProtectRWE) ? ALLOC_BUF : USER_BUF) , alloc_(allocator ? allocator : (Allocator*)&defaultAllocator_) @@ -1232,6 +1723,7 @@ public: , top_(type_ == USER_BUF ? reinterpret_cast(userPtr) : alloc_->alloc((std::max)(maxSize, 1))) , size_(0) , isCalledCalcJmpAddress_(false) + , curMode_(PROTECT_RW) { if (maxSize_ > 0 && top_ == 0) XBYAK_THROW(ERR_CANT_ALLOC) if ((type_ == ALLOC_BUF && userPtr != DontSetProtectRWE && useProtect()) && !setProtectMode(PROTECT_RWE, false)) { @@ -1249,7 +1741,10 @@ public: bool setProtectMode(ProtectMode mode, bool throwException = true) { bool isOK = protect(top_, maxSize_, mode); - if (isOK) return true; + if (isOK) { + curMode_ = mode; + return true; + } if (throwException) XBYAK_THROW_RET(ERR_CANT_PROTECT, false) return false; } @@ -1263,6 +1758,9 @@ public: } void db(int code) { +#ifdef XBYAK_NO_EXCEPTION + if (local::GetErrorRef()) return; +#endif if (size_ >= maxSize_) { if (type_ == AUTO_GROW) { growMemory(); @@ -1270,6 +1768,7 @@ public: XBYAK_THROW(ERR_CODE_IS_TOO_BIG) } } + if (top_ == 0) XBYAK_THROW(ERR_CANT_ALLOC) top_[size_++] = static_cast(code); } void db(const uint8_t *code, size_t codeSize) @@ -1337,6 +1836,7 @@ public: addrInfoList_.push_back(AddrInfo(offset, val, size, mode)); } bool isAutoGrow() const { return type_ == AUTO_GROW; } + bool isAllocType() const { return type_ == ALLOC_BUF || type_ == AUTO_GROW; } bool isCalledCalcJmpAddress() const { return isCalledCalcJmpAddress_; } /** change exec permission of memory @@ -1393,10 +1893,10 @@ class Address : public Operand { public: XBYAK_CONSTEXPR Address() : Operand(0, MEM, 0), e_(), label_(NULL), mode_(inner::M_ModRM), immSize(0), - disp8N(0), permitVsib(false), broadcast_(false), optimize_(true) { } + disp8N(0), permitVsib(false), broadcast_(false), optimized_(false) { } XBYAK_CONSTEXPR Address(uint32_t sizeBit, bool broadcast, const RegExp& e) : Operand(0, MEM, sizeBit), e_(e), label_(e.label_), mode_(), immSize(0), - disp8N(0), permitVsib(false), broadcast_(broadcast), optimize_(true) + disp8N(0), permitVsib(false), broadcast_(broadcast), optimized_(false) { if (e.rip_) { mode_ = (e.label_ || e.asPtr_) ? inner::M_ripAddr : inner::M_rip; @@ -1412,12 +1912,24 @@ public: } } e_.verify(); + // [reg * 2] => [reg + reg] to shorten the encoding (cloneNoOptimize() undoes this) + if (e_.index_.isBit(RegExp::i32e) && !e_.base_.getBit() && e_.scale_ == 2) { + e_.base_ = e_.index_; + e_.scale_ = 1; + optimized_ = true; + } } - RegExp getRegExp() const + const RegExp& getRegExp() const { return e_; } + Address cloneNoOptimize() const { - return optimize_ ? e_.optimize() : e_; + Address addr = *this; + if (addr.optimized_) { + addr.e_.base_ = Reg(); + addr.e_.scale_ = 2; + addr.optimized_ = false; + } + return addr; } - Address cloneNoOptimize() const { Address addr = *this; addr.optimize_ = false; return addr; } inner::AddressMode getMode() const { return mode_; } bool is32bit() const { return e_.getBase().getBit() == 32 || e_.getIndex().getBit() == 32; } bool isOnlyDisp() const { return e_.isOnlyDisp(); } @@ -1428,10 +1940,12 @@ public: const Label* getLabel() const { return label_; } bool operator==(const Address& rhs) const { - return getBit() == rhs.getBit() && e_ == rhs.e_ && label_ == rhs.label_ && mode_ == rhs.mode_ && immSize == rhs.immSize && disp8N == rhs.disp8N && permitVsib == rhs.permitVsib && broadcast_ == rhs.broadcast_ && optimize_ == rhs.optimize_; + return getBit() == rhs.getBit() && e_ == rhs.e_ && label_ == rhs.label_ && mode_ == rhs.mode_ && immSize == rhs.immSize && disp8N == rhs.disp8N && permitVsib == rhs.permitVsib && broadcast_ == rhs.broadcast_ && optimized_ == rhs.optimized_; } bool operator!=(const Address& rhs) const { return !operator==(rhs); } bool isVsib() const { return e_.isVsib(); } + // change byte to dword etc. + Address changeBit(int bit) const { Address addr(*this); addr.setBit(bit); return addr; } private: RegExp e_; const Label* label_; @@ -1442,7 +1956,7 @@ public: bool permitVsib; private: bool broadcast_; - bool optimize_; + bool optimized_; // e_ was rewritten from [reg * 2] to [reg + reg] }; inline const Address& Operand::getAddress() const @@ -1574,11 +2088,21 @@ class LabelManager { ClabelUndefList clabelUndefList_; LabelPtrList labelPtrList_; - int getId(const Label& label) const + // assign a new id at the first use of label (forward reference), so Label::id is mutable + int getOrAssignId(const Label& label) const { if (label.id == 0) label.id = labelId_++; return label.id; } + // label starting with '.' is local (stateList_.back()), otherwise global (stateList_.front()) + const SlabelState& getSlabelState(const std::string& label) const + { + return *label.c_str() == '.' ? stateList_.back() : stateList_.front(); + } + SlabelState& getSlabelState(const std::string& label) + { + return *label.c_str() == '.' ? stateList_.back() : stateList_.front(); + } template void define_inner(DefList& defList, UndefList& undefList, const T& labelId, size_t addrOffset) { @@ -1592,21 +2116,18 @@ class LabelManager { if (itr == undefList.end()) break; const JmpLabel *jmp = &itr->second; const size_t offset = jmp->endOfJmp - jmp->jmpSize; - size_t disp; + size_t disp = jmp->disp; if (jmp->mode == inner::LaddTop) { - disp = addrOffset; + disp += addrOffset; } else if (jmp->mode == inner::Labs) { - disp = size_t(base_->getCurr()); + disp += size_t(base_->getCode()) + addrOffset; // assign() defines a label at another offset } else { - disp = addrOffset - jmp->endOfJmp + jmp->disp; + disp += addrOffset - jmp->endOfJmp; #ifdef XBYAK64 if (jmp->jmpSize <= 4 && !inner::IsInInt32(disp)) XBYAK_THROW(ERR_OFFSET_IS_TOO_BIG) #endif if (jmp->jmpSize == 1 && !inner::IsInDisp8((uint32_t)disp)) XBYAK_THROW(ERR_LABEL_IS_TOO_FAR) } - if (jmp->mode != inner::LasIs) { - disp += jmp->disp; - } if (base_->isAutoGrow()) { base_->save(offset, disp, jmp->jmpSize, jmp->mode); } else { @@ -1706,12 +2227,12 @@ public: label = "@f"; } } - SlabelState& st = *label.c_str() == '.' ? stateList_.back() : stateList_.front(); + SlabelState& st = getSlabelState(label); define_inner(st.defList, st.undefList, label, base_->getSize()); } void defineClabel(Label& label) { - define_inner(clabelDefList_, clabelUndefList_, getId(label), base_->getSize()); + define_inner(clabelDefList_, clabelUndefList_, getOrAssignId(label), base_->getSize()); label.mgr = this; labelPtrList_.insert(&label); } @@ -1719,7 +2240,7 @@ public: { ClabelDefList::const_iterator i = clabelDefList_.find(src.id); if (i == clabelDefList_.end()) XBYAK_THROW(ERR_LABEL_ISNOT_SET_BY_L) - define_inner(clabelDefList_, clabelUndefList_, dst.id, i->second.offset); + define_inner(clabelDefList_, clabelUndefList_, getOrAssignId(dst), i->second.offset); dst.mgr = this; labelPtrList_.insert(&dst); } @@ -1737,16 +2258,16 @@ public: label = "@b"; } } - const SlabelState& st = *label.c_str() == '.' ? stateList_.back() : stateList_.front(); + const SlabelState& st = getSlabelState(label); return getOffset_inner(st.defList, offset, label); } bool getOffset(size_t *offset, const Label& label) const { - return getOffset_inner(clabelDefList_, offset, getId(label)); + return getOffset_inner(clabelDefList_, offset, getOrAssignId(label)); } void addUndefinedLabel(const std::string& label, const JmpLabel& jmp) { - SlabelState& st = *label.c_str() == '.' ? stateList_.back() : stateList_.front(); + SlabelState& st = getSlabelState(label); st.undefList.insert(SlabelUndefList::value_type(label, jmp)); } void addUndefinedLabel(const Label& label, const JmpLabel& jmp) @@ -1853,16 +2374,15 @@ private: { return op1.isREG(i32e) && (op2.isXMM() || op2.isMEM()); } - // (REG32, REG32|MEM) - static inline bool isREG32_REG32orMEM(const Operand& op1, const Operand& op2) - { - return op1.isREG(i32e) && ((op2.isREG(i32e) && op1.getBit() == op2.getBit()) || op2.isMEM()); - } static inline bool isValidSSE(const Operand& op) { // SSE instructions do not support XMM16 - XMM31 return !(op.isXMM() && op.getIdx() >= 16); } + void verifySSE(const Operand& op1, const Operand& op2 = Operand()) const + { + if (!isValidSSE(op1) || !isValidSSE(op2)) XBYAK_THROW(ERR_NOT_SUPPORTED) + } static inline uint8_t rexRXB(int bit, int bit3, const Reg& r, const Reg& b, const Reg& x = Reg()) { int v = bit3 ? 8 : 0; @@ -1871,17 +2391,30 @@ private: if (b.hasIdxBit(bit)) v |= 1; return uint8_t(v); } - void rex2(int bit3, int rex4bit, const Reg& r, const Reg& b, const Reg& x = Reg()) + void rex2(int bit3, int w, const Reg& r, const Reg& b, const Reg& x = Reg()) { db(0xD5); - db((rexRXB(4, bit3, r, b, x) << 4) | rex4bit); + db((rexRXB(4, bit3, r, b, x) << 4) | rexRXB(3, w, r, b, x)); + } + // emit REX2 or REX prefix for (reg, base, index) and return true if rex2 is selected + bool setRex(int w, const Reg& r, const Reg& b, const Reg& x, uint64_t type) + { + uint8_t rex = rexRXB(3, w, r, b, x); + if (r.hasRex2() || b.hasRex2() || x.hasRex2()) { + uint32_t map = getMap(type); + if (map == 2 || map == 3) XBYAK_THROW_RET(ERR_CANT_USE_REX2, false) + rex2(map == 1, w, r, b, x); + return true; + } + if (rex || r.isExt8bit() || b.isExt8bit() || x.isExt8bit()) rex |= 0x40; + if (rex) db(rex); + return false; } // return true if rex2 is selected bool rex(const Operand& op1, const Operand& op2 = Operand(), uint64_t type = 0) { if (op1.getNF() | op2.getNF()) XBYAK_THROW_RET(ERR_INVALID_NF, false) if (op1.getZU() | op2.getZU()) XBYAK_THROW_RET(ERR_INVALID_ZU, false) - uint8_t rex = 0; const Operand *p1 = &op1, *p2 = &op2; if (p1->isMEM()) std::swap(p1, p2); if (p1->isMEM()) XBYAK_THROW_RET(ERR_BAD_COMBINATION, false) @@ -1894,39 +2427,22 @@ private: if (type & T_F3) { db(0xF3); } - bool is0F = type & T_0F; if (p2->isMEM()) { const Reg& r = *static_cast(p1); const Address& addr = p2->getAddress(); - const RegExp e = addr.getRegExp(); - const Reg& base = e.getBase(); - const Reg& idx = e.getIndex(); + const RegExp& e = addr.getRegExp(); if (BIT == 64 && addr.is32bit()) db(0x67); - rex = rexRXB(3, r.isREG(64), r, base, idx); - if (r.hasRex2() || addr.hasRex2()) { - if (type & (T_0F38|T_0F3A)) XBYAK_THROW_RET(ERR_CANT_USE_REX2, false) - rex2(is0F, rex, r, base, idx); - return true; - } - if (rex || r.isExt8bit()) rex |= 0x40; + return setRex(r.isREG(64), r, e.getBase(), e.getIndex(), type); } else { const Reg& r1 = static_cast(op1); const Reg& r2 = static_cast(op2); // ModRM(reg, base); - rex = rexRXB(3, r1.isREG(64) || r2.isREG(64), r2, r1); - if (r1.hasRex2() || r2.hasRex2()) { - if (type & (T_0F38|T_0F3A)) XBYAK_THROW_RET(ERR_CANT_USE_REX2, 0) - rex2(is0F, rex, r2, r1); - return true; - } - if (rex || r1.isExt8bit() || r2.isExt8bit()) rex |= 0x40; + return setRex(r1.isREG(64) || r2.isREG(64), r2, r1, Reg(), type); } - if (rex) db(rex); - return false; } // @@@begin of avx_type_def.h static const uint64_t T_NONE = 0ull; - // low 3 bit + // N field (bit0-2) : disp8N = 1 << (value - 1), T_DUP is a sentinel static const uint64_t T_N1 = 1ull; static const uint64_t T_N2 = 2ull; static const uint64_t T_N4 = 3ull; @@ -1934,54 +2450,55 @@ private: static const uint64_t T_N16 = 5ull; static const uint64_t T_N32 = 6ull; static const uint64_t T_NX_MASK = 7ull; - static const uint64_t T_DUP = T_NX_MASK;//1 << 4, // N = (8, 32, 64) + static const uint64_t T_DUP = T_NX_MASK; // N = (8, 32, 64) static const uint64_t T_N_VL = 1ull << 3; // N * (1, 2, 4) for VL static const uint64_t T_APX = 1ull << 4; + // pp : one bit each (not a 2-bit field) because rex() emits 0x66 and 0xF2/0xF3 independently (e.g. crc32 uses T_66|T_F2) static const uint64_t T_66 = 1ull << 5; // pp = 1 static const uint64_t T_F3 = 1ull << 6; // pp = 2 - static const uint64_t T_ER_R = 1ull << 7; // reg{er} + static const uint64_t T_F2 = 1ull << 7; // pp = 3 + // map field (bit8-10) : the value is the same as the EVEX mmm field static const uint64_t T_0F = 1ull << 8; - static const uint64_t T_0F38 = 1ull << 9; - static const uint64_t T_0F3A = 1ull << 10; - static const uint64_t T_MAP5 = 1ull << 11; - static const uint64_t T_L1 = 1ull << 12; - static const uint64_t T_W0 = 1ull << 13; // T_EW0 = T_W0 - static const uint64_t T_W1 = 1ull << 14; // for VEX + static const uint64_t T_0F38 = 2ull << 8; + static const uint64_t T_0F3A = 3ull << 8; + static const uint64_t T_MAP5 = 5ull << 8; + static const uint64_t T_MAP6 = 6ull << 8; + static const uint64_t T_MAP_MASK = 7ull << 8; + // er/sae field (bit11-13) : an insn has at most one of these + static const uint64_t T_ER_X = 1ull << 11; // xmm{er} + static const uint64_t T_ER_Y = 2ull << 11; // ymm{er} + static const uint64_t T_ER_Z = 3ull << 11; // zmm{er} + static const uint64_t T_ER_R = 4ull << 11; // reg{er} + static const uint64_t T_SAE_X = 5ull << 11; // xmm{sae} + static const uint64_t T_SAE_Y = 6ull << 11; // ymm{sae} + static const uint64_t T_SAE_Z = 7ull << 11; // zmm{sae} + static const uint64_t T_ER_SAE_MASK = 7ull << 11; + static const uint64_t T_W0 = 1ull << 14; // T_EW0 = T_W0 + static const uint64_t T_W1 = 1ull << 15; // for VEX static const uint64_t T_EW1 = 1ull << 16; // for EVEX - static const uint64_t T_YMM = 1ull << 17; // support YMM, ZMM - static const uint64_t T_EVEX = 1ull << 18; - static const uint64_t T_ER_X = 1ull << 19; // xmm{er} - static const uint64_t T_ER_Y = 1ull << 20; // ymm{er} - static const uint64_t T_ER_Z = 1ull << 21; // zmm{er} - static const uint64_t T_SAE_X = 1ull << 22; // xmm{sae} - static const uint64_t T_SAE_Y = 1ull << 23; // ymm{sae} - static const uint64_t T_SAE_Z = 1ull << 24; // zmm{sae} - static const uint64_t T_MUST_EVEX = 1ull << 25; // contains T_EVEX - static const uint64_t T_B32 = 1ull << 26; // m32bcst - static const uint64_t T_B64 = 1ull << 27; // m64bcst - static const uint64_t T_B16 = T_B32 | T_B64; // m16bcst (Be careful) - static const uint64_t T_M_K = 1ull << 28; // mem{k} - static const uint64_t T_VSIB = 1ull << 29; - static const uint64_t T_MEM_EVEX = 1ull << 30; // use evex if mem - static const uint64_t T_MAP6 = 1ull << 31; - static const uint64_t T_NF = 1ull << 32; // T_nf - static const uint64_t T_CODE1_IF1 = 1ull << 33; // code|=1 if !r.isBit(8) - - static const uint64_t T_ND1 = 1ull << 35; // ND=1 - static const uint64_t T_ZU = 1ull << 36; // ND=ZU - static const uint64_t T_F2 = 1ull << 37; // pp = 3 - static const uint64_t T_SENTRY = (1ull << 38)-1; // attribute(>=T_SENTRY) is for error check - static const uint64_t T_ALLOW_DIFF_SIZE = 1ull << 38; // allow difference reg size - static const uint64_t T_ALLOW_ABCDH = 1ull << 39; // allow [abcd]h reg + static const uint64_t T_L1 = 1ull << 17; + static const uint64_t T_YMM = 1ull << 18; // support YMM, ZMM + // evex field (bit19-20) : which encodings the insn has + static const uint64_t T_EVEX = 1ull << 19; // both VEX and EVEX + static const uint64_t T_MUST_EVEX = 2ull << 19; // EVEX only + static const uint64_t T_EVEX_IF_MEM = 3ull << 19; // both, but the mem operand form exists only in EVEX + static const uint64_t T_EVEX_MASK = 3ull << 19; + // broadcast field (bit21-22) + static const uint64_t T_B32 = 1ull << 21; // m32bcst + static const uint64_t T_B64 = 2ull << 21; // m64bcst + static const uint64_t T_B16 = T_B32 | T_B64; // m16bcst + static const uint64_t T_M_K = 1ull << 23; // mem{k} + static const uint64_t T_VSIB = 1ull << 24; + static const uint64_t T_NF = 1ull << 25; // T_nf + static const uint64_t T_OP_W1 = 1ull << 26; // opcode bit0 is the w (operand-size) bit; code|=1 unless the operand is 8-bit + static const uint64_t T_ND1 = 1ull << 27; // ND=1 + static const uint64_t T_ZU = 1ull << 28; // ND=ZU + static const uint64_t T_ALLOW_DIFF_SIZE = 1ull << 29; // allow difference reg size + static const uint64_t T_ALLOW_ABCDH = 1ull << 30; // allow [abcd]h reg // T_66 = 1, T_F3 = 2, T_F2 = 3 static inline uint32_t getPP(uint64_t type) { return (type & T_66) ? 1 : (type & T_F3) ? 2 : (type & T_F2) ? 3 : 0; } // @@@end of avx_type_def.h - static inline uint32_t getMap(uint64_t type) - { - if (type & T_MAP6) return 6; - if (type & T_MAP5) return 5; - return (type & T_0F) ? 1 : (type & T_0F38) ? 2 : (type & T_0F3A) ? 3 : 0; - } + static inline uint32_t getMap(uint64_t type) { return uint32_t((type & T_MAP_MASK) >> 8); } void vex(const Reg& reg, const Reg& base, const Operand *v, uint64_t type, int code, bool x = false) { int w = (type & T_W1) ? 1 : 0; @@ -1992,7 +2509,7 @@ private: if ((idx | reg.getIdx() | base.getIdx()) >= 16) XBYAK_THROW(ERR_BAD_COMBINATION) uint32_t pp = getPP(type); uint32_t vvvv = (((~idx) & 15) << 3) | (is256 ? 4 : 0) | pp; - if (!b && !x && !w && (type & T_0F)) { + if (!b && !x && !w && getMap(type) == 1) { db(0xC5); db((r ? 0 : 0x80) | vvvv); } else { uint32_t mmmm = getMap(type); @@ -2002,13 +2519,15 @@ private: } void verifySAE(const Reg& r, uint64_t type) const { - if (((type & T_SAE_X) && r.isXMM()) || ((type & T_SAE_Y) && r.isYMM()) || ((type & T_SAE_Z) && r.isZMM())) return; + uint64_t v = type & T_ER_SAE_MASK; + if ((v == T_SAE_X && r.isXMM()) || (v == T_SAE_Y && r.isYMM()) || (v == T_SAE_Z && r.isZMM())) return; XBYAK_THROW(ERR_SAE_IS_INVALID) } void verifyER(const Reg& r, uint64_t type) const { - if ((type & T_ER_R) && r.isREG(32|64)) return; - if (((type & T_ER_X) && r.isXMM()) || ((type & T_ER_Y) && r.isYMM()) || ((type & T_ER_Z) && r.isZMM())) return; + uint64_t v = type & T_ER_SAE_MASK; + if (v == T_ER_R && r.isREG(32|64)) return; + if ((v == T_ER_X && r.isXMM()) || (v == T_ER_Y && r.isYMM()) || (v == T_ER_Z && r.isZMM())) return; XBYAK_THROW(ERR_ER_IS_INVALID) } // (a, b, c) contains non zero two or three values then err @@ -2018,9 +2537,17 @@ private: if ((a > 0 && a != v) + (b > 0 && b != v) + (c > 0 && c != v) > 0) XBYAK_THROW_RET(err, 0) return v; } - int evex(const Reg& reg, const Reg& base, const Operand *v, uint64_t type, int code, const Reg *x = 0, bool b = false, int aaa = 0, uint32_t VL = 0, bool Hi16Vidx = false) + int evex(const Reg& reg, const Reg& base, const Operand *v, uint64_t type, int code, const Address *addr = 0) { - if (!(type & (T_EVEX | T_MUST_EVEX))) XBYAK_THROW_RET(ERR_EVEX_IS_INVALID, 0) + const Reg *x = addr ? &addr->getRegExp().getIndex() : 0; + int aaa = addr ? addr->getOpmaskIdx() : 0; + if (aaa && !(type & T_M_K)) XBYAK_THROW_RET(ERR_INVALID_OPMASK_WITH_MEMORY, 0) + bool b = false; + if (addr && addr->isBroadcast()) { + if (!(type & (T_B32 | T_B64))) XBYAK_THROW_RET(ERR_INVALID_BROADCAST, 0) + b = true; + } + if (!(type & T_EVEX_MASK)) XBYAK_THROW_RET(ERR_EVEX_IS_INVALID, 0) int w = (type & T_EW1) ? 1 : 0; uint32_t mmm = getMap(type); uint32_t pp = getPP(type); @@ -2044,6 +2571,7 @@ private: } b = true; } else { + uint32_t VL = (x && x->isSIMD()) ? x->getBit() : 0; // vsib if (v) VL = (std::max)(VL, v->getBit()); VL = (std::max)((std::max)(reg.getBit(), base.getBit()), VL); LL = (VL >= 512 /* tmm */) ? 2 : (VL == 256) ? 1 : 0; @@ -2062,7 +2590,7 @@ private: } } } - bool V4 = ((v ? v->isExtIdx2() : 0) || Hi16Vidx); + bool V4 = (v && v->isExtIdx2()) || (x && x->isSIMD() && x->isExtIdx2()); bool z = reg.hasZero() || base.hasZero() || (v ? v->hasZero() : false); if (aaa == 0) aaa = verifyDuplicate(base.getOpmaskIdx(), reg.getOpmaskIdx(), (v ? v->getOpmaskIdx() : 0), ERR_OPMASK_IS_ALREADY_SET); if (aaa == 0) z = 0; // clear T_z if mask is not set @@ -2170,25 +2698,26 @@ private: db(disp); } else if (mod == mod10 || (mod == mod00 && !baseBit)) { if (label) { - putL_inner(*label, false, e.getDisp() - addr.immSize, 4); + putL_inner(*label, inner::Labs, e.getDisp(), 4); } else { dd(disp); } } } LabelManager labelMgr_; + // r is used only to determine the w bit (code|=1 unless r is 8-bit). + // opROO passes d here, which is often Reg() (bit=0), assuming that !r.isBit(8) is true then. void writeCode(uint64_t type, const Reg& r, int code, bool rex2 = false) { if (!(type&T_APX || rex2)) { - if (type & T_0F) { - db(0x0F); - } else if (type & T_0F38) { - db(0x0F); db(0x38); - } else if (type & T_0F3A) { - db(0x0F); db(0x3A); + switch (getMap(type)) { + case 1: db(0x0F); break; + case 2: db(0x0F); db(0x38); break; + case 3: db(0x0F); db(0x3A); break; + default: break; } } - db(code | (((type & T_SENTRY) == 0 || (type & T_CODE1_IF1)) && !r.isBit(8))); + db(code | (((type & T_OP_W1) != 0) && !r.isBit(8))); } void opRR(const Reg& r1, const Reg& r2, uint64_t type, int code) { @@ -2213,12 +2742,7 @@ private: void opLoadSeg(const Address& addr, const Reg& reg, uint64_t type, int code) { if (reg.isBit(8)) XBYAK_THROW(ERR_BAD_SIZE_OF_REGISTER) - if (addr.is64bitDisp()) XBYAK_THROW(ERR_CANT_USE_64BIT_DISP) - // can't use opMR - rex(addr, reg, type); - if (type & T_0F) db(0x0F); - db(code); - opAddr(addr, reg.getIdx()); + opMR(addr, reg, type, code); } // for only MPX(bnd*) void opMIB(const Address& addr, const Reg& reg, uint64_t type, int code) @@ -2244,7 +2768,7 @@ private: void opJmp(T& label, LabelType type, uint8_t shortCode, uint8_t longCode, uint8_t longPref) { if (type == T_FAR) XBYAK_THROW(ERR_NOT_SUPPORTED) - if (isAutoGrow() && size_ + 16 >= maxSize_) growMemory(); /* avoid splitting code of jmp */ + growMemoryForJmp(); size_t offset = 0; if (labelMgr_.getOffset(&offset, label)) { /* label exists */ makeJmp(inner::VerifyInInt32(offset - size_), type, shortCode, longCode, longPref); @@ -2267,11 +2791,11 @@ private: if (type == T_FAR) XBYAK_THROW(ERR_NOT_SUPPORTED) if (isAutoGrow()) { if (!isNEAR(type)) XBYAK_THROW(ERR_ONLY_T_NEAR_IS_SUPPORTED_IN_AUTO_GROW) - if (size_ + 16 >= maxSize_) growMemory(); + growMemoryForJmp(); if (longPref) db(longPref); db(longCode); dd(0); - save(size_ - 4, size_t(addr) - size_, 4, inner::Labs); + save(size_ - 4, size_t(addr) - size_, 4, inner::LsubTop); } else { makeJmp(inner::VerifyInInt32(reinterpret_cast(addr) - getCurr()), type, shortCode, longCode, longPref); } @@ -2297,7 +2821,7 @@ private: } else if (addr.getMode() == inner::M_rip || addr.getMode() == inner::M_ripAddr) { setModRM(0, reg, 5); if (addr.getLabel()) { // [rip + Label] - putL_inner(*addr.getLabel(), true, addr.getDisp() - addr.immSize, 4); + putL_inner(*addr.getLabel(), inner::LasIs, addr.getDisp() - addr.immSize, 4); } else { size_t disp = addr.getDisp(); if (addr.getMode() == inner::M_ripAddr) { @@ -2312,13 +2836,13 @@ private: void opSSE(const Reg& r, const Operand& op, uint64_t type, int code, bool isValid(const Operand&, const Operand&) = 0, int imm8 = NONE) { if (isValid && !isValid(r, op)) XBYAK_THROW(ERR_BAD_COMBINATION) - if (!isValidSSE(r) || !isValidSSE(op)) XBYAK_THROW(ERR_NOT_SUPPORTED) + verifySSE(r, op); opRO(r, op, type, code, true, (imm8 != NONE) ? 1 : 0); if (imm8 != NONE) db(imm8); } void opMMX_IMM(const Mmx& mmx, int imm8, int code, int ext) { - if (!isValidSSE(mmx)) XBYAK_THROW(ERR_NOT_SUPPORTED) + verifySSE(mmx); uint64_t type = T_0F; if (mmx.isXMM()) type |= T_66; opRR(Reg32(ext), mmx, type, code); @@ -2331,7 +2855,7 @@ private: } void opMovXMM(const Operand& op1, const Operand& op2, uint64_t type, int code) { - if (!isValidSSE(op1) || !isValidSSE(op2)) XBYAK_THROW(ERR_NOT_SUPPORTED) + verifySSE(op1, op2); if (op1.isXMM() && op2.isMEM()) { opMR(op2.getAddress(), op1.getReg(), type, code); } else if (op1.isMEM() && op2.isXMM()) { @@ -2343,7 +2867,7 @@ private: // pextr{w,b,d}, extractps void opExt(const Operand& op, const Mmx& mmx, int code, int imm, bool hasMMX2 = false) { - if (!isValidSSE(op) || !isValidSSE(mmx)) XBYAK_THROW(ERR_NOT_SUPPORTED) + verifySSE(op, mmx); if (hasMMX2 && op.isREG(i32e)) { /* pextrw is special */ if (mmx.isXMM()) db(0x66); opRR(op.getReg(), mmx, T_0F, 0xC5); db(imm); @@ -2361,14 +2885,14 @@ private: // (r, r, m) or (r, m, r) bool opROO(const Reg& d, const Operand& op1, const Operand& op2, uint64_t type, int code, int immSize = 0, int sc = NONE) { - if (!(type & T_MUST_EVEX) && !d.isREG() && !(d.hasRex2NFZU() || op1.hasRex2NFZU() || op2.hasRex2NFZU())) return false; + if ((type & T_EVEX_MASK) != T_MUST_EVEX && !d.isREG() && !(d.hasRex2NFZU() || op1.hasRex2NFZU() || op2.hasRex2NFZU())) return false; const Operand *p1 = &op1, *p2 = &op2; if (p1->isMEM()) { std::swap(p1, p2); } else { if (p2->isMEM()) code |= 2; } if (p1->isMEM()) XBYAK_THROW_RET(ERR_BAD_COMBINATION, false) if (p2->isMEM()) { const Reg& r = *static_cast(p1); Address addr = p2->getAddress(); - const RegExp e = addr.getRegExp(); + const RegExp& e = addr.getRegExp(); evexLeg(r, e.getBase(), e.getIndex(), d, type, sc); writeCode(type, d, code); addr.immSize = immSize; @@ -2385,7 +2909,8 @@ private: int opBit = op.getBit(); if (disableRex && opBit == 64) opBit = 32; const Reg r(ext, Operand::REG, opBit); - if ((type & T_APX) && (d != 0 || op.hasRex2NFZU()) && opROO(d ? *d : Reg(0, Operand::REG, opBit), op, r, type, code)) return; + // EVEX is required only for ND/NF/ZU; a plain EGPR is encodable with the shorter REX2 + if ((type & T_APX) && (d != 0 || op.getNF() || op.getZU()) && opROO(d ? *d : Reg(0, Operand::REG, opBit), op, r, type, code)) return; if (op.isMEM()) { opMR(op.getAddress(immSize), r, type, code); } else if (op.isREG(bit)) { @@ -2396,23 +2921,26 @@ private: } void opSetCC(const Operand& op, int ext) { - if (opROO(Reg(), op, Reg(), T_APX|T_ZU|T_F2, 0x40 | ext)) return; + // EVEX (opcode 0x40|ext) is required only for ZU; a plain EGPR is encodable with the shorter REX2 + if (op.getZU() && opROO(Reg(), op, Reg(), T_APX|T_ZU|T_F2, 0x40 | ext)) return; opRext(op, 8, 0, T_0F, 0x90 | ext); } + void opShiftCore(const Operand& op, int ext, const Reg *d, int code, int immSize) + { + if (d && op.getBit() != 0 && d->getBit() != op.getBit()) XBYAK_THROW(ERR_BAD_SIZE_OF_REGISTER) + uint64_t type = T_APX|T_OP_W1; if (ext & 8) type |= T_NF; if (d) type |= T_ND1; + opRext(op, 0, ext&7, type, code, false, immSize, d); + } void opShift(const Operand& op, int imm, int ext, const Reg *d = 0) { if (d == 0) verifyMemHasSize(op); - if (d && op.getBit() != 0 && d->getBit() != op.getBit()) XBYAK_THROW(ERR_BAD_SIZE_OF_REGISTER) - uint64_t type = T_APX|T_CODE1_IF1; if (ext & 8) type |= T_NF; if (d) type |= T_ND1; - opRext(op, 0, ext&7, type, (0xC0 | ((imm == 1 ? 1 : 0) << 4)), false, (imm != 1) ? 1 : 0, d); + opShiftCore(op, ext, d, (0xC0 | ((imm == 1 ? 1 : 0) << 4)), (imm != 1) ? 1 : 0); if (imm != 1) db(imm); } void opShift(const Operand& op, const Reg8& _cl, int ext, const Reg *d = 0) { if (_cl.getIdx() != Operand::CL) XBYAK_THROW(ERR_BAD_COMBINATION) - if (d && op.getBit() != 0 && d->getBit() != op.getBit()) XBYAK_THROW(ERR_BAD_SIZE_OF_REGISTER) - uint64_t type = T_APX|T_CODE1_IF1; if (ext & 8) type |= T_NF; if (d) type |= T_ND1; - opRext(op, 0, ext&7, type, 0xD2, false, 0, d); + opShiftCore(op, ext, d, 0xD2, 0); } // condR assumes that op.isREG() is true void opRO(const Reg& r, const Operand& op, uint64_t type, int code, bool condR = true, int immSize = 0) @@ -2443,52 +2971,49 @@ private: { if (op2.isMEM()) { if (!op1.isREG()) XBYAK_THROW(ERR_BAD_COMBINATION) - opMR(op2.getAddress(), op1.getReg(), 0, code | 2); + opMR(op2.getAddress(), op1.getReg(), T_OP_W1, code | 2); } else { - opRO(static_cast(op2), op1, 0, code, op1.getKind() == op2.getKind()); + opRO(static_cast(op2), op1, T_OP_W1, code, op1.getKind() == op2.getKind()); } } - bool isInDisp16(uint32_t x) const { return 0xFFFF8000 <= x || x <= 0x7FFF; } // allow add(ax, 0x8000); bool isInDisp16relaxed(uint32_t x) const { uint32_t v = x & 0xffff0000; return v == 0 || v == 0xffff0000; } + // size of imm to be encoded for op (8, 16 or 32) uint32_t getImmBit(const Operand& op, uint32_t imm) { verifyMemHasSize(op); + if (op.isBit(8)) return 8; uint32_t immBit = inner::IsInDisp8(imm) ? 8 : isInDisp16relaxed(imm) ? 16 : 32; - if (op.isBit(8)) immBit = 8; if (op.getBit() < immBit) XBYAK_THROW_RET(ERR_IMM_IS_TOO_BIG, 0) - if (op.isBit(32|64) && immBit == 16) immBit = 32; /* don't use MEM16 if 32/64bit mode */ + if (immBit == 16 && op.isBit(32|64)) immBit = 32; // don't use imm16 for 32/64-bit ops return immBit; } + // s bit of the 0x80 group : 0x83 = r/m, imm8 (sign-extended) + int getSbit(const Operand& op, uint32_t immBit) const { return (immBit == 8 && !op.isBit(8)) ? 2 : 0; } // (REG|MEM, IMM) void opOI(const Operand& op, uint32_t imm, int code, int ext) { uint32_t immBit = getImmBit(op, imm); - if (op.isREG() && op.getIdx() == 0 && (op.getBit() == immBit || (op.isBit(64) && immBit == 32))) { // rax, eax, ax, al + if (op.isREG() && op.getIdx() == 0 && immBit == (op.isBit(64) ? 32U : op.getBit())) { // short form for al/ax/eax/rax rex(op); db(code | 4 | (immBit == 8 ? 0 : 1)); } else { - int tmp = immBit < (std::min)(op.getBit(), 32U) ? 2 : 0; - opRext(op, 0, ext, 0, 0x80 | tmp, false, immBit / 8); + opRext(op, 0, ext, T_OP_W1, 0x80 | getSbit(op, immBit), false, immBit / 8); } db(imm, immBit / 8); } // (r, r/m, imm) - void opROI(const Reg& d, const Operand& op, uint32_t imm, uint64_t type, int ext) + void opROI(const Reg& d, const Operand& op, uint32_t imm, uint64_t type, int ext, int sc = NONE) { uint32_t immBit = getImmBit(d, imm); - int code = immBit < (std::min)(d.getBit(), 32U) ? 2 : 0; - opROO(d, op, Reg(ext, Operand::REG, d.getBit()), type, 0x80 | code, immBit / 8); + opROO(d, op, Reg(ext, Operand::REG, d.getBit()), type, 0x80 | getSbit(d, immBit), immBit / 8, sc); db(imm, immBit / 8); } void opIncDec(const Reg& d, const Operand& op, int ext) { #ifdef XBYAK64 if (d.isREG()) { - int code = d.isBit(8) ? 0xFE : 0xFF; - uint64_t type = T_APX|T_NF|T_ND1; - if (d.isBit(16)) type |= T_66; - opROO(d, op, Reg(ext, Operand::REG, d.getBit()), type, code); + opROO(d, op, Reg(ext, Operand::REG, d.getBit()), T_APX|T_NF|T_ND1|T_OP_W1, 0xFE); return; } #else @@ -2501,31 +3026,31 @@ private: return; } #endif - opRext(op, op.getBit(), ext, 0, 0xFE); + opRext(op, op.getBit(), ext, T_OP_W1, 0xFE); } void opPushPop(const Operand& op, int code, int ext, int alt) { - if (op.isREG() && op.hasRex2()) { - const Reg& r = static_cast(op); - rex2(0, rexRXB(3, 0, Reg(), r), Reg(), r); - db(alt | (r.getIdx() & 7)); - return; - } int bit = op.getBit(); - if (bit == 16 || bit == BIT) { - if (bit == 16) db(0x66); - if (op.isREG()) { - if (op.getReg().getIdx() >= 8) db(0x41); - db(alt | (op.getIdx() & 7)); - return; - } - if (op.isMEM()) { - opMR(op.getAddress(), Reg(ext, Operand::REG, 32), T_ALLOW_DIFF_SIZE, code); - return; - } + if (bit != 16 && bit != BIT) XBYAK_THROW(ERR_BAD_COMBINATION) + if (bit == 16) db(0x66); + if (op.isREG()) { + setRex(0, Reg(), op.getReg(), Reg(), 0); // 0x41 or REX2 if necessary + db(alt | (op.getIdx() & 7)); + } else if (op.isMEM()) { + opMR(op.getAddress(), Reg(ext, Operand::REG, 32), T_ALLOW_DIFF_SIZE, code); + } else { + XBYAK_THROW(ERR_BAD_COMBINATION) } - XBYAK_THROW(ERR_BAD_COMBINATION) } +#ifdef XBYAK64 + // PUSHP/POPP : REX2 is mandatory (carries the W=1 PPX hint), + // unlike ordinary push/pop where REX2 is only emitted for R16-31. + void opPushPopP(const Reg64& r, int alt) + { + rex2(0, 1, Reg(), r); + db(alt | (r.getIdx() & 7)); + } +#endif void verifyMemHasSize(const Operand& op) const { if (op.isMEM() && op.getBit() == 0) XBYAK_THROW(ERR_MEM_SIZE_IS_NOT_SPECIFIED) @@ -2552,26 +3077,30 @@ private: db(code | (idx & 7)); return bit / 8; } + /* + write (label + disp) as jmpSize bytes + mode : LasIs (rip-relative offset), Labs (absolute address; replaced by LaddTop in AutoGrow mode) + */ template - void putL_inner(T& label, bool relative = false, size_t disp = 0, int jmpSize = (int)sizeof(size_t)) + void putL_inner(T& label, inner::LabelMode mode, size_t disp, int jmpSize) { - if (relative) jmpSize = 4; - if (isAutoGrow() && size_ + 16 >= maxSize_) growMemory(); + growMemoryForJmp(); + if (mode == inner::Labs && isAutoGrow()) mode = inner::LaddTop; size_t offset = 0; if (labelMgr_.getOffset(&offset, label)) { - if (relative) { - db(inner::VerifyInInt32(offset + disp - size_ - jmpSize), jmpSize); - } else if (isAutoGrow()) { + offset += disp; + if (mode == inner::LasIs) { + db(inner::VerifyInInt32(offset - size_ - jmpSize), jmpSize); + } else if (mode == inner::LaddTop) { db(uint64_t(0), jmpSize); - save(size_ - jmpSize, offset, jmpSize, inner::LaddTop); + save(size_ - jmpSize, offset, jmpSize, mode); } else { db(size_t(top_) + offset, jmpSize); } return; } db(uint64_t(0), jmpSize); - JmpLabel jmp(size_, jmpSize, (relative ? inner::LasIs : isAutoGrow() ? inner::LaddTop : inner::Labs), disp); - labelMgr_.addUndefinedLabel(label, jmp); + labelMgr_.addUndefinedLabel(label, JmpLabel(size_, jmpSize, mode, disp)); } void opMovxx(const Reg& reg, const Operand& op, uint8_t code) { @@ -2605,33 +3134,26 @@ private: } void opVex(const Reg& r, const Operand *p1, const Operand& op2, uint64_t type, int code, int imm8 = NONE) { + const bool useEvex = (type & T_EVEX_MASK) == T_MUST_EVEX || r.hasEvex() || (p1 && p1->hasEvex()); if (op2.isMEM()) { + // zeroing-masking has no meaning when the destination is memory + if ((type & T_M_K) && (r.hasZero() || (p1 && p1->hasZero()) || op2.hasZero())) XBYAK_THROW(ERR_INVALID_ZERO) Address addr = op2.getAddress(); const RegExp& regExp = addr.getRegExp(); const Reg& base = regExp.getBase(); const Reg& index = regExp.getIndex(); if (BIT == 64 && addr.is32bit()) db(0x67); - int disp8N = 0; - if ((type & (T_MUST_EVEX|T_MEM_EVEX)) || r.hasEvex() || (p1 && p1->hasEvex()) || addr.isBroadcast() || addr.getOpmaskIdx() || addr.hasRex2()) { - int aaa = addr.getOpmaskIdx(); - if (aaa && !(type & T_M_K)) XBYAK_THROW(ERR_INVALID_OPMASK_WITH_MEMORY) - bool b = false; - if (addr.isBroadcast()) { - if (!(type & (T_B32 | T_B64))) XBYAK_THROW(ERR_INVALID_BROADCAST) - b = true; - } - int VL = regExp.isVsib() ? index.getBit() : 0; - disp8N = evex(r, base, p1, type, code, &index, b, aaa, VL, index.isSIMD() && index.isExtIdx2()); + if (useEvex || (type & T_EVEX_MASK) == T_EVEX_IF_MEM || addr.isBroadcast() || addr.getOpmaskIdx() || addr.hasRex2()) { + addr.disp8N = evex(r, base, p1, type, code, &addr); } else { vex(r, base, p1, type, code, index.isExtIdx()); } if (type & T_VSIB) addr.permitVsib = true; - if (disp8N) addr.disp8N = disp8N; if (imm8 != NONE) addr.immSize = 1; opAddr(addr, r.getIdx()); } else { const Reg& base = op2.getReg(); - if ((type & T_MUST_EVEX) || r.hasEvex() || (p1 && p1->hasEvex()) || base.hasEvex()) { + if (useEvex || base.hasEvex()) { evex(r, base, p1, type, code); } else { vex(r, base, p1, type, code); @@ -2671,56 +3193,64 @@ private: if (!op3.isMEM() && (x2.getKind() != op3.getKind())) XBYAK_THROW(ERR_BAD_COMBINATION) opVex(k, &x2, op3, type, code, imm8); } - // (x, x/m), (y, x/m256), (z, y/m) - void checkCvt1(const Operand& x, const Operand& op) const - { - if (!op.isMEM() && !(x.is(Operand::XMM | Operand::YMM) && op.isXMM()) && !(x.isZMM() && op.isYMM())) XBYAK_THROW(ERR_BAD_COMBINATION) - } - // (x, x/m), (x, y/m256), (y, z/m) - void checkCvt2(const Xmm& x, const Operand& op) const - { - if (!(x.isXMM() && op.is(Operand::XMM | Operand::YMM | Operand::MEM)) && !(x.isYMM() && op.is(Operand::ZMM | Operand::MEM))) XBYAK_THROW(ERR_BAD_COMBINATION) - } void opCvt(const Xmm& x, const Operand& op, uint64_t type, int code) { Operand::Kind kind = x.isXMM() ? (op.isBit(256) ? Operand::YMM : Operand::XMM) : Operand::ZMM; opVex(x.copyAndSetKind(kind), &xm0, op, type, code); } - void opCvt2(const Xmm& x, const Operand& op, uint64_t type, int code) + // xx_xy_xz + void opX_XM(const Operand& op, const Xmm& x, uint64_t type, uint8_t code) { - checkCvt2(x, op); + if (!op.isMEM() && !op.isXMM()) XBYAK_THROW(ERR_BAD_COMBINATION) + opVex(x, 0, op, type, code); + } + // opCvt_ab_cd_ef lists the accepted (r/m, reg) kind pairs with x/y/z = XMM/YMM/ZMM + // (x, x/m), (y, x/m256), (z, y/m) : e.g. vcvtdq2pd, vpmovdw + void opCvt_xx_xy_yz(const Xmm& x, const Operand& op, uint64_t type, int code, int imm8 = NONE) + { + if (!op.isMEM() && !(x.is(Operand::XMM | Operand::YMM) && op.isXMM()) && !(x.isZMM() && op.isYMM())) XBYAK_THROW(ERR_BAD_COMBINATION) + opVex(x, 0, op, type, code, imm8); + } + // (x, x/m), (x, y/m256), (y, z/m) : e.g. vcvtpd2dq + void opCvt_xx_yx_zy(const Xmm& x, const Operand& op, uint64_t type, int code) + { + if (!(x.isXMM() && op.is(Operand::XMM | Operand::YMM | Operand::MEM)) && !(x.isYMM() && op.is(Operand::ZMM | Operand::MEM))) XBYAK_THROW(ERR_BAD_COMBINATION) opCvt(x, op, type, code); } - void opCvt3(const Xmm& x1, const Xmm& x2, const Operand& op, uint64_t type, uint64_t type64, uint64_t type32, uint8_t code) + // (x, x, r32/r64/m) : vcvt(u)si2ss/sd/sh (scalar int to XMM) + // type is or-merged with type64/type32, so a packed field (N, map, er/sae, broadcast) must not have + // different non-zero values on both sides (checked by checkTypeMergeable() in the generator) + void opCvtSi2X(const Xmm& x1, const Xmm& x2, const Operand& op, uint64_t type, uint64_t type64, uint64_t type32, uint8_t code) { if (!(x1.isXMM() && x2.isXMM() && (op.isREG(i32e) || op.isMEM()))) XBYAK_THROW(ERR_BAD_SIZE_OF_REGISTER) - Xmm x(op.getIdx()); - const Operand *p = op.isREG() ? &x : &op; - opVex(x1, &x2, *p, type | (op.isBit(64) ? type64 : type32), code); + opVex(x1, &x2, op, type | (op.isBit(64) ? type64 : type32), code); } - // (x, x/y/xword/yword), (y, z/m) - void checkCvt4(const Xmm& x, const Operand& op) const + // (x, x/y/xword/yword), (y, z/m) : opCvt_xx_yx_zy with an explicitly sized mem for an xmm dst, e.g. vcvtdq2ph + void opCvt_xx_yx_zy_sized(const Xmm& x, const Operand& op, uint64_t type, int code) { - if (!(x.isXMM() && op.is(Operand::XMM | Operand::YMM | Operand::MEM) && op.isBit(128|256)) && !(x.isYMM() && op.is(Operand::ZMM | Operand::MEM))) XBYAK_THROW(ERR_BAD_COMBINATION) + if (x.isXMM() && !op.isBit(128|256)) XBYAK_THROW(ERR_BAD_COMBINATION) + opCvt_xx_yx_zy(x, op, type, code); } - // (x, x/y/z/xword/yword/zword) - void opCvt5(const Xmm& x, const Operand& op, uint64_t type, int code) + // (x, x/y/z/xword/yword/zword) : e.g. vcvtpd2ph + void opCvt_xx_yx_zx(const Xmm& x, const Operand& op, uint64_t type, int code) { if (!(x.isXMM() && op.isBit(128|256|512))) XBYAK_THROW(ERR_BAD_COMBINATION) Operand::Kind kind = op.isBit(128) ? Operand::XMM : op.isBit(256) ? Operand::YMM : Operand::ZMM; opVex(x.copyAndSetKind(kind), &xm0, op, type, code); } - // (x, x, x/m), (x, y, y/m), (y, z, z/m) - void opCvt6(const Xmm& x1, const Xmm& x2, const Operand& op, uint64_t type, int code) + // (x, x, x/m), (x, y, y/m), (y, z, z/m) : vcvtbias* + // dstXMM = true : dst is fixed XMM regardless of VL : (x, x, x/m), (x, y, y/m), (x, z, z/m) + void opCvtBias(const Xmm& x1, const Xmm& x2, const Operand& op, uint64_t type, int code, bool dstXMM = false) { - int b1 = x1.getBit(); - int b2 = x2.getBit(); - int b3 = op.getBit(); - if ((b1 == 128 && (b2 == 128 || b2 == 256) && (b2 == b3 || op.isMEM())) || (b1 == 256 && b2 == 512 && (b3 == b2 || op.isMEM()))) { - opVex(x1, &x2, op, type, code); - return; - } - XBYAK_THROW(ERR_BAD_COMBINATION); + uint32_t b2 = x2.getBit(); + uint32_t dstBit = (!dstXMM && b2 == 512) ? 256 : 128; + if (!(x1.getBit() == dstBit && (op.isMEM() || op.getBit() == b2))) XBYAK_THROW(ERR_BAD_COMBINATION) + opVex(x1, &x2, op, type, code); + } + // (r32/r64, x/m) : vcvt*2(u)si (XMM to scalar int), EVEX.W is set if r is 64-bit + void opCvtX2Si(const Reg& r, const Operand& op, uint64_t type, int code) + { + opVex(r, &xm0, op, type | (r.isREG(64) ? T_EW1 : T_W0), code); } const Xmm& cvtIdx0(const Operand& x) const { @@ -2792,25 +3322,14 @@ private: if (!(type & T_M_K) && x.getIdx() == regExp.getIndex().getIdx()) XBYAK_THROW(ERR_SAME_REGS_ARE_INVALID); opVex(x, 0, addr, type, code); } - /* - xx_xy_yz ; mode = true - xx_xy_xz ; mode = false - */ - void opVmov(const Operand& op, const Xmm& x, uint64_t type, uint8_t code, bool mode) - { - if (mode) { - if (!op.isMEM() && !((op.isXMM() && x.isXMM()) || (op.isXMM() && x.isYMM()) || (op.isYMM() && x.isZMM()))) XBYAK_THROW(ERR_BAD_COMBINATION) - } else { - if (!op.isMEM() && !op.isXMM()) XBYAK_THROW(ERR_BAD_COMBINATION) - } - opVex(x, 0, op, type, code); - } void opGatherFetch(const Address& addr, const Xmm& x, uint64_t type, uint8_t code, Operand::Kind kind) { if (addr.hasZero()) XBYAK_THROW(ERR_INVALID_ZERO) if (addr.getRegExp().getIndex().getKind() != kind) XBYAK_THROW(ERR_BAD_VSIB_ADDRESSING) opVex(x, 0, addr, type, code); } + // type is or-merged with typeVex/typeEvex, so a packed field (N, map, er/sae, broadcast) must not have + // different non-zero values on both sides (checked by checkTypeMergeable() in the generator) void opEncoding(const Xmm& x1, const Xmm& x2, const Operand& op, uint64_t type, int code, PreferredEncoding enc, int imm = NONE, uint64_t typeVex = 0, uint64_t typeEvex = 0, int sel = 0) { opAVX_X_X_XM(x1, x2, op, type | orEvexIf(enc, typeVex, typeEvex, sel), code, imm); @@ -2830,49 +3349,48 @@ private: enc = getEncoding(enc, sel); return ((sel == 0 && enc == VexEncoding) || (sel == 1 && enc != AVX10v2Encoding)) ? typeVex : (T_MUST_EVEX | typeEvex); } - void opInOut(const Reg& a, const Reg& d, uint8_t code) + void opInOut(const Reg& a, uint8_t code) { - if (a.getIdx() == Operand::AL && d.getIdx() == Operand::DX && d.getBit() == 16) { - switch (a.getBit()) { - case 8: db(code); return; - case 16: db(0x66); db(code + 1); return; - case 32: db(code + 1); return; - } + switch (a.getBit()) { + case 8: db(code); return; + case 16: db(0x66); db(code + 1); return; + case 32: db(code + 1); return; } XBYAK_THROW(ERR_BAD_COMBINATION) } + void opInOut(const Reg& a, const Reg& d, uint8_t code) + { + if (!(a.getIdx() == Operand::AL && d.getIdx() == Operand::DX && d.getBit() == 16)) XBYAK_THROW(ERR_BAD_COMBINATION) + opInOut(a, code); + } void opInOut(const Reg& a, uint8_t code, uint8_t v) { - if (a.getIdx() == Operand::AL) { - switch (a.getBit()) { - case 8: db(code); db(v); return; - case 16: db(0x66); db(code + 1); db(v); return; - case 32: db(code + 1); db(v); return; - } - } - XBYAK_THROW(ERR_BAD_COMBINATION) + if (a.getIdx() != Operand::AL) XBYAK_THROW(ERR_BAD_COMBINATION) + opInOut(a, code); + db(v); + } + void verifyDfv(int dfv) const + { + if (dfv < 0 || 15 < dfv) XBYAK_THROW(ERR_INVALID_DFV) } void opCcmp(const Operand& op1, const Operand& op2, int dfv, int code, int sc) // cmp = 0x38, test = 0x84 { - if (dfv < 0 || 15 < dfv) XBYAK_THROW(ERR_INVALID_DFV) - opROO(Reg(15 - dfv, Operand::REG, (op1.getBit() | op2.getBit())), op1, op2, T_APX|T_CODE1_IF1, code, 0, sc); + verifyDfv(dfv); + opROO(Reg(15 - dfv, Operand::REG, (op1.getBit() | op2.getBit())), op1, op2, T_APX|T_OP_W1, code, 0, sc); } void opCcmpi(const Operand& op, int imm, int dfv, int sc) { - if (dfv < 0 || 15 < dfv) XBYAK_THROW(ERR_INVALID_DFV) - uint32_t immBit = getImmBit(op, imm); - uint32_t opBit = op.getBit(); - int tmp = immBit < (std::min)(opBit, 32U) ? 2 : 0; - opROO(Reg(15 - dfv, Operand::REG, opBit), op, Reg(15, Operand::REG, opBit), T_APX|T_CODE1_IF1, 0x80 | tmp, immBit / 8, sc); - db(imm, immBit / 8); + verifyDfv(dfv); + verifyMemHasSize(op); + opROI(Reg(15 - dfv, Operand::REG, op.getBit()), op, imm, T_APX|T_OP_W1, 15, sc); } void opTesti(const Operand& op, int imm, int dfv, int sc) { - if (dfv < 0 || 15 < dfv) XBYAK_THROW(ERR_INVALID_DFV) + verifyDfv(dfv); uint32_t opBit = op.getBit(); if (opBit == 0) XBYAK_THROW(ERR_MEM_SIZE_IS_NOT_SPECIFIED); int immBit = (std::min)(opBit, 32U); - opROO(Reg(15 - dfv, Operand::REG, opBit), op, Reg(0, Operand::REG, opBit), T_APX|T_CODE1_IF1, 0xF6, immBit / 8, sc); + opROO(Reg(15 - dfv, Operand::REG, opBit), op, Reg(0, Operand::REG, opBit), T_APX|T_OP_W1, 0xF6, immBit / 8, sc); db(imm, immBit / 8); } void opCfcmov(const Reg& d, const Operand& op1, const Operand& op2, int code) @@ -2895,7 +3413,7 @@ private: Address addr2 = addr.cloneNoOptimize(); // require both base and index for all but opcode 0x49 (ldtilecfg/sttilecfg) if (code != 0x49) { - const RegExp exp = addr2.getRegExp(); + const RegExp& exp = addr2.getRegExp(); if (exp.getBase().getBit() == 0 || exp.getIndex().getBit() == 0) XBYAK_THROW(ERR_NOT_SUPPORTED) } if (opROO(Reg(), addr2, t1, T_APX|type, code)) return; @@ -2926,22 +3444,6 @@ private: if (opROO(Reg(), *p2, *p1, T_APX|type, code)) return; opVex(static_cast(*p1), 0, *p2, type, code); } - void opEncodeKey(const Reg32& r1, const Reg32& r2, uint8_t code1, uint8_t code2) - { - if (r1.getIdx() < 8 && r2.getIdx() < 8) { - db(0xF3); db(0x0F); db(0x38); db(code1); setModRM(3, r1.getIdx(), r2.getIdx()); - return; - } - opROO(Reg(), r2, r1, T_MUST_EVEX|T_F3, code2); - } - void opSSE_APX(const Xmm& x, const Operand& op, uint64_t type1, uint8_t code1, uint64_t type2, uint8_t code2, int imm = NONE) - { - if (x.getIdx() <= 15 && op.hasRex2() && opROO(Reg(), op, x, type2, code2, imm != NONE ? 1 : 0)) { - if (imm != NONE) db(imm); - return; - } - opSSE(x, op, type1, code1, isXMM_XMMorMEM, imm); - } // AVX10 zero-extending for vmovd, vmovw void opAVX10ZeroExt(const Operand& op1, const Operand& op2, const uint64_t typeTbl[4], const int codeTbl[4], PreferredEncoding enc, int bit) { @@ -2970,59 +3472,14 @@ private: public: unsigned int getVersion() const { return VERSION; } using CodeArray::db; - const Mmx mm0, mm1, mm2, mm3, mm4, mm5, mm6, mm7; - const Xmm xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7; - const Ymm ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7; - const Zmm zmm0, zmm1, zmm2, zmm3, zmm4, zmm5, zmm6, zmm7; - const Xmm &xm0, &xm1, &xm2, &xm3, &xm4, &xm5, &xm6, &xm7; - const Ymm &ym0, &ym1, &ym2, &ym3, &ym4, &ym5, &ym6, &ym7; - const Zmm &zm0, &zm1, &zm2, &zm3, &zm4, &zm5, &zm6, &zm7; - const Reg32 eax, ecx, edx, ebx, esp, ebp, esi, edi; - const Reg16 ax, cx, dx, bx, sp, bp, si, di; - const Reg8 al, cl, dl, bl, ah, ch, dh, bh; - const AddressFrame ptr, byte, word, dword, qword, xword, yword, zword; // xword is same as oword of NASM - const AddressFrame ptr_b, xword_b, yword_b, zword_b; // broadcast such as {1to2}, {1to4}, {1to8}, {1to16}, {b} - const Fpu st0, st1, st2, st3, st4, st5, st6, st7; - const Opmask k0, k1, k2, k3, k4, k5, k6, k7; - const BoundsReg bnd0, bnd1, bnd2, bnd3; - const EvexModifierRounding T_sae, T_rn_sae, T_rd_sae, T_ru_sae, T_rz_sae; // {sae}, {rn-sae}, {rd-sae}, {ru-sae}, {rz-sae} - const EvexModifierZero T_z; // {z} - const ApxFlagNF T_nf; - const ApxFlagZU T_zu; -#ifdef XBYAK64 - const Reg64 rax, rcx, rdx, rbx, rsp, rbp, rsi, rdi, r8, r9, r10, r11, r12, r13, r14, r15; - const Reg64 r16, r17, r18, r19, r20, r21, r22, r23, r24, r25, r26, r27, r28, r29, r30, r31; - const Reg32 r8d, r9d, r10d, r11d, r12d, r13d, r14d, r15d; - const Reg32 r16d, r17d, r18d, r19d, r20d, r21d, r22d, r23d, r24d, r25d, r26d, r27d, r28d, r29d, r30d, r31d; - const Reg16 r8w, r9w, r10w, r11w, r12w, r13w, r14w, r15w; - const Reg16 r16w, r17w, r18w, r19w, r20w, r21w, r22w, r23w, r24w, r25w, r26w, r27w, r28w, r29w, r30w, r31w; - const Reg8 r8b, r9b, r10b, r11b, r12b, r13b, r14b, r15b; - const Reg8 r16b, r17b, r18b, r19b, r20b, r21b, r22b, r23b, r24b, r25b, r26b, r27b, r28b, r29b, r30b, r31b; - const Reg8 spl, bpl, sil, dil; - const Xmm xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15; - const Xmm xmm16, xmm17, xmm18, xmm19, xmm20, xmm21, xmm22, xmm23; - const Xmm xmm24, xmm25, xmm26, xmm27, xmm28, xmm29, xmm30, xmm31; - const Ymm ymm8, ymm9, ymm10, ymm11, ymm12, ymm13, ymm14, ymm15; - const Ymm ymm16, ymm17, ymm18, ymm19, ymm20, ymm21, ymm22, ymm23; - const Ymm ymm24, ymm25, ymm26, ymm27, ymm28, ymm29, ymm30, ymm31; - const Zmm zmm8, zmm9, zmm10, zmm11, zmm12, zmm13, zmm14, zmm15; - const Zmm zmm16, zmm17, zmm18, zmm19, zmm20, zmm21, zmm22, zmm23; - const Zmm zmm24, zmm25, zmm26, zmm27, zmm28, zmm29, zmm30, zmm31; - const Tmm tmm0, tmm1, tmm2, tmm3, tmm4, tmm5, tmm6, tmm7; - const Xmm &xm8, &xm9, &xm10, &xm11, &xm12, &xm13, &xm14, &xm15; // for my convenience - const Xmm &xm16, &xm17, &xm18, &xm19, &xm20, &xm21, &xm22, &xm23; - const Xmm &xm24, &xm25, &xm26, &xm27, &xm28, &xm29, &xm30, &xm31; - const Ymm &ym8, &ym9, &ym10, &ym11, &ym12, &ym13, &ym14, &ym15; - const Ymm &ym16, &ym17, &ym18, &ym19, &ym20, &ym21, &ym22, &ym23; - const Ymm &ym24, &ym25, &ym26, &ym27, &ym28, &ym29, &ym30, &ym31; - const Zmm &zm8, &zm9, &zm10, &zm11, &zm12, &zm13, &zm14, &zm15; - const Zmm &zm16, &zm17, &zm18, &zm19, &zm20, &zm21, &zm22, &zm23; - const Zmm &zm24, &zm25, &zm26, &zm27, &zm28, &zm29, &zm30, &zm31; - const RegRip rip; -#endif -#ifndef XBYAK_DISABLE_SEGMENT - const Segment es, cs, ss, ds, fs, gs; +#ifdef XBYAK_USE_CONSTEXPR_REGISTERS + #define XBYAK_DEFINE_REGISTER(Type, name, ...) static constexpr Type name{__VA_ARGS__}; +#else + #define XBYAK_DEFINE_REGISTER(Type, name, ...) const Type name; #endif + XBYAK_FOR_EACH_REGISTER(XBYAK_DEFINE_REGISTER) + XBYAK_FOR_EACH_CONVENIENCE_ALL(XBYAK_DEFINE_REGISTER) + #undef XBYAK_DEFINE_REGISTER private: bool isDefaultJmpNEAR_; PreferredEncoding defaultEncoding_[2]; // 0:vnni, 1:vmpsadbw @@ -3043,8 +3500,8 @@ public: put address of label to buffer @note the put size is 4(32-bit), 8(64-bit) */ - void putL(std::string label) { putL_inner(label); } - void putL(const Label& label) { putL_inner(label); } + void putL(std::string label) { putL_inner(label, inner::Labs, 0, (int)sizeof(size_t)); } + void putL(const Label& label) { putL_inner(label, inner::Labs, 0, (int)sizeof(size_t)); } // set default type of `jmp` of undefined label to T_NEAR void setDefaultJmpNEAR(bool isNear) { isDefaultJmpNEAR_ = isNear; } @@ -3068,7 +3525,7 @@ public: void test(const Operand& op, const Reg& reg) { - opRO(reg, op, 0, 0x84, op.getKind() == reg.getKind()); + opRO(reg, op, T_OP_W1, 0x84, op.getKind() == reg.getKind()); } void test(const Operand& op, uint32_t imm) { @@ -3078,7 +3535,7 @@ public: rex(op); db(0xA8 | (op.isBit(8) ? 0 : 1)); } else { - opRext(op, 0, 0, 0, 0xF6, false, immSize); + opRext(op, 0, 0, T_OP_W1, 0xF6, false, immSize); } db(imm, immSize); } @@ -3134,7 +3591,7 @@ public: rex(*reg); db(op1.isREG(8) ? 0xA0 : op1.isREG() ? 0xA1 : op2.isREG(8) ? 0xA2 : 0xA3); if (addr->getLabel()) { - putL_inner(*addr->getLabel(), false, addr->getDisp() - addr->immSize, 8); + putL_inner(*addr->getLabel(), inner::Labs, addr->getDisp(), 8); } else { db(addr->getDisp(), 8); } @@ -3147,7 +3604,7 @@ public: rex(*reg, *addr); db(code | (reg->isBit(8) ? 0 : 1)); if (addr->getLabel()) { - putL_inner(*addr->getLabel(), false, addr->getDisp() - addr->immSize); + putL_inner(*addr->getLabel(), inner::Labs, addr->getDisp(), 4); } else { dd(static_cast(addr->getDisp())); } @@ -3172,7 +3629,7 @@ public: if (!inner::IsInInt32(imm)) XBYAK_THROW(ERR_IMM_IS_TOO_BIG) immSize = 4; } - opMR(op.getAddress(immSize), Reg(0, Operand::REG, op.getBit()), 0, 0xC6); + opMR(op.getAddress(immSize), Reg(0, Operand::REG, op.getBit()), T_OP_W1, 0xC6); db(static_cast(imm), immSize); } else { XBYAK_THROW(ERR_BAD_COMBINATION) @@ -3238,10 +3695,10 @@ public: void putSeg(const Segment& seg) { switch (seg.getIdx()) { - case Segment::es: db(0x2E); break; - case Segment::cs: db(0x36); break; - case Segment::ss: db(0x3E); break; - case Segment::ds: db(0x26); break; + case Segment::es: db(0x26); break; + case Segment::cs: db(0x2E); break; + case Segment::ss: db(0x36); break; + case Segment::ds: db(0x3E); break; case Segment::fs: db(0x64); break; case Segment::gs: db(0x65); break; default: @@ -3262,61 +3719,11 @@ public: // constructor CodeGenerator(size_t maxSize = DEFAULT_MAX_CODE_SIZE, void *userPtr = 0, Allocator *allocator = 0) : CodeArray(maxSize, userPtr, allocator) - , mm0(0), mm1(1), mm2(2), mm3(3), mm4(4), mm5(5), mm6(6), mm7(7) - , xmm0(0), xmm1(1), xmm2(2), xmm3(3), xmm4(4), xmm5(5), xmm6(6), xmm7(7) - , ymm0(0), ymm1(1), ymm2(2), ymm3(3), ymm4(4), ymm5(5), ymm6(6), ymm7(7) - , zmm0(0), zmm1(1), zmm2(2), zmm3(3), zmm4(4), zmm5(5), zmm6(6), zmm7(7) - // for my convenience - , xm0(xmm0), xm1(xmm1), xm2(xmm2), xm3(xmm3), xm4(xmm4), xm5(xmm5), xm6(xmm6), xm7(xmm7) - , ym0(ymm0), ym1(ymm1), ym2(ymm2), ym3(ymm3), ym4(ymm4), ym5(ymm5), ym6(ymm6), ym7(ymm7) - , zm0(zmm0), zm1(zmm1), zm2(zmm2), zm3(zmm3), zm4(zmm4), zm5(zmm5), zm6(zmm6), zm7(zmm7) - - , eax(Operand::EAX), ecx(Operand::ECX), edx(Operand::EDX), ebx(Operand::EBX), esp(Operand::ESP), ebp(Operand::EBP), esi(Operand::ESI), edi(Operand::EDI) - , ax(Operand::AX), cx(Operand::CX), dx(Operand::DX), bx(Operand::BX), sp(Operand::SP), bp(Operand::BP), si(Operand::SI), di(Operand::DI) - , al(Operand::AL), cl(Operand::CL), dl(Operand::DL), bl(Operand::BL), ah(Operand::AH), ch(Operand::CH), dh(Operand::DH), bh(Operand::BH) - , ptr(0), byte(8), word(16), dword(32), qword(64), xword(128), yword(256), zword(512) - , ptr_b(0, true), xword_b(128, true), yword_b(256, true), zword_b(512, true) - , st0(0), st1(1), st2(2), st3(3), st4(4), st5(5), st6(6), st7(7) - , k0(0), k1(1), k2(2), k3(3), k4(4), k5(5), k6(6), k7(7) - , bnd0(0), bnd1(1), bnd2(2), bnd3(3) - , T_sae(EvexModifierRounding::T_SAE), T_rn_sae(EvexModifierRounding::T_RN_SAE), T_rd_sae(EvexModifierRounding::T_RD_SAE), T_ru_sae(EvexModifierRounding::T_RU_SAE), T_rz_sae(EvexModifierRounding::T_RZ_SAE) - , T_z() - , T_nf() - , T_zu() -#ifdef XBYAK64 - , rax(Operand::RAX), rcx(Operand::RCX), rdx(Operand::RDX), rbx(Operand::RBX), rsp(Operand::RSP), rbp(Operand::RBP), rsi(Operand::RSI), rdi(Operand::RDI), r8(Operand::R8), r9(Operand::R9), r10(Operand::R10), r11(Operand::R11), r12(Operand::R12), r13(Operand::R13), r14(Operand::R14), r15(Operand::R15) - , r16(Operand::R16), r17(Operand::R17), r18(Operand::R18), r19(Operand::R19), r20(Operand::R20), r21(Operand::R21), r22(Operand::R22), r23(Operand::R23), r24(Operand::R24), r25(Operand::R25), r26(Operand::R26), r27(Operand::R27), r28(Operand::R28), r29(Operand::R29), r30(Operand::R30), r31(Operand::R31) - , r8d(8), r9d(9), r10d(10), r11d(11), r12d(12), r13d(13), r14d(14), r15d(15) - , r16d(Operand::R16D), r17d(Operand::R17D), r18d(Operand::R18D), r19d(Operand::R19D), r20d(Operand::R20D), r21d(Operand::R21D), r22d(Operand::R22D), r23d(Operand::R23D), r24d(Operand::R24D), r25d(Operand::R25D), r26d(Operand::R26D), r27d(Operand::R27D), r28d(Operand::R28D), r29d(Operand::R29D), r30d(Operand::R30D), r31d(Operand::R31D) - , r8w(8), r9w(9), r10w(10), r11w(11), r12w(12), r13w(13), r14w(14), r15w(15) - , r16w(Operand::R16W), r17w(Operand::R17W), r18w(Operand::R18W), r19w(Operand::R19W), r20w(Operand::R20W), r21w(Operand::R21W), r22w(Operand::R22W), r23w(Operand::R23W), r24w(Operand::R24W), r25w(Operand::R25W), r26w(Operand::R26W), r27w(Operand::R27W), r28w(Operand::R28W), r29w(Operand::R29W), r30w(Operand::R30W), r31w(Operand::R31W) - , r8b(8), r9b(9), r10b(10), r11b(11), r12b(12), r13b(13), r14b(14), r15b(15) - , r16b(Operand::R16B), r17b(Operand::R17B), r18b(Operand::R18B), r19b(Operand::R19B), r20b(Operand::R20B), r21b(Operand::R21B), r22b(Operand::R22B), r23b(Operand::R23B), r24b(Operand::R24B), r25b(Operand::R25B), r26b(Operand::R26B), r27b(Operand::R27B), r28b(Operand::R28B), r29b(Operand::R29B), r30b(Operand::R30B), r31b(Operand::R31B) - , spl(Operand::SPL, true), bpl(Operand::BPL, true), sil(Operand::SIL, true), dil(Operand::DIL, true) - , xmm8(8), xmm9(9), xmm10(10), xmm11(11), xmm12(12), xmm13(13), xmm14(14), xmm15(15) - , xmm16(16), xmm17(17), xmm18(18), xmm19(19), xmm20(20), xmm21(21), xmm22(22), xmm23(23) - , xmm24(24), xmm25(25), xmm26(26), xmm27(27), xmm28(28), xmm29(29), xmm30(30), xmm31(31) - , ymm8(8), ymm9(9), ymm10(10), ymm11(11), ymm12(12), ymm13(13), ymm14(14), ymm15(15) - , ymm16(16), ymm17(17), ymm18(18), ymm19(19), ymm20(20), ymm21(21), ymm22(22), ymm23(23) - , ymm24(24), ymm25(25), ymm26(26), ymm27(27), ymm28(28), ymm29(29), ymm30(30), ymm31(31) - , zmm8(8), zmm9(9), zmm10(10), zmm11(11), zmm12(12), zmm13(13), zmm14(14), zmm15(15) - , zmm16(16), zmm17(17), zmm18(18), zmm19(19), zmm20(20), zmm21(21), zmm22(22), zmm23(23) - , zmm24(24), zmm25(25), zmm26(26), zmm27(27), zmm28(28), zmm29(29), zmm30(30), zmm31(31) - , tmm0(0), tmm1(1), tmm2(2), tmm3(3), tmm4(4), tmm5(5), tmm6(6), tmm7(7) - // for my convenience - , xm8(xmm8), xm9(xmm9), xm10(xmm10), xm11(xmm11), xm12(xmm12), xm13(xmm13), xm14(xmm14), xm15(xmm15) - , xm16(xmm16), xm17(xmm17), xm18(xmm18), xm19(xmm19), xm20(xmm20), xm21(xmm21), xm22(xmm22), xm23(xmm23) - , xm24(xmm24), xm25(xmm25), xm26(xmm26), xm27(xmm27), xm28(xmm28), xm29(xmm29), xm30(xmm30), xm31(xmm31) - , ym8(ymm8), ym9(ymm9), ym10(ymm10), ym11(ymm11), ym12(ymm12), ym13(ymm13), ym14(ymm14), ym15(ymm15) - , ym16(ymm16), ym17(ymm17), ym18(ymm18), ym19(ymm19), ym20(ymm20), ym21(ymm21), ym22(ymm22), ym23(ymm23) - , ym24(ymm24), ym25(ymm25), ym26(ymm26), ym27(ymm27), ym28(ymm28), ym29(ymm29), ym30(ymm30), ym31(ymm31) - , zm8(zmm8), zm9(zmm9), zm10(zmm10), zm11(zmm11), zm12(zmm12), zm13(zmm13), zm14(zmm14), zm15(zmm15) - , zm16(zmm16), zm17(zmm17), zm18(zmm18), zm19(zmm19), zm20(zmm20), zm21(zmm21), zm22(zmm22), zm23(zmm23) - , zm24(zmm24), zm25(zmm25), zm26(zmm26), zm27(zmm27), zm28(zmm28), zm29(zmm29), zm30(zmm30), zm31(zmm31) - , rip() -#endif -#ifndef XBYAK_DISABLE_SEGMENT - , es(Segment::es), cs(Segment::cs), ss(Segment::ss), ds(Segment::ds), fs(Segment::fs), gs(Segment::gs) +#ifndef XBYAK_USE_CONSTEXPR_REGISTERS + #define XBYAK_INIT_REGISTER(Type, name, ...) , name(__VA_ARGS__) + XBYAK_FOR_EACH_REGISTER(XBYAK_INIT_REGISTER) + XBYAK_FOR_EACH_CONVENIENCE_ALL(XBYAK_INIT_REGISTER) + #undef XBYAK_INIT_REGISTER #endif , isDefaultJmpNEAR_(false) { @@ -3330,6 +3737,7 @@ public: resetSize(); labelMgr_.reset(); labelMgr_.set(this); + if (isAllocType() && useProtect() && curMode_ == PROTECT_RE) setProtectModeRW(); } bool hasUndefinedLabel() const { return labelMgr_.hasUndefSlabel() || labelMgr_.hasUndefClabel(); } /* @@ -3475,46 +3883,9 @@ inline void CodeGenerator::mov(const NativeReg& reg, const char *label) // can't } namespace util { -static const XBYAK_CONSTEXPR Mmx mm0(0), mm1(1), mm2(2), mm3(3), mm4(4), mm5(5), mm6(6), mm7(7); -static const XBYAK_CONSTEXPR Xmm xmm0(0), xmm1(1), xmm2(2), xmm3(3), xmm4(4), xmm5(5), xmm6(6), xmm7(7); -static const XBYAK_CONSTEXPR Ymm ymm0(0), ymm1(1), ymm2(2), ymm3(3), ymm4(4), ymm5(5), ymm6(6), ymm7(7); -static const XBYAK_CONSTEXPR Zmm zmm0(0), zmm1(1), zmm2(2), zmm3(3), zmm4(4), zmm5(5), zmm6(6), zmm7(7); -static const XBYAK_CONSTEXPR Reg32 eax(Operand::EAX), ecx(Operand::ECX), edx(Operand::EDX), ebx(Operand::EBX), esp(Operand::ESP), ebp(Operand::EBP), esi(Operand::ESI), edi(Operand::EDI); -static const XBYAK_CONSTEXPR Reg16 ax(Operand::AX), cx(Operand::CX), dx(Operand::DX), bx(Operand::BX), sp(Operand::SP), bp(Operand::BP), si(Operand::SI), di(Operand::DI); -static const XBYAK_CONSTEXPR Reg8 al(Operand::AL), cl(Operand::CL), dl(Operand::DL), bl(Operand::BL), ah(Operand::AH), ch(Operand::CH), dh(Operand::DH), bh(Operand::BH); -static const XBYAK_CONSTEXPR AddressFrame ptr(0), byte(8), word(16), dword(32), qword(64), xword(128), yword(256), zword(512); -static const XBYAK_CONSTEXPR AddressFrame ptr_b(0, true), xword_b(128, true), yword_b(256, true), zword_b(512, true); -static const XBYAK_CONSTEXPR Fpu st0(0), st1(1), st2(2), st3(3), st4(4), st5(5), st6(6), st7(7); -static const XBYAK_CONSTEXPR Opmask k0(0), k1(1), k2(2), k3(3), k4(4), k5(5), k6(6), k7(7); -static const XBYAK_CONSTEXPR BoundsReg bnd0(0), bnd1(1), bnd2(2), bnd3(3); -static const XBYAK_CONSTEXPR EvexModifierRounding T_sae(EvexModifierRounding::T_SAE), T_rn_sae(EvexModifierRounding::T_RN_SAE), T_rd_sae(EvexModifierRounding::T_RD_SAE), T_ru_sae(EvexModifierRounding::T_RU_SAE), T_rz_sae(EvexModifierRounding::T_RZ_SAE); -static const XBYAK_CONSTEXPR EvexModifierZero T_z; -#ifdef XBYAK64 -static const XBYAK_CONSTEXPR Reg64 rax(Operand::RAX), rcx(Operand::RCX), rdx(Operand::RDX), rbx(Operand::RBX), rsp(Operand::RSP), rbp(Operand::RBP), rsi(Operand::RSI), rdi(Operand::RDI), r8(Operand::R8), r9(Operand::R9), r10(Operand::R10), r11(Operand::R11), r12(Operand::R12), r13(Operand::R13), r14(Operand::R14), r15(Operand::R15); -static const XBYAK_CONSTEXPR Reg64 r16(16), r17(17), r18(18), r19(19), r20(20), r21(21), r22(22), r23(23), r24(24), r25(25), r26(26), r27(27), r28(28), r29(29), r30(30), r31(31); -static const XBYAK_CONSTEXPR Reg32 r8d(8), r9d(9), r10d(10), r11d(11), r12d(12), r13d(13), r14d(14), r15d(15); -static const XBYAK_CONSTEXPR Reg32 r16d(16), r17d(17), r18d(18), r19d(19), r20d(20), r21d(21), r22d(22), r23d(23), r24d(24), r25d(25), r26d(26), r27d(27), r28d(28), r29d(29), r30d(30), r31d(31); -static const XBYAK_CONSTEXPR Reg16 r8w(8), r9w(9), r10w(10), r11w(11), r12w(12), r13w(13), r14w(14), r15w(15); -static const XBYAK_CONSTEXPR Reg16 r16w(16), r17w(17), r18w(18), r19w(19), r20w(20), r21w(21), r22w(22), r23w(23), r24w(24), r25w(25), r26w(26), r27w(27), r28w(28), r29w(29), r30w(30), r31w(31); -static const XBYAK_CONSTEXPR Reg8 r8b(8), r9b(9), r10b(10), r11b(11), r12b(12), r13b(13), r14b(14), r15b(15), spl(Operand::SPL, true), bpl(Operand::BPL, true), sil(Operand::SIL, true), dil(Operand::DIL, true); -static const XBYAK_CONSTEXPR Reg8 r16b(16), r17b(17), r18b(18), r19b(19), r20b(20), r21b(21), r22b(22), r23b(23), r24b(24), r25b(25), r26b(26), r27b(27), r28b(28), r29b(29), r30b(30), r31b(31); -static const XBYAK_CONSTEXPR Xmm xmm8(8), xmm9(9), xmm10(10), xmm11(11), xmm12(12), xmm13(13), xmm14(14), xmm15(15); -static const XBYAK_CONSTEXPR Xmm xmm16(16), xmm17(17), xmm18(18), xmm19(19), xmm20(20), xmm21(21), xmm22(22), xmm23(23); -static const XBYAK_CONSTEXPR Xmm xmm24(24), xmm25(25), xmm26(26), xmm27(27), xmm28(28), xmm29(29), xmm30(30), xmm31(31); -static const XBYAK_CONSTEXPR Ymm ymm8(8), ymm9(9), ymm10(10), ymm11(11), ymm12(12), ymm13(13), ymm14(14), ymm15(15); -static const XBYAK_CONSTEXPR Ymm ymm16(16), ymm17(17), ymm18(18), ymm19(19), ymm20(20), ymm21(21), ymm22(22), ymm23(23); -static const XBYAK_CONSTEXPR Ymm ymm24(24), ymm25(25), ymm26(26), ymm27(27), ymm28(28), ymm29(29), ymm30(30), ymm31(31); -static const XBYAK_CONSTEXPR Zmm zmm8(8), zmm9(9), zmm10(10), zmm11(11), zmm12(12), zmm13(13), zmm14(14), zmm15(15); -static const XBYAK_CONSTEXPR Zmm zmm16(16), zmm17(17), zmm18(18), zmm19(19), zmm20(20), zmm21(21), zmm22(22), zmm23(23); -static const XBYAK_CONSTEXPR Zmm zmm24(24), zmm25(25), zmm26(26), zmm27(27), zmm28(28), zmm29(29), zmm30(30), zmm31(31); -static const XBYAK_CONSTEXPR Zmm tmm0(0), tmm1(1), tmm2(2), tmm3(3), tmm4(4), tmm5(5), tmm6(6), tmm7(7); -static const XBYAK_CONSTEXPR RegRip rip; -static const XBYAK_CONSTEXPR ApxFlagNF T_nf; -static const XBYAK_CONSTEXPR ApxFlagZU T_zu; -#endif -#ifndef XBYAK_DISABLE_SEGMENT -static const XBYAK_CONSTEXPR Segment es(Segment::es), cs(Segment::cs), ss(Segment::ss), ds(Segment::ds), fs(Segment::fs), gs(Segment::gs); -#endif +#define XBYAK_DEFINE_UTIL_REGISTER(Type, name, ...) static const XBYAK_CONSTEXPR Type name(__VA_ARGS__); +XBYAK_FOR_EACH_REGISTER(XBYAK_DEFINE_UTIL_REGISTER) +#undef XBYAK_DEFINE_UTIL_REGISTER } // util #ifdef _MSC_VER diff --git a/3rdparty/xbyak/xbyak/xbyak_mnemonic.h b/3rdparty/xbyak/xbyak/xbyak_mnemonic.h index 94288e7de8..c66fc40c8b 100644 --- a/3rdparty/xbyak/xbyak/xbyak_mnemonic.h +++ b/3rdparty/xbyak/xbyak/xbyak_mnemonic.h @@ -1,16 +1,16 @@ -const char *getVersionString() const { return "7.37"; } +const char *getVersionString() const { return "7.41.1"; } void aadd(const Address& addr, const Reg32e ®) { opMR(addr, reg, T_0F38, 0x0FC, T_APX); } void aand(const Address& addr, const Reg32e ®) { opMR(addr, reg, T_0F38|T_66, 0x0FC, T_APX|T_66); } void adc(const Operand& op, uint32_t imm) { opOI(op, imm, 0x10, 2); } void adc(const Operand& op1, const Operand& op2) { opRO_MR(op1, op2, 0x10); } -void adc(const Reg& d, const Operand& op, uint32_t imm) { opROI(d, op, imm, T_NONE, 2); } -void adc(const Reg& d, const Operand& op1, const Operand& op2) { opROO(d, op1, op2, T_NONE, 0x10); } +void adc(const Reg& d, const Operand& op, uint32_t imm) { opROI(d, op, imm, T_OP_W1, 2); } +void adc(const Reg& d, const Operand& op1, const Operand& op2) { opROO(d, op1, op2, T_OP_W1, 0x10); } void adcx(const Reg32e& d, const Reg32e& reg, const Operand& op) { opROO(d, op, reg, T_66, 0x66); } void adcx(const Reg32e& reg, const Operand& op) { if (!reg.isREG(16|i32e) && reg.getBit() == op.getBit()) XBYAK_THROW(ERR_BAD_SIZE_OF_REGISTER) if (opROO(Reg(), op, reg, T_66, 0x66)) return; opRO(reg, op, T_66 | T_0F38, 0xF6); } void add(const Operand& op, uint32_t imm) { opOI(op, imm, 0x00, 0); } void add(const Operand& op1, const Operand& op2) { opRO_MR(op1, op2, 0x00); } -void add(const Reg& d, const Operand& op, uint32_t imm) { opROI(d, op, imm, T_NF|T_CODE1_IF1, 0); } -void add(const Reg& d, const Operand& op1, const Operand& op2) { opROO(d, op1, op2, T_NF|T_CODE1_IF1, 0x00); } +void add(const Reg& d, const Operand& op, uint32_t imm) { opROI(d, op, imm, T_NF|T_OP_W1, 0); } +void add(const Reg& d, const Operand& op1, const Operand& op2) { opROO(d, op1, op2, T_NF|T_OP_W1, 0x00); } void addpd(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_0F | T_66, 0x58, isXMM_XMMorMEM); } void addps(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_0F, 0x58, isXMM_XMMorMEM); } void addsd(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_0F | T_F2, 0x58, isXMM_XMMorMEM); } @@ -27,8 +27,8 @@ void aesimc(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_66|T_0F38|T_W0 void aeskeygenassist(const Xmm& xmm, const Operand& op, uint8_t imm) { opSSE(xmm, op, T_66|T_0F3A, 0xDF, isXMM_XMMorMEM, imm); } void and_(const Operand& op, uint32_t imm) { opOI(op, imm, 0x20, 4); } void and_(const Operand& op1, const Operand& op2) { opRO_MR(op1, op2, 0x20); } -void and_(const Reg& d, const Operand& op, uint32_t imm) { opROI(d, op, imm, T_NF|T_CODE1_IF1, 4); } -void and_(const Reg& d, const Operand& op1, const Operand& op2) { opROO(d, op1, op2, T_NF|T_CODE1_IF1, 0x20); } +void and_(const Reg& d, const Operand& op, uint32_t imm) { opROI(d, op, imm, T_NF|T_OP_W1, 4); } +void and_(const Reg& d, const Operand& op1, const Operand& op2) { opROO(d, op1, op2, T_NF|T_OP_W1, 0x20); } void andn(const Reg32e& r1, const Reg32e& r2, const Operand& op) { opRRO(r1, r2, op, T_APX|T_0F38|T_NF, 0xf2); } void andnpd(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_0F | T_66, 0x55, isXMM_XMMorMEM); } void andnps(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_0F, 0x55, isXMM_XMMorMEM); } @@ -156,11 +156,11 @@ void cfcmovz(const Operand& op1, const Operand& op2) { opCfcmov(Reg(), op1, op2, void cfcmovz(const Reg& d, const Reg& r, const Operand& op) { opCfcmov(d|T_nf, op, r, 0x44); } void clc() { db(0xF8); } void cld() { db(0xFC); } -void cldemote(const Address& addr) { opMR(addr, eax, T_0F, 0x1C); } -void clflush(const Address& addr) { opMR(addr, Reg32(7), T_0F, 0xAE); } -void clflushopt(const Address& addr) { opMR(addr, Reg32(7), T_66 | T_0F, 0xAE); } +void cldemote(const Address& addr) { opMR(addr, eax, T_0F|T_ALLOW_DIFF_SIZE, 0x1C); } +void clflush(const Address& addr) { opMR(addr, Reg32(7), T_0F|T_ALLOW_DIFF_SIZE, 0xAE); } +void clflushopt(const Address& addr) { opMR(addr, Reg32(7), T_66|T_0F|T_ALLOW_DIFF_SIZE, 0xAE); } void cli() { db(0xFA); } -void clwb(const Address& addr) { opMR(addr, esi, T_66 | T_0F, 0xAE); } +void clwb(const Address& addr) { opMR(addr, esi, T_66|T_0F|T_ALLOW_DIFF_SIZE, 0xAE); } void clzero() { db(0x0F); db(0x01); db(0xFC); } void cmc() { db(0xF5); } void cmova(const Reg& d, const Reg& reg, const Operand& op) { opROO(d, op, reg, T_APX|T_ND1, 0x40 | 7); }//-V524 @@ -265,7 +265,7 @@ void cmpunordps(const Xmm& x, const Operand& op) { cmpps(x, op, 3); } void cmpunordsd(const Xmm& x, const Operand& op) { cmpsd(x, op, 3); } void cmpunordss(const Xmm& x, const Operand& op) { cmpss(x, op, 3); } void cmpxchg(const Operand& op, const Reg& reg) { opRO(reg, op, T_0F, 0xB0 | (reg.isBit(8) ? 0 : 1), op.getBit() == reg.getBit()); } -void cmpxchg8b(const Address& addr) { opMR(addr, Reg32(1), T_0F, 0xC7); } +void cmpxchg8b(const Address& addr) { opMR(addr, Reg32(1), T_0F|T_ALLOW_DIFF_SIZE, 0xC7); } void comisd(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_66|T_0F, 0x2F, isXMM_XMMorMEM); } void comiss(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_0F, 0x2F, isXMM_XMMorMEM); } void cpuid() { db(0x0F); db(0xA2); } @@ -352,7 +352,7 @@ void cwd() { db(0x66); db(0x99); } void cwde() { db(0x98); } void dec(const Operand& op) { opIncDec(Reg(), op, 1); } void dec(const Reg& d, const Operand& op) { opIncDec(d, op, 1); } -void div(const Operand& op) { opRext(op, 0, 6, T_APX|T_NF|T_CODE1_IF1, 0xF6); } +void div(const Operand& op) { opRext(op, 0, 6, T_APX|T_NF|T_OP_W1, 0xF6); } void divpd(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_0F | T_66, 0x5E, isXMM_XMMorMEM); } void divps(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_0F, 0x5E, isXMM_XMMorMEM); } void divsd(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_0F | T_F2, 0x5E, isXMM_XMMorMEM); } @@ -372,8 +372,8 @@ void fadd(const Fpu& reg1, const Fpu& reg2) { opFpuFpu(reg1, reg2, 0xD8C0, 0xDCC void faddp() { db(0xDE); db(0xC1); } void faddp(const Fpu& reg1) { opFpuFpu(reg1, st0, 0x0000, 0xDEC0); } void faddp(const Fpu& reg1, const Fpu& reg2) { opFpuFpu(reg1, reg2, 0x0000, 0xDEC0); } -void fbld(const Address& addr) { opMR(addr, Reg32(4), 0, 0xDF); } -void fbstp(const Address& addr) { opMR(addr, Reg32(6), 0, 0xDF); } +void fbld(const Address& addr) { opMR(addr, Reg32(4), T_ALLOW_DIFF_SIZE, 0xDF); } +void fbstp(const Address& addr) { opMR(addr, Reg32(6), T_ALLOW_DIFF_SIZE, 0xDF); } void fchs() { db(0xD9); db(0xE0); } void fclex() { db(0x9B); db(0xDB); db(0xE2); } void fcmovb(const Fpu& reg1) { opFpuFpu(st0, reg1, 0xDAC0, 0x00C0); } @@ -435,8 +435,8 @@ void fisubr(const Address& addr) { opFpuMem(addr, 0xDE, 0xDA, 0x00, 5, 0); } void fld(const Address& addr) { opFpuMem(addr, 0x00, 0xD9, 0xDD, 0, 0); } void fld(const Fpu& reg) { opFpu(reg, 0xD9, 0xC0); } void fld1() { db(0xD9); db(0xE8); } -void fldcw(const Address& addr) { opMR(addr, Reg32(5), 0, 0xD9); } -void fldenv(const Address& addr) { opMR(addr, Reg32(4), 0, 0xD9); } +void fldcw(const Address& addr) { opMR(addr, Reg32(5), T_ALLOW_DIFF_SIZE, 0xD9); } +void fldenv(const Address& addr) { opMR(addr, Reg32(4), T_ALLOW_DIFF_SIZE, 0xD9); } void fldl2e() { db(0xD9); db(0xEA); } void fldl2t() { db(0xD9); db(0xE9); } void fldlg2() { db(0xD9); db(0xEC); } @@ -452,29 +452,29 @@ void fmulp(const Fpu& reg1, const Fpu& reg2) { opFpuFpu(reg1, reg2, 0x0000, 0xDE void fnclex() { db(0xDB); db(0xE2); } void fninit() { db(0xDB); db(0xE3); } void fnop() { db(0xD9); db(0xD0); } -void fnsave(const Address& addr) { opMR(addr, Reg32(6), 0, 0xDD); } -void fnstcw(const Address& addr) { opMR(addr, Reg32(7), 0, 0xD9); } -void fnstenv(const Address& addr) { opMR(addr, Reg32(6), 0, 0xD9); } -void fnstsw(const Address& addr) { opMR(addr, Reg32(7), 0, 0xDD); } +void fnsave(const Address& addr) { opMR(addr, Reg32(6), T_ALLOW_DIFF_SIZE, 0xDD); } +void fnstcw(const Address& addr) { opMR(addr, Reg32(7), T_ALLOW_DIFF_SIZE, 0xD9); } +void fnstenv(const Address& addr) { opMR(addr, Reg32(6), T_ALLOW_DIFF_SIZE, 0xD9); } +void fnstsw(const Address& addr) { opMR(addr, Reg32(7), T_ALLOW_DIFF_SIZE, 0xDD); } void fnstsw(const Reg16& r) { if (r.getIdx() != Operand::AX) XBYAK_THROW(ERR_BAD_PARAMETER) db(0xDF); db(0xE0); } void fpatan() { db(0xD9); db(0xF3); } void fprem() { db(0xD9); db(0xF8); } void fprem1() { db(0xD9); db(0xF5); } void fptan() { db(0xD9); db(0xF2); } void frndint() { db(0xD9); db(0xFC); } -void frstor(const Address& addr) { opMR(addr, Reg32(4), 0, 0xDD); } -void fsave(const Address& addr) { db(0x9B); opMR(addr, Reg32(6), 0, 0xDD); } +void frstor(const Address& addr) { opMR(addr, Reg32(4), T_ALLOW_DIFF_SIZE, 0xDD); } +void fsave(const Address& addr) { db(0x9B); opMR(addr, Reg32(6), T_ALLOW_DIFF_SIZE, 0xDD); } void fscale() { db(0xD9); db(0xFD); } void fsin() { db(0xD9); db(0xFE); } void fsincos() { db(0xD9); db(0xFB); } void fsqrt() { db(0xD9); db(0xFA); } void fst(const Address& addr) { opFpuMem(addr, 0x00, 0xD9, 0xDD, 2, 0); } void fst(const Fpu& reg) { opFpu(reg, 0xDD, 0xD0); } -void fstcw(const Address& addr) { db(0x9B); opMR(addr, Reg32(7), 0, 0xD9); } -void fstenv(const Address& addr) { db(0x9B); opMR(addr, Reg32(6), 0, 0xD9); } +void fstcw(const Address& addr) { db(0x9B); opMR(addr, Reg32(7), T_ALLOW_DIFF_SIZE, 0xD9); } +void fstenv(const Address& addr) { db(0x9B); opMR(addr, Reg32(6), T_ALLOW_DIFF_SIZE, 0xD9); } void fstp(const Address& addr) { opFpuMem(addr, 0x00, 0xD9, 0xDD, 3, 0); } void fstp(const Fpu& reg) { opFpu(reg, 0xDD, 0xD8); } -void fstsw(const Address& addr) { db(0x9B); opMR(addr, Reg32(7), 0, 0xDD); } +void fstsw(const Address& addr) { db(0x9B); opMR(addr, Reg32(7), T_ALLOW_DIFF_SIZE, 0xDD); } void fstsw(const Reg16& r) { if (r.getIdx() != Operand::AX) XBYAK_THROW(ERR_BAD_PARAMETER) db(0x9B); db(0xDF); db(0xE0); } void fsub(const Address& addr) { opFpuMem(addr, 0x00, 0xD8, 0xDC, 4, 0); } void fsub(const Fpu& reg1) { opFpuFpu(st0, reg1, 0xD8E0, 0xDCE8); } @@ -502,7 +502,7 @@ void fwait() { db(0x9B); } void fxam() { db(0xD9); db(0xE5); } void fxch() { db(0xD9); db(0xC9); } void fxch(const Fpu& reg) { opFpu(reg, 0xD9, 0xC8); } -void fxrstor(const Address& addr) { opMR(addr, Reg32(1), T_0F, 0xAE); } +void fxrstor(const Address& addr) { opMR(addr, Reg32(1), T_0F|T_ALLOW_DIFF_SIZE, 0xAE); } void fxtract() { db(0xD9); db(0xF4); } void fyl2x() { db(0xD9); db(0xF1); } void fyl2xp1() { db(0xD9); db(0xF9); } @@ -514,8 +514,8 @@ void haddps(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_F2|T_0F|T_YMM, void hlt() { db(0xF4); } void hsubpd(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_66|T_0F|T_YMM, 0x7D, isXMM_XMMorMEM); } void hsubps(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_F2|T_0F|T_YMM, 0x7D, isXMM_XMMorMEM); } -void idiv(const Operand& op) { opRext(op, 0, 7, T_APX|T_NF|T_CODE1_IF1, 0xF6); } -void imul(const Operand& op) { opRext(op, 0, 5, T_APX|T_NF|T_CODE1_IF1, 0xF6); } +void idiv(const Operand& op) { opRext(op, 0, 7, T_APX|T_NF|T_OP_W1, 0xF6); } +void imul(const Operand& op) { opRext(op, 0, 5, T_APX|T_NF|T_OP_W1, 0xF6); } void imul(const Reg& d, const Reg& reg, const Operand& op) { opROO(d, op, reg, T_APX|T_ND1|T_NF, 0xAF); } void imul(const Reg& reg, const Operand& op) { if (opROO(Reg(), op, reg, T_APX|T_NF, 0xAF)) return; opRO(reg, op, T_0F, 0xAF, reg.getKind() == op.getKind()); } void in_(const Reg& a, const Reg& d) { opInOut(a, d, 0xEC); } @@ -648,11 +648,11 @@ void jz(std::string label, LabelType type = T_AUTO) { opJmp(label, type, 0x74, 0 void lahf() { db(0x9F); } void lddqu(const Xmm& xmm, const Address& addr) { opSSE(xmm, addr, T_F2 | T_0F, 0xF0); } void ldmxcsr(const Address& addr) { opMR(addr, Reg32(2), T_0F, 0xAE); } -void lea(const Reg& reg, const Address& addr) { if (!reg.isBit(16 | i32e)) XBYAK_THROW(ERR_BAD_SIZE_OF_REGISTER) opMR(addr, reg, 0, 0x8D); } +void lea(const Reg& reg, const Address& addr) { if (!reg.isBit(16 | i32e)) XBYAK_THROW(ERR_BAD_SIZE_OF_REGISTER) opMR(addr, reg, T_ALLOW_DIFF_SIZE, 0x8D); } void leave() { db(0xC9); } void lfence() { db(0x0F); db(0xAE); db(0xE8); } -void lfs(const Reg& reg, const Address& addr) { opLoadSeg(addr, reg, T_0F, 0xB4); } -void lgs(const Reg& reg, const Address& addr) { opLoadSeg(addr, reg, T_0F, 0xB5); } +void lfs(const Reg& reg, const Address& addr) { opLoadSeg(addr, reg, T_0F|T_ALLOW_DIFF_SIZE, 0xB4); } +void lgs(const Reg& reg, const Address& addr) { opLoadSeg(addr, reg, T_0F|T_ALLOW_DIFF_SIZE, 0xB5); } void lock() { db(0xF0); } void lodsb() { db(0xAC); } void lodsd() { db(0xAD); } @@ -666,7 +666,7 @@ void loope(std::string label) { opJmp(label, T_SHORT, 0xE1, 0, 0); } void loopne(const Label& label) { opJmp(label, T_SHORT, 0xE0, 0, 0); } void loopne(const char *label) { loopne(std::string(label)); } void loopne(std::string label) { opJmp(label, T_SHORT, 0xE0, 0, 0); } -void lss(const Reg& reg, const Address& addr) { opLoadSeg(addr, reg, T_0F, 0xB2); } +void lss(const Reg& reg, const Address& addr) { opLoadSeg(addr, reg, T_0F|T_ALLOW_DIFF_SIZE, 0xB2); } void lzcnt(const Reg®, const Operand& op) { if (opROO(Reg(), op, reg, T_APX|T_NF, 0xF5)) return; opCnt(reg, op, 0xBD); } void maskmovdqu(const Xmm& reg1, const Xmm& reg2) { opSSE(reg1, reg2, T_66|T_0F, 0xF7); } void maskmovq(const Mmx& reg1, const Mmx& reg2) { opSSE(reg1, reg2, T_0F, 0xF7); } @@ -690,7 +690,7 @@ void movbe(const Reg& reg, const Address& addr) { opMR(addr, reg, T_0F38, 0xF0, void movd(const Mmx& mmx, const Operand& op) { if (!(op.isMEM() || op.isREG(32))) XBYAK_THROW(ERR_BAD_COMBINATION) if (mmx.isXMM()) db(0x66); opSSE(mmx, op, T_0F | T_ALLOW_DIFF_SIZE, 0x6E); } void movd(const Operand& op, const Mmx& mmx) { if (!(op.isMEM() || op.isREG(32))) XBYAK_THROW(ERR_BAD_COMBINATION) if (mmx.isXMM()) db(0x66); opSSE(mmx, op, T_0F | T_ALLOW_DIFF_SIZE, 0x7E); } void movddup(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_DUP|T_F2|T_0F|T_EW1|T_YMM|T_EVEX, 0x12, isXMM_XMMorMEM, NONE); } -void movdir64b(const Reg& reg, const Address& addr) { opMR(addr, reg.cvt32(), T_66|T_0F38, 0xF8, T_APX|T_66); } +void movdir64b(const Reg& reg, const Address& addr) { opMR(addr, reg.cvt32(), T_66|T_0F38|T_ALLOW_DIFF_SIZE, 0xF8, T_APX|T_66); } void movdiri(const Address& addr, const Reg32e& reg) { opMR(addr, reg, T_0F38, 0xF9, T_APX); } void movdq2q(const Mmx& mmx, const Xmm& xmm) { opSSE(mmx, xmm, T_F2 | T_0F, 0xD6); } void movdqa(const Address& addr, const Xmm& xmm) { opSSE(xmm, addr, T_0F|T_66, 0x7F); } @@ -730,7 +730,7 @@ void movups(const Address& addr, const Xmm& xmm) { opSSE(xmm, addr, T_0F|T_NONE, void movups(const Xmm& xmm, const Operand& op) { opMMX(xmm, op, 0x10, T_0F, T_NONE); } void movzx(const Reg& reg, const Operand& op) { opMovxx(reg, op, 0xB6); } void mpsadbw(const Xmm& xmm, const Operand& op, int imm) { opSSE(xmm, op, T_66 | T_0F3A, 0x42, isXMM_XMMorMEM, static_cast(imm)); } -void mul(const Operand& op) { opRext(op, 0, 4, T_APX|T_NF|T_CODE1_IF1, 0xF6); } +void mul(const Operand& op) { opRext(op, 0, 4, T_APX|T_NF|T_OP_W1, 0xF6); } void mulpd(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_0F | T_66, 0x59, isXMM_XMMorMEM); } void mulps(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_0F, 0x59, isXMM_XMMorMEM); } void mulsd(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_0F | T_F2, 0x59, isXMM_XMMorMEM); } @@ -738,14 +738,14 @@ void mulss(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_0F | T_F3, 0x59 void mulx(const Reg32e& r1, const Reg32e& r2, const Operand& op) { opRRO(r1, r2, op, T_APX|T_F2|T_0F38, 0xf6); } void mwait() { db(0x0F); db(0x01); db(0xC9); } void mwaitx() { db(0x0F); db(0x01); db(0xFB); } -void neg(const Operand& op) { opRext(op, 0, 3, T_APX|T_NF|T_CODE1_IF1, 0xF6); } -void neg(const Reg& d, const Operand& op) { opROO(d, op, Reg(3, Operand::REG, d.getBit()), T_APX|T_NF|T_CODE1_IF1|T_ND1, 0xF6); } -void not_(const Operand& op) { opRext(op, 0, 2, T_APX|T_CODE1_IF1, 0xF6); } -void not_(const Reg& d, const Operand& op) { opROO(d, op, Reg(2, Operand::REG, d.getBit()), T_APX|T_CODE1_IF1|T_ND1, 0xF6); } +void neg(const Operand& op) { opRext(op, 0, 3, T_APX|T_NF|T_OP_W1, 0xF6); } +void neg(const Reg& d, const Operand& op) { opROO(d, op, Reg(3, Operand::REG, d.getBit()), T_APX|T_NF|T_OP_W1|T_ND1, 0xF6); } +void not_(const Operand& op) { opRext(op, 0, 2, T_APX|T_OP_W1, 0xF6); } +void not_(const Reg& d, const Operand& op) { opROO(d, op, Reg(2, Operand::REG, d.getBit()), T_APX|T_OP_W1|T_ND1, 0xF6); } void or_(const Operand& op, uint32_t imm) { opOI(op, imm, 0x08, 1); } void or_(const Operand& op1, const Operand& op2) { opRO_MR(op1, op2, 0x08); } -void or_(const Reg& d, const Operand& op, uint32_t imm) { opROI(d, op, imm, T_NF|T_CODE1_IF1, 1); } -void or_(const Reg& d, const Operand& op1, const Operand& op2) { opROO(d, op1, op2, T_NF|T_CODE1_IF1, 0x08); } +void or_(const Reg& d, const Operand& op, uint32_t imm) { opROI(d, op, imm, T_NF|T_OP_W1, 1); } +void or_(const Reg& d, const Operand& op1, const Operand& op2) { opROO(d, op1, op2, T_NF|T_OP_W1, 0x08); } void orpd(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_0F | T_66, 0x56, isXMM_XMMorMEM); } void orps(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_0F, 0x56, isXMM_XMMorMEM); } void out_(const Reg& d, const Reg& a) { opInOut(a, d, 0xEE); } @@ -842,17 +842,17 @@ void pmulhw(const Mmx& mmx, const Operand& op) { opMMX(mmx, op, 0xE5); } void pmulld(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_66 | T_0F38, 0x40, isXMM_XMMorMEM); } void pmullw(const Mmx& mmx, const Operand& op) { opMMX(mmx, op, 0xD5); } void pmuludq(const Mmx& mmx, const Operand& op) { opMMX(mmx, op, 0xF4); } -void popcnt(const Reg®, const Operand& op) { opCnt(reg, op, 0xB8); } +void popcnt(const Reg®, const Operand& op) { if (opROO(Reg(), op, reg, T_APX|T_NF, 0x88)) return; opCnt(reg, op, 0xB8); } void popf() { db(0x9D); } void por(const Mmx& mmx, const Operand& op) { opMMX(mmx, op, 0xEB); } -void prefetchit0(const Address& addr) { opMR(addr, Reg32(7), T_0F, 0x18); } -void prefetchit1(const Address& addr) { opMR(addr, Reg32(6), T_0F, 0x18); } -void prefetchnta(const Address& addr) { opMR(addr, Reg32(0), T_0F, 0x18); } -void prefetchrst2(const Address& addr) { opMR(addr, Reg32(4), T_0F, 0x18); } -void prefetcht0(const Address& addr) { opMR(addr, Reg32(1), T_0F, 0x18); } -void prefetcht1(const Address& addr) { opMR(addr, Reg32(2), T_0F, 0x18); } -void prefetcht2(const Address& addr) { opMR(addr, Reg32(3), T_0F, 0x18); } -void prefetchw(const Address& addr) { opMR(addr, Reg32(1), T_0F, 0x0D); } +void prefetchit0(const Address& addr) { opMR(addr, Reg32(7), T_0F|T_ALLOW_DIFF_SIZE, 0x18); } +void prefetchit1(const Address& addr) { opMR(addr, Reg32(6), T_0F|T_ALLOW_DIFF_SIZE, 0x18); } +void prefetchnta(const Address& addr) { opMR(addr, Reg32(0), T_0F|T_ALLOW_DIFF_SIZE, 0x18); } +void prefetchrst2(const Address& addr) { opMR(addr, Reg32(4), T_0F|T_ALLOW_DIFF_SIZE, 0x18); } +void prefetcht0(const Address& addr) { opMR(addr, Reg32(1), T_0F|T_ALLOW_DIFF_SIZE, 0x18); } +void prefetcht1(const Address& addr) { opMR(addr, Reg32(2), T_0F|T_ALLOW_DIFF_SIZE, 0x18); } +void prefetcht2(const Address& addr) { opMR(addr, Reg32(3), T_0F|T_ALLOW_DIFF_SIZE, 0x18); } +void prefetchw(const Address& addr) { opMR(addr, Reg32(1), T_0F|T_ALLOW_DIFF_SIZE, 0x0D); } void psadbw(const Mmx& mmx, const Operand& op) { opMMX(mmx, op, 0xF6); } void pshufb(const Mmx& mmx, const Operand& op) { opMMX(mmx, op, 0x00, T_0F38, T_66); } void pshufd(const Mmx& mmx, const Operand& op, uint8_t imm8) { opMMX(mmx, op, 0x70, T_0F, T_66, imm8); } @@ -949,8 +949,8 @@ void sar(const Reg& d, const Operand& op, int imm) { opShift(op, imm, 15, &d); } void sarx(const Reg32e& r1, const Operand& op, const Reg32e& r2) { opRRO(r1, r2, op, T_APX|T_F3|T_0F38, 0xf7); } void sbb(const Operand& op, uint32_t imm) { opOI(op, imm, 0x18, 3); } void sbb(const Operand& op1, const Operand& op2) { opRO_MR(op1, op2, 0x18); } -void sbb(const Reg& d, const Operand& op, uint32_t imm) { opROI(d, op, imm, T_NONE, 3); } -void sbb(const Reg& d, const Operand& op1, const Operand& op2) { opROO(d, op1, op2, T_NONE, 0x18); } +void sbb(const Reg& d, const Operand& op, uint32_t imm) { opROI(d, op, imm, T_OP_W1, 3); } +void sbb(const Reg& d, const Operand& op1, const Operand& op2) { opROO(d, op1, op2, T_OP_W1, 0x18); } void scasb() { db(0xAE); } void scasd() { db(0xAF); } void scasw() { db(0x66); db(0xAF); } @@ -986,13 +986,13 @@ void setpo(const Operand& op) { opSetCC(op, 11); }//-V524 void sets(const Operand& op) { opSetCC(op, 8); }//-V524 void setz(const Operand& op) { opSetCC(op, 4); }//-V524 void sfence() { db(0x0F); db(0xAE); db(0xF8); } -void sha1msg1(const Xmm& x, const Operand& op) { opSSE_APX(x, op, T_0F38, 0xC9, T_MUST_EVEX, 0xD9); } -void sha1msg2(const Xmm& x, const Operand& op) { opSSE_APX(x, op, T_0F38, 0xCA, T_MUST_EVEX, 0xDA); } -void sha1nexte(const Xmm& x, const Operand& op) { opSSE_APX(x, op, T_0F38, 0xC8, T_MUST_EVEX, 0xD8); } -void sha1rnds4(const Xmm& x, const Operand& op, uint8_t imm) { opSSE_APX(x, op, T_0F3A, 0xCC, T_MUST_EVEX, 0xD4, imm); } -void sha256msg1(const Xmm& x, const Operand& op) { opSSE_APX(x, op, T_0F38, 0xCC, T_MUST_EVEX, 0xDC); } -void sha256msg2(const Xmm& x, const Operand& op) { opSSE_APX(x, op, T_0F38, 0xCD, T_MUST_EVEX, 0xDD); } -void sha256rnds2(const Xmm& x, const Operand& op) { opSSE_APX(x, op, T_0F38, 0xCB, T_MUST_EVEX, 0xDB); } +void sha1msg1(const Xmm& x, const Operand& op) { opSSE(x, op, T_0F38, 0xC9, isXMM_XMMorMEM); } +void sha1msg2(const Xmm& x, const Operand& op) { opSSE(x, op, T_0F38, 0xCA, isXMM_XMMorMEM); } +void sha1nexte(const Xmm& x, const Operand& op) { opSSE(x, op, T_0F38, 0xC8, isXMM_XMMorMEM); } +void sha1rnds4(const Xmm& x, const Operand& op, uint8_t imm) { opSSE(x, op, T_0F3A, 0xCC, isXMM_XMMorMEM, imm); } +void sha256msg1(const Xmm& x, const Operand& op) { opSSE(x, op, T_0F38, 0xCC, isXMM_XMMorMEM); } +void sha256msg2(const Xmm& x, const Operand& op) { opSSE(x, op, T_0F38, 0xCD, isXMM_XMMorMEM); } +void sha256rnds2(const Xmm& x, const Operand& op) { opSSE(x, op, T_0F38, 0xCB, isXMM_XMMorMEM); } void shl(const Operand& op, const Reg8& _cl) { opShift(op, _cl, 12); } void shl(const Operand& op, int imm) { opShift(op, imm, 12); } void shl(const Reg& d, const Operand& op, const Reg8& _cl) { opShift(op, _cl, 12, &d); } @@ -1027,21 +1027,21 @@ void stosd() { db(0xAB); } void stosw() { db(0x66); db(0xAB); } void sub(const Operand& op, uint32_t imm) { opOI(op, imm, 0x28, 5); } void sub(const Operand& op1, const Operand& op2) { opRO_MR(op1, op2, 0x28); } -void sub(const Reg& d, const Operand& op, uint32_t imm) { opROI(d, op, imm, T_NF|T_CODE1_IF1, 5); } -void sub(const Reg& d, const Operand& op1, const Operand& op2) { opROO(d, op1, op2, T_NF|T_CODE1_IF1, 0x28); } +void sub(const Reg& d, const Operand& op, uint32_t imm) { opROI(d, op, imm, T_NF|T_OP_W1, 5); } +void sub(const Reg& d, const Operand& op1, const Operand& op2) { opROO(d, op1, op2, T_NF|T_OP_W1, 0x28); } void subpd(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_0F | T_66, 0x5C, isXMM_XMMorMEM); } void subps(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_0F, 0x5C, isXMM_XMMorMEM); } void subsd(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_0F | T_F2, 0x5C, isXMM_XMMorMEM); } void subss(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_0F | T_F3, 0x5C, isXMM_XMMorMEM); } void sysenter() { db(0x0F); db(0x34); } void sysexit() { db(0x0F); db(0x35); } -void tpause(const Reg32& r) { int idx = r.getIdx(); if (idx > 7) XBYAK_THROW(ERR_BAD_PARAMETER) db(0x66); db(0x0F); db(0xAE); setModRM(3, 6, idx); } +void tpause(const Reg32& r) { opRR(esi, r, T_66 | T_0F, 0xAE); } void tzcnt(const Reg®, const Operand& op) { if (opROO(Reg(), op, reg, T_APX|T_NF, 0xF4)) return; opCnt(reg, op, 0xBC); } void ucomisd(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_66|T_0F, 0x2E, isXMM_XMMorMEM); } void ucomiss(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_0F, 0x2E, isXMM_XMMorMEM); } void ud2() { db(0x0F); db(0x0B); } -void umonitor(const Reg& r) { int idx = r.getIdx(); if (idx > 7) XBYAK_THROW(ERR_BAD_PARAMETER) int bit = r.getBit(); if (BIT != bit) { if ((BIT == 32 && bit == 16) || (BIT == 64 && bit == 32)) { db(0x67); } else { XBYAK_THROW(ERR_BAD_SIZE_OF_REGISTER) } } db(0xF3); db(0x0F); db(0xAE); setModRM(3, 6, idx); } -void umwait(const Reg32& r) { int idx = r.getIdx(); if (idx > 7) XBYAK_THROW(ERR_BAD_PARAMETER) db(0xF2); db(0x0F); db(0xAE); setModRM(3, 6, idx); } +void umonitor(const Reg& r) { int bit = r.getBit(); if (bit == 8) XBYAK_THROW(ERR_BAD_SIZE_OF_REGISTER); if (BIT == bit * 2) db(0x67); opRR(esi, r.cvt32(), T_F3|T_0F, 0xAE); } +void umwait(const Reg32& r) { opRR(esi, r, T_F2|T_0F, 0xAE); } void unpckhpd(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_0F | T_66, 0x15, isXMM_XMMorMEM); } void unpckhps(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_0F, 0x15, isXMM_XMMorMEM); } void unpcklpd(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_0F | T_66, 0x14, isXMM_XMMorMEM); } @@ -1206,26 +1206,26 @@ void vcmpunordsd(const Xmm& x1, const Xmm& x2, const Operand& op) { vcmpsd(x1, x void vcmpunordss(const Xmm& x1, const Xmm& x2, const Operand& op) { vcmpss(x1, x2, op, 3); } void vcomisd(const Xmm& xm, const Operand& op) { opAVX_X_XM_IMM(xm, op, T_N8|T_66|T_0F|T_EW1|T_EVEX|T_SAE_X, 0x2F); } void vcomiss(const Xmm& xm, const Operand& op) { opAVX_X_XM_IMM(xm, op, T_N4|T_0F|T_W0|T_EVEX|T_SAE_X, 0x2F); } -void vcvtdq2pd(const Xmm& x, const Operand& op) { checkCvt1(x, op); opVex(x, 0, op, T_0F | T_F3 | T_YMM | T_EVEX | T_W0 | T_B32 | T_N8 | T_N_VL, 0xE6); } +void vcvtdq2pd(const Xmm& x, const Operand& op) { opCvt_xx_xy_yz(x, op, T_0F | T_F3 | T_YMM | T_EVEX | T_W0 | T_B32 | T_N8 | T_N_VL, 0xE6); } void vcvtdq2ps(const Xmm& xm, const Operand& op) { opAVX_X_XM_IMM(xm, op, T_0F|T_W0|T_YMM|T_EVEX|T_ER_Z|T_B32, 0x5B); } void vcvtneebf162ps(const Xmm& x, const Address& addr) { opVex(x, 0, addr, T_F3|T_0F38|T_W0|T_YMM, 0xB0); } void vcvtneeph2ps(const Xmm& x, const Address& addr) { opVex(x, 0, addr, T_66|T_0F38|T_W0|T_YMM, 0xB0); } void vcvtneobf162ps(const Xmm& x, const Address& addr) { opVex(x, 0, addr, T_F2|T_0F38|T_W0|T_YMM, 0xB0); } void vcvtneoph2ps(const Xmm& x, const Address& addr) { opVex(x, 0, addr, T_0F38|T_W0|T_YMM, 0xB0); } -void vcvtneps2bf16(const Xmm& x, const Operand& op, PreferredEncoding encoding = DefaultEncoding) { opCvt2(x, op, T_F3|T_0F38|T_W0|T_YMM|T_SAE_Z|T_B32|orEvexIf(encoding, 0, T_MUST_EVEX, 0), 0x72); } -void vcvtpd2dq(const Xmm& x, const Operand& op) { opCvt2(x, op, T_0F | T_F2 | T_YMM | T_EVEX | T_EW1 | T_B64 | T_ER_Z, 0xE6); } -void vcvtpd2ps(const Xmm& x, const Operand& op) { opCvt2(x, op, T_0F | T_66 | T_YMM | T_EVEX | T_EW1 | T_B64 | T_ER_Z, 0x5A); } -void vcvtph2ps(const Xmm& x, const Operand& op) { checkCvt1(x, op); opVex(x, 0, op, T_0F38 | T_66 | T_W0 | T_EVEX | T_N8 | T_N_VL | T_SAE_Y, 0x13); } +void vcvtneps2bf16(const Xmm& x, const Operand& op, PreferredEncoding encoding = DefaultEncoding) { opCvt_xx_yx_zy(x, op, T_F3|T_0F38|T_W0|T_YMM|T_SAE_Z|T_B32|orEvexIf(encoding, 0, T_MUST_EVEX, 0), 0x72); } +void vcvtpd2dq(const Xmm& x, const Operand& op) { opCvt_xx_yx_zy(x, op, T_0F | T_F2 | T_YMM | T_EVEX | T_EW1 | T_B64 | T_ER_Z, 0xE6); } +void vcvtpd2ps(const Xmm& x, const Operand& op) { opCvt_xx_yx_zy(x, op, T_0F | T_66 | T_YMM | T_EVEX | T_EW1 | T_B64 | T_ER_Z, 0x5A); } +void vcvtph2ps(const Xmm& x, const Operand& op) { opCvt_xx_xy_yz(x, op, T_0F38 | T_66 | T_W0 | T_EVEX | T_N8 | T_N_VL | T_SAE_Y, 0x13); } void vcvtps2dq(const Xmm& xm, const Operand& op) { opAVX_X_XM_IMM(xm, op, T_66|T_0F|T_W0|T_YMM|T_EVEX|T_ER_Z|T_B32, 0x5B); } -void vcvtps2pd(const Xmm& x, const Operand& op) { checkCvt1(x, op); opVex(x, 0, op, T_0F | T_YMM | T_EVEX | T_W0 | T_B32 | T_N8 | T_N_VL | T_SAE_Y, 0x5A); } -void vcvtps2ph(const Operand& op, const Xmm& x, uint8_t imm) { checkCvt1(x, op); opVex(x, 0, op, T_0F3A | T_66 | T_W0 | T_EVEX | T_N8 | T_N_VL | T_SAE_Y | T_M_K, 0x1D, imm); } +void vcvtps2pd(const Xmm& x, const Operand& op) { opCvt_xx_xy_yz(x, op, T_0F | T_YMM | T_EVEX | T_W0 | T_B32 | T_N8 | T_N_VL | T_SAE_Y, 0x5A); } +void vcvtps2ph(const Operand& op, const Xmm& x, uint8_t imm) { opCvt_xx_xy_yz(x, op, T_0F3A | T_66 | T_W0 | T_EVEX | T_N8 | T_N_VL | T_SAE_Y | T_M_K, 0x1D, imm); } void vcvtsd2si(const Reg32& r, const Operand& op) { opAVX_X_X_XM(Xmm(r.getIdx()), xm0, op, T_0F | T_F2 | T_W0 | T_EVEX | T_N4 | T_ER_X, 0x2D); } void vcvtsd2ss(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_N8|T_F2|T_0F|T_EW1|T_EVEX|T_ER_X, 0x5A); } -void vcvtsi2sd(const Xmm& x1, const Xmm& x2, const Operand& op) { opCvt3(x1, x2, op, T_0F | T_F2 | T_EVEX, T_W1 | T_EW1 | T_ER_X | T_N8, T_W0 | T_N4, 0x2A); } -void vcvtsi2ss(const Xmm& x1, const Xmm& x2, const Operand& op) { opCvt3(x1, x2, op, T_0F | T_F3 | T_EVEX | T_ER_X, T_W1 | T_EW1 | T_N8, T_W0 | T_N4, 0x2A); } +void vcvtsi2sd(const Xmm& x1, const Xmm& x2, const Operand& op) { opCvtSi2X(x1, x2, op, T_0F | T_F2 | T_EVEX, T_W1 | T_EW1 | T_ER_R | T_N8, T_W0 | T_N4, 0x2A); } +void vcvtsi2ss(const Xmm& x1, const Xmm& x2, const Operand& op) { opCvtSi2X(x1, x2, op, T_0F | T_F3 | T_EVEX | T_ER_R, T_W1 | T_EW1 | T_N8, T_W0 | T_N4, 0x2A); } void vcvtss2sd(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_N4|T_F3|T_0F|T_W0|T_EVEX|T_SAE_X, 0x5A); } void vcvtss2si(const Reg32& r, const Operand& op) { opAVX_X_X_XM(Xmm(r.getIdx()), xm0, op, T_0F | T_F3 | T_W0 | T_EVEX | T_ER_X | T_N8, 0x2D); } -void vcvttpd2dq(const Xmm& x, const Operand& op) { opCvt2(x, op, T_66 | T_0F | T_YMM | T_EVEX |T_EW1 | T_B64 | T_SAE_Z, 0xE6); } +void vcvttpd2dq(const Xmm& x, const Operand& op) { opCvt_xx_yx_zy(x, op, T_66 | T_0F | T_YMM | T_EVEX |T_EW1 | T_B64 | T_SAE_Z, 0xE6); } void vcvttps2dq(const Xmm& xm, const Operand& op) { opAVX_X_XM_IMM(xm, op, T_F3|T_0F|T_W0|T_YMM|T_EVEX|T_SAE_Z|T_B32, 0x5B); } void vcvttsd2si(const Reg32& r, const Operand& op) { opAVX_X_X_XM(Xmm(r.getIdx()), xm0, op, T_0F | T_F2 | T_W0 | T_EVEX | T_N4 | T_SAE_X, 0x2C); } void vcvttss2si(const Reg32& r, const Operand& op) { opAVX_X_X_XM(Xmm(r.getIdx()), xm0, op, T_0F | T_F3 | T_W0 | T_EVEX | T_SAE_X | T_N8, 0x2C); } @@ -1436,7 +1436,7 @@ void vpermq(const Ymm& y1, const Ymm& y2, const Operand& op) { opAVX_X_X_XM(y1, void vpextrb(const Operand& op, const Xmm& x, uint8_t imm) { if (!((op.isREG(8|16|i32e) || op.isMEM()) && x.isXMM())) XBYAK_THROW(ERR_BAD_COMBINATION) opVex(x, 0, op, T_0F3A | T_66 | T_EVEX | T_N1, 0x14, imm); } void vpextrd(const Operand& op, const Xmm& x, uint8_t imm) { if (!((op.isREG(32) || op.isMEM()) && x.isXMM())) XBYAK_THROW(ERR_BAD_COMBINATION) opVex(x, 0, op, T_0F3A | T_66 | T_W0 | T_EVEX | T_N4, 0x16, imm); } void vpextrq(const Operand& op, const Xmm& x, uint8_t imm) { if (!((op.isREG(64) || op.isMEM()) && x.isXMM())) XBYAK_THROW(ERR_BAD_COMBINATION) opVex(x, 0, op, T_0F3A | T_66 | T_W1 | T_EVEX | T_EW1 | T_N8, 0x16, imm); } -void vpextrw(const Operand& op, const Xmm& x, uint8_t imm) { if (!((op.isREG(16|i32e) || op.isMEM()) && x.isXMM())) XBYAK_THROW(ERR_BAD_COMBINATION) if (op.isREG() && x.getIdx() < 16) { opAVX_X_X_XM(Xmm(op.getIdx()), xm0, x, T_0F | T_66, 0xC5, imm); } else { opVex(x, 0, op, T_0F3A | T_66 | T_EVEX | T_N2, 0x15, imm); } } +void vpextrw(const Operand& op, const Xmm& x, uint8_t imm) { if (!((op.isREG(16|i32e) || op.isMEM()) && x.isXMM())) XBYAK_THROW(ERR_BAD_COMBINATION) if (op.isREG() && x.getIdx() < 16 && op.getIdx() < 16) { opAVX_X_X_XM(Xmm(op.getIdx()), xm0, x, T_0F | T_66, 0xC5, imm); } else { opVex(x, 0, op, T_0F3A | T_66 | T_EVEX | T_N2, 0x15, imm); } } void vpgatherdd(const Xmm& x1, const Address& addr, const Xmm& x2) { opGather(x1, addr, x2, T_0F38 | T_66 | T_YMM | T_VSIB | T_W0, 0x90, 1); } void vpgatherdq(const Xmm& x1, const Address& addr, const Xmm& x2) { opGather(x1, addr, x2, T_0F38 | T_66 | T_YMM | T_VSIB | T_W1, 0x90, 0); } void vpgatherqd(const Xmm& x1, const Address& addr, const Xmm& x2) { opGather(x1, addr, x2, T_0F38 | T_66 | T_YMM | T_VSIB | T_W0, 0x91, 2); } @@ -1499,28 +1499,28 @@ void vpshuflw(const Xmm& xm, const Operand& op, uint8_t imm) { opAVX_X_XM_IMM(xm void vpsignb(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_66|T_0F38|T_YMM, 0x08); } void vpsignd(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_66|T_0F38|T_YMM, 0x0A); } void vpsignw(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_66|T_0F38|T_YMM, 0x09); } -void vpslld(const Xmm& x, const Operand& op, uint8_t imm) { opAVX_X_X_XM(Xmm(x.getKind(), 6), x, op, T_66|T_0F|T_W0|T_YMM|T_EVEX|T_B32|T_MEM_EVEX, 0x72, imm); } +void vpslld(const Xmm& x, const Operand& op, uint8_t imm) { opAVX_X_X_XM(Xmm(x.getKind(), 6), x, op, T_66|T_0F|T_W0|T_YMM|T_EVEX_IF_MEM|T_B32, 0x72, imm); } void vpslld(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_N16|T_66|T_0F|T_W0|T_YMM|T_EVEX, 0xF2); } -void vpslldq(const Xmm& x, const Operand& op, uint8_t imm) { opAVX_X_X_XM(Xmm(x.getKind(), 7), x, op, T_66|T_0F|T_YMM|T_EVEX|T_MEM_EVEX, 0x73, imm); } -void vpsllq(const Xmm& x, const Operand& op, uint8_t imm) { opAVX_X_X_XM(Xmm(x.getKind(), 6), x, op, T_66|T_0F|T_EW1|T_YMM|T_EVEX|T_B64|T_MEM_EVEX, 0x73, imm); } +void vpslldq(const Xmm& x, const Operand& op, uint8_t imm) { opAVX_X_X_XM(Xmm(x.getKind(), 7), x, op, T_66|T_0F|T_YMM|T_EVEX_IF_MEM, 0x73, imm); } +void vpsllq(const Xmm& x, const Operand& op, uint8_t imm) { opAVX_X_X_XM(Xmm(x.getKind(), 6), x, op, T_66|T_0F|T_EW1|T_YMM|T_EVEX_IF_MEM|T_B64, 0x73, imm); } void vpsllq(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_N16|T_66|T_0F|T_EW1|T_YMM|T_EVEX, 0xF3); } void vpsllvd(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_66|T_0F38|T_W0|T_YMM|T_EVEX|T_B32, 0x47); } void vpsllvq(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_66|T_0F38|T_W1|T_EW1|T_YMM|T_EVEX|T_B64, 0x47); } -void vpsllw(const Xmm& x, const Operand& op, uint8_t imm) { opAVX_X_X_XM(Xmm(x.getKind(), 6), x, op, T_66|T_0F|T_YMM|T_EVEX|T_MEM_EVEX, 0x71, imm); } +void vpsllw(const Xmm& x, const Operand& op, uint8_t imm) { opAVX_X_X_XM(Xmm(x.getKind(), 6), x, op, T_66|T_0F|T_YMM|T_EVEX_IF_MEM, 0x71, imm); } void vpsllw(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_N16|T_66|T_0F|T_YMM|T_EVEX, 0xF1); } -void vpsrad(const Xmm& x, const Operand& op, uint8_t imm) { opAVX_X_X_XM(Xmm(x.getKind(), 4), x, op, T_66|T_0F|T_W0|T_YMM|T_EVEX|T_B32|T_MEM_EVEX, 0x72, imm); } +void vpsrad(const Xmm& x, const Operand& op, uint8_t imm) { opAVX_X_X_XM(Xmm(x.getKind(), 4), x, op, T_66|T_0F|T_W0|T_YMM|T_EVEX_IF_MEM|T_B32, 0x72, imm); } void vpsrad(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_N16|T_66|T_0F|T_W0|T_YMM|T_EVEX, 0xE2); } void vpsravd(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_66|T_0F38|T_W0|T_YMM|T_EVEX|T_B32, 0x46); } -void vpsraw(const Xmm& x, const Operand& op, uint8_t imm) { opAVX_X_X_XM(Xmm(x.getKind(), 4), x, op, T_66|T_0F|T_YMM|T_EVEX|T_MEM_EVEX, 0x71, imm); } +void vpsraw(const Xmm& x, const Operand& op, uint8_t imm) { opAVX_X_X_XM(Xmm(x.getKind(), 4), x, op, T_66|T_0F|T_YMM|T_EVEX_IF_MEM, 0x71, imm); } void vpsraw(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_N16|T_66|T_0F|T_YMM|T_EVEX, 0xE1); } -void vpsrld(const Xmm& x, const Operand& op, uint8_t imm) { opAVX_X_X_XM(Xmm(x.getKind(), 2), x, op, T_66|T_0F|T_W0|T_YMM|T_EVEX|T_B32|T_MEM_EVEX, 0x72, imm); } +void vpsrld(const Xmm& x, const Operand& op, uint8_t imm) { opAVX_X_X_XM(Xmm(x.getKind(), 2), x, op, T_66|T_0F|T_W0|T_YMM|T_EVEX_IF_MEM|T_B32, 0x72, imm); } void vpsrld(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_N16|T_66|T_0F|T_W0|T_YMM|T_EVEX, 0xD2); } -void vpsrldq(const Xmm& x, const Operand& op, uint8_t imm) { opAVX_X_X_XM(Xmm(x.getKind(), 3), x, op, T_66|T_0F|T_YMM|T_EVEX|T_MEM_EVEX, 0x73, imm); } -void vpsrlq(const Xmm& x, const Operand& op, uint8_t imm) { opAVX_X_X_XM(Xmm(x.getKind(), 2), x, op, T_66|T_0F|T_EW1|T_YMM|T_EVEX|T_B64|T_MEM_EVEX, 0x73, imm); } +void vpsrldq(const Xmm& x, const Operand& op, uint8_t imm) { opAVX_X_X_XM(Xmm(x.getKind(), 3), x, op, T_66|T_0F|T_YMM|T_EVEX_IF_MEM, 0x73, imm); } +void vpsrlq(const Xmm& x, const Operand& op, uint8_t imm) { opAVX_X_X_XM(Xmm(x.getKind(), 2), x, op, T_66|T_0F|T_EW1|T_YMM|T_EVEX_IF_MEM|T_B64, 0x73, imm); } void vpsrlq(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_N16|T_66|T_0F|T_EW1|T_YMM|T_EVEX, 0xD3); } void vpsrlvd(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_66|T_0F38|T_W0|T_YMM|T_EVEX|T_B32, 0x45); } void vpsrlvq(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_66|T_0F38|T_W1|T_EW1|T_YMM|T_EVEX|T_B64, 0x45); } -void vpsrlw(const Xmm& x, const Operand& op, uint8_t imm) { opAVX_X_X_XM(Xmm(x.getKind(), 2), x, op, T_66|T_0F|T_YMM|T_EVEX|T_MEM_EVEX, 0x71, imm); } +void vpsrlw(const Xmm& x, const Operand& op, uint8_t imm) { opAVX_X_X_XM(Xmm(x.getKind(), 2), x, op, T_66|T_0F|T_YMM|T_EVEX_IF_MEM, 0x71, imm); } void vpsrlw(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_N16|T_66|T_0F|T_YMM|T_EVEX, 0xD1); } void vpsubb(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_66|T_0F|T_YMM|T_EVEX, 0xF8); } void vpsubd(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_66|T_0F|T_W0|T_YMM|T_EVEX|T_B32, 0xFA); } @@ -1590,8 +1590,8 @@ void xgetbv() { db(0x0F); db(0x01); db(0xD0); } void xlatb() { db(0xD7); } void xor_(const Operand& op, uint32_t imm) { opOI(op, imm, 0x30, 6); } void xor_(const Operand& op1, const Operand& op2) { opRO_MR(op1, op2, 0x30); } -void xor_(const Reg& d, const Operand& op, uint32_t imm) { opROI(d, op, imm, T_NF|T_CODE1_IF1, 6); } -void xor_(const Reg& d, const Operand& op1, const Operand& op2) { opROO(d, op1, op2, T_NF|T_CODE1_IF1, 0x30); } +void xor_(const Reg& d, const Operand& op, uint32_t imm) { opROI(d, op, imm, T_NF|T_OP_W1, 6); } +void xor_(const Reg& d, const Operand& op1, const Operand& op2) { opROO(d, op1, op2, T_NF|T_OP_W1, 0x30); } void xorpd(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_0F | T_66, 0x57, isXMM_XMMorMEM); } void xorps(const Xmm& xmm, const Operand& op) { opSSE(xmm, op, T_0F, 0x57, isXMM_XMMorMEM); } void xresldtrk() { db(0xF2); db(0x0F); db(0x01); db(0xE9); } @@ -1875,7 +1875,7 @@ void stui() { db(0xF3); db(0x0F); db(0x01); db(0xEF); } void testui() { db(0xF3); db(0x0F); db(0x01); db(0xED); } void uiret() { db(0xF3); db(0x0F); db(0x01); db(0xEC); } void cmpxchg16b(const Address& addr) { opMR(addr, Reg64(1), T_0F|T_ALLOW_DIFF_SIZE, 0xC7); } -void fxrstor64(const Address& addr) { opMR(addr, Reg64(1), T_0F, 0xAE); } +void fxrstor64(const Address& addr) { opMR(addr, Reg64(1), T_0F|T_ALLOW_DIFF_SIZE, 0xAE); } void movq(const Reg64& reg, const Mmx& mmx) { if (mmx.isXMM()) db(0x66); opSSE(mmx, reg, T_0F, 0x7E); } void movq(const Mmx& mmx, const Reg64& reg) { if (mmx.isXMM()) db(0x66); opSSE(mmx, reg, T_0F, 0x6E); } void movrs(const Reg& reg, const Address& addr) { opMR(addr, reg, T_0F38, reg.isBit(8) ? 0x8A : 0x8B); } @@ -1887,13 +1887,15 @@ void vcvtss2si(const Reg64& r, const Operand& op) { opAVX_X_X_XM(Xmm(r.getIdx()) void vcvttss2si(const Reg64& r, const Operand& op) { opAVX_X_X_XM(Xmm(r.getIdx()), xm0, op, T_0F | T_F3 | T_W1 | T_EVEX | T_EW1 | T_SAE_X | T_N8, 0x2C); } void vcvtsd2si(const Reg64& r, const Operand& op) { opAVX_X_X_XM(Xmm(r.getIdx()), xm0, op, T_0F | T_F2 | T_W1 | T_EVEX | T_EW1 | T_N4 | T_ER_X, 0x2D); } void vcvttsd2si(const Reg64& r, const Operand& op) { opAVX_X_X_XM(Xmm(r.getIdx()), xm0, op, T_0F | T_F2 | T_W1 | T_EVEX | T_EW1 | T_N4 | T_SAE_X, 0x2C); } -void vmovq(const Xmm& x, const Reg64& r) { opAVX_X_X_XM(x, xm0, Xmm(r.getIdx()), T_66 | T_0F | T_W1 | T_EVEX | T_EW1, 0x6E); } -void vmovq(const Reg64& r, const Xmm& x) { opAVX_X_X_XM(x, xm0, Xmm(r.getIdx()), T_66 | T_0F | T_W1 | T_EVEX | T_EW1, 0x7E); } +void vmovq(const Xmm& x, const Reg64& r) { opAVX_X_X_XM(x, xm0, r, T_66 | T_0F | T_W1 | T_EVEX | T_EW1, 0x6E); } +void vmovq(const Reg64& r, const Xmm& x) { opAVX_X_X_XM(x, xm0, r, T_66 | T_0F | T_W1 | T_EVEX | T_EW1, 0x7E); } void jmpabs(uint64_t addr) { db(0xD5); db(0x00); db(0xA1); dq(addr); } void push2(const Reg64& r1, const Reg64& r2) { opROO(r1, r2, Reg64(6), T_APX|T_ND1|T_W0, 0xFF); } void push2p(const Reg64& r1, const Reg64& r2) { opROO(r1, r2, Reg64(6), T_APX|T_ND1|T_W1, 0xFF); } void pop2(const Reg64& r1, const Reg64& r2) { opROO(r1, r2, Reg64(0), T_APX|T_ND1|T_W0, 0x8F); } void pop2p(const Reg64& r1, const Reg64& r2) { opROO(r1, r2, Reg64(0), T_APX|T_ND1|T_W1, 0x8F); } +void pushp(const Reg64& r) { opPushPopP(r, 0x50); } +void popp(const Reg64& r) { opPushPopP(r, 0x58); } void cmpbexadd(const Address& addr, const Reg32e& r1, const Reg32e& r2) { opRRO(r1, r2, addr, T_APX|T_66|T_0F38, 0xE6); } void cmpbxadd(const Address& addr, const Reg32e& r1, const Reg32e& r2) { opRRO(r1, r2, addr, T_APX|T_66|T_0F38, 0xE2); } void cmplexadd(const Address& addr, const Reg32e& r1, const Reg32e& r2) { opRRO(r1, r2, addr, T_APX|T_66|T_0F38, 0xEE); } @@ -1910,16 +1912,16 @@ void cmpoxadd(const Address& addr, const Reg32e& r1, const Reg32e& r2) { opRRO(r void cmppxadd(const Address& addr, const Reg32e& r1, const Reg32e& r2) { opRRO(r1, r2, addr, T_APX|T_66|T_0F38, 0xEA); } void cmpsxadd(const Address& addr, const Reg32e& r1, const Reg32e& r2) { opRRO(r1, r2, addr, T_APX|T_66|T_0F38, 0xE8); } void cmpzxadd(const Address& addr, const Reg32e& r1, const Reg32e& r2) { opRRO(r1, r2, addr, T_APX|T_66|T_0F38, 0xE4); } -void aesdec128kl(const Xmm& x, const Address& addr) { opSSE_APX(x, addr, T_F3|T_0F38, 0xDD, T_F3|T_MUST_EVEX, 0xDD); } -void aesdec256kl(const Xmm& x, const Address& addr) { opSSE_APX(x, addr, T_F3|T_0F38, 0xDF, T_F3|T_MUST_EVEX, 0xDF); } -void aesdecwide128kl(const Address& addr) { opSSE_APX(xmm1, addr, T_F3|T_0F38, 0xD8, T_F3|T_MUST_EVEX, 0xD8); } -void aesdecwide256kl(const Address& addr) { opSSE_APX(xmm3, addr, T_F3|T_0F38, 0xD8, T_F3|T_MUST_EVEX, 0xD8); } -void aesenc128kl(const Xmm& x, const Address& addr) { opSSE_APX(x, addr, T_F3|T_0F38, 0xDC, T_F3|T_MUST_EVEX, 0xDC); } -void aesenc256kl(const Xmm& x, const Address& addr) { opSSE_APX(x, addr, T_F3|T_0F38, 0xDE, T_F3|T_MUST_EVEX, 0xDE); } -void aesencwide128kl(const Address& addr) { opSSE_APX(xmm0, addr, T_F3|T_0F38, 0xD8, T_F3|T_MUST_EVEX, 0xD8); } -void aesencwide256kl(const Address& addr) { opSSE_APX(xmm2, addr, T_F3|T_0F38, 0xD8, T_F3|T_MUST_EVEX, 0xD8); } -void encodekey128(const Reg32& r1, const Reg32& r2) { opEncodeKey(r1, r2, 0xFA, 0xDA); } -void encodekey256(const Reg32& r1, const Reg32& r2) { opEncodeKey(r1, r2, 0xFB, 0xDB); } +void aesdec128kl(const Xmm& x, const Address& addr) { opSSE(x, addr, T_F3|T_0F38, 0xDD, isXMM_XMMorMEM); } +void aesdec256kl(const Xmm& x, const Address& addr) { opSSE(x, addr, T_F3|T_0F38, 0xDF, isXMM_XMMorMEM); } +void aesdecwide128kl(const Address& addr) { opSSE(xmm1, addr, T_F3|T_0F38, 0xD8, isXMM_XMMorMEM); } +void aesdecwide256kl(const Address& addr) { opSSE(xmm3, addr, T_F3|T_0F38, 0xD8, isXMM_XMMorMEM); } +void aesenc128kl(const Xmm& x, const Address& addr) { opSSE(x, addr, T_F3|T_0F38, 0xDC, isXMM_XMMorMEM); } +void aesenc256kl(const Xmm& x, const Address& addr) { opSSE(x, addr, T_F3|T_0F38, 0xDE, isXMM_XMMorMEM); } +void aesencwide128kl(const Address& addr) { opSSE(xmm0, addr, T_F3|T_0F38, 0xD8, isXMM_XMMorMEM); } +void aesencwide256kl(const Address& addr) { opSSE(xmm2, addr, T_F3|T_0F38, 0xD8, isXMM_XMMorMEM); } +void encodekey128(const Reg32& r1, const Reg32& r2) { opRR(r1, r2, T_F3|T_0F38, 0xFA); } +void encodekey256(const Reg32& r1, const Reg32& r2) { opRR(r1, r2, T_F3|T_0F38, 0xFB); } void rdfsbase(const Reg32e& r) { opRR(eax, r, T_F3|T_0F|T_ALLOW_DIFF_SIZE, 0xAE); } void rdgsbase(const Reg32e& r) { opRR(ecx, r, T_F3|T_0F|T_ALLOW_DIFF_SIZE, 0xAE); } void wrfsbase(const Reg32e& r) { opRR(edx, r, T_F3|T_0F|T_ALLOW_DIFF_SIZE, 0xAE); } @@ -1934,7 +1936,6 @@ void tdpbf8ps(const Tmm& x1, const Tmm& x2, const Tmm& x3) { opVex(x1, &x3, x2, void tdpbhf8ps(const Tmm& x1, const Tmm& x2, const Tmm& x3) { opVex(x1, &x3, x2, T_F2|T_MAP5|T_W0, 0xFD); } void tdphbf8ps(const Tmm& x1, const Tmm& x2, const Tmm& x3) { opVex(x1, &x3, x2, T_F3|T_MAP5|T_W0, 0xFD); } void tdphf8ps(const Tmm& x1, const Tmm& x2, const Tmm& x3) { opVex(x1, &x3, x2, T_66|T_MAP5|T_W0, 0xFD); } -void tmmultf32ps(const Tmm& x1, const Tmm& x2, const Tmm& x3) { opVex(x1, &x3, x2, T_66|T_0F38|T_W0, 0x48); } void tcmmimfp16ps(const Tmm& x1, const Tmm& x2, const Tmm& x3) { opVex(x1, &x3, x2, T_66|T_0F38|T_W0, 0x6C); } void tcmmrlfp16ps(const Tmm& x1, const Tmm& x2, const Tmm& x3) { opVex(x1, &x3, x2, T_0F38|T_W0, 0x6C); } void tileloadd(const Tmm& tm, const Address& addr) { opAMX(tm, addr, T_F2|T_0F38|T_W0, 0x4B); } @@ -1964,8 +1965,8 @@ void pusha() { db(0x60); } void pushad() { db(0x60); } void pushfd() { db(0x9C); } void popa() { db(0x61); } -void lds(const Reg& reg, const Address& addr) { opLoadSeg(addr, reg, T_NONE, 0xC5); } -void les(const Reg& reg, const Address& addr) { opLoadSeg(addr, reg, T_NONE, 0xC4); } +void lds(const Reg& reg, const Address& addr) { opLoadSeg(addr, reg, T_ALLOW_DIFF_SIZE, 0xC5); } +void les(const Reg& reg, const Address& addr) { opLoadSeg(addr, reg, T_ALLOW_DIFF_SIZE, 0xC4); } #endif #ifndef XBYAK_NO_OP_NAMES void and(const Operand& op1, const Operand& op2) { and_(op1, op2); } @@ -2189,8 +2190,8 @@ void vcmpunordsd(const Opmask& k, const Xmm& x, const Operand& op) { vcmpsd(k, x void vcmpunordss(const Opmask& k, const Xmm& x, const Operand& op) { vcmpss(k, x, op, 3); } void vcomisbf16(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_N2|T_66|T_MAP5|T_W0|T_MUST_EVEX, 0x2F); } void vcomish(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_N2|T_MAP5|T_W0|T_SAE_X|T_MUST_EVEX, 0x2F); } -void vcompresspd(const Operand& op, const Xmm& x) { opAVX_X_XM_IMM(x, op, T_N8|T_66|T_0F38|T_EW1|T_YMM|T_MUST_EVEX, 0x8A); } -void vcompressps(const Operand& op, const Xmm& x) { opAVX_X_XM_IMM(x, op, T_N4|T_66|T_0F38|T_W0|T_YMM|T_MUST_EVEX, 0x8A); } +void vcompresspd(const Operand& op, const Xmm& x) { opAVX_X_XM_IMM(x, op, T_N8|T_66|T_0F38|T_EW1|T_YMM|T_MUST_EVEX|T_M_K, 0x8A); } +void vcompressps(const Operand& op, const Xmm& x) { opAVX_X_XM_IMM(x, op, T_N4|T_66|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x8A); } void vcomxsd(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_N8|T_F2|T_0F|T_EW1|T_SAE_X|T_MUST_EVEX, 0x2F); } void vcomxsh(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_N2|T_F3|T_MAP5|T_W0|T_SAE_X|T_MUST_EVEX, 0x2F); } void vcomxss(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_N4|T_F3|T_0F|T_W0|T_SAE_X|T_MUST_EVEX, 0x2F); } @@ -2201,92 +2202,92 @@ void vcvt2ph2hf8s(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_X void vcvt2ps2phx(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_66|T_0F38|T_W0|T_YMM|T_ER_Z|T_MUST_EVEX|T_B32, 0x67); } void vcvtbf162ibs(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_F2|T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x69); } void vcvtbf162iubs(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_F2|T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x6B); } -void vcvtbiasph2bf8(const Xmm& x1, const Xmm& x2, const Operand& op) { opCvt6(x1, x2, op, T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x74); } -void vcvtbiasph2bf8s(const Xmm& x1, const Xmm& x2, const Operand& op) { opCvt6(x1, x2, op, T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x74); } -void vcvtbiasph2hf8(const Xmm& x1, const Xmm& x2, const Operand& op) { opCvt6(x1, x2, op, T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x18); } -void vcvtbiasph2hf8s(const Xmm& x1, const Xmm& x2, const Operand& op) { opCvt6(x1, x2, op, T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x1B); } -void vcvtdq2ph(const Xmm& x, const Operand& op) { checkCvt4(x, op); opCvt(x, op, T_N16|T_N_VL|T_MAP5|T_W0|T_YMM|T_ER_Z|T_MUST_EVEX|T_B32, 0x5B); } -void vcvthf82ph(const Xmm& x, const Operand& op) { checkCvt1(x, op); opVex(x, 0, op, T_MUST_EVEX|T_F2|T_MAP5|T_W0|T_YMM|T_N8|T_N_VL, 0x1E); } +void vcvtbiasph2bf8(const Xmm& x1, const Xmm& x2, const Operand& op) { opCvtBias(x1, x2, op, T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x74); } +void vcvtbiasph2bf8s(const Xmm& x1, const Xmm& x2, const Operand& op) { opCvtBias(x1, x2, op, T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x74); } +void vcvtbiasph2hf8(const Xmm& x1, const Xmm& x2, const Operand& op) { opCvtBias(x1, x2, op, T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x18); } +void vcvtbiasph2hf8s(const Xmm& x1, const Xmm& x2, const Operand& op) { opCvtBias(x1, x2, op, T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x1B); } +void vcvtdq2ph(const Xmm& x, const Operand& op) { opCvt_xx_yx_zy_sized(x, op, T_N16|T_N_VL|T_MAP5|T_W0|T_YMM|T_ER_Z|T_MUST_EVEX|T_B32, 0x5B); } +void vcvthf82ph(const Xmm& x, const Operand& op) { opCvt_xx_xy_yz(x, op, T_MUST_EVEX|T_F2|T_MAP5|T_W0|T_YMM|T_N8|T_N_VL, 0x1E); } void vcvtne2ps2bf16(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_F2|T_0F38|T_W0|T_YMM|T_SAE_Z|T_MUST_EVEX|T_B32, 0x72); } -void vcvtpd2ph(const Xmm& x, const Operand& op) { opCvt5(x, op, T_N16|T_N_VL|T_66|T_MAP5|T_EW1|T_ER_Z|T_MUST_EVEX|T_B64, 0x5A); } +void vcvtpd2ph(const Xmm& x, const Operand& op) { opCvt_xx_yx_zx(x, op, T_N16|T_N_VL|T_66|T_MAP5|T_EW1|T_ER_Z|T_MUST_EVEX|T_B64, 0x5A); } void vcvtpd2qq(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_66|T_0F|T_EW1|T_YMM|T_ER_Z|T_MUST_EVEX|T_B64, 0x7B); } -void vcvtpd2udq(const Xmm& x, const Operand& op) { opCvt2(x, op, T_0F|T_EW1|T_YMM|T_ER_Z|T_MUST_EVEX|T_B64, 0x79); } +void vcvtpd2udq(const Xmm& x, const Operand& op) { opCvt_xx_yx_zy(x, op, T_0F|T_EW1|T_YMM|T_ER_Z|T_MUST_EVEX|T_B64, 0x79); } void vcvtpd2uqq(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_66|T_0F|T_EW1|T_YMM|T_ER_Z|T_MUST_EVEX|T_B64, 0x79); } -void vcvtph2bf8(const Xmm& x, const Operand& op) { opCvt2(x, op, T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x74); } -void vcvtph2bf8s(const Xmm& x, const Operand& op) { opCvt2(x, op, T_F3|T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x74); } -void vcvtph2dq(const Xmm& x, const Operand& op) { checkCvt1(x, op); opVex(x, 0, op, T_N8|T_N_VL|T_66|T_MAP5|T_W0|T_YMM|T_ER_Y|T_MUST_EVEX|T_B16, 0x5B); } -void vcvtph2hf8(const Xmm& x, const Operand& op) { opCvt2(x, op, T_F3|T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x18); } -void vcvtph2hf8s(const Xmm& x, const Operand& op) { opCvt2(x, op, T_F3|T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x1B); } +void vcvtph2bf8(const Xmm& x, const Operand& op) { opCvt_xx_yx_zy(x, op, T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x74); } +void vcvtph2bf8s(const Xmm& x, const Operand& op) { opCvt_xx_yx_zy(x, op, T_F3|T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x74); } +void vcvtph2dq(const Xmm& x, const Operand& op) { opCvt_xx_xy_yz(x, op, T_N8|T_N_VL|T_66|T_MAP5|T_W0|T_YMM|T_ER_Y|T_MUST_EVEX|T_B16, 0x5B); } +void vcvtph2hf8(const Xmm& x, const Operand& op) { opCvt_xx_yx_zy(x, op, T_F3|T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x18); } +void vcvtph2hf8s(const Xmm& x, const Operand& op) { opCvt_xx_yx_zy(x, op, T_F3|T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x1B); } void vcvtph2ibs(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_MAP5|T_W0|T_YMM|T_ER_Z|T_MUST_EVEX|T_B16, 0x69); } void vcvtph2iubs(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_MAP5|T_W0|T_YMM|T_ER_Z|T_MUST_EVEX|T_B16, 0x6B); } -void vcvtph2pd(const Xmm& x, const Operand& op) { if (!op.isXMM() && !op.isMEM()) XBYAK_THROW(ERR_BAD_MEM_SIZE) opVex(x, 0, op, T_N4|T_N_VL|T_MAP5|T_W0|T_YMM|T_SAE_X|T_MUST_EVEX|T_B16, 0x5A); } -void vcvtph2psx(const Xmm& x, const Operand& op) { checkCvt1(x, op); opVex(x, 0, op, T_N8|T_N_VL|T_66|T_MAP6|T_W0|T_YMM|T_SAE_Y|T_MUST_EVEX|T_B16, 0x13); } -void vcvtph2qq(const Xmm& x, const Operand& op) { if (!op.isXMM() && !op.isMEM()) XBYAK_THROW(ERR_BAD_MEM_SIZE) opVex(x, 0, op, T_N4|T_N_VL|T_66|T_MAP5|T_W0|T_YMM|T_ER_X|T_MUST_EVEX|T_B16, 0x7B); } -void vcvtph2udq(const Xmm& x, const Operand& op) { checkCvt1(x, op); opVex(x, 0, op, T_N8|T_N_VL|T_MAP5|T_W0|T_YMM|T_ER_Y|T_MUST_EVEX|T_B16, 0x79); } -void vcvtph2uqq(const Xmm& x, const Operand& op) { if (!op.isXMM() && !op.isMEM()) XBYAK_THROW(ERR_BAD_MEM_SIZE) opVex(x, 0, op, T_N4|T_N_VL|T_66|T_MAP5|T_W0|T_YMM|T_ER_X|T_MUST_EVEX|T_B16, 0x79); } +void vcvtph2pd(const Xmm& x, const Operand& op) { opX_XM(op, x, T_N4|T_N_VL|T_MAP5|T_W0|T_YMM|T_SAE_X|T_MUST_EVEX|T_B16, 0x5A); } +void vcvtph2psx(const Xmm& x, const Operand& op) { opCvt_xx_xy_yz(x, op, T_N8|T_N_VL|T_66|T_MAP6|T_W0|T_YMM|T_SAE_Y|T_MUST_EVEX|T_B16, 0x13); } +void vcvtph2qq(const Xmm& x, const Operand& op) { opX_XM(op, x, T_N4|T_N_VL|T_66|T_MAP5|T_W0|T_YMM|T_ER_X|T_MUST_EVEX|T_B16, 0x7B); } +void vcvtph2udq(const Xmm& x, const Operand& op) { opCvt_xx_xy_yz(x, op, T_N8|T_N_VL|T_MAP5|T_W0|T_YMM|T_ER_Y|T_MUST_EVEX|T_B16, 0x79); } +void vcvtph2uqq(const Xmm& x, const Operand& op) { opX_XM(op, x, T_N4|T_N_VL|T_66|T_MAP5|T_W0|T_YMM|T_ER_X|T_MUST_EVEX|T_B16, 0x79); } void vcvtph2uw(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_MAP5|T_W0|T_YMM|T_ER_Z|T_MUST_EVEX|T_B16, 0x7D); } void vcvtph2w(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_66|T_MAP5|T_W0|T_YMM|T_ER_Z|T_MUST_EVEX|T_B16, 0x7D); } void vcvtps2ibs(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_66|T_MAP5|T_W0|T_YMM|T_ER_Z|T_MUST_EVEX|T_B32, 0x69); } void vcvtps2iubs(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_66|T_MAP5|T_W0|T_YMM|T_ER_Z|T_MUST_EVEX|T_B32, 0x6B); } -void vcvtps2phx(const Xmm& x, const Operand& op) { checkCvt4(x, op); opCvt(x, op, T_N16|T_N_VL|T_66|T_MAP5|T_W0|T_ER_Z|T_MUST_EVEX|T_B32, 0x1D); } -void vcvtps2qq(const Xmm& x, const Operand& op) { checkCvt1(x, op); opVex(x, 0, op, T_N8|T_N_VL|T_66|T_0F|T_W0|T_YMM|T_ER_Y|T_MUST_EVEX|T_B32, 0x7B); } +void vcvtps2phx(const Xmm& x, const Operand& op) { opCvt_xx_yx_zy_sized(x, op, T_N16|T_N_VL|T_66|T_MAP5|T_W0|T_ER_Z|T_MUST_EVEX|T_B32, 0x1D); } +void vcvtps2qq(const Xmm& x, const Operand& op) { opCvt_xx_xy_yz(x, op, T_N8|T_N_VL|T_66|T_0F|T_W0|T_YMM|T_ER_Y|T_MUST_EVEX|T_B32, 0x7B); } void vcvtps2udq(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_0F|T_W0|T_YMM|T_ER_Z|T_MUST_EVEX|T_B32, 0x79); } -void vcvtps2uqq(const Xmm& x, const Operand& op) { checkCvt1(x, op); opVex(x, 0, op, T_N8|T_N_VL|T_66|T_0F|T_W0|T_YMM|T_ER_Y|T_MUST_EVEX|T_B32, 0x79); } +void vcvtps2uqq(const Xmm& x, const Operand& op) { opCvt_xx_xy_yz(x, op, T_N8|T_N_VL|T_66|T_0F|T_W0|T_YMM|T_ER_Y|T_MUST_EVEX|T_B32, 0x79); } void vcvtqq2pd(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_F3|T_0F|T_EW1|T_YMM|T_ER_Z|T_MUST_EVEX|T_B64, 0xE6); } -void vcvtqq2ph(const Xmm& x, const Operand& op) { opCvt5(x, op, T_N16|T_N_VL|T_MAP5|T_EW1|T_ER_Z|T_MUST_EVEX|T_B64, 0x5B); } -void vcvtqq2ps(const Xmm& x, const Operand& op) { opCvt2(x, op, T_0F|T_EW1|T_YMM|T_ER_Z|T_MUST_EVEX|T_B64, 0x5B); } +void vcvtqq2ph(const Xmm& x, const Operand& op) { opCvt_xx_yx_zx(x, op, T_N16|T_N_VL|T_MAP5|T_EW1|T_ER_Z|T_MUST_EVEX|T_B64, 0x5B); } +void vcvtqq2ps(const Xmm& x, const Operand& op) { opCvt_xx_yx_zy(x, op, T_0F|T_EW1|T_YMM|T_ER_Z|T_MUST_EVEX|T_B64, 0x5B); } void vcvtsd2sh(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_N8|T_F2|T_MAP5|T_EW1|T_ER_X|T_MUST_EVEX, 0x5A); } -void vcvtsd2usi(const Reg32e& r, const Operand& op) { uint64_t type = (T_N8|T_F2|T_0F|T_ER_X|T_MUST_EVEX) | (r.isREG(64) ? T_EW1 : T_W0); opVex(r, &xm0, op, type, 0x79); } +void vcvtsd2usi(const Reg32e& r, const Operand& op) { opCvtX2Si(r, op, T_N8|T_F2|T_0F|T_ER_X|T_MUST_EVEX, 0x79); } void vcvtsh2sd(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_N2|T_F3|T_MAP5|T_W0|T_SAE_X|T_MUST_EVEX, 0x5A); } -void vcvtsh2si(const Reg32e& r, const Operand& op) { uint64_t type = (T_N2|T_F3|T_MAP5|T_ER_X|T_MUST_EVEX) | (r.isREG(64) ? T_EW1 : T_W0); opVex(r, &xm0, op, type, 0x2D); } +void vcvtsh2si(const Reg32e& r, const Operand& op) { opCvtX2Si(r, op, T_N2|T_F3|T_MAP5|T_ER_X|T_MUST_EVEX, 0x2D); } void vcvtsh2ss(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_N2|T_MAP6|T_W0|T_SAE_X|T_MUST_EVEX, 0x13); } -void vcvtsh2usi(const Reg32e& r, const Operand& op) { uint64_t type = (T_N2|T_F3|T_MAP5|T_ER_X|T_MUST_EVEX) | (r.isREG(64) ? T_EW1 : T_W0); opVex(r, &xm0, op, type, 0x79); } -void vcvtsi2sh(const Xmm& x1, const Xmm& x2, const Operand& op) { if (!(x1.isXMM() && x2.isXMM() && op.isBit(32|64))) XBYAK_THROW(ERR_BAD_COMBINATION) uint64_t type = (T_F3|T_MAP5|T_ER_R|T_MUST_EVEX|T_M_K) | (op.isBit(32) ? (T_W0 | T_N4) : (T_EW1 | T_N8)); opVex(x1, &x2, op, type, 0x2A); } +void vcvtsh2usi(const Reg32e& r, const Operand& op) { opCvtX2Si(r, op, T_N2|T_F3|T_MAP5|T_ER_X|T_MUST_EVEX, 0x79); } +void vcvtsi2sh(const Xmm& x1, const Xmm& x2, const Operand& op) { opCvtSi2X(x1, x2, op, T_F3|T_MAP5|T_ER_R|T_MUST_EVEX|T_M_K, T_EW1 | T_N8, T_W0 | T_N4, 0x2A); } void vcvtss2sh(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_N4|T_MAP5|T_W0|T_ER_X|T_MUST_EVEX, 0x1D); } -void vcvtss2usi(const Reg32e& r, const Operand& op) { uint64_t type = (T_N4|T_F3|T_0F|T_ER_X|T_MUST_EVEX) | (r.isREG(64) ? T_EW1 : T_W0); opVex(r, &xm0, op, type, 0x79); } +void vcvtss2usi(const Reg32e& r, const Operand& op) { opCvtX2Si(r, op, T_N4|T_F3|T_0F|T_ER_X|T_MUST_EVEX, 0x79); } void vcvttbf162ibs(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_F2|T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x68); } void vcvttbf162iubs(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_F2|T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x6A); } -void vcvttpd2dqs(const Xmm& x, const Operand& op) { opCvt2(x, op, T_MAP5|T_EW1|T_YMM|T_SAE_Z|T_MUST_EVEX|T_B64, 0x6D); } +void vcvttpd2dqs(const Xmm& x, const Operand& op) { opCvt_xx_yx_zy(x, op, T_MAP5|T_EW1|T_YMM|T_SAE_Z|T_MUST_EVEX|T_B64, 0x6D); } void vcvttpd2qq(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_66|T_0F|T_EW1|T_YMM|T_SAE_Z|T_MUST_EVEX|T_B64, 0x7A); } void vcvttpd2qqs(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_66|T_MAP5|T_EW1|T_YMM|T_SAE_Z|T_MUST_EVEX|T_B64, 0x6D); } -void vcvttpd2udq(const Xmm& x, const Operand& op) { opCvt2(x, op, T_0F|T_EW1|T_YMM|T_SAE_Z|T_MUST_EVEX|T_B64, 0x78); } -void vcvttpd2udqs(const Xmm& x, const Operand& op) { opCvt2(x, op, T_MAP5|T_EW1|T_YMM|T_SAE_Z|T_MUST_EVEX|T_B64, 0x6C); } +void vcvttpd2udq(const Xmm& x, const Operand& op) { opCvt_xx_yx_zy(x, op, T_0F|T_EW1|T_YMM|T_SAE_Z|T_MUST_EVEX|T_B64, 0x78); } +void vcvttpd2udqs(const Xmm& x, const Operand& op) { opCvt_xx_yx_zy(x, op, T_MAP5|T_EW1|T_YMM|T_SAE_Z|T_MUST_EVEX|T_B64, 0x6C); } void vcvttpd2uqq(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_66|T_0F|T_EW1|T_YMM|T_SAE_Z|T_MUST_EVEX|T_B64, 0x78); } void vcvttpd2uqqs(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_66|T_MAP5|T_EW1|T_YMM|T_SAE_Z|T_MUST_EVEX|T_B64, 0x6C); } -void vcvttph2dq(const Xmm& x, const Operand& op) { checkCvt1(x, op); opVex(x, 0, op, T_N8|T_N_VL|T_F3|T_MAP5|T_W0|T_YMM|T_SAE_Y|T_MUST_EVEX|T_B16, 0x5B); } -void vcvttph2ibs(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_MAP5|T_W0|T_YMM|T_ER_Z|T_MUST_EVEX|T_B16, 0x68); } -void vcvttph2iubs(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_MAP5|T_W0|T_YMM|T_ER_Z|T_MUST_EVEX|T_B16, 0x6A); } -void vcvttph2qq(const Xmm& x, const Operand& op) { if (!op.isXMM() && !op.isMEM()) XBYAK_THROW(ERR_BAD_MEM_SIZE) opVex(x, 0, op, T_N4|T_N_VL|T_66|T_MAP5|T_W0|T_YMM|T_SAE_X|T_MUST_EVEX|T_B16, 0x7A); } -void vcvttph2udq(const Xmm& x, const Operand& op) { checkCvt1(x, op); opVex(x, 0, op, T_N8|T_N_VL|T_MAP5|T_W0|T_YMM|T_SAE_Y|T_MUST_EVEX|T_B16, 0x78); } -void vcvttph2uqq(const Xmm& x, const Operand& op) { if (!op.isXMM() && !op.isMEM()) XBYAK_THROW(ERR_BAD_MEM_SIZE) opVex(x, 0, op, T_N4|T_N_VL|T_66|T_MAP5|T_W0|T_YMM|T_SAE_X|T_MUST_EVEX|T_B16, 0x78); } +void vcvttph2dq(const Xmm& x, const Operand& op) { opCvt_xx_xy_yz(x, op, T_N8|T_N_VL|T_F3|T_MAP5|T_W0|T_YMM|T_SAE_Y|T_MUST_EVEX|T_B16, 0x5B); } +void vcvttph2ibs(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_MAP5|T_W0|T_YMM|T_SAE_Z|T_MUST_EVEX|T_B16, 0x68); } +void vcvttph2iubs(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_MAP5|T_W0|T_YMM|T_SAE_Z|T_MUST_EVEX|T_B16, 0x6A); } +void vcvttph2qq(const Xmm& x, const Operand& op) { opX_XM(op, x, T_N4|T_N_VL|T_66|T_MAP5|T_W0|T_YMM|T_SAE_X|T_MUST_EVEX|T_B16, 0x7A); } +void vcvttph2udq(const Xmm& x, const Operand& op) { opCvt_xx_xy_yz(x, op, T_N8|T_N_VL|T_MAP5|T_W0|T_YMM|T_SAE_Y|T_MUST_EVEX|T_B16, 0x78); } +void vcvttph2uqq(const Xmm& x, const Operand& op) { opX_XM(op, x, T_N4|T_N_VL|T_66|T_MAP5|T_W0|T_YMM|T_SAE_X|T_MUST_EVEX|T_B16, 0x78); } void vcvttph2uw(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_MAP5|T_W0|T_YMM|T_SAE_Z|T_MUST_EVEX|T_B16, 0x7C); } void vcvttph2w(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_66|T_MAP5|T_W0|T_YMM|T_SAE_Z|T_MUST_EVEX|T_B16, 0x7C); } void vcvttps2dqs(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_MAP5|T_W0|T_YMM|T_SAE_Z|T_MUST_EVEX|T_B32, 0x6D); } -void vcvttps2ibs(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_66|T_MAP5|T_W0|T_YMM|T_ER_Z|T_MUST_EVEX|T_B32, 0x68); } -void vcvttps2iubs(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_66|T_MAP5|T_W0|T_YMM|T_ER_Z|T_MUST_EVEX|T_B32, 0x6A); } -void vcvttps2qq(const Xmm& x, const Operand& op) { checkCvt1(x, op); opVex(x, 0, op, T_N8|T_N_VL|T_66|T_0F|T_W0|T_YMM|T_SAE_Y|T_MUST_EVEX|T_B32, 0x7A); } -void vcvttps2qqs(const Xmm& x, const Operand& op) { checkCvt1(x, op); opVex(x, 0, op, T_N8|T_N_VL|T_66|T_MAP5|T_W0|T_YMM|T_SAE_Y|T_MUST_EVEX|T_B32, 0x6D); } +void vcvttps2ibs(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_66|T_MAP5|T_W0|T_YMM|T_SAE_Z|T_MUST_EVEX|T_B32, 0x68); } +void vcvttps2iubs(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_66|T_MAP5|T_W0|T_YMM|T_SAE_Z|T_MUST_EVEX|T_B32, 0x6A); } +void vcvttps2qq(const Xmm& x, const Operand& op) { opCvt_xx_xy_yz(x, op, T_N8|T_N_VL|T_66|T_0F|T_W0|T_YMM|T_SAE_Y|T_MUST_EVEX|T_B32, 0x7A); } +void vcvttps2qqs(const Xmm& x, const Operand& op) { opCvt_xx_xy_yz(x, op, T_N8|T_N_VL|T_66|T_MAP5|T_W0|T_YMM|T_SAE_Y|T_MUST_EVEX|T_B32, 0x6D); } void vcvttps2udq(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_0F|T_W0|T_YMM|T_SAE_Z|T_MUST_EVEX|T_B32, 0x78); } void vcvttps2udqs(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_MAP5|T_W0|T_YMM|T_SAE_Z|T_MUST_EVEX|T_B32, 0x6C); } -void vcvttps2uqq(const Xmm& x, const Operand& op) { checkCvt1(x, op); opVex(x, 0, op, T_N8|T_N_VL|T_66|T_0F|T_W0|T_YMM|T_SAE_Y|T_MUST_EVEX|T_B32, 0x78); } -void vcvttps2uqqs(const Xmm& x, const Operand& op) { checkCvt1(x, op); opVex(x, 0, op, T_N8|T_N_VL|T_66|T_MAP5|T_W0|T_YMM|T_SAE_Y|T_MUST_EVEX|T_B32, 0x6C); } -void vcvttsd2sis(const Reg32e& r, const Operand& op) { uint64_t type = (T_N8|T_F2|T_MAP5|T_SAE_X|T_MUST_EVEX) | (r.isREG(64) ? T_EW1 : T_W0); opVex(r, &xm0, op, type, 0x6D); } -void vcvttsd2usi(const Reg32e& r, const Operand& op) { uint64_t type = (T_N8|T_F2|T_0F|T_SAE_X|T_MUST_EVEX) | (r.isREG(64) ? T_EW1 : T_W0); opVex(r, &xm0, op, type, 0x78); } -void vcvttsd2usis(const Reg32e& r, const Operand& op) { uint64_t type = (T_N8|T_F2|T_MAP5|T_SAE_X|T_MUST_EVEX) | (r.isREG(64) ? T_EW1 : T_W0); opVex(r, &xm0, op, type, 0x6C); } -void vcvttsh2si(const Reg32e& r, const Operand& op) { uint64_t type = (T_N2|T_F3|T_MAP5|T_SAE_X|T_MUST_EVEX) | (r.isREG(64) ? T_EW1 : T_W0); opVex(r, &xm0, op, type, 0x2C); } -void vcvttsh2usi(const Reg32e& r, const Operand& op) { uint64_t type = (T_N2|T_F3|T_MAP5|T_SAE_X|T_MUST_EVEX) | (r.isREG(64) ? T_EW1 : T_W0); opVex(r, &xm0, op, type, 0x78); } -void vcvttss2sis(const Reg32e& r, const Operand& op) { uint64_t type = (T_N4|T_F3|T_MAP5|T_SAE_X|T_MUST_EVEX) | (r.isREG(64) ? T_EW1 : T_W0); opVex(r, &xm0, op, type, 0x6D); } -void vcvttss2usi(const Reg32e& r, const Operand& op) { uint64_t type = (T_N4|T_F3|T_0F|T_SAE_X|T_MUST_EVEX) | (r.isREG(64) ? T_EW1 : T_W0); opVex(r, &xm0, op, type, 0x78); } -void vcvttss2usis(const Reg32e& r, const Operand& op) { uint64_t type = (T_N4|T_F3|T_MAP5|T_SAE_X|T_MUST_EVEX) | (r.isREG(64) ? T_EW1 : T_W0); opVex(r, &xm0, op, type, 0x6C); } -void vcvtudq2pd(const Xmm& x, const Operand& op) { checkCvt1(x, op); opVex(x, 0, op, T_N8|T_N_VL|T_F3|T_0F|T_W0|T_YMM|T_MUST_EVEX|T_B32, 0x7A); } -void vcvtudq2ph(const Xmm& x, const Operand& op) { checkCvt4(x, op); opCvt(x, op, T_N16|T_N_VL|T_F2|T_MAP5|T_W0|T_ER_Z|T_MUST_EVEX|T_B32, 0x7A); } +void vcvttps2uqq(const Xmm& x, const Operand& op) { opCvt_xx_xy_yz(x, op, T_N8|T_N_VL|T_66|T_0F|T_W0|T_YMM|T_SAE_Y|T_MUST_EVEX|T_B32, 0x78); } +void vcvttps2uqqs(const Xmm& x, const Operand& op) { opCvt_xx_xy_yz(x, op, T_N8|T_N_VL|T_66|T_MAP5|T_W0|T_YMM|T_SAE_Y|T_MUST_EVEX|T_B32, 0x6C); } +void vcvttsd2sis(const Reg32e& r, const Operand& op) { opCvtX2Si(r, op, T_N8|T_F2|T_MAP5|T_SAE_X|T_MUST_EVEX, 0x6D); } +void vcvttsd2usi(const Reg32e& r, const Operand& op) { opCvtX2Si(r, op, T_N8|T_F2|T_0F|T_SAE_X|T_MUST_EVEX, 0x78); } +void vcvttsd2usis(const Reg32e& r, const Operand& op) { opCvtX2Si(r, op, T_N8|T_F2|T_MAP5|T_SAE_X|T_MUST_EVEX, 0x6C); } +void vcvttsh2si(const Reg32e& r, const Operand& op) { opCvtX2Si(r, op, T_N2|T_F3|T_MAP5|T_SAE_X|T_MUST_EVEX, 0x2C); } +void vcvttsh2usi(const Reg32e& r, const Operand& op) { opCvtX2Si(r, op, T_N2|T_F3|T_MAP5|T_SAE_X|T_MUST_EVEX, 0x78); } +void vcvttss2sis(const Reg32e& r, const Operand& op) { opCvtX2Si(r, op, T_N4|T_F3|T_MAP5|T_SAE_X|T_MUST_EVEX, 0x6D); } +void vcvttss2usi(const Reg32e& r, const Operand& op) { opCvtX2Si(r, op, T_N4|T_F3|T_0F|T_SAE_X|T_MUST_EVEX, 0x78); } +void vcvttss2usis(const Reg32e& r, const Operand& op) { opCvtX2Si(r, op, T_N4|T_F3|T_MAP5|T_SAE_X|T_MUST_EVEX, 0x6C); } +void vcvtudq2pd(const Xmm& x, const Operand& op) { opCvt_xx_xy_yz(x, op, T_N8|T_N_VL|T_F3|T_0F|T_W0|T_YMM|T_MUST_EVEX|T_B32, 0x7A); } +void vcvtudq2ph(const Xmm& x, const Operand& op) { opCvt_xx_yx_zy_sized(x, op, T_N16|T_N_VL|T_F2|T_MAP5|T_W0|T_ER_Z|T_MUST_EVEX|T_B32, 0x7A); } void vcvtudq2ps(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_F2|T_0F|T_W0|T_YMM|T_ER_Z|T_MUST_EVEX|T_B32, 0x7A); } void vcvtuqq2pd(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_F3|T_0F|T_EW1|T_YMM|T_ER_Z|T_MUST_EVEX|T_B64, 0x7A); } -void vcvtuqq2ph(const Xmm& x, const Operand& op) { opCvt5(x, op, T_N16|T_N_VL|T_F2|T_MAP5|T_EW1|T_ER_Z|T_MUST_EVEX|T_B64, 0x7A); } -void vcvtuqq2ps(const Xmm& x, const Operand& op) { opCvt2(x, op, T_F2|T_0F|T_EW1|T_YMM|T_ER_Z|T_MUST_EVEX|T_B64, 0x7A); } -void vcvtusi2sd(const Xmm& x1, const Xmm& x2, const Operand& op) { opCvt3(x1, x2, op, T_F2 | T_0F | T_MUST_EVEX, T_W1 | T_EW1 | T_ER_X | T_N8, T_W0 | T_N4, 0x7B); } -void vcvtusi2sh(const Xmm& x1, const Xmm& x2, const Operand& op) { if (!(x1.isXMM() && x2.isXMM() && op.isBit(32|64))) XBYAK_THROW(ERR_BAD_COMBINATION) uint64_t type = (T_F3|T_MAP5|T_ER_R|T_MUST_EVEX|T_M_K) | (op.isBit(32) ? (T_W0 | T_N4) : (T_EW1 | T_N8)); opVex(x1, &x2, op, type, 0x7B); } -void vcvtusi2ss(const Xmm& x1, const Xmm& x2, const Operand& op) { opCvt3(x1, x2, op, T_F3 | T_0F | T_MUST_EVEX | T_ER_X, T_W1 | T_EW1 | T_N8, T_W0 | T_N4, 0x7B); } +void vcvtuqq2ph(const Xmm& x, const Operand& op) { opCvt_xx_yx_zx(x, op, T_N16|T_N_VL|T_F2|T_MAP5|T_EW1|T_ER_Z|T_MUST_EVEX|T_B64, 0x7A); } +void vcvtuqq2ps(const Xmm& x, const Operand& op) { opCvt_xx_yx_zy(x, op, T_F2|T_0F|T_EW1|T_YMM|T_ER_Z|T_MUST_EVEX|T_B64, 0x7A); } +void vcvtusi2sd(const Xmm& x1, const Xmm& x2, const Operand& op) { opCvtSi2X(x1, x2, op, T_F2 | T_0F | T_MUST_EVEX, T_W1 | T_EW1 | T_ER_R | T_N8, T_W0 | T_N4, 0x7B); } +void vcvtusi2sh(const Xmm& x1, const Xmm& x2, const Operand& op) { opCvtSi2X(x1, x2, op, T_F3|T_MAP5|T_ER_R|T_MUST_EVEX|T_M_K, T_EW1 | T_N8, T_W0 | T_N4, 0x7B); } +void vcvtusi2ss(const Xmm& x1, const Xmm& x2, const Operand& op) { opCvtSi2X(x1, x2, op, T_F3 | T_0F | T_MUST_EVEX | T_ER_R, T_W1 | T_EW1 | T_N8, T_W0 | T_N4, 0x7B); } void vcvtuw2ph(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_F2|T_MAP5|T_W0|T_YMM|T_ER_Z|T_MUST_EVEX|T_B16, 0x7D); } void vcvtw2ph(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_F3|T_MAP5|T_W0|T_YMM|T_ER_Z|T_MUST_EVEX|T_B16, 0x7D); } void vdbpsadbw(const Xmm& x1, const Xmm& x2, const Operand& op, uint8_t imm) { opAVX_X_X_XM(x1, x2, op, T_66|T_0F3A|T_W0|T_YMM|T_MUST_EVEX, 0x42, imm); } @@ -2299,14 +2300,14 @@ void vexp2pd(const Zmm& z, const Operand& op) { opAVX_X_XM_IMM(z, op, T_66 | T_0 void vexp2ps(const Zmm& z, const Operand& op) { opAVX_X_XM_IMM(z, op, T_66 | T_0F38 | T_MUST_EVEX | T_YMM | T_W0 | T_B32 | T_SAE_Z, 0xC8); } void vexpandpd(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_N8|T_66|T_0F38|T_EW1|T_YMM|T_MUST_EVEX, 0x88); } void vexpandps(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_N4|T_66|T_0F38|T_W0|T_YMM|T_MUST_EVEX, 0x88); } -void vextractf32x4(const Operand& op, const Ymm& r, uint8_t imm) { if (!op.is(Operand::MEM | Operand::XMM)) XBYAK_THROW(ERR_BAD_COMBINATION) opVex(r, 0, op, T_N16|T_66|T_0F3A|T_W0|T_YMM|T_MUST_EVEX, 0x19, imm); } -void vextractf32x8(const Operand& op, const Zmm& r, uint8_t imm) { if (!op.is(Operand::MEM | Operand::YMM)) XBYAK_THROW(ERR_BAD_COMBINATION) opVex(r, 0, op, T_N32|T_66|T_0F3A|T_W0|T_YMM|T_MUST_EVEX, 0x1B, imm); } -void vextractf64x2(const Operand& op, const Ymm& r, uint8_t imm) { if (!op.is(Operand::MEM | Operand::XMM)) XBYAK_THROW(ERR_BAD_COMBINATION) opVex(r, 0, op, T_N16|T_66|T_0F3A|T_EW1|T_YMM|T_MUST_EVEX, 0x19, imm); } -void vextractf64x4(const Operand& op, const Zmm& r, uint8_t imm) { if (!op.is(Operand::MEM | Operand::YMM)) XBYAK_THROW(ERR_BAD_COMBINATION) opVex(r, 0, op, T_N32|T_66|T_0F3A|T_EW1|T_YMM|T_MUST_EVEX, 0x1B, imm); } -void vextracti32x4(const Operand& op, const Ymm& r, uint8_t imm) { if (!op.is(Operand::MEM | Operand::XMM)) XBYAK_THROW(ERR_BAD_COMBINATION) opVex(r, 0, op, T_N16|T_66|T_0F3A|T_W0|T_YMM|T_MUST_EVEX, 0x39, imm); } -void vextracti32x8(const Operand& op, const Zmm& r, uint8_t imm) { if (!op.is(Operand::MEM | Operand::YMM)) XBYAK_THROW(ERR_BAD_COMBINATION) opVex(r, 0, op, T_N32|T_66|T_0F3A|T_W0|T_YMM|T_MUST_EVEX, 0x3B, imm); } -void vextracti64x2(const Operand& op, const Ymm& r, uint8_t imm) { if (!op.is(Operand::MEM | Operand::XMM)) XBYAK_THROW(ERR_BAD_COMBINATION) opVex(r, 0, op, T_N16|T_66|T_0F3A|T_EW1|T_YMM|T_MUST_EVEX, 0x39, imm); } -void vextracti64x4(const Operand& op, const Zmm& r, uint8_t imm) { if (!op.is(Operand::MEM | Operand::YMM)) XBYAK_THROW(ERR_BAD_COMBINATION) opVex(r, 0, op, T_N32|T_66|T_0F3A|T_EW1|T_YMM|T_MUST_EVEX, 0x3B, imm); } +void vextractf32x4(const Operand& op, const Ymm& r, uint8_t imm) { if (!op.is(Operand::MEM | Operand::XMM)) XBYAK_THROW(ERR_BAD_COMBINATION) opVex(r, 0, op, T_N16|T_66|T_0F3A|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x19, imm); } +void vextractf32x8(const Operand& op, const Zmm& r, uint8_t imm) { if (!op.is(Operand::MEM | Operand::YMM)) XBYAK_THROW(ERR_BAD_COMBINATION) opVex(r, 0, op, T_N32|T_66|T_0F3A|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x1B, imm); } +void vextractf64x2(const Operand& op, const Ymm& r, uint8_t imm) { if (!op.is(Operand::MEM | Operand::XMM)) XBYAK_THROW(ERR_BAD_COMBINATION) opVex(r, 0, op, T_N16|T_66|T_0F3A|T_EW1|T_YMM|T_MUST_EVEX|T_M_K, 0x19, imm); } +void vextractf64x4(const Operand& op, const Zmm& r, uint8_t imm) { if (!op.is(Operand::MEM | Operand::YMM)) XBYAK_THROW(ERR_BAD_COMBINATION) opVex(r, 0, op, T_N32|T_66|T_0F3A|T_EW1|T_YMM|T_MUST_EVEX|T_M_K, 0x1B, imm); } +void vextracti32x4(const Operand& op, const Ymm& r, uint8_t imm) { if (!op.is(Operand::MEM | Operand::XMM)) XBYAK_THROW(ERR_BAD_COMBINATION) opVex(r, 0, op, T_N16|T_66|T_0F3A|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x39, imm); } +void vextracti32x8(const Operand& op, const Zmm& r, uint8_t imm) { if (!op.is(Operand::MEM | Operand::YMM)) XBYAK_THROW(ERR_BAD_COMBINATION) opVex(r, 0, op, T_N32|T_66|T_0F3A|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x3B, imm); } +void vextracti64x2(const Operand& op, const Ymm& r, uint8_t imm) { if (!op.is(Operand::MEM | Operand::XMM)) XBYAK_THROW(ERR_BAD_COMBINATION) opVex(r, 0, op, T_N16|T_66|T_0F3A|T_EW1|T_YMM|T_MUST_EVEX|T_M_K, 0x39, imm); } +void vextracti64x4(const Operand& op, const Zmm& r, uint8_t imm) { if (!op.is(Operand::MEM | Operand::YMM)) XBYAK_THROW(ERR_BAD_COMBINATION) opVex(r, 0, op, T_N32|T_66|T_0F3A|T_EW1|T_YMM|T_MUST_EVEX|T_M_K, 0x3B, imm); } void vfcmaddcph(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_F2|T_MAP6|T_W0|T_YMM|T_ER_Z|T_MUST_EVEX|T_B32, 0x56); } void vfcmulcph(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_F2|T_MAP6|T_W0|T_YMM|T_ER_Z|T_MUST_EVEX|T_B32, 0xD6); } void vfixupimmpd(const Xmm& x1, const Xmm& x2, const Operand& op, uint8_t imm) { opAVX_X_X_XM(x1, x2, op, T_66|T_0F3A|T_EW1|T_YMM|T_SAE_Z|T_MUST_EVEX|T_B64, 0x54, imm); } @@ -2462,10 +2463,10 @@ void vpcmpud(const Opmask& k, const Xmm& x, const Operand& op, uint8_t imm) { op void vpcmpuq(const Opmask& k, const Xmm& x, const Operand& op, uint8_t imm) { opAVX_K_X_XM(k, x, op, T_66|T_0F3A|T_EW1|T_YMM|T_MUST_EVEX|T_B64, 0x1E, imm); } void vpcmpuw(const Opmask& k, const Xmm& x, const Operand& op, uint8_t imm) { opAVX_K_X_XM(k, x, op, T_66|T_0F3A|T_EW1|T_YMM|T_MUST_EVEX, 0x3E, imm); } void vpcmpw(const Opmask& k, const Xmm& x, const Operand& op, uint8_t imm) { opAVX_K_X_XM(k, x, op, T_66|T_0F3A|T_EW1|T_YMM|T_MUST_EVEX, 0x3F, imm); } -void vpcompressb(const Operand& op, const Xmm& x) { opAVX_X_XM_IMM(x, op, T_N1|T_66|T_0F38|T_W0|T_YMM|T_MUST_EVEX, 0x63); } -void vpcompressd(const Operand& op, const Xmm& x) { opAVX_X_XM_IMM(x, op, T_N4|T_66|T_0F38|T_W0|T_YMM|T_MUST_EVEX, 0x8B); } -void vpcompressq(const Operand& op, const Xmm& x) { opAVX_X_XM_IMM(x, op, T_N8|T_66|T_0F38|T_EW1|T_YMM|T_MUST_EVEX, 0x8B); } -void vpcompressw(const Operand& op, const Xmm& x) { opAVX_X_XM_IMM(x, op, T_N2|T_66|T_0F38|T_EW1|T_YMM|T_MUST_EVEX, 0x63); } +void vpcompressb(const Operand& op, const Xmm& x) { opAVX_X_XM_IMM(x, op, T_N1|T_66|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x63); } +void vpcompressd(const Operand& op, const Xmm& x) { opAVX_X_XM_IMM(x, op, T_N4|T_66|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x8B); } +void vpcompressq(const Operand& op, const Xmm& x) { opAVX_X_XM_IMM(x, op, T_N8|T_66|T_0F38|T_EW1|T_YMM|T_MUST_EVEX|T_M_K, 0x8B); } +void vpcompressw(const Operand& op, const Xmm& x) { opAVX_X_XM_IMM(x, op, T_N2|T_66|T_0F38|T_EW1|T_YMM|T_MUST_EVEX|T_M_K, 0x63); } void vpconflictd(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_66|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_B32, 0xC4); } void vpconflictq(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_66|T_0F38|T_EW1|T_YMM|T_MUST_EVEX|T_B64, 0xC4); } void vpdpbssd(const Xmm& x1, const Xmm& x2, const Operand& op, PreferredEncoding encoding = DefaultEncoding) { opEncoding(x1, x2, op, T_F2|T_0F38|T_W0|T_YMM, 0x50, encoding, NONE, T_NONE, T_B32, 1); } @@ -2512,30 +2513,30 @@ void vpminsq(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, void vpminuq(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_66|T_0F38|T_EW1|T_YMM|T_MUST_EVEX|T_B64, 0x3B); } void vpmovb2m(const Opmask& k, const Xmm& x) { opVex(k, 0, x, T_F3 | T_0F38 | T_MUST_EVEX | T_YMM | T_W0, 0x29); } void vpmovd2m(const Opmask& k, const Xmm& x) { opVex(k, 0, x, T_F3 | T_0F38 | T_MUST_EVEX | T_YMM | T_W0, 0x39); } -void vpmovdb(const Operand& op, const Xmm& x) { opVmov(op, x, T_N4|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x31, false); } -void vpmovdw(const Operand& op, const Xmm& x) { opVmov(op, x, T_N8|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x33, true); } +void vpmovdb(const Operand& op, const Xmm& x) { opX_XM(op, x, T_N4|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x31); } +void vpmovdw(const Operand& op, const Xmm& x) { opCvt_xx_xy_yz(x, op, T_N8|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x33); } void vpmovm2b(const Xmm& x, const Opmask& k) { opVex(x, 0, k, T_F3 | T_0F38 | T_MUST_EVEX | T_YMM | T_W0, 0x28); } void vpmovm2d(const Xmm& x, const Opmask& k) { opVex(x, 0, k, T_F3 | T_0F38 | T_MUST_EVEX | T_YMM | T_W0, 0x38); } void vpmovm2q(const Xmm& x, const Opmask& k) { opVex(x, 0, k, T_F3 | T_0F38 | T_MUST_EVEX | T_YMM | T_EW1, 0x38); } void vpmovm2w(const Xmm& x, const Opmask& k) { opVex(x, 0, k, T_F3 | T_0F38 | T_MUST_EVEX | T_YMM | T_EW1, 0x28); } void vpmovq2m(const Opmask& k, const Xmm& x) { opVex(k, 0, x, T_F3 | T_0F38 | T_MUST_EVEX | T_YMM | T_EW1, 0x39); } -void vpmovqb(const Operand& op, const Xmm& x) { opVmov(op, x, T_N2|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x32, false); } -void vpmovqd(const Operand& op, const Xmm& x) { opVmov(op, x, T_N8|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x35, true); } -void vpmovqw(const Operand& op, const Xmm& x) { opVmov(op, x, T_N4|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x34, false); } -void vpmovsdb(const Operand& op, const Xmm& x) { opVmov(op, x, T_N4|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x21, false); } -void vpmovsdw(const Operand& op, const Xmm& x) { opVmov(op, x, T_N8|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x23, true); } -void vpmovsqb(const Operand& op, const Xmm& x) { opVmov(op, x, T_N2|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x22, false); } -void vpmovsqd(const Operand& op, const Xmm& x) { opVmov(op, x, T_N8|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x25, true); } -void vpmovsqw(const Operand& op, const Xmm& x) { opVmov(op, x, T_N4|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x24, false); } -void vpmovswb(const Operand& op, const Xmm& x) { opVmov(op, x, T_N8|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x20, true); } -void vpmovusdb(const Operand& op, const Xmm& x) { opVmov(op, x, T_N4|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x11, false); } -void vpmovusdw(const Operand& op, const Xmm& x) { opVmov(op, x, T_N8|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x13, true); } -void vpmovusqb(const Operand& op, const Xmm& x) { opVmov(op, x, T_N2|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x12, false); } -void vpmovusqd(const Operand& op, const Xmm& x) { opVmov(op, x, T_N8|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x15, true); } -void vpmovusqw(const Operand& op, const Xmm& x) { opVmov(op, x, T_N4|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x14, false); } -void vpmovuswb(const Operand& op, const Xmm& x) { opVmov(op, x, T_N8|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x10, true); } +void vpmovqb(const Operand& op, const Xmm& x) { opX_XM(op, x, T_N2|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x32); } +void vpmovqd(const Operand& op, const Xmm& x) { opCvt_xx_xy_yz(x, op, T_N8|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x35); } +void vpmovqw(const Operand& op, const Xmm& x) { opX_XM(op, x, T_N4|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x34); } +void vpmovsdb(const Operand& op, const Xmm& x) { opX_XM(op, x, T_N4|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x21); } +void vpmovsdw(const Operand& op, const Xmm& x) { opCvt_xx_xy_yz(x, op, T_N8|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x23); } +void vpmovsqb(const Operand& op, const Xmm& x) { opX_XM(op, x, T_N2|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x22); } +void vpmovsqd(const Operand& op, const Xmm& x) { opCvt_xx_xy_yz(x, op, T_N8|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x25); } +void vpmovsqw(const Operand& op, const Xmm& x) { opX_XM(op, x, T_N4|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x24); } +void vpmovswb(const Operand& op, const Xmm& x) { opCvt_xx_xy_yz(x, op, T_N8|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x20); } +void vpmovusdb(const Operand& op, const Xmm& x) { opX_XM(op, x, T_N4|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x11); } +void vpmovusdw(const Operand& op, const Xmm& x) { opCvt_xx_xy_yz(x, op, T_N8|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x13); } +void vpmovusqb(const Operand& op, const Xmm& x) { opX_XM(op, x, T_N2|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x12); } +void vpmovusqd(const Operand& op, const Xmm& x) { opCvt_xx_xy_yz(x, op, T_N8|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x15); } +void vpmovusqw(const Operand& op, const Xmm& x) { opX_XM(op, x, T_N4|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x14); } +void vpmovuswb(const Operand& op, const Xmm& x) { opCvt_xx_xy_yz(x, op, T_N8|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x10); } void vpmovw2m(const Opmask& k, const Xmm& x) { opVex(k, 0, x, T_F3 | T_0F38 | T_MUST_EVEX | T_YMM | T_EW1, 0x29); } -void vpmovwb(const Operand& op, const Xmm& x) { opVmov(op, x, T_N8|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x30, true); } +void vpmovwb(const Operand& op, const Xmm& x) { opCvt_xx_xy_yz(x, op, T_N8|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x30); } void vpmullq(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_66|T_0F38|T_EW1|T_YMM|T_MUST_EVEX|T_B64, 0x40); } void vpmultishiftqb(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_66|T_0F38|T_EW1|T_YMM|T_MUST_EVEX|T_B64, 0x83); } void vpopcntb(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_66|T_0F38|T_W0|T_YMM|T_SAE_Z|T_MUST_EVEX, 0x54); } @@ -2662,6 +2663,12 @@ void vucomxsd(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_N8|T_F2 void vucomxsh(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_N2|T_F3|T_MAP5|T_W0|T_SAE_X|T_MUST_EVEX, 0x2E); } void vucomxss(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_N4|T_F3|T_0F|T_W0|T_SAE_X|T_MUST_EVEX, 0x2E); } #ifdef XBYAK64 +void bsrinit(const Bsr& b) { vex(b, b, 0, T_F2|T_0F38|T_W1, 0x49); setModRM(3, b.getIdx(), 0); } +void bsrmovf(const Bsr& b, const Zmm& z1, const Operand& op) { opVex(b, &z1, op, T_MUST_EVEX|T_MAP6|T_EW1|T_N1, 0x95); } +void bsrmovh(const Bsr& b, const Operand& op) { opVex(b, 0, op, T_N1|T_F2|T_MAP6|T_EW1|T_MUST_EVEX, 0x95); } +void bsrmovh(const Operand& op, const Bsr& b) { opVex(b, 0, op, T_N1|T_F2|T_MAP6|T_W0|T_MUST_EVEX, 0x95); } +void bsrmovl(const Bsr& b, const Operand& op) { opVex(b, 0, op, T_N1|T_F3|T_MAP6|T_EW1|T_MUST_EVEX, 0x95); } +void bsrmovl(const Operand& op, const Bsr& b) { opVex(b, 0, op, T_N1|T_F3|T_MAP6|T_W0|T_MUST_EVEX, 0x95); } void kmovq(const Reg64& r, const Opmask& k) { opKmov(k, r, true, 64); } void tcvtrowd2ps(const Zmm& z, const Tmm& t, const Reg32& r) { opVex(z, &r, t, T_F3|T_0F38|T_W0|T_MUST_EVEX, 0x4A); } void tcvtrowd2ps(const Zmm& z, const Tmm& t, uint8_t imm) { opVex(z, 0, t, T_F3|T_0F3A|T_W0|T_MUST_EVEX, 0x07, imm); } @@ -2673,12 +2680,47 @@ void tcvtrowps2phh(const Zmm& z, const Tmm& t, const Reg32& r) { opVex(z, &r, t, void tcvtrowps2phh(const Zmm& z, const Tmm& t, uint8_t imm) { opVex(z, 0, t, T_0F3A|T_W0|T_MUST_EVEX, 0x07, imm); } void tcvtrowps2phl(const Zmm& z, const Tmm& t, const Reg32& r) { opVex(z, &r, t, T_66|T_0F38|T_W0|T_MUST_EVEX, 0x6D); } void tcvtrowps2phl(const Zmm& z, const Tmm& t, uint8_t imm) { opVex(z, 0, t, T_F2|T_0F3A|T_W0|T_MUST_EVEX, 0x77, imm); } +void tilemovcol(const Tmm& t1, const Zmm& z2, const Reg32& r) { opVex(t1, &r, z2, T_66|T_0F38|T_EW1|T_MUST_EVEX, 0x4B); } +void tilemovcol(const Tmm& t1, const Zmm& z2, uint8_t imm) { opVex(t1, 0, z2, T_66|T_0F3A|T_EW1|T_MUST_EVEX, 0x2F, imm); } +void tilemovrow(const Tmm& t1, const Zmm& z2, const Reg32& r) { opVex(t1, &r, z2, T_66|T_0F38|T_EW1|T_MUST_EVEX, 0x4A); } +void tilemovrow(const Tmm& t1, const Zmm& z2, uint8_t imm) { opVex(t1, 0, z2, T_66|T_0F3A|T_EW1|T_MUST_EVEX, 0x07, imm); } void tilemovrow(const Zmm& z, const Tmm& t, const Reg32& r) { opVex(z, &r, t, T_66|T_0F38|T_W0|T_MUST_EVEX, 0x4A); } void tilemovrow(const Zmm& z, const Tmm& t, uint8_t imm) { opVex(z, 0, t, T_66|T_0F3A|T_W0|T_MUST_EVEX, 0x07, imm); } +void top2bf16ps(const Tmm& t1, const Zmm& z2, const Zmm& z3) { opVex(t1, &z3, z2, T_F3|T_0F38|T_W0|T_MUST_EVEX, 0x5C); } +void top4bssd(const Tmm& t1, const Zmm& z2, const Zmm& z3) { opVex(t1, &z3, z2, T_F2|T_0F38|T_W0|T_MUST_EVEX, 0x5E); } +void top4bsud(const Tmm& t1, const Zmm& z2, const Zmm& z3) { opVex(t1, &z3, z2, T_F3|T_0F38|T_W0|T_MUST_EVEX, 0x5E); } +void top4busd(const Tmm& t1, const Zmm& z2, const Zmm& z3) { opVex(t1, &z3, z2, T_66|T_0F38|T_W0|T_MUST_EVEX, 0x5E); } +void top4buud(const Tmm& t1, const Zmm& z2, const Zmm& z3) { opVex(t1, &z3, z2, T_0F38|T_W0|T_MUST_EVEX, 0x5E); } +void top4mxbf8ps(const Tmm& t1, const Zmm& z2, const Zmm& z3, uint8_t imm) { opVex(t1, &z3, z2, T_0F3A|T_W0|T_MUST_EVEX, 0x8D, imm); } +void top4mxbhf8ps(const Tmm& t1, const Zmm& z2, const Zmm& z3, uint8_t imm) { opVex(t1, &z3, z2, T_F2|T_0F3A|T_W0|T_MUST_EVEX, 0x8D, imm); } +void top4mxbssps(const Tmm& t1, const Zmm& z2, const Zmm& z3, uint8_t imm) { opVex(t1, &z3, z2, T_F2|T_0F3A|T_W0|T_MUST_EVEX, 0x8F, imm); } +void top4mxhbf8ps(const Tmm& t1, const Zmm& z2, const Zmm& z3, uint8_t imm) { opVex(t1, &z3, z2, T_F3|T_0F3A|T_W0|T_MUST_EVEX, 0x8D, imm); } +void top4mxhf8ps(const Tmm& t1, const Zmm& z2, const Zmm& z3, uint8_t imm) { opVex(t1, &z3, z2, T_66|T_0F3A|T_W0|T_MUST_EVEX, 0x8D, imm); } +void vcvtbf42hf8(const Xmm& x, const Operand& op) { opCvt_xx_xy_yz(x, op, T_N8|T_N_VL|T_MAP5|T_W0|T_YMM|T_MUST_EVEX, 0x37); } +void vcvtbf62hf8(const Xmm& x, const Xmm& op) { opVex(x, 0, op, T_66|T_MAP5|T_EW1|T_YMM|T_MUST_EVEX, 0x37); } +void vcvtbf82bf4s(const Operand& op, const Xmm& x) { opCvt_xx_xy_yz(x, op, T_N8|T_N_VL|T_F3|T_MAP5|T_EW1|T_YMM|T_MUST_EVEX, 0x3D); } +void vcvtbf82bf6s(const Xmm& x, const Xmm& op) { opVex(op, 0, x, T_F3|T_MAP5|T_EW1|T_YMM|T_MUST_EVEX, 0x3E); } +void vcvtbf82ps(const Xmm& x, const Operand& op) { opX_XM(op, x, T_N4|T_N_VL|T_MAP5|T_EW1|T_YMM|T_MUST_EVEX, 0x36); } +void vcvtbiasps2bf8(const Xmm& x1, const Xmm& x2, const Operand& op) { opCvtBias(x1, x2, op, T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B32, 0x39, true); } +void vcvtbiasps2bf8s(const Xmm& x1, const Xmm& x2, const Operand& op) { opCvtBias(x1, x2, op, T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B32, 0x3B, true); } +void vcvtbiasps2hf8(const Xmm& x1, const Xmm& x2, const Operand& op) { opCvtBias(x1, x2, op, T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B32, 0x38, true); } +void vcvtbiasps2hf8s(const Xmm& x1, const Xmm& x2, const Operand& op) { opCvtBias(x1, x2, op, T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B32, 0x3A, true); } +void vcvthf62hf8(const Xmm& x, const Xmm& op) { opVex(x, 0, op, T_66|T_MAP5|T_W0|T_YMM|T_MUST_EVEX, 0x37); } +void vcvthf82bf4s(const Operand& op, const Xmm& x) { opCvt_xx_xy_yz(x, op, T_N8|T_N_VL|T_F3|T_MAP5|T_W0|T_YMM|T_MUST_EVEX, 0x3D); } +void vcvthf82hf6s(const Xmm& x, const Xmm& op) { opVex(op, 0, x, T_F3|T_MAP5|T_W0|T_YMM|T_MUST_EVEX, 0x3C); } +void vcvthf82ps(const Xmm& x, const Operand& op) { opX_XM(op, x, T_N4|T_N_VL|T_MAP5|T_W0|T_YMM|T_MUST_EVEX, 0x36); } +void vcvtps2bf8(const Xmm& x, const Operand& op) { opCvt_xx_yx_zx(x, op, T_F3|T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B32, 0x39); } +void vcvtps2bf8s(const Xmm& x, const Operand& op) { opCvt_xx_yx_zx(x, op, T_F3|T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B32, 0x3B); } +void vcvtps2hf8(const Xmm& x, const Operand& op) { opCvt_xx_yx_zx(x, op, T_F3|T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B32, 0x38); } +void vcvtps2hf8s(const Xmm& x, const Operand& op) { opCvt_xx_yx_zx(x, op, T_F3|T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B32, 0x3A); } +void vcvtrops2hf8(const Xmm& x, const Operand& op) { opCvt_xx_yx_zx(x, op, T_66|T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B32, 0x38); } +void vcvtrops2hf8s(const Xmm& x, const Operand& op) { opCvt_xx_yx_zx(x, op, T_66|T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B32, 0x3A); } void vmovrsb(const Xmm& x, const Address& addr) { opVex(x, 0, addr, T_F2|T_MAP5|T_W0|T_MUST_EVEX, 0x6F); } void vmovrsd(const Xmm& x, const Address& addr) { opVex(x, 0, addr, T_F3|T_MAP5|T_W0|T_MUST_EVEX, 0x6F); } void vmovrsq(const Xmm& x, const Address& addr) { opVex(x, 0, addr, T_F3|T_MAP5|T_EW1|T_MUST_EVEX, 0x6F); } void vmovrsw(const Xmm& x, const Address& addr) { opVex(x, 0, addr, T_F2|T_MAP5|T_EW1|T_MUST_EVEX, 0x6F); } void vpbroadcastq(const Xmm& x, const Reg64& r) { opVex(x, 0, r, T_66|T_0F38|T_EW1|T_YMM|T_MUST_EVEX, 0x7C); } +void vpmovssdb(const Operand& op, const Xmm& x) { opX_XM(op, x, T_N4|T_N_VL|T_F3|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_M_K, 0x41); } +void vunpackb(const Xmm& x, const Operand& op, uint8_t imm) { opAVX_X_XM_IMM(x, op, T_0F3A|T_W0|T_YMM|T_MUST_EVEX, 0x3D, imm); } #endif #endif diff --git a/3rdparty/xbyak/xbyak/xbyak_util.h b/3rdparty/xbyak/xbyak/xbyak_util.h index a75d3f937b..befd29e999 100644 --- a/3rdparty/xbyak/xbyak/xbyak_util.h +++ b/3rdparty/xbyak/xbyak/xbyak_util.h @@ -179,6 +179,8 @@ private: uint32_t coresSharingDataCache_[maxNumberCacheLevels]; uint32_t dataCacheLevels_; uint32_t avx10version_; + uint32_t aceVersion_; + uint32_t maxPalette_; uint32_t get32bitAsBE(const char *x) const { @@ -582,7 +584,7 @@ public: XBYAK_DEFINE_TYPE(88, tSSE4a); XBYAK_DEFINE_TYPE(89, tCLWB); XBYAK_DEFINE_TYPE(90, tTSXLDTRK); - XBYAK_DEFINE_TYPE(91, tAMX_TRANSPOSE); +// XBYAK_DEFINE_TYPE(91, tAMX_TRANSPOSE); XBYAK_DEFINE_TYPE(92, tAMX_TF32); XBYAK_DEFINE_TYPE(93, tAMX_AVX512); XBYAK_DEFINE_TYPE(94, tAMX_MOVRS); @@ -590,6 +592,9 @@ public: XBYAK_DEFINE_TYPE(96, tMOVRS); XBYAK_DEFINE_TYPE(97, tHYBRID); XBYAK_DEFINE_TYPE(98, tAMX_COMPLEX); + XBYAK_DEFINE_TYPE(99, tACE); + XBYAK_DEFINE_TYPE(100, tAVX10_V1_AUX); + XBYAK_DEFINE_TYPE(101, tAVX10_V2_AUX); #undef XBYAK_SPLIT_ID #undef XBYAK_DEFINE_TYPE @@ -601,6 +606,8 @@ public: , coresSharingDataCache_() , dataCacheLevels_(0) , avx10version_(0) + , aceVersion_(0) + , maxPalette_(0) { uint32_t data[4] = {}; const uint32_t& eax = data[0]; @@ -748,13 +755,19 @@ public: if (edx & (1U << 14)) type_ |= tPREFETCHITI; if (edx & (1U << 19)) type_ |= tAVX10; if (edx & (1U << 21)) type_ |= tAPX_F; - - getCpuidEx(0x1e, 1, data); - if (eax & (1U << 4)) type_ |= tAMX_FP8; - if (eax & (1U << 5)) type_ |= tAMX_TRANSPOSE; - if (eax & (1U << 6)) type_ |= tAMX_TF32; - if (eax & (1U << 7)) type_ |= tAMX_AVX512; - if (eax & (1U << 8)) type_ |= tAMX_MOVRS; + if (ecx & (1U << 11)) type_ |= tACE; + } + if (maxNum >= 0x1e) { + getCpuidEx(0x1e, 0, data); + if (eax /* maxNumSubLeaves */ >= 1) { // 0 on SPR/EMR + getCpuidEx(0x1e, 1, data); + // eax bits 0-3 (AMX-INT8/BF16/COMPLEX/FP16) mirror the leaf 7 bits, so use leaf 7 + if (eax & (1U << 4)) type_ |= tAMX_FP8; +// if (eax & (1U << 5)) type_ |= tAMX_TRANSPOSE; // removed at 319433-059 + if (eax & (1U << 6)) type_ |= tAMX_TF32; + if (eax & (1U << 7)) type_ |= tAMX_AVX512; + if (eax & (1U << 8)) type_ |= tAMX_MOVRS; + } } } if (maxNum >= 0x19) { @@ -765,7 +778,21 @@ public: } if (has(tAVX10) && maxNum >= 0x24) { getCpuidEx(0x24, 0, data); + const uint32_t maxNumSubLeaves = eax; avx10version_ = ebx & mask(7); + if (maxNumSubLeaves >= 1) { + getCpuidEx(0x24, 1, data); + if (ecx & (1U << 2)) type_ |= tAVX10_V1_AUX; + if (ecx & (1U << 3)) type_ |= tAVX10_V2_AUX; + } + } + if (has(tAMX_TILE) && maxNum >= 0x1d) { + getCpuidEx(0x1d, 0, data); + maxPalette_ = eax; + if (has(tACE) && maxPalette_ >= 2) { + getCpuidEx(0x1d, 2, data); + aceVersion_ = eax & mask(8); + } } setFamily(); setNumCores(); @@ -784,6 +811,8 @@ public: return (type & type_) == type; } int getAVX10version() const { return avx10version_; } + int getACEVersion() const { return aceVersion_; } + int getMaxPalette() const { return maxPalette_; } }; #ifdef _MSC_VER #pragma warning(pop) @@ -1834,7 +1863,22 @@ const int UseRCX = 1 << 6; const int UseRDX = 1 << 7; const int UseRSI = 1 << 8; const int UseRDI = 1 << 9; -const int UseRBPAsFramePointer = UseRBP | (1 << 10); +const int UseR30R31 = 1 << 10; // reserve r30/r31 (APX EGPRs), pushed/popped unconditionally +const int UseRBX = 1 << 11; +const int UseRBPAsFramePointer = UseRBP | (1 << 30); +const int UsePUSH2 = 1 << 28; // use push2/pop2 where RSP is 16-byte aligned, push/pop otherwise +const int UsePPX = 1 << 29; // use pushp/popp (or push2p/pop2p with UsePUSH2) with the PPX store-forwarding hint + +namespace local { +const int UseVecNumShift = 16; // bits 16..21 : vector register count for UseSSE/UseAVX +const int UseVecSSE = 1 << 22; +const int UseVecAVX = 1 << 23; +} // local +const int NoVzeroupper = 1 << 24; // suppress vzeroupper in close() (UseAVX required) +// declare the use of xmm0, ..., xmm(n-1) with SSE instructions (0 <= n <= 16) +inline int UseSSE(int n) { return local::UseVecSSE | (n << local::UseVecNumShift); } +// declare the use of xmm/ymm/zmm 0, ..., n-1 with AVX instructions (0 <= n <= 32) +inline int UseAVX(int n) { return local::UseVecAVX | (n << local::UseVecNumShift); } class StackFrame { #ifdef XBYAK64_WIN @@ -1845,7 +1889,9 @@ class StackFrame { static const int maxPnum = 4; static const int maxRegNum = 14; // maxRegNum = 16 - rsp - rax static const int calleeSaveNum = maxRegNum - noSaveNum; - static const int UseMASK = UseRCX|UseRDX|UseRSI|UseRDI|UseRBP; + static const int maxSaveRegNum = calleeSaveNum + 2; // +2 for r30/r31 (UseR30R31) + static const int UseMASK = UseRBX|UseRCX|UseRDX|UseRSI|UseRDI|UseRBP|UseR30R31|UsePUSH2|UsePPX; + static const int UseVecMASK = (63 << local::UseVecNumShift)|local::UseVecSSE|local::UseVecAVX|NoVzeroupper; Xbyak::CodeGenerator *code_; Xbyak::Reg64 pTbl_[maxPnum]; Xbyak::Reg64 tTbl_[maxRegNum]; @@ -1855,8 +1901,12 @@ class StackFrame { int tNum_; int useRegs_; int saveNum_; - int saveRegs_[calleeSaveNum]; + int saveRegs_[maxSaveRegNum]; int P_; + int vecSaveNum_; // number of saved xmm registers (Win64 only) + int vecPos_; // offset of the xmm save area from rsp after the prolog + bool vzeroupper_; // emit vzeroupper at the top of close() + bool useVmovaps_; // save/restore with vmovaps instead of movaps bool makeEpilog_; StackFrame(const StackFrame&); void operator=(const StackFrame&); @@ -1867,7 +1917,7 @@ public: make stack frame @param sf [in] this @param pNum [in] number of function parameters(0 <= pNum <= 4) - @param tNum [in] number of temporary registers(0 <= tNum, can be OR-ed with Use{RCX,RDX,RSI,RDI,RBP}, e.g., 3|UseRCX) + @param tNum [in] number of temporary registers(0 <= tNum, can be OR-ed with Use{RBX,RCX,RDX,RSI,RDI,RBP,R30R31}, e.g., 3|UseRCX) @param stackSizeByte [in] local stack size @param makeEpilog [in] automatically call close() if true @@ -1877,22 +1927,53 @@ public: rax p[0], ..., p[pNum-1] as function parameters t[0], ..., t[tNum-1] as temporary registers - {rcx,rdx,rsi,rdi,rbp} are explicitly available by specifying Use{RCX,RDX,RSI,RDI,RBP} in tNum + {rbx,rcx,rdx,rsi,rdi,rbp} are explicitly available by specifying Use{RBX,RCX,RDX,RSI,RDI,RBP} in tNum + r30, r31 are explicitly available by specifying UseR30R31 in tNum rsp[0..stackSizeByte-1] if stackSizeByte > 0 + xmm0, ..., xmm(n-1) are declared by UseSSE(n) (0 <= n <= 16) : only SSE instructions are emitted + xmm/ymm/zmm 0, ..., n-1 are declared by UseAVX(n) (0 <= n <= 32) : vzeroupper is emitted at the top of close() unless NoVzeroupper is specified + on Win64 the lower 128 bits of xmm6, ..., xmm(min(n,16)-1) are saved/restored automatically (xmm16-31 are volatile everywhere and need not be counted in n) */ StackFrame(Xbyak::CodeGenerator *code, int pNum, int tNum = 0, int stackSizeByte = 0, bool makeEpilog = true) : code_(code) , pNum_(pNum) - , tNum_(tNum & ~(UseMASK|UseRBPAsFramePointer)) + , tNum_(tNum & ~(UseMASK|UseRBPAsFramePointer|UseVecMASK)) , useRegs_(tNum & UseMASK) // drop UseRBPAsFramePointer bit , saveNum_(0) , P_(0) + , vecSaveNum_(0) + , vecPos_(0) + , vzeroupper_(false) + , useVmovaps_(false) , makeEpilog_(makeEpilog) , p(p_) , t(t_) { if (pNum < 0 || pNum > 4) XBYAK_THROW(ERR_BAD_PNUM) - if (tNum < 0) XBYAK_THROW(ERR_BAD_TNUM) + if (tNum_ < 0) XBYAK_THROW(ERR_BAD_TNUM) + const int vecKind = tNum & (local::UseVecSSE|local::UseVecAVX); + const int vecNum = (tNum >> local::UseVecNumShift) & 63; + if (vecKind == (local::UseVecSSE|local::UseVecAVX)) XBYAK_THROW(ERR_BAD_TNUM) + // NoVzeroupper requires UseAVX + if ((tNum & NoVzeroupper) && vecKind != local::UseVecAVX) XBYAK_THROW(ERR_BAD_TNUM) + if (vecKind == 0) { + if (vecNum > 0) XBYAK_THROW(ERR_BAD_TNUM) + } else { + // UseSSE rejects n > 16 because SSE encodings cannot reach xmm16+ + if (vecNum > ((vecKind == local::UseVecAVX) ? 32 : 16)) XBYAK_THROW(ERR_BAD_TNUM) + if (vecKind == local::UseVecAVX) { + if (tNum & NoVzeroupper) { + // the upper state may be dirty at the prolog/epilog; avoid legacy SSE movaps + useVmovaps_ = true; + } else { + vzeroupper_ = true; + } + } +#ifdef XBYAK64_WIN + // Win64 requires saving the lower 128 bits of xmm6-15; xmm16+ are volatile everywhere + if (vecNum > 6) vecSaveNum_ = local::min_(vecNum, 16) - 6; +#endif + } const int *const fullTbl = getRegEntryTbl(); const int *const calleeTbl = fullTbl + noSaveNum; int callerUseNum = 0; @@ -1911,24 +1992,64 @@ public: const int baseSaveNum = local::max_(0, pNum + tNum_ + useNum - noSaveNum); bool pushedRbp = false; if (useRegs_ & UseRBP) { - code->push(rbp); + // keep the pushp/popp pair matched because close() pops rbp with popp + if (useRegs_ & UsePPX) { + code->pushp(rbp); + } else { + code->push(rbp); + } saveRegs_[saveNum_++] = Operand::RBP; pushedRbp = true; if ((tNum & UseRBPAsFramePointer) == UseRBPAsFramePointer) code->mov(rbp, rsp); } + if (useRegs_ & UseR30R31) { + saveRegs_[saveNum_++] = Operand::R30; + saveRegs_[saveNum_++] = Operand::R31; + } for (int i = 0; i < calleeSaveNum; i++) { int r = calleeTbl[i]; if (i < baseSaveNum || isUseReg(r)) { if (pushedRbp && r == Operand::RBP) continue; saveRegs_[saveNum_++] = r; - code->push(Reg64(r)); } } - P_ = (stackSizeByte + 7) / 8; - // (rsp % 16) == 8, then increment P_ for 16 byte alignment - if (P_ > 0 && (P_ & 1) == (saveNum_ & 1)) P_++; - P_ *= 8; - if (P_ > 0) code->sub(rsp, P_); + // RSP is 8 mod 16 at function entry; each push subtracts 8, so an odd + // loop index means RSP is 16-byte aligned before saveRegs_[i] is pushed. + for (int i = pushedRbp ? 1 : 0; i < saveNum_; i++) { + if ((useRegs_ & UsePUSH2) && (i & 1) && i + 1 < saveNum_) { + if (useRegs_ & UsePPX) { + code->push2p(Reg64(saveRegs_[i]), Reg64(saveRegs_[i + 1])); + } else { + code->push2(Reg64(saveRegs_[i]), Reg64(saveRegs_[i + 1])); + } + i++; + } else if (useRegs_ & UsePPX) { + code->pushp(Reg64(saveRegs_[i])); + } else { + code->push(Reg64(saveRegs_[i])); + } + } + if (vecSaveNum_ > 0) { + // layout from the lower address : local stack (stackSizeByte) / xmm save area (16-byte aligned) / padding (0 or 8) + vecPos_ = (stackSizeByte + 15) & ~15; + P_ = vecPos_ + vecSaveNum_ * 16; + // after the pushes (rsp % 16) == 8 * ((1 + saveNum_) % 2), so make rsp 16-byte aligned for movaps + if ((saveNum_ & 1) == 0) P_ += 8; + code->sub(rsp, P_); + for (int i = 0; i < vecSaveNum_; i++) { + if (useVmovaps_) { + code->vmovaps(ptr[rsp + (vecPos_ + i * 16)], Xmm(6 + i)); + } else { + code->movaps(ptr[rsp + (vecPos_ + i * 16)], Xmm(6 + i)); + } + } + } else { + P_ = (stackSizeByte + 7) / 8; + // (rsp % 16) == 8, then increment P_ for 16 byte alignment + if (P_ > 0 && (P_ & 1) == (saveNum_ & 1)) P_++; + P_ *= 8; + if (P_ > 0) code->sub(rsp, P_); + } int pos = 0; for (int i = 0; i < pNum; i++) { pTbl_[i] = Xbyak::Reg64(getRegIdx(pos)); @@ -1952,9 +2073,30 @@ public: */ void close(bool callRet = true) { + // vzeroupper comes before the restores so that legacy SSE movaps does not run with a dirty upper state + if (vzeroupper_) code_->vzeroupper(); + for (int i = 0; i < vecSaveNum_; i++) { + if (useVmovaps_) { + code_->vmovaps(Xmm(6 + i), ptr[rsp + (vecPos_ + i * 16)]); + } else { + code_->movaps(Xmm(6 + i), ptr[rsp + (vecPos_ + i * 16)]); + } + } if (P_ > 0) code_->add(code_->rsp, P_); + const int start = (useRegs_ & UseRBP) ? 1 : 0; for (int i = saveNum_ - 1; i >= 0; i--) { - code_->pop(Reg64(saveRegs_[i])); + if ((useRegs_ & UsePUSH2) && !(i & 1) && i - 1 >= start) { + if (useRegs_ & UsePPX) { + code_->pop2p(Reg64(saveRegs_[i]), Reg64(saveRegs_[i - 1])); + } else { + code_->pop2(Reg64(saveRegs_[i]), Reg64(saveRegs_[i - 1])); + } + i--; + } else if (useRegs_ & UsePPX) { + code_->popp(Reg64(saveRegs_[i])); + } else { + code_->pop(Reg64(saveRegs_[i])); + } } if (callRet) code_->ret(); } @@ -1967,6 +2109,7 @@ private: static int useFlagOf(int r) { switch (r) { + case Operand::RBX: return UseRBX; case Operand::RCX: return UseRCX; case Operand::RDX: return UseRDX; case Operand::RSI: return UseRSI;