xemu-project/xemu
1
0
Fork 0
mirror of https://github.com/xemu-project/xemu.git synced 2024-11-23 11:39:53 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

target-arm queue:

Browse Source 
* Fix regression booting Trusted Firmware
  * Honor HCR_E2H and HCR_TGE in ats_write64()
  * Copy the entire vector in DO_ZIP
  * Fix Privileged Access Never (PAN) for aarch32
  * Make TLBIOS and TLBIRANGE ops trap on HCR_EL2.TTLB
  * Set SCR_EL3.HXEn when direct booting kernel
  * Set SME and SVE EL3 vector lengths when direct booting kernel
 -----BEGIN PGP SIGNATURE-----
 
 iQJNBAABCAA3FiEE4aXFk81BneKOgxXPPCUl7RQ2DN4FAmNk+KkZHHBldGVyLm1h
 eWRlbGxAbGluYXJvLm9yZwAKCRA8JSXtFDYM3vUsD/9SYZP3ne2OZxBe8he98jJ5
 6apODiBksBLUM+1bKEoYW8Kw4XpS10I1Tbnxe7n0bNAfIiZlsZ7HJAJaYWy4MX4k
 Bq0v1EIFo+Obumocc14ZzWcw9yPpHOGavKHXfPxTtIw0amtOmh3aMBPuOZKiMSaq
 TdI/8654DbAOY3Hp/r6WnXwEgAc23kx/PtGhQFdU4iWhzTdeQeFkgCCsVMO02zFQ
 ZM4wiAATpfNfgf5+Wxoin6RQ8nI9PF+Xf7HhN3d1CiXju3vOl+geYNkubJzIopv1
 itLcnvduYE6+5oJsnXZ4FDNO6/nnqWRNqtyDf0/NjLROfj84BPJpZqMX+FR6Q0I0
 d+4/oEw4A46qfaS5b4/YelbJOiUgiViWU1Xs3g2dkeTMT8CyGfDrJ2HRDKN7AaHo
 llL7s1calkX2oSs+gU0BAw8xRETGwMBSOpF6JmPVh277LjvWfN1vsJzVUG3wrSXL
 G7qa2h+fHV5Xu876sc/i0+d4qHuqcE/EU86VQ6X40f+dRzN02rkSCPAxzGFwLXOr
 8fl5MsX6z5pqcubnzxkhi66ZHc6fXsvtUjKBxyrVpMyjMlV9PTJ2Q1RCgVctErXk
 lDzsLuplzPSjZBy3Peib/rLnmYUxJHyPe0RFYIumzZv/UHwL4GjZgkI842UVBpAL
 FvIGblcCXHhdP4UFvqgZhw==
 =Fcb4
 -----END PGP SIGNATURE-----

Merge tag 'pull-target-arm-20221104' of https://git.linaro.org/people/pmaydell/qemu-arm into staging

target-arm queue:
 * Fix regression booting Trusted Firmware
 * Honor HCR_E2H and HCR_TGE in ats_write64()
 * Copy the entire vector in DO_ZIP
 * Fix Privileged Access Never (PAN) for aarch32
 * Make TLBIOS and TLBIRANGE ops trap on HCR_EL2.TTLB
 * Set SCR_EL3.HXEn when direct booting kernel
 * Set SME and SVE EL3 vector lengths when direct booting kernel

# -----BEGIN PGP SIGNATURE-----
#
# iQJNBAABCAA3FiEE4aXFk81BneKOgxXPPCUl7RQ2DN4FAmNk+KkZHHBldGVyLm1h
# eWRlbGxAbGluYXJvLm9yZwAKCRA8JSXtFDYM3vUsD/9SYZP3ne2OZxBe8he98jJ5
# 6apODiBksBLUM+1bKEoYW8Kw4XpS10I1Tbnxe7n0bNAfIiZlsZ7HJAJaYWy4MX4k
# Bq0v1EIFo+Obumocc14ZzWcw9yPpHOGavKHXfPxTtIw0amtOmh3aMBPuOZKiMSaq
# TdI/8654DbAOY3Hp/r6WnXwEgAc23kx/PtGhQFdU4iWhzTdeQeFkgCCsVMO02zFQ
# ZM4wiAATpfNfgf5+Wxoin6RQ8nI9PF+Xf7HhN3d1CiXju3vOl+geYNkubJzIopv1
# itLcnvduYE6+5oJsnXZ4FDNO6/nnqWRNqtyDf0/NjLROfj84BPJpZqMX+FR6Q0I0
# d+4/oEw4A46qfaS5b4/YelbJOiUgiViWU1Xs3g2dkeTMT8CyGfDrJ2HRDKN7AaHo
# llL7s1calkX2oSs+gU0BAw8xRETGwMBSOpF6JmPVh277LjvWfN1vsJzVUG3wrSXL
# G7qa2h+fHV5Xu876sc/i0+d4qHuqcE/EU86VQ6X40f+dRzN02rkSCPAxzGFwLXOr
# 8fl5MsX6z5pqcubnzxkhi66ZHc6fXsvtUjKBxyrVpMyjMlV9PTJ2Q1RCgVctErXk
# lDzsLuplzPSjZBy3Peib/rLnmYUxJHyPe0RFYIumzZv/UHwL4GjZgkI842UVBpAL
# FvIGblcCXHhdP4UFvqgZhw==
# =Fcb4
# -----END PGP SIGNATURE-----
# gpg: Signature made Fri 04 Nov 2022 07:34:01 EDT
# gpg:                using RSA key E1A5C593CD419DE28E8315CF3C2525ED14360CDE
# gpg:                issuer "peter.maydell@linaro.org"
# gpg: Good signature from "Peter Maydell <peter.maydell@linaro.org>" [full]
# gpg:                 aka "Peter Maydell <pmaydell@gmail.com>" [full]
# gpg:                 aka "Peter Maydell <pmaydell@chiark.greenend.org.uk>" [full]
# gpg:                 aka "Peter Maydell <peter@archaic.org.uk>" [unknown]
# Primary key fingerprint: E1A5 C593 CD41 9DE2 8E83  15CF 3C25 25ED 1436 0CDE

* tag 'pull-target-arm-20221104' of https://git.linaro.org/people/pmaydell/qemu-arm:
  target/arm: Two fixes for secure ptw
  target/arm: Honor HCR_E2H and HCR_TGE in ats_write64()
  target/arm: Copy the entire vector in DO_ZIP
  target/arm: Fix Privileged Access Never (PAN) for aarch32
  target/arm: Make TLBIOS and TLBIRANGE ops trap on HCR_EL2.TTLB
  hw/arm/boot: Set SCR_EL3.HXEn when booting kernel
  hw/arm/boot: Set SME and SVE EL3 vector lengths when booting kernel

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
This commit is contained in:
Stefan Hajnoczi 2022-11-04 11:01:17 -04:00
commit 6295a58ad1
4 changed files with 83 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hw/arm/boot.c
View File

@ -764,10 +764,15 @@ static void do_cpu_reset(void *opaque)
}
if (cpu_isar_feature(aa64_sve, cpu)) {
env->cp15.cptr_el[3] |= R_CPTR_EL3_EZ_MASK;
env->vfp.zcr_el[3] = 0xf;
}
if (cpu_isar_feature(aa64_sme, cpu)) {
env->cp15.cptr_el[3] |= R_CPTR_EL3_ESM_MASK;
env->cp15.scr_el3 |= SCR_ENTP2;
env->vfp.smcr_el[3] = 0xf;
}
if (cpu_isar_feature(aa64_hcx, cpu)) {
env->cp15.scr_el3 |= SCR_HXEN;
}
/* AArch64 kernels never boot in secure mode */
assert(!info->secure_boot);

64
target/arm/helper.c
View File

@ -3501,19 +3501,22 @@ static void ats_write64(CPUARMState *env, const ARMCPRegInfo *ri,
MMUAccessType access_type = ri->opc2 & 1 ? MMU_DATA_STORE : MMU_DATA_LOAD;
ARMMMUIdx mmu_idx;
int secure = arm_is_secure_below_el3(env);
uint64_t hcr_el2 = arm_hcr_el2_eff(env);
bool regime_e20 = (hcr_el2 & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE);
switch (ri->opc2 & 6) {
case 0:
switch (ri->opc1) {
case 0: /* AT S1E1R, AT S1E1W, AT S1E1RP, AT S1E1WP */
if (ri->crm == 9 && (env->pstate & PSTATE_PAN)) {
mmu_idx = ARMMMUIdx_Stage1_E1_PAN;
mmu_idx = regime_e20 ?
ARMMMUIdx_E20_2_PAN : ARMMMUIdx_Stage1_E1_PAN;
} else {
mmu_idx = ARMMMUIdx_Stage1_E1;
mmu_idx = regime_e20 ? ARMMMUIdx_E20_2 : ARMMMUIdx_Stage1_E1;
}
break;
case 4: /* AT S1E2R, AT S1E2W */
mmu_idx = ARMMMUIdx_E2;
mmu_idx = hcr_el2 & HCR_E2H ? ARMMMUIdx_E20_2 : ARMMMUIdx_E2;
break;
case 6: /* AT S1E3R, AT S1E3W */
mmu_idx = ARMMMUIdx_E3;
@ -3524,13 +3527,13 @@ static void ats_write64(CPUARMState *env, const ARMCPRegInfo *ri,
}
break;
case 2: /* AT S1E0R, AT S1E0W */
mmu_idx = ARMMMUIdx_Stage1_E0;
mmu_idx = regime_e20 ? ARMMMUIdx_E20_0 : ARMMMUIdx_Stage1_E0;
break;
case 4: /* AT S12E1R, AT S12E1W */
mmu_idx = ARMMMUIdx_E10_1;
mmu_idx = regime_e20 ? ARMMMUIdx_E20_2 : ARMMMUIdx_E10_1;
break;
case 6: /* AT S12E0R, AT S12E0W */
mmu_idx = ARMMMUIdx_E10_0;
mmu_idx = regime_e20 ? ARMMMUIdx_E20_0 : ARMMMUIdx_E10_0;
break;
default:
g_assert_not_reached();
@ -6717,51 +6720,51 @@ static const ARMCPRegInfo pauth_reginfo[] = {
static const ARMCPRegInfo tlbirange_reginfo[] = {
{ .name = "TLBI_RVAE1IS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 2, .opc2 = 1,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae1is_write },
{ .name = "TLBI_RVAAE1IS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 2, .opc2 = 3,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae1is_write },
{ .name = "TLBI_RVALE1IS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 2, .opc2 = 5,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae1is_write },
{ .name = "TLBI_RVAALE1IS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 2, .opc2 = 7,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae1is_write },
{ .name = "TLBI_RVAE1OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 1,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae1is_write },
{ .name = "TLBI_RVAAE1OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 3,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae1is_write },
{ .name = "TLBI_RVALE1OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 5,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae1is_write },
{ .name = "TLBI_RVAALE1OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 7,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae1is_write },
{ .name = "TLBI_RVAE1", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 1,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae1_write },
{ .name = "TLBI_RVAAE1", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 3,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae1_write },
{ .name = "TLBI_RVALE1", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 5,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae1_write },
{ .name = "TLBI_RVAALE1", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 7,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae1_write },
{ .name = "TLBI_RIPAS2E1IS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 2,
@ -6832,27 +6835,27 @@ static const ARMCPRegInfo tlbirange_reginfo[] = {
static const ARMCPRegInfo tlbios_reginfo[] = {
{ .name = "TLBI_VMALLE1OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 1, .opc2 = 0,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_vmalle1is_write },
{ .name = "TLBI_VAE1OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 1, .opc2 = 1,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_vae1is_write },
{ .name = "TLBI_ASIDE1OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 1, .opc2 = 2,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_vmalle1is_write },
{ .name = "TLBI_VAAE1OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 1, .opc2 = 3,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_vae1is_write },
{ .name = "TLBI_VALE1OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 1, .opc2 = 5,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_vae1is_write },
{ .name = "TLBI_VAALE1OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 1, .opc2 = 7,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_vae1is_write },
{ .name = "TLBI_ALLE2OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 4, .crn = 8, .crm = 1, .opc2 = 0,
@ -11003,6 +11006,15 @@ ARMMMUIdx arm_v7m_mmu_idx_for_secstate(CPUARMState *env, bool secstate)
}
#endif
static bool arm_pan_enabled(CPUARMState *env)
{
if (is_a64(env)) {
return env->pstate & PSTATE_PAN;
} else {
return env->uncached_cpsr & CPSR_PAN;
}
}
ARMMMUIdx arm_mmu_idx_el(CPUARMState *env, int el)
{
ARMMMUIdx idx;
@ -11023,7 +11035,7 @@ ARMMMUIdx arm_mmu_idx_el(CPUARMState *env, int el)
}
break;
case 1:
if (env->pstate & PSTATE_PAN) {
if (arm_pan_enabled(env)) {
idx = ARMMMUIdx_E10_1_PAN;
} else {
idx = ARMMMUIdx_E10_1;
@ -11032,7 +11044,7 @@ ARMMMUIdx arm_mmu_idx_el(CPUARMState *env, int el)
case 2:
/* Note that TGE does not apply at EL2. */
if (arm_hcr_el2_eff(env) & HCR_E2H) {
if (env->pstate & PSTATE_PAN) {
if (arm_pan_enabled(env)) {
idx = ARMMMUIdx_E20_2_PAN;
} else {
idx = ARMMMUIdx_E20_2;

50
target/arm/ptw.c
View File

@ -503,12 +503,11 @@ static bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx,
* @mmu_idx: MMU index indicating required translation regime
* @ap: The 3-bit access permissions (AP[2:0])
* @domain_prot: The 2-bit domain access permissions
* @is_user: TRUE if accessing from PL0
*/
static int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx,
int ap, int domain_prot)
static int ap_to_rw_prot_is_user(CPUARMState *env, ARMMMUIdx mmu_idx,
int ap, int domain_prot, bool is_user)
{
bool is_user = regime_is_user(env, mmu_idx);
if (domain_prot == 3) {
return PAGE_READ | PAGE_WRITE;
}
@ -552,6 +551,20 @@ static int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx,
}
}
/*
* Translate section/page access permissions to page R/W protection flags
* @env: CPUARMState
* @mmu_idx: MMU index indicating required translation regime
* @ap: The 3-bit access permissions (AP[2:0])
* @domain_prot: The 2-bit domain access permissions
*/
static int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx,
int ap, int domain_prot)
{
return ap_to_rw_prot_is_user(env, mmu_idx, ap, domain_prot,
regime_is_user(env, mmu_idx));
}
/*
* Translate section/page access permissions to page R/W protection flags.
* @ap: The 2-bit simple AP (AP[2:1])
@ -720,6 +733,7 @@ static bool get_phys_addr_v6(CPUARMState *env, S1Translate *ptw,
hwaddr phys_addr;
uint32_t dacr;
bool ns;
int user_prot;
/* Pagetable walk. */
/* Lookup l1 descriptor. */
@ -831,8 +845,10 @@ static bool get_phys_addr_v6(CPUARMState *env, S1Translate *ptw,
goto do_fault;
}
result->f.prot = simple_ap_to_rw_prot(env, mmu_idx, ap >> 1);
user_prot = simple_ap_to_rw_prot_is_user(ap >> 1, 1);
} else {
result->f.prot = ap_to_rw_prot(env, mmu_idx, ap, domain_prot);
user_prot = ap_to_rw_prot_is_user(env, mmu_idx, ap, domain_prot, 1);
}
if (result->f.prot && !xn) {
result->f.prot |= PAGE_EXEC;
@ -842,6 +858,14 @@ static bool get_phys_addr_v6(CPUARMState *env, S1Translate *ptw,
fi->type = ARMFault_Permission;
goto do_fault;
}
if (regime_is_pan(env, mmu_idx) &&
!regime_is_user(env, mmu_idx) &&
user_prot &&
access_type != MMU_INST_FETCH) {
/* Privileged Access Never fault */
fi->type = ARMFault_Permission;
goto do_fault;
}
}
if (ns) {
/* The NS bit will (as required by the architecture) have no effect if
@ -1357,7 +1381,7 @@ static bool get_phys_addr_lpae(CPUARMState *env, S1Translate *ptw,
descaddr |= (address >> (stride * (4 - level))) & indexmask;
descaddr &= ~7ULL;
nstable = extract32(tableattrs, 4, 1);
if (!nstable) {
if (nstable) {
/*
* Stage2_S -> Stage2 or Phys_S -> Phys_NS
* Assert that the non-secure idx are even, and relative order.
@ -2671,6 +2695,13 @@ static bool get_phys_addr_with_struct(CPUARMState *env, S1Translate *ptw,
bool is_secure = ptw->in_secure;
ARMMMUIdx s1_mmu_idx;
/*
* The page table entries may downgrade secure to non-secure, but
* cannot upgrade an non-secure translation regime's attributes
* to secure.
*/
result->f.attrs.secure = is_secure;
switch (mmu_idx) {
case ARMMMUIdx_Phys_S:
case ARMMMUIdx_Phys_NS:
@ -2712,12 +2743,6 @@ static bool get_phys_addr_with_struct(CPUARMState *env, S1Translate *ptw,
break;
}
/*
* The page table entries may downgrade secure to non-secure, but
* cannot upgrade an non-secure translation regime's attributes
* to secure.
*/
result->f.attrs.secure = is_secure;
result->f.attrs.user = regime_is_user(env, mmu_idx);
/*
@ -2773,7 +2798,8 @@ static bool get_phys_addr_with_struct(CPUARMState *env, S1Translate *ptw,
if (regime_using_lpae_format(env, mmu_idx)) {
return get_phys_addr_lpae(env, ptw, address, access_type, false,
result, fi);
} else if (regime_sctlr(env, mmu_idx) & SCTLR_XP) {
} else if (arm_feature(env, ARM_FEATURE_V7) ||
regime_sctlr(env, mmu_idx) & SCTLR_XP) {
return get_phys_addr_v6(env, ptw, address, access_type, result, fi);
} else {
return get_phys_addr_v5(env, ptw, address, access_type, result, fi);

4
target/arm/sve_helper.c
View File

@ -3366,10 +3366,10 @@ void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
/* We produce output faster than we consume input. \
Therefore we must be mindful of possible overlap. */ \
if (unlikely((vn - vd) < (uintptr_t)oprsz)) { \
vn = memcpy(&tmp_n, vn, oprsz_2); \
vn = memcpy(&tmp_n, vn, oprsz); \
} \
if (unlikely((vm - vd) < (uintptr_t)oprsz)) { \
vm = memcpy(&tmp_m, vm, oprsz_2); \
vm = memcpy(&tmp_m, vm, oprsz); \
} \
for (i = 0; i < oprsz_2; i += sizeof(TYPE)) { \
*(TYPE *)(vd + H(2 * i + 0)) = *(TYPE *)(vn + odd_ofs + H(i)); \