xemu-project/xemu
1
0
Fork 0
mirror of https://github.com/xemu-project/xemu.git synced 2024-11-23 19:49:43 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

target-arm queue:

Browse Source 
* Implement SVE2 emulation
  * Implement integer matrix multiply accumulate
  * Implement FEAT_TLBIOS
  * Implement FEAT_TLBRANGE
  * disas/libvixl: Protect C system header for C++ compiler
  * Use correct SP in M-profile exception return
  * AN524, AN547: Correct modelling of internal SRAMs
  * hw/intc/arm_gicv3_cpuif: Fix EOIR write access check logic
  * hw/arm/smmuv3: Another range invalidation fix
 -----BEGIN PGP SIGNATURE-----
 
 iQJNBAABCAA3FiEE4aXFk81BneKOgxXPPCUl7RQ2DN4FAmCtEYEZHHBldGVyLm1h
 eWRlbGxAbGluYXJvLm9yZwAKCRA8JSXtFDYM3mTGD/9udRUbfe6gfYGakZQL69t4
 iDGxREE6dlEwma0WxP00CGNWQTleh+TwEHxv02ITk4Ni8L62yJGrTvAUcsTrTtSa
 OQN1p6IxS2bVNo0nAH+VT+Ry3Ttg0OEKZo3tzT0ICERWwmz0sxnq5CBp1s5mFJp1
 OIxGaHLdjrkPt4OHKbMekoU7IJomsxHPb0D7+LyNVFq4wxxg03NfscfyfacYtAVI
 kfhO9/fniH4YZbEhQ6YUSnRjFpQBiuH/8AgOSki2VD+vaIZnXXMkrVhUU4V6qIfU
 SVg1cy0XWxxNTNVXsqPzQlwK/eBgYe9dpmnOndAfAls7v3+BHUa9kKq/wOx11WVu
 kehjrLss5c5y2Vev3CqopC/htXAWDDAPUgGDDNpoqT6rvRnRUDQEfA+jTL3Srsyq
 PEShmATq36hoE1mDMAtkv98JAtfj/Uzy2bBlJ34DXdAE+/erZax9HF57TmxN2NYn
 2y1Xp7JTJVz8xK13Nz/dlsXSuaUH7WI/WG/zzEJE8Kain+RBKfm0OmXIjM3cJVTk
 T45nMTZCXejSXliIThKhNc3abC2xsF/X2R1bZKH/kRQ2/eNPVIPjMN4oO/iRXQ1Q
 fGB68wA80XM5EOGiuqpE2GZnCpvh3iMRwa+4Ps7HjfetTfM068jV5HX5+nYIgE8H
 VpA02UbPU0RzsHW/E3/blQ==
 =BwQN
 -----END PGP SIGNATURE-----

Merge remote-tracking branch 'remotes/pmaydell/tags/pull-target-arm-20210525' into staging

target-arm queue:
 * Implement SVE2 emulation
 * Implement integer matrix multiply accumulate
 * Implement FEAT_TLBIOS
 * Implement FEAT_TLBRANGE
 * disas/libvixl: Protect C system header for C++ compiler
 * Use correct SP in M-profile exception return
 * AN524, AN547: Correct modelling of internal SRAMs
 * hw/intc/arm_gicv3_cpuif: Fix EOIR write access check logic
 * hw/arm/smmuv3: Another range invalidation fix

# gpg: Signature made Tue 25 May 2021 16:02:25 BST
# gpg:                using RSA key E1A5C593CD419DE28E8315CF3C2525ED14360CDE
# gpg:                issuer "peter.maydell@linaro.org"
# gpg: Good signature from "Peter Maydell <peter.maydell@linaro.org>" [ultimate]
# gpg:                 aka "Peter Maydell <pmaydell@gmail.com>" [ultimate]
# gpg:                 aka "Peter Maydell <pmaydell@chiark.greenend.org.uk>" [ultimate]
# Primary key fingerprint: E1A5 C593 CD41 9DE2 8E83  15CF 3C25 25ED 1436 0CDE

* remotes/pmaydell/tags/pull-target-arm-20210525: (114 commits)
  target/arm: Enable SVE2 and related extensions
  linux-user/aarch64: Enable hwcap bits for sve2 and related extensions
  target/arm: Implement integer matrix multiply accumulate
  target/arm: Implement aarch32 VSUDOT, VUSDOT
  target/arm: Split decode of VSDOT and VUDOT
  target/arm: Split out do_neon_ddda
  target/arm: Fix decode for VDOT (indexed)
  target/arm: Remove unused fpst from VDOT_scalar
  target/arm: Split out do_neon_ddda_fpst
  target/arm: Implement aarch64 SUDOT, USDOT
  target/arm: Implement SVE2 fp multiply-add long
  target/arm: Move endian adjustment macros to vec_internal.h
  target/arm: Implement SVE2 bitwise shift immediate
  target/arm: Implement 128-bit ZIP, UZP, TRN
  target/arm: Implement SVE2 LD1RO
  target/arm: Tidy do_ldrq
  target/arm: Share table of sve load functions
  target/arm: Implement SVE2 FLOGB
  target/arm: Implement SVE2 FCVTXNT, FCVTX
  target/arm: Implement SVE2 FCVTLT
  ...

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
This commit is contained in:
Peter Maydell 2021-05-25 16:17:06 +01:00
commit 92f8c6fef1
33 changed files with 8275 additions and 1300 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

231
accel/tcg/cputlb.c
View File

@ -707,8 +707,9 @@ void tlb_flush_page_all_cpus_synced(CPUState *src, target_ulong addr)
tlb_flush_page_by_mmuidx_all_cpus_synced(src, addr, ALL_MMUIDX_BITS);
}
static void tlb_flush_page_bits_locked(CPUArchState *env, int midx,
target_ulong page, unsigned bits)
static void tlb_flush_range_locked(CPUArchState *env, int midx,
target_ulong addr, target_ulong len,
unsigned bits)
{
CPUTLBDesc *d = &env_tlb(env)->d[midx];
CPUTLBDescFast *f = &env_tlb(env)->f[midx];
@ -718,20 +719,26 @@ static void tlb_flush_page_bits_locked(CPUArchState *env, int midx,
* If @bits is smaller than the tlb size, there may be multiple entries
* within the TLB; otherwise all addresses that match under @mask hit
* the same TLB entry.
*
* TODO: Perhaps allow bits to be a few bits less than the size.
* For now, just flush the entire TLB.
*
* If @len is larger than the tlb size, then it will take longer to
* test all of the entries in the TLB than it will to flush it all.
*/
if (mask < f->mask) {
if (mask < f->mask || len > f->mask) {
tlb_debug("forcing full flush midx %d ("
TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
midx, page, mask);
TARGET_FMT_lx "/" TARGET_FMT_lx "+" TARGET_FMT_lx ")\n",
midx, addr, mask, len);
tlb_flush_one_mmuidx_locked(env, midx, get_clock_realtime());
return;
}
/* Check if we need to flush due to large pages. */
if ((page & d->large_page_mask) == d->large_page_addr) {
/*
* Check if we need to flush due to large pages.
* Because large_page_mask contains all 1's from the msb,
* we only need to test the end of the range.
*/
if (((addr + len - 1) & d->large_page_mask) == d->large_page_addr) {
tlb_debug("forcing full flush midx %d ("
TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
midx, d->large_page_addr, d->large_page_mask);
@ -739,85 +746,67 @@ static void tlb_flush_page_bits_locked(CPUArchState *env, int midx,
return;
}
if (tlb_flush_entry_mask_locked(tlb_entry(env, midx, page), page, mask)) {
tlb_n_used_entries_dec(env, midx);
for (target_ulong i = 0; i < len; i += TARGET_PAGE_SIZE) {
target_ulong page = addr + i;
CPUTLBEntry *entry = tlb_entry(env, midx, page);
if (tlb_flush_entry_mask_locked(entry, page, mask)) {
tlb_n_used_entries_dec(env, midx);
}
tlb_flush_vtlb_page_mask_locked(env, midx, page, mask);
}
tlb_flush_vtlb_page_mask_locked(env, midx, page, mask);
}
typedef struct {
target_ulong addr;
target_ulong len;
uint16_t idxmap;
uint16_t bits;
} TLBFlushPageBitsByMMUIdxData;
} TLBFlushRangeData;
static void
tlb_flush_page_bits_by_mmuidx_async_0(CPUState *cpu,
TLBFlushPageBitsByMMUIdxData d)
static void tlb_flush_range_by_mmuidx_async_0(CPUState *cpu,
TLBFlushRangeData d)
{
CPUArchState *env = cpu->env_ptr;
int mmu_idx;
assert_cpu_is_self(cpu);
tlb_debug("page addr:" TARGET_FMT_lx "/%u mmu_map:0x%x\n",
d.addr, d.bits, d.idxmap);
tlb_debug("range:" TARGET_FMT_lx "/%u+" TARGET_FMT_lx " mmu_map:0x%x\n",
d.addr, d.bits, d.len, d.idxmap);
qemu_spin_lock(&env_tlb(env)->c.lock);
for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
if ((d.idxmap >> mmu_idx) & 1) {
tlb_flush_page_bits_locked(env, mmu_idx, d.addr, d.bits);
tlb_flush_range_locked(env, mmu_idx, d.addr, d.len, d.bits);
}
}
qemu_spin_unlock(&env_tlb(env)->c.lock);
tb_flush_jmp_cache(cpu, d.addr);
}
static bool encode_pbm_to_runon(run_on_cpu_data *out,
TLBFlushPageBitsByMMUIdxData d)
{
/* We need 6 bits to hold to hold @bits up to 63. */
if (d.idxmap <= MAKE_64BIT_MASK(0, TARGET_PAGE_BITS - 6)) {
*out = RUN_ON_CPU_TARGET_PTR(d.addr | (d.idxmap << 6) | d.bits);
return true;
for (target_ulong i = 0; i < d.len; i += TARGET_PAGE_SIZE) {
tb_flush_jmp_cache(cpu, d.addr + i);
}
return false;
}
static TLBFlushPageBitsByMMUIdxData
decode_runon_to_pbm(run_on_cpu_data data)
static void tlb_flush_range_by_mmuidx_async_1(CPUState *cpu,
run_on_cpu_data data)
{
target_ulong addr_map_bits = (target_ulong) data.target_ptr;
return (TLBFlushPageBitsByMMUIdxData){
.addr = addr_map_bits & TARGET_PAGE_MASK,
.idxmap = (addr_map_bits & ~TARGET_PAGE_MASK) >> 6,
.bits = addr_map_bits & 0x3f
};
}
static void tlb_flush_page_bits_by_mmuidx_async_1(CPUState *cpu,
run_on_cpu_data runon)
{
tlb_flush_page_bits_by_mmuidx_async_0(cpu, decode_runon_to_pbm(runon));
}
static void tlb_flush_page_bits_by_mmuidx_async_2(CPUState *cpu,
run_on_cpu_data data)
{
TLBFlushPageBitsByMMUIdxData *d = data.host_ptr;
tlb_flush_page_bits_by_mmuidx_async_0(cpu, *d);
TLBFlushRangeData *d = data.host_ptr;
tlb_flush_range_by_mmuidx_async_0(cpu, *d);
g_free(d);
}
void tlb_flush_page_bits_by_mmuidx(CPUState *cpu, target_ulong addr,
uint16_t idxmap, unsigned bits)
void tlb_flush_range_by_mmuidx(CPUState *cpu, target_ulong addr,
target_ulong len, uint16_t idxmap,
unsigned bits)
{
TLBFlushPageBitsByMMUIdxData d;
run_on_cpu_data runon;
TLBFlushRangeData d;
/* If all bits are significant, this devolves to tlb_flush_page. */
if (bits >= TARGET_LONG_BITS) {
/*
* If all bits are significant, and len is small,
* this devolves to tlb_flush_page.
*/
if (bits >= TARGET_LONG_BITS && len <= TARGET_PAGE_SIZE) {
tlb_flush_page_by_mmuidx(cpu, addr, idxmap);
return;
}
@ -829,34 +818,38 @@ void tlb_flush_page_bits_by_mmuidx(CPUState *cpu, target_ulong addr,
/* This should already be page aligned */
d.addr = addr & TARGET_PAGE_MASK;
d.len = len;
d.idxmap = idxmap;
d.bits = bits;
if (qemu_cpu_is_self(cpu)) {
tlb_flush_page_bits_by_mmuidx_async_0(cpu, d);
} else if (encode_pbm_to_runon(&runon, d)) {
async_run_on_cpu(cpu, tlb_flush_page_bits_by_mmuidx_async_1, runon);
tlb_flush_range_by_mmuidx_async_0(cpu, d);
} else {
TLBFlushPageBitsByMMUIdxData *p
= g_new(TLBFlushPageBitsByMMUIdxData, 1);
/* Otherwise allocate a structure, freed by the worker. */
*p = d;
async_run_on_cpu(cpu, tlb_flush_page_bits_by_mmuidx_async_2,
TLBFlushRangeData *p = g_memdup(&d, sizeof(d));
async_run_on_cpu(cpu, tlb_flush_range_by_mmuidx_async_1,
RUN_ON_CPU_HOST_PTR(p));
}
}
void tlb_flush_page_bits_by_mmuidx_all_cpus(CPUState *src_cpu,
target_ulong addr,
uint16_t idxmap,
unsigned bits)
void tlb_flush_page_bits_by_mmuidx(CPUState *cpu, target_ulong addr,
uint16_t idxmap, unsigned bits)
{
TLBFlushPageBitsByMMUIdxData d;
run_on_cpu_data runon;
tlb_flush_range_by_mmuidx(cpu, addr, TARGET_PAGE_SIZE, idxmap, bits);
}
/* If all bits are significant, this devolves to tlb_flush_page. */
if (bits >= TARGET_LONG_BITS) {
void tlb_flush_range_by_mmuidx_all_cpus(CPUState *src_cpu,
target_ulong addr, target_ulong len,
uint16_t idxmap, unsigned bits)
{
TLBFlushRangeData d;
CPUState *dst_cpu;
/*
* If all bits are significant, and len is small,
* this devolves to tlb_flush_page.
*/
if (bits >= TARGET_LONG_BITS && len <= TARGET_PAGE_SIZE) {
tlb_flush_page_by_mmuidx_all_cpus(src_cpu, addr, idxmap);
return;
}
@ -868,40 +861,45 @@ void tlb_flush_page_bits_by_mmuidx_all_cpus(CPUState *src_cpu,
/* This should already be page aligned */
d.addr = addr & TARGET_PAGE_MASK;
d.len = len;
d.idxmap = idxmap;
d.bits = bits;
if (encode_pbm_to_runon(&runon, d)) {
flush_all_helper(src_cpu, tlb_flush_page_bits_by_mmuidx_async_1, runon);
} else {
CPUState *dst_cpu;
TLBFlushPageBitsByMMUIdxData *p;
/* Allocate a separate data block for each destination cpu. */
CPU_FOREACH(dst_cpu) {
if (dst_cpu != src_cpu) {
p = g_new(TLBFlushPageBitsByMMUIdxData, 1);
*p = d;
async_run_on_cpu(dst_cpu,
tlb_flush_page_bits_by_mmuidx_async_2,
RUN_ON_CPU_HOST_PTR(p));
}
/* Allocate a separate data block for each destination cpu. */
CPU_FOREACH(dst_cpu) {
if (dst_cpu != src_cpu) {
TLBFlushRangeData *p = g_memdup(&d, sizeof(d));
async_run_on_cpu(dst_cpu,
tlb_flush_range_by_mmuidx_async_1,
RUN_ON_CPU_HOST_PTR(p));
}
}
tlb_flush_page_bits_by_mmuidx_async_0(src_cpu, d);
tlb_flush_range_by_mmuidx_async_0(src_cpu, d);
}
void tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
target_ulong addr,
uint16_t idxmap,
unsigned bits)
void tlb_flush_page_bits_by_mmuidx_all_cpus(CPUState *src_cpu,
target_ulong addr,
uint16_t idxmap, unsigned bits)
{
TLBFlushPageBitsByMMUIdxData d;
run_on_cpu_data runon;
tlb_flush_range_by_mmuidx_all_cpus(src_cpu, addr, TARGET_PAGE_SIZE,
idxmap, bits);
}
/* If all bits are significant, this devolves to tlb_flush_page. */
if (bits >= TARGET_LONG_BITS) {
void tlb_flush_range_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
target_ulong addr,
target_ulong len,
uint16_t idxmap,
unsigned bits)
{
TLBFlushRangeData d, *p;
CPUState *dst_cpu;
/*
* If all bits are significant, and len is small,
* this devolves to tlb_flush_page.
*/
if (bits >= TARGET_LONG_BITS && len <= TARGET_PAGE_SIZE) {
tlb_flush_page_by_mmuidx_all_cpus_synced(src_cpu, addr, idxmap);
return;
}
@ -913,32 +911,31 @@ void tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
/* This should already be page aligned */
d.addr = addr & TARGET_PAGE_MASK;
d.len = len;
d.idxmap = idxmap;
d.bits = bits;
if (encode_pbm_to_runon(&runon, d)) {
flush_all_helper(src_cpu, tlb_flush_page_bits_by_mmuidx_async_1, runon);
async_safe_run_on_cpu(src_cpu, tlb_flush_page_bits_by_mmuidx_async_1,
runon);
} else {
CPUState *dst_cpu;
TLBFlushPageBitsByMMUIdxData *p;
/* Allocate a separate data block for each destination cpu. */
CPU_FOREACH(dst_cpu) {
if (dst_cpu != src_cpu) {
p = g_new(TLBFlushPageBitsByMMUIdxData, 1);
*p = d;
async_run_on_cpu(dst_cpu, tlb_flush_page_bits_by_mmuidx_async_2,
RUN_ON_CPU_HOST_PTR(p));
}
/* Allocate a separate data block for each destination cpu. */
CPU_FOREACH(dst_cpu) {
if (dst_cpu != src_cpu) {
p = g_memdup(&d, sizeof(d));
async_run_on_cpu(dst_cpu, tlb_flush_range_by_mmuidx_async_1,
RUN_ON_CPU_HOST_PTR(p));
}
p = g_new(TLBFlushPageBitsByMMUIdxData, 1);
*p = d;
async_safe_run_on_cpu(src_cpu, tlb_flush_page_bits_by_mmuidx_async_2,
RUN_ON_CPU_HOST_PTR(p));
}
p = g_memdup(&d, sizeof(d));
async_safe_run_on_cpu(src_cpu, tlb_flush_range_by_mmuidx_async_1,
RUN_ON_CPU_HOST_PTR(p));
}
void tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
target_ulong addr,
uint16_t idxmap,
unsigned bits)
{
tlb_flush_range_by_mmuidx_all_cpus_synced(src_cpu, addr, TARGET_PAGE_SIZE,
idxmap, bits);
}
/* update the TLBs so that writes to code in the virtual page 'addr'

2
disas/libvixl/vixl/code-buffer.h
View File

@ -27,7 +27,7 @@
#ifndef VIXL_CODE_BUFFER_H
#define VIXL_CODE_BUFFER_H
#include <string.h>
#include <cstring>
#include "vixl/globals.h"
namespace vixl {

16
disas/libvixl/vixl/globals.h
View File

@ -40,15 +40,17 @@
#define __STDC_FORMAT_MACROS
#endif
#include <stdint.h>
extern "C" {
#include <inttypes.h>
#include <assert.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <stddef.h>
}
#include <cassert>
#include <cstdarg>
#include <cstddef>
#include <cstdio>
#include <cstdlib>
#include "vixl/platform.h"

2
disas/libvixl/vixl/invalset.h
View File

@ -27,7 +27,7 @@
#ifndef VIXL_INVALSET_H_
#define VIXL_INVALSET_H_
#include <string.h>
#include <cstring>
#include <algorithm>
#include <vector>

2
disas/libvixl/vixl/platform.h
View File

@ -28,7 +28,9 @@
#define PLATFORM_H
// Define platform specific functionalities.
extern "C" {
#include <signal.h>
}
namespace vixl {
inline void HostBreakpoint() { raise(SIGINT); }

2
disas/libvixl/vixl/utils.cc
View File

@ -25,7 +25,7 @@
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "vixl/utils.h"
#include <stdio.h>
#include <cstdio>
namespace vixl {

2
disas/libvixl/vixl/utils.h
View File

@ -27,8 +27,8 @@
#ifndef VIXL_UTILS_H
#define VIXL_UTILS_H
#include <string.h>
#include <cmath>
#include <cstring>
#include "vixl/globals.h"
#include "vixl/compiler-intrinsics.h"

35
hw/arm/armsse.c
View File

@ -13,6 +13,7 @@
#include "qemu/log.h"
#include "qemu/module.h"
#include "qemu/bitops.h"
#include "qemu/units.h"
#include "qapi/error.h"
#include "trace.h"
#include "hw/sysbus.h"
@ -59,6 +60,7 @@ struct ARMSSEInfo {
const char *cpu_type;
uint32_t sse_version;
int sram_banks;
uint32_t sram_bank_base;
int num_cpus;
uint32_t sys_version;
uint32_t iidr;
@ -69,6 +71,7 @@ struct ARMSSEInfo {
bool has_cpuid;
bool has_cpu_pwrctrl;
bool has_sse_counter;
bool has_tcms;
Property *props;
const ARMSSEDeviceInfo *devinfo;
const bool *irq_is_common;
@ -102,7 +105,7 @@ static Property sse300_properties[] = {
DEFINE_PROP_LINK("memory", ARMSSE, board_memory, TYPE_MEMORY_REGION,
MemoryRegion *),
DEFINE_PROP_UINT32("EXP_NUMIRQ", ARMSSE, exp_numirq, 64),
DEFINE_PROP_UINT32("SRAM_ADDR_WIDTH", ARMSSE, sram_addr_width, 15),
DEFINE_PROP_UINT32("SRAM_ADDR_WIDTH", ARMSSE, sram_addr_width, 18),
DEFINE_PROP_UINT32("init-svtor", ARMSSE, init_svtor, 0x10000000),
DEFINE_PROP_BOOL("CPU0_FPU", ARMSSE, cpu_fpu[0], true),
DEFINE_PROP_BOOL("CPU0_DSP", ARMSSE, cpu_dsp[0], true),
@ -504,6 +507,7 @@ static const ARMSSEInfo armsse_variants[] = {
.sse_version = ARMSSE_IOTKIT,
.cpu_type = ARM_CPU_TYPE_NAME("cortex-m33"),
.sram_banks = 1,
.sram_bank_base = 0x20000000,
.num_cpus = 1,
.sys_version = 0x41743,
.iidr = 0,
@ -514,6 +518,7 @@ static const ARMSSEInfo armsse_variants[] = {
.has_cpuid = false,
.has_cpu_pwrctrl = false,
.has_sse_counter = false,
.has_tcms = false,
.props = iotkit_properties,
.devinfo = iotkit_devices,
.irq_is_common = sse200_irq_is_common,
@ -523,6 +528,7 @@ static const ARMSSEInfo armsse_variants[] = {
.sse_version = ARMSSE_SSE200,
.cpu_type = ARM_CPU_TYPE_NAME("cortex-m33"),
.sram_banks = 4,
.sram_bank_base = 0x20000000,
.num_cpus = 2,
.sys_version = 0x22041743,
.iidr = 0,
@ -533,6 +539,7 @@ static const ARMSSEInfo armsse_variants[] = {
.has_cpuid = true,
.has_cpu_pwrctrl = false,
.has_sse_counter = false,
.has_tcms = false,
.props = sse200_properties,
.devinfo = sse200_devices,
.irq_is_common = sse200_irq_is_common,
@ -542,6 +549,7 @@ static const ARMSSEInfo armsse_variants[] = {
.sse_version = ARMSSE_SSE300,
.cpu_type = ARM_CPU_TYPE_NAME("cortex-m55"),
.sram_banks = 2,
.sram_bank_base = 0x21000000,
.num_cpus = 1,
.sys_version = 0x7e00043b,
.iidr = 0x74a0043b,
@ -552,6 +560,7 @@ static const ARMSSEInfo armsse_variants[] = {
.has_cpuid = true,
.has_cpu_pwrctrl = true,
.has_sse_counter = true,
.has_tcms = true,
.props = sse300_properties,
.devinfo = sse300_devices,
.irq_is_common = sse300_irq_is_common,
@ -909,7 +918,6 @@ static void armsse_realize(DeviceState *dev, Error **errp)
const ARMSSEDeviceInfo *devinfo;
int i;
MemoryRegion *mr;
Error *err = NULL;
SysBusDevice *sbd_apb_ppc0;
SysBusDevice *sbd_secctl;
DeviceState *dev_apb_ppc0;
@ -918,6 +926,8 @@ static void armsse_realize(DeviceState *dev, Error **errp)
DeviceState *dev_splitter;
uint32_t addr_width_max;
ERRP_GUARD();
if (!s->board_memory) {
error_setg(errp, "memory property was not set");
return;
@ -1147,10 +1157,9 @@ static void armsse_realize(DeviceState *dev, Error **errp)
uint32_t sram_bank_size = 1 << s->sram_addr_width;
memory_region_init_ram(&s->sram[i], NULL, ramname,
sram_bank_size, &err);
sram_bank_size, errp);
g_free(ramname);
if (err) {
error_propagate(errp, err);
if (*errp) {
return;
}
object_property_set_link(OBJECT(&s->mpc[i]), "downstream",
@ -1161,7 +1170,7 @@ static void armsse_realize(DeviceState *dev, Error **errp)
/* Map the upstream end of the MPC into the right place... */
sbd_mpc = SYS_BUS_DEVICE(&s->mpc[i]);
memory_region_add_subregion(&s->container,
0x20000000 + i * sram_bank_size,
info->sram_bank_base + i * sram_bank_size,
sysbus_mmio_get_region(sbd_mpc, 1));
/* ...and its register interface */
memory_region_add_subregion(&s->container, 0x50083000 + i * 0x1000,
@ -1210,6 +1219,20 @@ static void armsse_realize(DeviceState *dev, Error **errp)
sysbus_mmio_get_region(sbd, 1));
}
if (info->has_tcms) {
/* The SSE-300 has an ITCM at 0x0000_0000 and a DTCM at 0x2000_0000 */
memory_region_init_ram(&s->itcm, NULL, "sse300-itcm", 512 * KiB, errp);
if (*errp) {
return;
}
memory_region_init_ram(&s->dtcm, NULL, "sse300-dtcm", 512 * KiB, errp);
if (*errp) {
return;
}
memory_region_add_subregion(&s->container, 0x00000000, &s->itcm);
memory_region_add_subregion(&s->container, 0x20000000, &s->dtcm);
}
/* Devices behind APB PPC0:
* 0x40000000: timer0
* 0x40001000: timer1

39
hw/arm/mps2-tz.c
View File

@ -123,8 +123,10 @@ struct MPS2TZMachineClass {
int numirq; /* Number of external interrupts */
int uart_overflow_irq; /* number of the combined UART overflow IRQ */
uint32_t init_svtor; /* init-svtor setting for SSE */
uint32_t sram_addr_width; /* SRAM_ADDR_WIDTH setting for SSE */
const RAMInfo *raminfo;
const char *armsse_type;
uint32_t boot_ram_size; /* size of ram at address 0; 0 == find in raminfo */
};
struct MPS2TZMachineState {
@ -243,19 +245,13 @@ static const RAMInfo an524_raminfo[] = { {
.size = 512 * KiB,
.mpc = 0,
.mrindex = 0,
}, {
.name = "sram",
.base = 0x20000000,
.size = 32 * 4 * KiB,
.mpc = -1,
.mrindex = 1,
}, {
/* We don't model QSPI flash yet; for now expose it as simple ROM */
.name = "QSPI",
.base = 0x28000000,
.size = 8 * MiB,
.mpc = 1,
.mrindex = 2,
.mrindex = 1,
.flags = IS_ROM,
}, {
.name = "DDR",
@ -269,23 +265,11 @@ static const RAMInfo an524_raminfo[] = { {
};
static const RAMInfo an547_raminfo[] = { {
.name = "itcm",
.base = 0x00000000,
.size = 512 * KiB,
.mpc = -1,
.mrindex = 0,
}, {
.name = "sram",
.base = 0x01000000,
.size = 2 * MiB,
.mpc = 0,
.mrindex = 1,
}, {
.name = "dtcm",
.base = 0x20000000,
.size = 4 * 128 * KiB,
.mpc = -1,
.mrindex = 2,
}, {
.name = "sram 2",
.base = 0x21000000,
@ -766,6 +750,14 @@ static uint32_t boot_ram_size(MPS2TZMachineState *mms)
const RAMInfo *p;
MPS2TZMachineClass *mmc = MPS2TZ_MACHINE_GET_CLASS(mms);
/*
* Use a per-board specification (for when the boot RAM is in
* the SSE and so doesn't have a RAMInfo list entry)
*/
if (mmc->boot_ram_size) {
return mmc->boot_ram_size;
}
for (p = mmc->raminfo; p->name; p++) {
if (p->base == boot_mem_base(mms)) {
return p->size;
@ -812,6 +804,7 @@ static void mps2tz_common_init(MachineState *machine)
OBJECT(system_memory), &error_abort);
qdev_prop_set_uint32(iotkitdev, "EXP_NUMIRQ", mmc->numirq);
qdev_prop_set_uint32(iotkitdev, "init-svtor", mmc->init_svtor);
qdev_prop_set_uint32(iotkitdev, "SRAM_ADDR_WIDTH", mmc->sram_addr_width);
qdev_connect_clock_in(iotkitdev, "MAINCLK", mms->sysclk);
qdev_connect_clock_in(iotkitdev, "S32KCLK", mms->s32kclk);
sysbus_realize(SYS_BUS_DEVICE(&mms->iotkit), &error_fatal);
@ -1269,8 +1262,10 @@ static void mps2tz_an505_class_init(ObjectClass *oc, void *data)
mmc->numirq = 92;
mmc->uart_overflow_irq = 47;
mmc->init_svtor = 0x10000000;
mmc->sram_addr_width = 15;
mmc->raminfo = an505_raminfo;
mmc->armsse_type = TYPE_IOTKIT;
mmc->boot_ram_size = 0;
mps2tz_set_default_ram_info(mmc);
}
@ -1296,8 +1291,10 @@ static void mps2tz_an521_class_init(ObjectClass *oc, void *data)
mmc->numirq = 92;
mmc->uart_overflow_irq = 47;
mmc->init_svtor = 0x10000000;
mmc->sram_addr_width = 15;
mmc->raminfo = an505_raminfo; /* AN521 is the same as AN505 here */
mmc->armsse_type = TYPE_SSE200;
mmc->boot_ram_size = 0;
mps2tz_set_default_ram_info(mmc);
}
@ -1323,8 +1320,10 @@ static void mps3tz_an524_class_init(ObjectClass *oc, void *data)
mmc->numirq = 95;
mmc->uart_overflow_irq = 47;
mmc->init_svtor = 0x10000000;
mmc->sram_addr_width = 15;
mmc->raminfo = an524_raminfo;
mmc->armsse_type = TYPE_SSE200;
mmc->boot_ram_size = 0;
mps2tz_set_default_ram_info(mmc);
object_class_property_add_str(oc, "remap", mps2_get_remap, mps2_set_remap);
@ -1355,8 +1354,10 @@ static void mps3tz_an547_class_init(ObjectClass *oc, void *data)
mmc->numirq = 96;
mmc->uart_overflow_irq = 48;
mmc->init_svtor = 0x00000000;
mmc->sram_addr_width = 21;
mmc->raminfo = an547_raminfo;
mmc->armsse_type = TYPE_SSE300;
mmc->boot_ram_size = 512 * KiB;
mps2tz_set_default_ram_info(mmc);
}

50
hw/arm/smmuv3.c
View File

@ -857,43 +857,45 @@ static void smmuv3_inv_notifiers_iova(SMMUState *s, int asid, dma_addr_t iova,
static void smmuv3_s1_range_inval(SMMUState *s, Cmd *cmd)
{
uint8_t scale = 0, num = 0, ttl = 0;
dma_addr_t addr = CMD_ADDR(cmd);
dma_addr_t end, addr = CMD_ADDR(cmd);
uint8_t type = CMD_TYPE(cmd);
uint16_t vmid = CMD_VMID(cmd);
uint8_t scale = CMD_SCALE(cmd);
uint8_t num = CMD_NUM(cmd);
uint8_t ttl = CMD_TTL(cmd);
bool leaf = CMD_LEAF(cmd);
uint8_t tg = CMD_TG(cmd);
uint64_t first_page = 0, last_page;
uint64_t num_pages = 1;
uint64_t num_pages;
uint8_t granule;
int asid = -1;
if (tg) {
scale = CMD_SCALE(cmd);
num = CMD_NUM(cmd);
ttl = CMD_TTL(cmd);
num_pages = (num + 1) * BIT_ULL(scale);
}
if (type == SMMU_CMD_TLBI_NH_VA) {
asid = CMD_ASID(cmd);
}
if (!tg) {
trace_smmuv3_s1_range_inval(vmid, asid, addr, tg, 1, ttl, leaf);
smmuv3_inv_notifiers_iova(s, asid, addr, tg, 1);
smmu_iotlb_inv_iova(s, asid, addr, tg, 1, ttl);
return;
}
/* RIL in use */
num_pages = (num + 1) * BIT_ULL(scale);
granule = tg * 2 + 10;
/* Split invalidations into ^2 range invalidations */
last_page = num_pages - 1;
while (num_pages) {
uint8_t granule = tg * 2 + 10;
uint64_t mask, count;
end = addr + (num_pages << granule) - 1;
mask = dma_aligned_pow2_mask(first_page, last_page, 64 - granule);
count = mask + 1;
while (addr != end + 1) {
uint64_t mask = dma_aligned_pow2_mask(addr, end, 64);
trace_smmuv3_s1_range_inval(vmid, asid, addr, tg, count, ttl, leaf);
smmuv3_inv_notifiers_iova(s, asid, addr, tg, count);
smmu_iotlb_inv_iova(s, asid, addr, tg, count, ttl);
num_pages -= count;
first_page += count;
addr += count * BIT_ULL(granule);
num_pages = (mask + 1) >> granule;
trace_smmuv3_s1_range_inval(vmid, asid, addr, tg, num_pages, ttl, leaf);
smmuv3_inv_notifiers_iova(s, asid, addr, tg, num_pages);
smmu_iotlb_inv_iova(s, asid, addr, tg, num_pages, ttl);
addr += mask + 1;
}
}

48
hw/intc/arm_gicv3_cpuif.c
View File

@ -1307,27 +1307,16 @@ static void icc_eoir_write(CPUARMState *env, const ARMCPRegInfo *ri,
GICv3CPUState *cs = icc_cs_from_env(env);
int irq = value & 0xffffff;
int grp;
bool is_eoir0 = ri->crm == 8;
if (icv_access(env, ri->crm == 8 ? HCR_FMO : HCR_IMO)) {
if (icv_access(env, is_eoir0 ? HCR_FMO : HCR_IMO)) {
icv_eoir_write(env, ri, value);
return;
}
trace_gicv3_icc_eoir_write(ri->crm == 8 ? 0 : 1,
trace_gicv3_icc_eoir_write(is_eoir0 ? 0 : 1,
gicv3_redist_affid(cs), value);
if (ri->crm == 8) {
/* EOIR0 */
grp = GICV3_G0;
} else {
/* EOIR1 */
if (arm_is_secure(env)) {
grp = GICV3_G1;
} else {
grp = GICV3_G1NS;
}
}
if (irq >= cs->gic->num_irq) {
/* This handles two cases:
* 1. If software writes the ID of a spurious interrupt [ie 1020-1023]
@ -1340,8 +1329,35 @@ static void icc_eoir_write(CPUARMState *env, const ARMCPRegInfo *ri,
return;
}
if (icc_highest_active_group(cs) != grp) {
return;
grp = icc_highest_active_group(cs);
switch (grp) {
case GICV3_G0:
if (!is_eoir0) {
return;
}
if (!(cs->gic->gicd_ctlr & GICD_CTLR_DS)
&& arm_feature(env, ARM_FEATURE_EL3) && !arm_is_secure(env)) {
return;
}
break;
case GICV3_G1:
if (is_eoir0) {
return;
}
if (!arm_is_secure(env)) {
return;
}
break;
case GICV3_G1NS:
if (is_eoir0) {
return;
}
if (!arm_is_el3_or_mon(env) && arm_is_secure(env)) {
return;
}
break;
default:
g_assert_not_reached();
}
icc_drop_prio(cs, grp);

44
include/exec/exec-all.h
View File

@ -262,6 +262,31 @@ void tlb_flush_page_bits_by_mmuidx_all_cpus(CPUState *cpu, target_ulong addr,
void tlb_flush_page_bits_by_mmuidx_all_cpus_synced
(CPUState *cpu, target_ulong addr, uint16_t idxmap, unsigned bits);
/**
* tlb_flush_range_by_mmuidx
* @cpu: CPU whose TLB should be flushed
* @addr: virtual address of the start of the range to be flushed
* @len: length of range to be flushed
* @idxmap: bitmap of mmu indexes to flush
* @bits: number of significant bits in address
*
* For each mmuidx in @idxmap, flush all pages within [@addr,@addr+@len),
* comparing only the low @bits worth of each virtual page.
*/
void tlb_flush_range_by_mmuidx(CPUState *cpu, target_ulong addr,
target_ulong len, uint16_t idxmap,
unsigned bits);
/* Similarly, with broadcast and syncing. */
void tlb_flush_range_by_mmuidx_all_cpus(CPUState *cpu, target_ulong addr,
target_ulong len, uint16_t idxmap,
unsigned bits);
void tlb_flush_range_by_mmuidx_all_cpus_synced(CPUState *cpu,
target_ulong addr,
target_ulong len,
uint16_t idxmap,
unsigned bits);
/**
* tlb_set_page_with_attrs:
* @cpu: CPU to add this TLB entry for
@ -365,6 +390,25 @@ tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *cpu, target_ulong addr,
uint16_t idxmap, unsigned bits)
{
}
static inline void tlb_flush_range_by_mmuidx(CPUState *cpu, target_ulong addr,
target_ulong len, uint16_t idxmap,
unsigned bits)
{
}
static inline void tlb_flush_range_by_mmuidx_all_cpus(CPUState *cpu,
target_ulong addr,
target_ulong len,
uint16_t idxmap,
unsigned bits)
{
}
static inline void tlb_flush_range_by_mmuidx_all_cpus_synced(CPUState *cpu,
target_ulong addr,
target_long len,
uint16_t idxmap,
unsigned bits)
{
}
#endif
/**
* probe_access:

2
include/hw/arm/armsse.h
View File

@ -198,6 +198,8 @@ struct ARMSSE {
MemoryRegion alias2;
MemoryRegion alias3[SSE_MAX_CPUS];
MemoryRegion sram[MAX_SRAM_BANKS];
MemoryRegion itcm;
MemoryRegion dtcm;
qemu_irq *exp_irqs[SSE_MAX_CPUS];
qemu_irq ppc0_irq;

10
linux-user/elfload.c
View File

@ -648,8 +648,18 @@ static uint32_t get_elf_hwcap2(void)
uint32_t hwcaps = 0;
GET_FEATURE_ID(aa64_dcpodp, ARM_HWCAP2_A64_DCPODP);
GET_FEATURE_ID(aa64_sve2, ARM_HWCAP2_A64_SVE2);
GET_FEATURE_ID(aa64_sve2_aes, ARM_HWCAP2_A64_SVEAES);
GET_FEATURE_ID(aa64_sve2_pmull128, ARM_HWCAP2_A64_SVEPMULL);
GET_FEATURE_ID(aa64_sve2_bitperm, ARM_HWCAP2_A64_SVEBITPERM);
GET_FEATURE_ID(aa64_sve2_sha3, ARM_HWCAP2_A64_SVESHA3);
GET_FEATURE_ID(aa64_sve2_sm4, ARM_HWCAP2_A64_SVESM4);
GET_FEATURE_ID(aa64_condm_5, ARM_HWCAP2_A64_FLAGM2);
GET_FEATURE_ID(aa64_frint, ARM_HWCAP2_A64_FRINT);
GET_FEATURE_ID(aa64_sve_i8mm, ARM_HWCAP2_A64_SVEI8MM);
GET_FEATURE_ID(aa64_sve_f32mm, ARM_HWCAP2_A64_SVEF32MM);
GET_FEATURE_ID(aa64_sve_f64mm, ARM_HWCAP2_A64_SVEF64MM);
GET_FEATURE_ID(aa64_i8mm, ARM_HWCAP2_A64_I8MM);
GET_FEATURE_ID(aa64_rndr, ARM_HWCAP2_A64_RNG);
GET_FEATURE_ID(aa64_bti, ARM_HWCAP2_A64_BTI);
GET_FEATURE_ID(aa64_mte, ARM_HWCAP2_A64_MTE);

2
target/arm/cpu.c
View File

@ -1503,6 +1503,7 @@ static void arm_cpu_realizefn(DeviceState *dev, Error **errp)
t = cpu->isar.id_aa64isar1;
t = FIELD_DP64(t, ID_AA64ISAR1, FCMA, 0);
t = FIELD_DP64(t, ID_AA64ISAR1, I8MM, 0);
cpu->isar.id_aa64isar1 = t;
t = cpu->isar.id_aa64pfr0;
@ -1517,6 +1518,7 @@ static void arm_cpu_realizefn(DeviceState *dev, Error **errp)
u = cpu->isar.id_isar6;
u = FIELD_DP32(u, ID_ISAR6, DP, 0);
u = FIELD_DP32(u, ID_ISAR6, FHM, 0);
u = FIELD_DP32(u, ID_ISAR6, I8MM, 0);
cpu->isar.id_isar6 = u;
if (!arm_feature(env, ARM_FEATURE_M)) {

76
target/arm/cpu.h
View File

@ -947,6 +947,7 @@ struct ARMCPU {
uint64_t id_aa64mmfr2;
uint64_t id_aa64dfr0;
uint64_t id_aa64dfr1;
uint64_t id_aa64zfr0;
} isar;
uint64_t midr;
uint32_t revidr;
@ -2034,6 +2035,16 @@ FIELD(ID_AA64DFR0, DOUBLELOCK, 36, 4)
FIELD(ID_AA64DFR0, TRACEFILT, 40, 4)
FIELD(ID_AA64DFR0, MTPMU, 48, 4)
FIELD(ID_AA64ZFR0, SVEVER, 0, 4)
FIELD(ID_AA64ZFR0, AES, 4, 4)
FIELD(ID_AA64ZFR0, BITPERM, 16, 4)
FIELD(ID_AA64ZFR0, BFLOAT16, 20, 4)
FIELD(ID_AA64ZFR0, SHA3, 32, 4)
FIELD(ID_AA64ZFR0, SM4, 40, 4)
FIELD(ID_AA64ZFR0, I8MM, 44, 4)
FIELD(ID_AA64ZFR0, F32MM, 52, 4)
FIELD(ID_AA64ZFR0, F64MM, 56, 4)
FIELD(ID_DFR0, COPDBG, 0, 4)
FIELD(ID_DFR0, COPSDBG, 4, 4)
FIELD(ID_DFR0, MMAPDBG, 8, 4)
@ -3772,6 +3783,11 @@ static inline bool isar_feature_aa32_predinv(const ARMISARegisters *id)
return FIELD_EX32(id->id_isar6, ID_ISAR6, SPECRES) != 0;
}
static inline bool isar_feature_aa32_i8mm(const ARMISARegisters *id)
{
return FIELD_EX32(id->id_isar6, ID_ISAR6, I8MM) != 0;
}
static inline bool isar_feature_aa32_ras(const ARMISARegisters *id)
{
return FIELD_EX32(id->id_pfr0, ID_PFR0, RAS) != 0;
@ -4071,6 +4087,16 @@ static inline bool isar_feature_aa64_pauth_arch(const ARMISARegisters *id)
return FIELD_EX64(id->id_aa64isar1, ID_AA64ISAR1, APA) != 0;
}
static inline bool isar_feature_aa64_tlbirange(const ARMISARegisters *id)
{
return FIELD_EX64(id->id_aa64isar0, ID_AA64ISAR0, TLB) == 2;
}
static inline bool isar_feature_aa64_tlbios(const ARMISARegisters *id)
{
return FIELD_EX64(id->id_aa64isar0, ID_AA64ISAR0, TLB) != 0;
}
static inline bool isar_feature_aa64_sb(const ARMISARegisters *id)
{
return FIELD_EX64(id->id_aa64isar1, ID_AA64ISAR1, SB) != 0;
@ -4195,6 +4221,11 @@ static inline bool isar_feature_aa64_rcpc_8_4(const ARMISARegisters *id)
return FIELD_EX64(id->id_aa64isar1, ID_AA64ISAR1, LRCPC) >= 2;
}
static inline bool isar_feature_aa64_i8mm(const ARMISARegisters *id)
{
return FIELD_EX64(id->id_aa64isar1, ID_AA64ISAR1, I8MM) != 0;
}
static inline bool isar_feature_aa64_ccidx(const ARMISARegisters *id)
{
return FIELD_EX64(id->id_aa64mmfr2, ID_AA64MMFR2, CCIDX) != 0;
@ -4215,6 +4246,51 @@ static inline bool isar_feature_aa64_ssbs(const ARMISARegisters *id)
return FIELD_EX64(id->id_aa64pfr1, ID_AA64PFR1, SSBS) != 0;
}
static inline bool isar_feature_aa64_sve2(const ARMISARegisters *id)
{
return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, SVEVER) != 0;
}
static inline bool isar_feature_aa64_sve2_aes(const ARMISARegisters *id)
{
return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, AES) != 0;
}
static inline bool isar_feature_aa64_sve2_pmull128(const ARMISARegisters *id)
{
return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, AES) >= 2;
}
static inline bool isar_feature_aa64_sve2_bitperm(const ARMISARegisters *id)
{
return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, BITPERM) != 0;
}
static inline bool isar_feature_aa64_sve2_sha3(const ARMISARegisters *id)
{
return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, SHA3) != 0;
}
static inline bool isar_feature_aa64_sve2_sm4(const ARMISARegisters *id)
{
return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, SM4) != 0;
}
static inline bool isar_feature_aa64_sve_i8mm(const ARMISARegisters *id)
{
return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, I8MM) != 0;
}
static inline bool isar_feature_aa64_sve_f32mm(const ARMISARegisters *id)
{
return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, F32MM) != 0;
}
static inline bool isar_feature_aa64_sve_f64mm(const ARMISARegisters *id)
{
return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, F64MM) != 0;
}
/*
* Feature tests for "does this exist in either 32-bit or 64-bit?"
*/

14
target/arm/cpu64.c
View File

@ -651,6 +651,7 @@ static void aarch64_max_initfn(Object *obj)
t = FIELD_DP64(t, ID_AA64ISAR0, DP, 1);
t = FIELD_DP64(t, ID_AA64ISAR0, FHM, 1);
t = FIELD_DP64(t, ID_AA64ISAR0, TS, 2); /* v8.5-CondM */
t = FIELD_DP64(t, ID_AA64ISAR0, TLB, 2); /* FEAT_TLBIRANGE */
t = FIELD_DP64(t, ID_AA64ISAR0, RNDR, 1);
cpu->isar.id_aa64isar0 = t;
@ -662,6 +663,7 @@ static void aarch64_max_initfn(Object *obj)
t = FIELD_DP64(t, ID_AA64ISAR1, SPECRES, 1);
t = FIELD_DP64(t, ID_AA64ISAR1, FRINTTS, 1);
t = FIELD_DP64(t, ID_AA64ISAR1, LRCPC, 2); /* ARMv8.4-RCPC */
t = FIELD_DP64(t, ID_AA64ISAR1, I8MM, 1);
cpu->isar.id_aa64isar1 = t;
t = cpu->isar.id_aa64pfr0;
@ -702,6 +704,17 @@ static void aarch64_max_initfn(Object *obj)
t = FIELD_DP64(t, ID_AA64MMFR2, ST, 1); /* TTST */
cpu->isar.id_aa64mmfr2 = t;
t = cpu->isar.id_aa64zfr0;
t = FIELD_DP64(t, ID_AA64ZFR0, SVEVER, 1);
t = FIELD_DP64(t, ID_AA64ZFR0, AES, 2); /* PMULL */
t = FIELD_DP64(t, ID_AA64ZFR0, BITPERM, 1);
t = FIELD_DP64(t, ID_AA64ZFR0, SHA3, 1);
t = FIELD_DP64(t, ID_AA64ZFR0, SM4, 1);
t = FIELD_DP64(t, ID_AA64ZFR0, I8MM, 1);
t = FIELD_DP64(t, ID_AA64ZFR0, F32MM, 1);
t = FIELD_DP64(t, ID_AA64ZFR0, F64MM, 1);
cpu->isar.id_aa64zfr0 = t;
/* Replicate the same data to the 32-bit id registers. */
u = cpu->isar.id_isar5;
u = FIELD_DP32(u, ID_ISAR5, AES, 2); /* AES + PMULL */
@ -718,6 +731,7 @@ static void aarch64_max_initfn(Object *obj)
u = FIELD_DP32(u, ID_ISAR6, FHM, 1);
u = FIELD_DP32(u, ID_ISAR6, SB, 1);
u = FIELD_DP32(u, ID_ISAR6, SPECRES, 1);
u = FIELD_DP32(u, ID_ISAR6, I8MM, 1);
cpu->isar.id_isar6 = u;
u = cpu->isar.id_pfr0;

1
target/arm/cpu_tcg.c
View File

@ -968,6 +968,7 @@ static void arm_max_initfn(Object *obj)
t = FIELD_DP32(t, ID_ISAR6, FHM, 1);
t = FIELD_DP32(t, ID_ISAR6, SB, 1);
t = FIELD_DP32(t, ID_ISAR6, SPECRES, 1);
t = FIELD_DP32(t, ID_ISAR6, I8MM, 1);
cpu->isar.id_isar6 = t;
t = cpu->isar.mvfr1;

722
target/arm/helper-sve.h
View File

@ -158,6 +158,128 @@ DEF_HELPER_FLAGS_5(sve_umulh_zpzz_s, TCG_CALL_NO_RWG,
DEF_HELPER_FLAGS_5(sve_umulh_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sadalp_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sadalp_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sadalp_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uadalp_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uadalp_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uadalp_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_srshl_zpzz_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_srshl_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_srshl_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_srshl_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_urshl_zpzz_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_urshl_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_urshl_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_urshl_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqshl_zpzz_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqshl_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqshl_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqshl_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uqshl_zpzz_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uqshl_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uqshl_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uqshl_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrshl_zpzz_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrshl_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrshl_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrshl_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uqrshl_zpzz_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uqrshl_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uqrshl_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uqrshl_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_shadd_zpzz_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_shadd_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_shadd_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_shadd_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uhadd_zpzz_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uhadd_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uhadd_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uhadd_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_srhadd_zpzz_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_srhadd_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_srhadd_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_srhadd_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_urhadd_zpzz_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_urhadd_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_urhadd_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_urhadd_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_shsub_zpzz_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_shsub_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_shsub_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_shsub_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uhsub_zpzz_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uhsub_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uhsub_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uhsub_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve_sdiv_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve_sdiv_zpzz_d, TCG_CALL_NO_RWG,
@ -204,6 +326,105 @@ DEF_HELPER_FLAGS_5(sve_sel_zpzz_s, TCG_CALL_NO_RWG,
DEF_HELPER_FLAGS_5(sve_sel_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_addp_zpzz_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_addp_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_addp_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_addp_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_smaxp_zpzz_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_smaxp_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_smaxp_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_smaxp_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_umaxp_zpzz_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_umaxp_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_umaxp_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_umaxp_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sminp_zpzz_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sminp_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sminp_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sminp_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uminp_zpzz_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uminp_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uminp_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uminp_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqadd_zpzz_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqadd_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqadd_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqadd_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uqadd_zpzz_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uqadd_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uqadd_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uqadd_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqsub_zpzz_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqsub_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqsub_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqsub_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uqsub_zpzz_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uqsub_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uqsub_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uqsub_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_suqadd_zpzz_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_suqadd_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_suqadd_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_suqadd_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_usqadd_zpzz_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_usqadd_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_usqadd_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_usqadd_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve_asr_zpzw_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve_asr_zpzw_h, TCG_CALL_NO_RWG,
@ -440,6 +661,16 @@ DEF_HELPER_FLAGS_4(sve_tbl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_tbl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_tbl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_tbl_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_tbl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_tbl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_tbl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_tbx_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_tbx_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_tbx_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_tbx_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve_sunpk_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve_sunpk_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve_sunpk_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
@ -458,16 +689,19 @@ DEF_HELPER_FLAGS_4(sve_zip_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_zip_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_zip_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_zip_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_zip_q, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_uzp_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_uzp_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_uzp_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_uzp_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_uzp_q, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_trn_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_trn_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_trn_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_trn_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_trn_q, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_compact_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_compact_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
@ -488,6 +722,19 @@ DEF_HELPER_FLAGS_4(sve_rbit_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_rbit_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_rbit_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqabs_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqabs_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqabs_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqabs_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqneg_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqneg_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqneg_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqneg_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_urecpe_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_ursqrte_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve_splice, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve_cmpeq_ppzz_b, TCG_CALL_NO_RWG,
@ -679,7 +926,8 @@ DEF_HELPER_FLAGS_4(sve_brkns, TCG_CALL_NO_RWG, i32, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve_cntp, TCG_CALL_NO_RWG, i64, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve_while, TCG_CALL_NO_RWG, i32, ptr, i32, i32)
DEF_HELPER_FLAGS_3(sve_whilel, TCG_CALL_NO_RWG, i32, ptr, i32, i32)
DEF_HELPER_FLAGS_3(sve_whileg, TCG_CALL_NO_RWG, i32, ptr, i32, i32)
DEF_HELPER_FLAGS_4(sve_subri_b, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32)
DEF_HELPER_FLAGS_4(sve_subri_h, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32)
@ -1133,6 +1381,46 @@ DEF_HELPER_FLAGS_5(sve_ftmad_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve_ftmad_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve_ftmad_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_saddl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_saddl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_saddl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_ssubl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_ssubl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_ssubl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sabdl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sabdl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sabdl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_uaddl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_uaddl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_uaddl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_usubl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_usubl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_usubl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_uabdl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_uabdl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_uabdl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_saddw_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_saddw_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_saddw_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_ssubw_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_ssubw_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_ssubw_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_uaddw_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_uaddw_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_uaddw_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_usubw_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_usubw_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_usubw_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_ld1bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
DEF_HELPER_FLAGS_4(sve_ld2bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
DEF_HELPER_FLAGS_4(sve_ld3bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
@ -2073,4 +2361,436 @@ DEF_HELPER_FLAGS_6(sve_stdd_le_zd_mte, TCG_CALL_NO_WG,
DEF_HELPER_FLAGS_6(sve_stdd_be_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_4(sve2_sqdmull_zzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqdmull_zzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqdmull_zzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_smull_zzz_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_smull_zzz_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_smull_zzz_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_umull_zzz_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_umull_zzz_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_umull_zzz_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_pmull_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_pmull_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sshll_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sshll_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sshll_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_ushll_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_ushll_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_ushll_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_eoril_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_eoril_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_eoril_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_eoril_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_bext_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_bext_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_bext_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_bext_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_bdep_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_bdep_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_bdep_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_bdep_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_bgrp_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_bgrp_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_bgrp_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_bgrp_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_cadd_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_cadd_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_cadd_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_cadd_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqcadd_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqcadd_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqcadd_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqcadd_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sabal_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sabal_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sabal_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uabal_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uabal_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uabal_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_adcl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_adcl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqxtnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqxtnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqxtnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_uqxtnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_uqxtnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_uqxtnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqxtunb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqxtunb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqxtunb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqxtnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqxtnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqxtnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_uqxtnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_uqxtnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_uqxtnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqxtunt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqxtunt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqxtunt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_shrnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_shrnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_shrnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_shrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_shrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_shrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_rshrnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_rshrnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_rshrnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_rshrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_rshrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_rshrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqshrunb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqshrunb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqshrunb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqshrunt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqshrunt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqshrunt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqrshrunb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqrshrunb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqrshrunb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqrshrunt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqrshrunt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqrshrunt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqshrnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqshrnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqshrnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqshrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqshrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqshrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqrshrnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqrshrnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqrshrnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqrshrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqrshrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqrshrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_uqshrnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_uqshrnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_uqshrnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_uqshrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_uqshrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_uqshrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_uqrshrnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_uqrshrnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_uqrshrnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_uqrshrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_uqrshrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_uqrshrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_addhnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_addhnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_addhnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_addhnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_addhnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_addhnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_raddhnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_raddhnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_raddhnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_raddhnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_raddhnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_raddhnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_subhnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_subhnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_subhnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_subhnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_subhnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_subhnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_rsubhnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_rsubhnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_rsubhnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_rsubhnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_rsubhnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_rsubhnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_match_ppzz_b, TCG_CALL_NO_RWG,
i32, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_match_ppzz_h, TCG_CALL_NO_RWG,
i32, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_nmatch_ppzz_b, TCG_CALL_NO_RWG,
i32, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_nmatch_ppzz_h, TCG_CALL_NO_RWG,
i32, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_histcnt_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_histcnt_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_histseg, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_xar_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_xar_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_xar_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sve2_fmaxnmp_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sve2_fmaxnmp_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sve2_fmaxnmp_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sve2_fminnmp_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sve2_fminnmp_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sve2_fminnmp_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sve2_fmaxp_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sve2_fmaxp_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sve2_fmaxp_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sve2_fminp_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sve2_fminp_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sve2_fminp_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_eor3, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_bcax, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_bsl1n, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_bsl2n, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_nbsl, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqdmlal_zzzw_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqdmlal_zzzw_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqdmlal_zzzw_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqdmlsl_zzzw_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqdmlsl_zzzw_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqdmlsl_zzzw_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_smlal_zzzw_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_smlal_zzzw_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_smlal_zzzw_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_umlal_zzzw_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_umlal_zzzw_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_umlal_zzzw_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_smlsl_zzzw_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_smlsl_zzzw_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_smlsl_zzzw_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_umlsl_zzzw_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_umlsl_zzzw_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_umlsl_zzzw_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_cmla_zzzz_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_cmla_zzzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_cmla_zzzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_cmla_zzzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_zzzz_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_zzzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_zzzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_zzzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(fmmla_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(fmmla_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrdmlah_idx_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrdmlah_idx_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrdmlah_idx_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_idx_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_idx_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_idx_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqdmlal_idx_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqdmlal_idx_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqdmlsl_idx_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqdmlsl_idx_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqdmull_idx_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqdmull_idx_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_smlal_idx_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_smlal_idx_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_smlsl_idx_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_smlsl_idx_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_umlal_idx_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_umlal_idx_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_umlsl_idx_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_umlsl_idx_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_smull_idx_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_smull_idx_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_umull_idx_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_umull_idx_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_cmla_idx_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_cmla_idx_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_idx_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_idx_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_cdot_zzzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_cdot_zzzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_cdot_idx_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_cdot_idx_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_fcvtnt_sh, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_fcvtnt_ds, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_fcvtlt_hs, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_fcvtlt_sd, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(flogb_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(flogb_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(flogb_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqshl_zpzi_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqshl_zpzi_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqshl_zpzi_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqshl_zpzi_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_uqshl_zpzi_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_uqshl_zpzi_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_uqshl_zpzi_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_uqshl_zpzi_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_srshr_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_srshr_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_srshr_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_srshr_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_urshr_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_urshr_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_urshr_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_urshr_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqshlu_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqshlu_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqshlu_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqshlu_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)

327
target/arm/helper.c
View File

@ -4759,6 +4759,172 @@ static void tlbi_aa64_vae3is_write(CPUARMState *env, const ARMCPRegInfo *ri,
ARMMMUIdxBit_SE3, bits);
}
#ifdef TARGET_AARCH64
static uint64_t tlbi_aa64_range_get_length(CPUARMState *env,
uint64_t value)
{
unsigned int page_shift;
unsigned int page_size_granule;
uint64_t num;
uint64_t scale;
uint64_t exponent;
uint64_t length;
num = extract64(value, 39, 4);
scale = extract64(value, 44, 2);
page_size_granule = extract64(value, 46, 2);
page_shift = page_size_granule * 2 + 12;
if (page_size_granule == 0) {
qemu_log_mask(LOG_GUEST_ERROR, "Invalid page size granule %d\n",
page_size_granule);
return 0;
}
exponent = (5 * scale) + 1;
length = (num + 1) << (exponent + page_shift);
return length;
}
static uint64_t tlbi_aa64_range_get_base(CPUARMState *env, uint64_t value,
bool two_ranges)
{
/* TODO: ARMv8.7 FEAT_LPA2 */
uint64_t pageaddr;
if (two_ranges) {
pageaddr = sextract64(value, 0, 37) << TARGET_PAGE_BITS;
} else {
pageaddr = extract64(value, 0, 37) << TARGET_PAGE_BITS;
}
return pageaddr;
}
static void do_rvae_write(CPUARMState *env, uint64_t value,
int idxmap, bool synced)
{
ARMMMUIdx one_idx = ARM_MMU_IDX_A | ctz32(idxmap);
bool two_ranges = regime_has_2_ranges(one_idx);
uint64_t baseaddr, length;
int bits;
baseaddr = tlbi_aa64_range_get_base(env, value, two_ranges);
length = tlbi_aa64_range_get_length(env, value);
bits = tlbbits_for_regime(env, one_idx, baseaddr);
if (synced) {
tlb_flush_range_by_mmuidx_all_cpus_synced(env_cpu(env),
baseaddr,
length,
idxmap,
bits);
} else {
tlb_flush_range_by_mmuidx(env_cpu(env), baseaddr,
length, idxmap, bits);
}
}
static void tlbi_aa64_rvae1_write(CPUARMState *env,
const ARMCPRegInfo *ri,
uint64_t value)
{
/*
* Invalidate by VA range, EL1&0.
* Currently handles all of RVAE1, RVAAE1, RVAALE1 and RVALE1,
* since we don't support flush-for-specific-ASID-only or
* flush-last-level-only.
*/
do_rvae_write(env, value, vae1_tlbmask(env),
tlb_force_broadcast(env));
}
static void tlbi_aa64_rvae1is_write(CPUARMState *env,
const ARMCPRegInfo *ri,
uint64_t value)
{
/*
* Invalidate by VA range, Inner/Outer Shareable EL1&0.
* Currently handles all of RVAE1IS, RVAE1OS, RVAAE1IS, RVAAE1OS,
* RVAALE1IS, RVAALE1OS, RVALE1IS and RVALE1OS, since we don't support
* flush-for-specific-ASID-only, flush-last-level-only or inner/outer
* shareable specific flushes.
*/
do_rvae_write(env, value, vae1_tlbmask(env), true);
}
static int vae2_tlbmask(CPUARMState *env)
{
return (arm_is_secure_below_el3(env)
? ARMMMUIdxBit_SE2 : ARMMMUIdxBit_E2);
}
static void tlbi_aa64_rvae2_write(CPUARMState *env,
const ARMCPRegInfo *ri,
uint64_t value)
{
/*
* Invalidate by VA range, EL2.
* Currently handles all of RVAE2 and RVALE2,
* since we don't support flush-for-specific-ASID-only or
* flush-last-level-only.
*/
do_rvae_write(env, value, vae2_tlbmask(env),
tlb_force_broadcast(env));
}
static void tlbi_aa64_rvae2is_write(CPUARMState *env,
const ARMCPRegInfo *ri,
uint64_t value)
{
/*
* Invalidate by VA range, Inner/Outer Shareable, EL2.
* Currently handles all of RVAE2IS, RVAE2OS, RVALE2IS and RVALE2OS,
* since we don't support flush-for-specific-ASID-only,
* flush-last-level-only or inner/outer shareable specific flushes.
*/
do_rvae_write(env, value, vae2_tlbmask(env), true);
}
static void tlbi_aa64_rvae3_write(CPUARMState *env,
const ARMCPRegInfo *ri,
uint64_t value)
{
/*
* Invalidate by VA range, EL3.
* Currently handles all of RVAE3 and RVALE3,
* since we don't support flush-for-specific-ASID-only or
* flush-last-level-only.
*/
do_rvae_write(env, value, ARMMMUIdxBit_SE3,
tlb_force_broadcast(env));
}
static void tlbi_aa64_rvae3is_write(CPUARMState *env,
const ARMCPRegInfo *ri,
uint64_t value)
{
/*
* Invalidate by VA range, EL3, Inner/Outer Shareable.
* Currently handles all of RVAE3IS, RVAE3OS, RVALE3IS and RVALE3OS,
* since we don't support flush-for-specific-ASID-only,
* flush-last-level-only or inner/outer specific flushes.
*/
do_rvae_write(env, value, ARMMMUIdxBit_SE3, true);
}
#endif
static CPAccessResult aa64_zva_access(CPUARMState *env, const ARMCPRegInfo *ri,
bool isread)
{
@ -6920,6 +7086,158 @@ static const ARMCPRegInfo pauth_reginfo[] = {
REGINFO_SENTINEL
};
static const ARMCPRegInfo tlbirange_reginfo[] = {
{ .name = "TLBI_RVAE1IS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 2, .opc2 = 1,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae1is_write },
{ .name = "TLBI_RVAAE1IS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 2, .opc2 = 3,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae1is_write },
{ .name = "TLBI_RVALE1IS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 2, .opc2 = 5,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae1is_write },
{ .name = "TLBI_RVAALE1IS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 2, .opc2 = 7,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae1is_write },
{ .name = "TLBI_RVAE1OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 1,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae1is_write },
{ .name = "TLBI_RVAAE1OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 3,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae1is_write },
{ .name = "TLBI_RVALE1OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 5,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae1is_write },
{ .name = "TLBI_RVAALE1OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 7,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae1is_write },
{ .name = "TLBI_RVAE1", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 1,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae1_write },
{ .name = "TLBI_RVAAE1", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 3,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae1_write },
{ .name = "TLBI_RVALE1", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 5,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae1_write },
{ .name = "TLBI_RVAALE1", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 7,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae1_write },
{ .name = "TLBI_RIPAS2E1IS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 2,
.access = PL2_W, .type = ARM_CP_NOP },
{ .name = "TLBI_RIPAS2LE1IS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 6,
.access = PL2_W, .type = ARM_CP_NOP },
{ .name = "TLBI_RVAE2IS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 4, .crn = 8, .crm = 2, .opc2 = 1,
.access = PL2_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae2is_write },
{ .name = "TLBI_RVALE2IS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 4, .crn = 8, .crm = 2, .opc2 = 5,
.access = PL2_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae2is_write },
{ .name = "TLBI_RIPAS2E1", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 2,
.access = PL2_W, .type = ARM_CP_NOP },
{ .name = "TLBI_RIPAS2LE1", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 6,
.access = PL2_W, .type = ARM_CP_NOP },
{ .name = "TLBI_RVAE2OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 4, .crn = 8, .crm = 5, .opc2 = 1,
.access = PL2_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae2is_write },
{ .name = "TLBI_RVALE2OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 4, .crn = 8, .crm = 5, .opc2 = 5,
.access = PL2_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae2is_write },
{ .name = "TLBI_RVAE2", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 4, .crn = 8, .crm = 6, .opc2 = 1,
.access = PL2_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae2_write },
{ .name = "TLBI_RVALE2", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 4, .crn = 8, .crm = 6, .opc2 = 5,
.access = PL2_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae2_write },
{ .name = "TLBI_RVAE3IS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 6, .crn = 8, .crm = 2, .opc2 = 1,
.access = PL3_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae3is_write },
{ .name = "TLBI_RVALE3IS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 6, .crn = 8, .crm = 2, .opc2 = 5,
.access = PL3_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae3is_write },
{ .name = "TLBI_RVAE3OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 6, .crn = 8, .crm = 5, .opc2 = 1,
.access = PL3_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae3is_write },
{ .name = "TLBI_RVALE3OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 6, .crn = 8, .crm = 5, .opc2 = 5,
.access = PL3_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae3is_write },
{ .name = "TLBI_RVAE3", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 6, .crn = 8, .crm = 6, .opc2 = 1,
.access = PL3_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae3_write },
{ .name = "TLBI_RVALE3", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 6, .crn = 8, .crm = 6, .opc2 = 5,
.access = PL3_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_rvae3_write },
REGINFO_SENTINEL
};
static const ARMCPRegInfo tlbios_reginfo[] = {
{ .name = "TLBI_VMALLE1OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 1, .opc2 = 0,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_vmalle1is_write },
{ .name = "TLBI_ASIDE1OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 0, .crn = 8, .crm = 1, .opc2 = 2,
.access = PL1_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_vmalle1is_write },
{ .name = "TLBI_ALLE2OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 4, .crn = 8, .crm = 1, .opc2 = 0,
.access = PL2_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_alle2is_write },
{ .name = "TLBI_ALLE1OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 4, .crn = 8, .crm = 1, .opc2 = 4,
.access = PL2_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_alle1is_write },
{ .name = "TLBI_VMALLS12E1OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 4, .crn = 8, .crm = 1, .opc2 = 6,
.access = PL2_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_alle1is_write },
{ .name = "TLBI_IPAS2E1OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 0,
.access = PL2_W, .type = ARM_CP_NOP },
{ .name = "TLBI_RIPAS2E1OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 3,
.access = PL2_W, .type = ARM_CP_NOP },
{ .name = "TLBI_IPAS2LE1OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 4,
.access = PL2_W, .type = ARM_CP_NOP },
{ .name = "TLBI_RIPAS2LE1OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 7,
.access = PL2_W, .type = ARM_CP_NOP },
{ .name = "TLBI_ALLE3OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 6, .crn = 8, .crm = 1, .opc2 = 0,
.access = PL3_W, .type = ARM_CP_NO_RAW,
.writefn = tlbi_aa64_alle3is_write },
REGINFO_SENTINEL
};
static uint64_t rndr_readfn(CPUARMState *env, const ARMCPRegInfo *ri)
{
Error *err = NULL;
@ -7561,8 +7879,7 @@ void register_cp_regs_for_features(ARMCPU *cpu)
.opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 4,
.access = PL1_R, .type = ARM_CP_CONST,
.accessfn = access_aa64_tid3,
/* At present, only SVEver == 0 is defined anyway. */
.resetvalue = 0 },
.resetvalue = cpu->isar.id_aa64zfr0 },
{ .name = "ID_AA64PFR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 5,
.access = PL1_R, .type = ARM_CP_CONST,
@ -8289,6 +8606,12 @@ void register_cp_regs_for_features(ARMCPU *cpu)
if (cpu_isar_feature(aa64_rndr, cpu)) {
define_arm_cp_regs(cpu, rndr_reginfo);
}
if (cpu_isar_feature(aa64_tlbirange, cpu)) {
define_arm_cp_regs(cpu, tlbirange_reginfo);
}
if (cpu_isar_feature(aa64_tlbios, cpu)) {
define_arm_cp_regs(cpu, tlbios_reginfo);
}
#ifndef CONFIG_USER_ONLY
/* Data Cache clean instructions up to PoP */
if (cpu_isar_feature(aa64_dcpop, cpu)) {

110
target/arm/helper.h
View File

@ -591,15 +591,41 @@ DEF_HELPER_FLAGS_5(gvec_qrdmlah_s32, TCG_CALL_NO_RWG,
DEF_HELPER_FLAGS_5(gvec_qrdmlsh_s32, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(gvec_sdot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(gvec_udot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(gvec_sdot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(gvec_udot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrdmlah_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrdmlah_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrdmlah_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrdmlah_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(gvec_sdot_idx_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(gvec_udot_idx_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(gvec_sdot_idx_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(gvec_udot_idx_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_sdot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_udot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_sdot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_udot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_usdot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_sdot_idx_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_udot_idx_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_sdot_idx_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_udot_idx_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_sudot_idx_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_usdot_idx_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_fcaddh, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
@ -608,16 +634,16 @@ DEF_HELPER_FLAGS_5(gvec_fcadds, TCG_CALL_NO_RWG,
DEF_HELPER_FLAGS_5(gvec_fcaddd, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_fcmlah, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_fcmlah_idx, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_fcmlas, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_fcmlas_idx, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_fcmlad, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(gvec_fcmlah, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(gvec_fcmlah_idx, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(gvec_fcmlas, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(gvec_fcmlas_idx, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(gvec_fcmlad, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(neon_paddh, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(neon_pmaxh, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
@ -828,6 +854,16 @@ DEF_HELPER_FLAGS_3(gvec_cgt0_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(gvec_cge0_b, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(gvec_cge0_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(gvec_smulh_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(gvec_smulh_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(gvec_smulh_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(gvec_smulh_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(gvec_umulh_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(gvec_umulh_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(gvec_umulh_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(gvec_umulh_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(gvec_sshl_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(gvec_sshl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(gvec_ushl_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
@ -926,6 +962,44 @@ DEF_HELPER_FLAGS_5(neon_sqrdmulh_h, TCG_CALL_NO_RWG,
DEF_HELPER_FLAGS_5(neon_sqrdmulh_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqdmulh_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqdmulh_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqdmulh_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqdmulh_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqrdmulh_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqrdmulh_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqrdmulh_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqrdmulh_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqdmulh_idx_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqdmulh_idx_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqdmulh_idx_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqrdmulh_idx_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqrdmulh_idx_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqrdmulh_idx_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sve2_fmlal_zzzw_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sve2_fmlal_zzxw_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(gvec_xar_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_smmla_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_ummla_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_usmmla_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
#ifdef TARGET_AARCH64
#include "helper-a64.h"
#include "helper-sve.h"

21
target/arm/kvm64.c
View File

@ -647,17 +647,26 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
sve_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION, KVM_CAP_ARM_SVE) > 0;
kvm_arm_destroy_scratch_host_vcpu(fdarray);
if (err < 0) {
return false;
}
/* Add feature bits that can't appear until after VCPU init. */
if (sve_supported) {
t = ahcf->isar.id_aa64pfr0;
t = FIELD_DP64(t, ID_AA64PFR0, SVE, 1);
ahcf->isar.id_aa64pfr0 = t;
/*
* Before v5.1, KVM did not support SVE and did not expose
* ID_AA64ZFR0_EL1 even as RAZ. After v5.1, KVM still does
* not expose the register to "user" requests like this
* unless the host supports SVE.
*/
err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64zfr0,
ARM64_SYS_REG(3, 0, 0, 4, 4));
}
kvm_arm_destroy_scratch_host_vcpu(fdarray);
if (err < 0) {
return false;
}
/*

3
target/arm/m_helper.c
View File

@ -1597,10 +1597,11 @@ static void do_v7m_exception_exit(ARMCPU *cpu)
* We use this limited C variable scope so we don't accidentally
* use 'frame_sp_p' after we do something that makes it invalid.
*/
bool spsel = env->v7m.control[return_to_secure] & R_V7M_CONTROL_SPSEL_MASK;
uint32_t *frame_sp_p = get_v7m_sp_ptr(env,
return_to_secure,
!return_to_handler,
return_to_sp_process);
spsel);
uint32_t frameptr = *frame_sp_p;
bool pop_ok = true;
ARMMMUIdx mmu_idx;

24
target/arm/neon-shared.decode
View File

@ -46,8 +46,11 @@ VCMLA 1111 110 rot:2 . 1 . .... .... 1000 . q:1 . 0 .... \
VCADD 1111 110 rot:1 1 . 0 . .... .... 1000 . q:1 . 0 .... \
vm=%vm_dp vn=%vn_dp vd=%vd_dp size=%vcadd_size
# VUDOT and VSDOT
VDOT 1111 110 00 . 10 .... .... 1101 . q:1 . u:1 .... \
VSDOT 1111 110 00 . 10 .... .... 1101 . q:1 . 0 .... \
vm=%vm_dp vn=%vn_dp vd=%vd_dp
VUDOT 1111 110 00 . 10 .... .... 1101 . q:1 . 1 .... \
vm=%vm_dp vn=%vn_dp vd=%vd_dp
VUSDOT 1111 110 01 . 10 .... .... 1101 . q:1 . 0 .... \
vm=%vm_dp vn=%vn_dp vd=%vd_dp
# VFM[AS]L
@ -56,13 +59,26 @@ VFML 1111 110 0 s:1 . 10 .... .... 1000 . 0 . 1 .... \
VFML 1111 110 0 s:1 . 10 .... .... 1000 . 1 . 1 .... \
vm=%vm_dp vn=%vn_dp vd=%vd_dp q=1
VSMMLA 1111 1100 0.10 .... .... 1100 .1.0 .... \
vm=%vm_dp vn=%vn_dp vd=%vd_dp
VUMMLA 1111 1100 0.10 .... .... 1100 .1.1 .... \
vm=%vm_dp vn=%vn_dp vd=%vd_dp
VUSMMLA 1111 1100 1.10 .... .... 1100 .1.0 .... \
vm=%vm_dp vn=%vn_dp vd=%vd_dp
VCMLA_scalar 1111 1110 0 . rot:2 .... .... 1000 . q:1 index:1 0 vm:4 \
vn=%vn_dp vd=%vd_dp size=1
VCMLA_scalar 1111 1110 1 . rot:2 .... .... 1000 . q:1 . 0 .... \
vm=%vm_dp vn=%vn_dp vd=%vd_dp size=2 index=0
VDOT_scalar 1111 1110 0 . 10 .... .... 1101 . q:1 index:1 u:1 rm:4 \
vm=%vm_dp vn=%vn_dp vd=%vd_dp
VSDOT_scalar 1111 1110 0 . 10 .... .... 1101 . q:1 index:1 0 vm:4 \
vn=%vn_dp vd=%vd_dp
VUDOT_scalar 1111 1110 0 . 10 .... .... 1101 . q:1 index:1 1 vm:4 \
vn=%vn_dp vd=%vd_dp
VUSDOT_scalar 1111 1110 1 . 00 .... .... 1101 . q:1 index:1 0 vm:4 \
vn=%vn_dp vd=%vd_dp
VSUDOT_scalar 1111 1110 1 . 00 .... .... 1101 . q:1 index:1 1 vm:4 \
vn=%vn_dp vd=%vd_dp
%vfml_scalar_q0_rm 0:3 5:1
%vfml_scalar_q1_index 5:1 3:1

507
target/arm/neon_helper.c
View File

@ -11,6 +11,7 @@
#include "cpu.h"
#include "exec/helper-proto.h"
#include "fpu/softfloat.h"
#include "vec_internal.h"
#define SIGNBIT (uint32_t)0x80000000
#define SIGNBIT64 ((uint64_t)1 << 63)
@ -576,496 +577,154 @@ NEON_POP(pmax_s16, neon_s16, 2)
NEON_POP(pmax_u16, neon_u16, 2)
#undef NEON_FN
#define NEON_FN(dest, src1, src2) do { \
int8_t tmp; \
tmp = (int8_t)src2; \
if (tmp >= (ssize_t)sizeof(src1) * 8 || \
tmp <= -(ssize_t)sizeof(src1) * 8) { \
dest = 0; \
} else if (tmp < 0) { \
dest = src1 >> -tmp; \
} else { \
dest = src1 << tmp; \
}} while (0)
#define NEON_FN(dest, src1, src2) \
(dest = do_uqrshl_bhs(src1, (int8_t)src2, 16, false, NULL))
NEON_VOP(shl_u16, neon_u16, 2)
#undef NEON_FN
#define NEON_FN(dest, src1, src2) do { \
int8_t tmp; \
tmp = (int8_t)src2; \
if (tmp >= (ssize_t)sizeof(src1) * 8) { \
dest = 0; \
} else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
dest = src1 >> (sizeof(src1) * 8 - 1); \
} else if (tmp < 0) { \
dest = src1 >> -tmp; \
} else { \
dest = src1 << tmp; \
}} while (0)
#define NEON_FN(dest, src1, src2) \
(dest = do_sqrshl_bhs(src1, (int8_t)src2, 16, false, NULL))
NEON_VOP(shl_s16, neon_s16, 2)
#undef NEON_FN
#define NEON_FN(dest, src1, src2) do { \
int8_t tmp; \
tmp = (int8_t)src2; \
if ((tmp >= (ssize_t)sizeof(src1) * 8) \
|| (tmp <= -(ssize_t)sizeof(src1) * 8)) { \
dest = 0; \
} else if (tmp < 0) { \
dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
} else { \
dest = src1 << tmp; \
}} while (0)
#define NEON_FN(dest, src1, src2) \
(dest = do_sqrshl_bhs(src1, (int8_t)src2, 8, true, NULL))
NEON_VOP(rshl_s8, neon_s8, 4)
#undef NEON_FN
#define NEON_FN(dest, src1, src2) \
(dest = do_sqrshl_bhs(src1, (int8_t)src2, 16, true, NULL))
NEON_VOP(rshl_s16, neon_s16, 2)
#undef NEON_FN
/* The addition of the rounding constant may overflow, so we use an
* intermediate 64 bit accumulator. */
uint32_t HELPER(neon_rshl_s32)(uint32_t valop, uint32_t shiftop)
uint32_t HELPER(neon_rshl_s32)(uint32_t val, uint32_t shift)
{
int32_t dest;
int32_t val = (int32_t)valop;
int8_t shift = (int8_t)shiftop;
if ((shift >= 32) || (shift <= -32)) {
dest = 0;
} else if (shift < 0) {
int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
dest = big_dest >> -shift;
} else {
dest = val << shift;
}
return dest;
return do_sqrshl_bhs(val, (int8_t)shift, 32, true, NULL);
}
/* Handling addition overflow with 64 bit input values is more
* tricky than with 32 bit values. */
uint64_t HELPER(neon_rshl_s64)(uint64_t valop, uint64_t shiftop)
uint64_t HELPER(neon_rshl_s64)(uint64_t val, uint64_t shift)
{
int8_t shift = (int8_t)shiftop;
int64_t val = valop;
if ((shift >= 64) || (shift <= -64)) {
val = 0;
} else if (shift < 0) {
val >>= (-shift - 1);
if (val == INT64_MAX) {
/* In this case, it means that the rounding constant is 1,
* and the addition would overflow. Return the actual
* result directly. */
val = 0x4000000000000000LL;
} else {
val++;
val >>= 1;
}
} else {
val <<= shift;
}
return val;
return do_sqrshl_d(val, (int8_t)shift, true, NULL);
}
#define NEON_FN(dest, src1, src2) do { \
int8_t tmp; \
tmp = (int8_t)src2; \
if (tmp >= (ssize_t)sizeof(src1) * 8 || \
tmp < -(ssize_t)sizeof(src1) * 8) { \
dest = 0; \
} else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
dest = src1 >> (-tmp - 1); \
} else if (tmp < 0) { \
dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
} else { \
dest = src1 << tmp; \
}} while (0)
#define NEON_FN(dest, src1, src2) \
(dest = do_uqrshl_bhs(src1, (int8_t)src2, 8, true, NULL))
NEON_VOP(rshl_u8, neon_u8, 4)
#undef NEON_FN
#define NEON_FN(dest, src1, src2) \
(dest = do_uqrshl_bhs(src1, (int8_t)src2, 16, true, NULL))
NEON_VOP(rshl_u16, neon_u16, 2)
#undef NEON_FN
/* The addition of the rounding constant may overflow, so we use an
* intermediate 64 bit accumulator. */
uint32_t HELPER(neon_rshl_u32)(uint32_t val, uint32_t shiftop)
uint32_t HELPER(neon_rshl_u32)(uint32_t val, uint32_t shift)
{
uint32_t dest;
int8_t shift = (int8_t)shiftop;
if (shift >= 32 || shift < -32) {
dest = 0;
} else if (shift == -32) {
dest = val >> 31;
} else if (shift < 0) {
uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
dest = big_dest >> -shift;
} else {
dest = val << shift;
}
return dest;
return do_uqrshl_bhs(val, (int8_t)shift, 32, true, NULL);
}
/* Handling addition overflow with 64 bit input values is more
* tricky than with 32 bit values. */
uint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shiftop)
uint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shift)
{
int8_t shift = (uint8_t)shiftop;
if (shift >= 64 || shift < -64) {
val = 0;
} else if (shift == -64) {
/* Rounding a 1-bit result just preserves that bit. */
val >>= 63;
} else if (shift < 0) {
val >>= (-shift - 1);
if (val == UINT64_MAX) {
/* In this case, it means that the rounding constant is 1,
* and the addition would overflow. Return the actual
* result directly. */
val = 0x8000000000000000ULL;
} else {
val++;
val >>= 1;
}
} else {
val <<= shift;
}
return val;
return do_uqrshl_d(val, (int8_t)shift, true, NULL);
}
#define NEON_FN(dest, src1, src2) do { \
int8_t tmp; \
tmp = (int8_t)src2; \
if (tmp >= (ssize_t)sizeof(src1) * 8) { \
if (src1) { \
SET_QC(); \
dest = ~0; \
} else { \
dest = 0; \
} \
} else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
dest = 0; \
} else if (tmp < 0) { \
dest = src1 >> -tmp; \
} else { \
dest = src1 << tmp; \
if ((dest >> tmp) != src1) { \
SET_QC(); \
dest = ~0; \
} \
}} while (0)
#define NEON_FN(dest, src1, src2) \
(dest = do_uqrshl_bhs(src1, (int8_t)src2, 8, false, env->vfp.qc))
NEON_VOP_ENV(qshl_u8, neon_u8, 4)
#undef NEON_FN
#define NEON_FN(dest, src1, src2) \
(dest = do_uqrshl_bhs(src1, (int8_t)src2, 16, false, env->vfp.qc))
NEON_VOP_ENV(qshl_u16, neon_u16, 2)
NEON_VOP_ENV(qshl_u32, neon_u32, 1)
#undef NEON_FN
uint64_t HELPER(neon_qshl_u64)(CPUARMState *env, uint64_t val, uint64_t shiftop)
uint32_t HELPER(neon_qshl_u32)(CPUARMState *env, uint32_t val, uint32_t shift)
{
int8_t shift = (int8_t)shiftop;
if (shift >= 64) {
if (val) {
val = ~(uint64_t)0;
SET_QC();
}
} else if (shift <= -64) {
val = 0;
} else if (shift < 0) {
val >>= -shift;
} else {
uint64_t tmp = val;
val <<= shift;
if ((val >> shift) != tmp) {
SET_QC();
val = ~(uint64_t)0;
}
}
return val;
return do_uqrshl_bhs(val, (int8_t)shift, 32, false, env->vfp.qc);
}
#define NEON_FN(dest, src1, src2) do { \
int8_t tmp; \
tmp = (int8_t)src2; \
if (tmp >= (ssize_t)sizeof(src1) * 8) { \
if (src1) { \
SET_QC(); \
dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
if (src1 > 0) { \
dest--; \
} \
} else { \
dest = src1; \
} \
} else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
dest = src1 >> 31; \
} else if (tmp < 0) { \
dest = src1 >> -tmp; \
} else { \
dest = src1 << tmp; \
if ((dest >> tmp) != src1) { \
SET_QC(); \
dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
if (src1 > 0) { \
dest--; \
} \
} \
}} while (0)
uint64_t HELPER(neon_qshl_u64)(CPUARMState *env, uint64_t val, uint64_t shift)
{
return do_uqrshl_d(val, (int8_t)shift, false, env->vfp.qc);
}
#define NEON_FN(dest, src1, src2) \
(dest = do_sqrshl_bhs(src1, (int8_t)src2, 8, false, env->vfp.qc))
NEON_VOP_ENV(qshl_s8, neon_s8, 4)
#undef NEON_FN
#define NEON_FN(dest, src1, src2) \
(dest = do_sqrshl_bhs(src1, (int8_t)src2, 16, false, env->vfp.qc))
NEON_VOP_ENV(qshl_s16, neon_s16, 2)
NEON_VOP_ENV(qshl_s32, neon_s32, 1)
#undef NEON_FN
uint64_t HELPER(neon_qshl_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
uint32_t HELPER(neon_qshl_s32)(CPUARMState *env, uint32_t val, uint32_t shift)
{
int8_t shift = (uint8_t)shiftop;
int64_t val = valop;
if (shift >= 64) {
if (val) {
SET_QC();
val = (val >> 63) ^ ~SIGNBIT64;
}
} else if (shift <= -64) {
val >>= 63;
} else if (shift < 0) {
val >>= -shift;
} else {
int64_t tmp = val;
val <<= shift;
if ((val >> shift) != tmp) {
SET_QC();
val = (tmp >> 63) ^ ~SIGNBIT64;
}
}
return val;
return do_sqrshl_bhs(val, (int8_t)shift, 32, false, env->vfp.qc);
}
#define NEON_FN(dest, src1, src2) do { \
if (src1 & (1 << (sizeof(src1) * 8 - 1))) { \
SET_QC(); \
dest = 0; \
} else { \
int8_t tmp; \
tmp = (int8_t)src2; \
if (tmp >= (ssize_t)sizeof(src1) * 8) { \
if (src1) { \
SET_QC(); \
dest = ~0; \
} else { \
dest = 0; \
} \
} else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
dest = 0; \
} else if (tmp < 0) { \
dest = src1 >> -tmp; \
} else { \
dest = src1 << tmp; \
if ((dest >> tmp) != src1) { \
SET_QC(); \
dest = ~0; \
} \
} \
}} while (0)
NEON_VOP_ENV(qshlu_s8, neon_u8, 4)
NEON_VOP_ENV(qshlu_s16, neon_u16, 2)
uint64_t HELPER(neon_qshl_s64)(CPUARMState *env, uint64_t val, uint64_t shift)
{
return do_sqrshl_d(val, (int8_t)shift, false, env->vfp.qc);
}
#define NEON_FN(dest, src1, src2) \
(dest = do_suqrshl_bhs(src1, (int8_t)src2, 8, false, env->vfp.qc))
NEON_VOP_ENV(qshlu_s8, neon_s8, 4)
#undef NEON_FN
uint32_t HELPER(neon_qshlu_s32)(CPUARMState *env, uint32_t valop, uint32_t shiftop)
#define NEON_FN(dest, src1, src2) \
(dest = do_suqrshl_bhs(src1, (int8_t)src2, 16, false, env->vfp.qc))
NEON_VOP_ENV(qshlu_s16, neon_s16, 2)
#undef NEON_FN
uint32_t HELPER(neon_qshlu_s32)(CPUARMState *env, uint32_t val, uint32_t shift)
{
if ((int32_t)valop < 0) {
SET_QC();
return 0;
}
return helper_neon_qshl_u32(env, valop, shiftop);
return do_suqrshl_bhs(val, (int8_t)shift, 32, false, env->vfp.qc);
}
uint64_t HELPER(neon_qshlu_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
uint64_t HELPER(neon_qshlu_s64)(CPUARMState *env, uint64_t val, uint64_t shift)
{
if ((int64_t)valop < 0) {
SET_QC();
return 0;
}
return helper_neon_qshl_u64(env, valop, shiftop);
return do_suqrshl_d(val, (int8_t)shift, false, env->vfp.qc);
}
#define NEON_FN(dest, src1, src2) do { \
int8_t tmp; \
tmp = (int8_t)src2; \
if (tmp >= (ssize_t)sizeof(src1) * 8) { \
if (src1) { \
SET_QC(); \
dest = ~0; \
} else { \
dest = 0; \
} \
} else if (tmp < -(ssize_t)sizeof(src1) * 8) { \
dest = 0; \
} else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
dest = src1 >> (sizeof(src1) * 8 - 1); \
} else if (tmp < 0) { \
dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
} else { \
dest = src1 << tmp; \
if ((dest >> tmp) != src1) { \
SET_QC(); \
dest = ~0; \
} \
}} while (0)
#define NEON_FN(dest, src1, src2) \
(dest = do_uqrshl_bhs(src1, (int8_t)src2, 8, true, env->vfp.qc))
NEON_VOP_ENV(qrshl_u8, neon_u8, 4)
#undef NEON_FN
#define NEON_FN(dest, src1, src2) \
(dest = do_uqrshl_bhs(src1, (int8_t)src2, 16, true, env->vfp.qc))
NEON_VOP_ENV(qrshl_u16, neon_u16, 2)
#undef NEON_FN
/* The addition of the rounding constant may overflow, so we use an
* intermediate 64 bit accumulator. */
uint32_t HELPER(neon_qrshl_u32)(CPUARMState *env, uint32_t val, uint32_t shiftop)
uint32_t HELPER(neon_qrshl_u32)(CPUARMState *env, uint32_t val, uint32_t shift)
{
uint32_t dest;
int8_t shift = (int8_t)shiftop;
if (shift >= 32) {
if (val) {
SET_QC();
dest = ~0;
} else {
dest = 0;
}
} else if (shift < -32) {
dest = 0;
} else if (shift == -32) {
dest = val >> 31;
} else if (shift < 0) {
uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
dest = big_dest >> -shift;
} else {
dest = val << shift;
if ((dest >> shift) != val) {
SET_QC();
dest = ~0;
}
}
return dest;
return do_uqrshl_bhs(val, (int8_t)shift, 32, true, env->vfp.qc);
}
/* Handling addition overflow with 64 bit input values is more
* tricky than with 32 bit values. */
uint64_t HELPER(neon_qrshl_u64)(CPUARMState *env, uint64_t val, uint64_t shiftop)
uint64_t HELPER(neon_qrshl_u64)(CPUARMState *env, uint64_t val, uint64_t shift)
{
int8_t shift = (int8_t)shiftop;
if (shift >= 64) {
if (val) {
SET_QC();
val = ~0;
}
} else if (shift < -64) {
val = 0;
} else if (shift == -64) {
val >>= 63;
} else if (shift < 0) {
val >>= (-shift - 1);
if (val == UINT64_MAX) {
/* In this case, it means that the rounding constant is 1,
* and the addition would overflow. Return the actual
* result directly. */
val = 0x8000000000000000ULL;
} else {
val++;
val >>= 1;
}
} else { \
uint64_t tmp = val;
val <<= shift;
if ((val >> shift) != tmp) {
SET_QC();
val = ~0;
}
}
return val;
return do_uqrshl_d(val, (int8_t)shift, true, env->vfp.qc);
}
#define NEON_FN(dest, src1, src2) do { \
int8_t tmp; \
tmp = (int8_t)src2; \
if (tmp >= (ssize_t)sizeof(src1) * 8) { \
if (src1) { \
SET_QC(); \
dest = (typeof(dest))(1 << (sizeof(src1) * 8 - 1)); \
if (src1 > 0) { \
dest--; \
} \
} else { \
dest = 0; \
} \
} else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
dest = 0; \
} else if (tmp < 0) { \
dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
} else { \
dest = src1 << tmp; \
if ((dest >> tmp) != src1) { \
SET_QC(); \
dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
if (src1 > 0) { \
dest--; \
} \
} \
}} while (0)
#define NEON_FN(dest, src1, src2) \
(dest = do_sqrshl_bhs(src1, (int8_t)src2, 8, true, env->vfp.qc))
NEON_VOP_ENV(qrshl_s8, neon_s8, 4)
#undef NEON_FN
#define NEON_FN(dest, src1, src2) \
(dest = do_sqrshl_bhs(src1, (int8_t)src2, 16, true, env->vfp.qc))
NEON_VOP_ENV(qrshl_s16, neon_s16, 2)
#undef NEON_FN
/* The addition of the rounding constant may overflow, so we use an
* intermediate 64 bit accumulator. */
uint32_t HELPER(neon_qrshl_s32)(CPUARMState *env, uint32_t valop, uint32_t shiftop)
uint32_t HELPER(neon_qrshl_s32)(CPUARMState *env, uint32_t val, uint32_t shift)
{
int32_t dest;
int32_t val = (int32_t)valop;
int8_t shift = (int8_t)shiftop;
if (shift >= 32) {
if (val) {
SET_QC();
dest = (val >> 31) ^ ~SIGNBIT;
} else {
dest = 0;
}
} else if (shift <= -32) {
dest = 0;
} else if (shift < 0) {
int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
dest = big_dest >> -shift;
} else {
dest = val << shift;
if ((dest >> shift) != val) {
SET_QC();
dest = (val >> 31) ^ ~SIGNBIT;
}
}
return dest;
return do_sqrshl_bhs(val, (int8_t)shift, 32, true, env->vfp.qc);
}
/* Handling addition overflow with 64 bit input values is more
* tricky than with 32 bit values. */
uint64_t HELPER(neon_qrshl_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
uint64_t HELPER(neon_qrshl_s64)(CPUARMState *env, uint64_t val, uint64_t shift)
{
int8_t shift = (uint8_t)shiftop;
int64_t val = valop;
if (shift >= 64) {
if (val) {
SET_QC();
val = (val >> 63) ^ ~SIGNBIT64;
}
} else if (shift <= -64) {
val = 0;
} else if (shift < 0) {
val >>= (-shift - 1);
if (val == INT64_MAX) {
/* In this case, it means that the rounding constant is 1,
* and the addition would overflow. Return the actual
* result directly. */
val = 0x4000000000000000ULL;
} else {
val++;
val >>= 1;
}
} else {
int64_t tmp = val;
val <<= shift;
if ((val >> shift) != tmp) {
SET_QC();
val = (tmp >> 63) ^ ~SIGNBIT64;
}
}
return val;
return do_sqrshl_d(val, (int8_t)shift, true, env->vfp.qc);
}
uint32_t HELPER(neon_add_u8)(uint32_t a, uint32_t b)

574
target/arm/sve.decode
View File

@ -30,6 +30,8 @@
%size_23 23:2
%dtype_23_13 23:2 13:2
%index3_22_19 22:1 19:2
%index3_19_11 19:2 11:1
%index2_20_11 20:1 11:1
# A combination of tsz:imm3 -- extract esize.
%tszimm_esz 22:2 5:5 !function=tszimm_esz
@ -65,11 +67,15 @@
&rr_dbm rd rn dbm
&rrri rd rn rm imm
&rri_esz rd rn imm esz
&rrri_esz rd rn rm imm esz
&rrr_esz rd rn rm esz
&rrx_esz rd rn rm index esz
&rpr_esz rd pg rn esz
&rpr_s rd pg rn s
&rprr_s rd pg rn rm s
&rprr_esz rd pg rn rm esz
&rrrr_esz rd ra rn rm esz
&rrxr_esz rd rn rm ra index esz
&rprrr_esz rd pg rn rm ra esz
&rpri_esz rd pg rn imm esz
&ptrue rd esz pat s
@ -112,6 +118,8 @@
@pd_pn_pm ........ esz:2 .. rm:4 ....... rn:4 . rd:4 &rrr_esz
@rdn_rm ........ esz:2 ...... ...... rm:5 rd:5 \
&rrr_esz rn=%reg_movprfx
@rdn_rm_e0 ........ .. ...... ...... rm:5 rd:5 \
&rrr_esz rn=%reg_movprfx esz=0
@rdn_sh_i8u ........ esz:2 ...... ...... ..... rd:5 \
&rri_esz rn=%reg_movprfx imm=%sh8_i8u
@rdn_i8u ........ esz:2 ...... ... imm:8 rd:5 \
@ -119,6 +127,16 @@
@rdn_i8s ........ esz:2 ...... ... imm:s8 rd:5 \
&rri_esz rn=%reg_movprfx
# Four operand, vector element size
@rda_rn_rm ........ esz:2 . rm:5 ... ... rn:5 rd:5 \
&rrrr_esz ra=%reg_movprfx
# Four operand with unused vector element size
@rda_rn_rm_e0 ........ ... rm:5 ... ... rn:5 rd:5 \
&rrrr_esz esz=0 ra=%reg_movprfx
@rdn_ra_rm_e0 ........ ... rm:5 ... ... ra:5 rd:5 \
&rrrr_esz esz=0 rn=%reg_movprfx
# Three operand with "memory" size, aka immediate left shift
@rd_rn_msz_rm ........ ... rm:5 .... imm:2 rn:5 rd:5 &rrri
@ -137,6 +155,7 @@
&rprrr_esz rn=%reg_movprfx
@rdn_pg_rm_ra ........ esz:2 . ra:5 ... pg:3 rm:5 rd:5 \
&rprrr_esz rn=%reg_movprfx
@rd_pg_rn_rm ........ esz:2 . rm:5 ... pg:3 rn:5 rd:5 &rprr_esz
# One register operand, with governing predicate, vector element size
@rd_pg_rn ........ esz:2 ... ... ... pg:3 rn:5 rd:5 &rpr_esz
@ -234,6 +253,32 @@
@rpri_scatter_store ....... msz:2 .. imm:5 ... pg:3 rn:5 rd:5 \
&rpri_scatter_store
# Two registers and a scalar by N-bit index
@rrx_3 ........ .. . .. rm:3 ...... rn:5 rd:5 \
&rrx_esz index=%index3_22_19
@rrx_2 ........ .. . index:2 rm:3 ...... rn:5 rd:5 &rrx_esz
@rrx_1 ........ .. . index:1 rm:4 ...... rn:5 rd:5 &rrx_esz
# Two registers and a scalar by N-bit index, alternate
@rrx_3a ........ .. . .. rm:3 ...... rn:5 rd:5 \
&rrx_esz index=%index3_19_11
@rrx_2a ........ .. . . rm:4 ...... rn:5 rd:5 \
&rrx_esz index=%index2_20_11
# Three registers and a scalar by N-bit index
@rrxr_3 ........ .. . .. rm:3 ...... rn:5 rd:5 \
&rrxr_esz ra=%reg_movprfx index=%index3_22_19
@rrxr_2 ........ .. . index:2 rm:3 ...... rn:5 rd:5 \
&rrxr_esz ra=%reg_movprfx
@rrxr_1 ........ .. . index:1 rm:4 ...... rn:5 rd:5 \
&rrxr_esz ra=%reg_movprfx
# Three registers and a scalar by N-bit index, alternate
@rrxr_3a ........ .. ... rm:3 ...... rn:5 rd:5 \
&rrxr_esz ra=%reg_movprfx index=%index3_19_11
@rrxr_2a ........ .. .. rm:4 ...... rn:5 rd:5 \
&rrxr_esz ra=%reg_movprfx index=%index2_20_11
###########################################################################
# Instruction patterns. Grouped according to the SVE encodingindex.xhtml.
@ -297,6 +342,11 @@ ASR_zpzi 00000100 .. 000 000 100 ... .. ... ..... @rdn_pg_tszimm_shr
LSR_zpzi 00000100 .. 000 001 100 ... .. ... ..... @rdn_pg_tszimm_shr
LSL_zpzi 00000100 .. 000 011 100 ... .. ... ..... @rdn_pg_tszimm_shl
ASRD 00000100 .. 000 100 100 ... .. ... ..... @rdn_pg_tszimm_shr
SQSHL_zpzi 00000100 .. 000 110 100 ... .. ... ..... @rdn_pg_tszimm_shl
UQSHL_zpzi 00000100 .. 000 111 100 ... .. ... ..... @rdn_pg_tszimm_shl
SRSHR 00000100 .. 001 100 100 ... .. ... ..... @rdn_pg_tszimm_shr
URSHR 00000100 .. 001 101 100 ... .. ... ..... @rdn_pg_tszimm_shr
SQSHLU 00000100 .. 001 111 100 ... .. ... ..... @rdn_pg_tszimm_shl
# SVE bitwise shift by vector (predicated)
ASR_zpzz 00000100 .. 010 000 100 ... ..... ..... @rdn_pg_rm
@ -374,6 +424,17 @@ ORR_zzz 00000100 01 1 ..... 001 100 ..... ..... @rd_rn_rm_e0
EOR_zzz 00000100 10 1 ..... 001 100 ..... ..... @rd_rn_rm_e0
BIC_zzz 00000100 11 1 ..... 001 100 ..... ..... @rd_rn_rm_e0
XAR 00000100 .. 1 ..... 001 101 rm:5 rd:5 &rrri_esz \
rn=%reg_movprfx esz=%tszimm16_esz imm=%tszimm16_shr
# SVE2 bitwise ternary operations
EOR3 00000100 00 1 ..... 001 110 ..... ..... @rdn_ra_rm_e0
BSL 00000100 00 1 ..... 001 111 ..... ..... @rdn_ra_rm_e0
BCAX 00000100 01 1 ..... 001 110 ..... ..... @rdn_ra_rm_e0
BSL1N 00000100 01 1 ..... 001 111 ..... ..... @rdn_ra_rm_e0
BSL2N 00000100 10 1 ..... 001 111 ..... ..... @rdn_ra_rm_e0
NBSL 00000100 11 1 ..... 001 111 ..... ..... @rdn_ra_rm_e0
### SVE Index Generation Group
# SVE index generation (immediate start, immediate increment)
@ -472,10 +533,14 @@ CPY_z_i 00000101 .. 01 .... 00 . ........ ..... @rdn_pg4 imm=%sh8_i8s
### SVE Permute - Extract Group
# SVE extract vector (immediate offset)
# SVE extract vector (destructive)
EXT 00000101 001 ..... 000 ... rm:5 rd:5 \
&rrri rn=%reg_movprfx imm=%imm8_16_10
# SVE2 extract vector (constructive)
EXT_sve2 00000101 011 ..... 000 ... rn:5 rd:5 \
&rri imm=%imm8_16_10
### SVE Permute - Unpredicated Group
# SVE broadcast general register
@ -500,6 +565,11 @@ TBL 00000101 .. 1 ..... 001100 ..... ..... @rd_rn_rm
# SVE unpack vector elements
UNPK 00000101 esz:2 1100 u:1 h:1 001110 rn:5 rd:5
# SVE2 Table Lookup (three sources)
TBL_sve2 00000101 .. 1 ..... 001010 ..... ..... @rd_rn_rm
TBX 00000101 .. 1 ..... 001011 ..... ..... @rd_rn_rm
### SVE Permute - Predicates Group
# SVE permute predicate elements
@ -527,6 +597,14 @@ UZP2_z 00000101 .. 1 ..... 011 011 ..... ..... @rd_rn_rm
TRN1_z 00000101 .. 1 ..... 011 100 ..... ..... @rd_rn_rm
TRN2_z 00000101 .. 1 ..... 011 101 ..... ..... @rd_rn_rm
# SVE2 permute vector segments
ZIP1_q 00000101 10 1 ..... 000 000 ..... ..... @rd_rn_rm_e0
ZIP2_q 00000101 10 1 ..... 000 001 ..... ..... @rd_rn_rm_e0
UZP1_q 00000101 10 1 ..... 000 010 ..... ..... @rd_rn_rm_e0
UZP2_q 00000101 10 1 ..... 000 011 ..... ..... @rd_rn_rm_e0
TRN1_q 00000101 10 1 ..... 000 110 ..... ..... @rd_rn_rm_e0
TRN2_q 00000101 10 1 ..... 000 111 ..... ..... @rd_rn_rm_e0
### SVE Permute - Predicated Group
# SVE compress active elements
@ -566,9 +644,12 @@ REVH 00000101 .. 1001 01 100 ... ..... ..... @rd_pg_rn
REVW 00000101 .. 1001 10 100 ... ..... ..... @rd_pg_rn
RBIT 00000101 .. 1001 11 100 ... ..... ..... @rd_pg_rn
# SVE vector splice (predicated)
# SVE vector splice (predicated, destructive)
SPLICE 00000101 .. 101 100 100 ... ..... ..... @rdn_pg_rm
# SVE2 vector splice (predicated, constructive)
SPLICE_sve2 00000101 .. 101 101 100 ... ..... ..... @rd_pg_rn
### SVE Select Vectors Group
# SVE select vector elements (predicated)
@ -695,7 +776,10 @@ SINCDECP_z 00100101 .. 1010 d:1 u:1 10000 00 .... ..... @incdec2_pred
CTERM 00100101 1 sf:1 1 rm:5 001000 rn:5 ne:1 0000
# SVE integer compare scalar count and limit
WHILE 00100101 esz:2 1 rm:5 000 sf:1 u:1 1 rn:5 eq:1 rd:4
WHILE 00100101 esz:2 1 rm:5 000 sf:1 u:1 lt:1 rn:5 eq:1 rd:4
# SVE2 pointer conflict compare
WHILE_ptr 00100101 esz:2 1 rm:5 001 100 rn:5 rw:1 rd:4
### SVE Integer Wide Immediate - Unpredicated Group
@ -724,13 +808,114 @@ UMIN_zzi 00100101 .. 101 011 110 ........ ..... @rdn_i8u
MUL_zzi 00100101 .. 110 000 110 ........ ..... @rdn_i8s
# SVE integer dot product (unpredicated)
DOT_zzz 01000100 1 sz:1 0 rm:5 00000 u:1 rn:5 rd:5 ra=%reg_movprfx
DOT_zzzz 01000100 1 sz:1 0 rm:5 00000 u:1 rn:5 rd:5 \
ra=%reg_movprfx
# SVE2 complex dot product (vectors)
CDOT_zzzz 01000100 esz:2 0 rm:5 0001 rot:2 rn:5 rd:5 ra=%reg_movprfx
#### SVE Multiply - Indexed
# SVE integer dot product (indexed)
DOT_zzx 01000100 101 index:2 rm:3 00000 u:1 rn:5 rd:5 \
sz=0 ra=%reg_movprfx
DOT_zzx 01000100 111 index:1 rm:4 00000 u:1 rn:5 rd:5 \
sz=1 ra=%reg_movprfx
SDOT_zzxw_s 01000100 10 1 ..... 000000 ..... ..... @rrxr_2 esz=2
SDOT_zzxw_d 01000100 11 1 ..... 000000 ..... ..... @rrxr_1 esz=3
UDOT_zzxw_s 01000100 10 1 ..... 000001 ..... ..... @rrxr_2 esz=2
UDOT_zzxw_d 01000100 11 1 ..... 000001 ..... ..... @rrxr_1 esz=3
# SVE2 integer multiply-add (indexed)
MLA_zzxz_h 01000100 0. 1 ..... 000010 ..... ..... @rrxr_3 esz=1
MLA_zzxz_s 01000100 10 1 ..... 000010 ..... ..... @rrxr_2 esz=2
MLA_zzxz_d 01000100 11 1 ..... 000010 ..... ..... @rrxr_1 esz=3
MLS_zzxz_h 01000100 0. 1 ..... 000011 ..... ..... @rrxr_3 esz=1
MLS_zzxz_s 01000100 10 1 ..... 000011 ..... ..... @rrxr_2 esz=2
MLS_zzxz_d 01000100 11 1 ..... 000011 ..... ..... @rrxr_1 esz=3
# SVE2 saturating multiply-add high (indexed)
SQRDMLAH_zzxz_h 01000100 0. 1 ..... 000100 ..... ..... @rrxr_3 esz=1
SQRDMLAH_zzxz_s 01000100 10 1 ..... 000100 ..... ..... @rrxr_2 esz=2
SQRDMLAH_zzxz_d 01000100 11 1 ..... 000100 ..... ..... @rrxr_1 esz=3
SQRDMLSH_zzxz_h 01000100 0. 1 ..... 000101 ..... ..... @rrxr_3 esz=1
SQRDMLSH_zzxz_s 01000100 10 1 ..... 000101 ..... ..... @rrxr_2 esz=2
SQRDMLSH_zzxz_d 01000100 11 1 ..... 000101 ..... ..... @rrxr_1 esz=3
# SVE mixed sign dot product (indexed)
USDOT_zzxw_s 01000100 10 1 ..... 000110 ..... ..... @rrxr_2 esz=2
SUDOT_zzxw_s 01000100 10 1 ..... 000111 ..... ..... @rrxr_2 esz=2
# SVE2 saturating multiply-add (indexed)
SQDMLALB_zzxw_s 01000100 10 1 ..... 0010.0 ..... ..... @rrxr_3a esz=2
SQDMLALB_zzxw_d 01000100 11 1 ..... 0010.0 ..... ..... @rrxr_2a esz=3
SQDMLALT_zzxw_s 01000100 10 1 ..... 0010.1 ..... ..... @rrxr_3a esz=2
SQDMLALT_zzxw_d 01000100 11 1 ..... 0010.1 ..... ..... @rrxr_2a esz=3
SQDMLSLB_zzxw_s 01000100 10 1 ..... 0011.0 ..... ..... @rrxr_3a esz=2
SQDMLSLB_zzxw_d 01000100 11 1 ..... 0011.0 ..... ..... @rrxr_2a esz=3
SQDMLSLT_zzxw_s 01000100 10 1 ..... 0011.1 ..... ..... @rrxr_3a esz=2
SQDMLSLT_zzxw_d 01000100 11 1 ..... 0011.1 ..... ..... @rrxr_2a esz=3
# SVE2 complex integer dot product (indexed)
CDOT_zzxw_s 01000100 10 1 index:2 rm:3 0100 rot:2 rn:5 rd:5 \
ra=%reg_movprfx
CDOT_zzxw_d 01000100 11 1 index:1 rm:4 0100 rot:2 rn:5 rd:5 \
ra=%reg_movprfx
# SVE2 complex integer multiply-add (indexed)
CMLA_zzxz_h 01000100 10 1 index:2 rm:3 0110 rot:2 rn:5 rd:5 \
ra=%reg_movprfx
CMLA_zzxz_s 01000100 11 1 index:1 rm:4 0110 rot:2 rn:5 rd:5 \
ra=%reg_movprfx
# SVE2 complex saturating integer multiply-add (indexed)
SQRDCMLAH_zzxz_h 01000100 10 1 index:2 rm:3 0111 rot:2 rn:5 rd:5 \
ra=%reg_movprfx
SQRDCMLAH_zzxz_s 01000100 11 1 index:1 rm:4 0111 rot:2 rn:5 rd:5 \
ra=%reg_movprfx
# SVE2 multiply-add long (indexed)
SMLALB_zzxw_s 01000100 10 1 ..... 1000.0 ..... ..... @rrxr_3a esz=2
SMLALB_zzxw_d 01000100 11 1 ..... 1000.0 ..... ..... @rrxr_2a esz=3
SMLALT_zzxw_s 01000100 10 1 ..... 1000.1 ..... ..... @rrxr_3a esz=2
SMLALT_zzxw_d 01000100 11 1 ..... 1000.1 ..... ..... @rrxr_2a esz=3
UMLALB_zzxw_s 01000100 10 1 ..... 1001.0 ..... ..... @rrxr_3a esz=2
UMLALB_zzxw_d 01000100 11 1 ..... 1001.0 ..... ..... @rrxr_2a esz=3
UMLALT_zzxw_s 01000100 10 1 ..... 1001.1 ..... ..... @rrxr_3a esz=2
UMLALT_zzxw_d 01000100 11 1 ..... 1001.1 ..... ..... @rrxr_2a esz=3
SMLSLB_zzxw_s 01000100 10 1 ..... 1010.0 ..... ..... @rrxr_3a esz=2
SMLSLB_zzxw_d 01000100 11 1 ..... 1010.0 ..... ..... @rrxr_2a esz=3
SMLSLT_zzxw_s 01000100 10 1 ..... 1010.1 ..... ..... @rrxr_3a esz=2
SMLSLT_zzxw_d 01000100 11 1 ..... 1010.1 ..... ..... @rrxr_2a esz=3
UMLSLB_zzxw_s 01000100 10 1 ..... 1011.0 ..... ..... @rrxr_3a esz=2
UMLSLB_zzxw_d 01000100 11 1 ..... 1011.0 ..... ..... @rrxr_2a esz=3
UMLSLT_zzxw_s 01000100 10 1 ..... 1011.1 ..... ..... @rrxr_3a esz=2
UMLSLT_zzxw_d 01000100 11 1 ..... 1011.1 ..... ..... @rrxr_2a esz=3
# SVE2 integer multiply long (indexed)
SMULLB_zzx_s 01000100 10 1 ..... 1100.0 ..... ..... @rrx_3a esz=2
SMULLB_zzx_d 01000100 11 1 ..... 1100.0 ..... ..... @rrx_2a esz=3
SMULLT_zzx_s 01000100 10 1 ..... 1100.1 ..... ..... @rrx_3a esz=2
SMULLT_zzx_d 01000100 11 1 ..... 1100.1 ..... ..... @rrx_2a esz=3
UMULLB_zzx_s 01000100 10 1 ..... 1101.0 ..... ..... @rrx_3a esz=2
UMULLB_zzx_d 01000100 11 1 ..... 1101.0 ..... ..... @rrx_2a esz=3
UMULLT_zzx_s 01000100 10 1 ..... 1101.1 ..... ..... @rrx_3a esz=2
UMULLT_zzx_d 01000100 11 1 ..... 1101.1 ..... ..... @rrx_2a esz=3
# SVE2 saturating multiply (indexed)
SQDMULLB_zzx_s 01000100 10 1 ..... 1110.0 ..... ..... @rrx_3a esz=2
SQDMULLB_zzx_d 01000100 11 1 ..... 1110.0 ..... ..... @rrx_2a esz=3
SQDMULLT_zzx_s 01000100 10 1 ..... 1110.1 ..... ..... @rrx_3a esz=2
SQDMULLT_zzx_d 01000100 11 1 ..... 1110.1 ..... ..... @rrx_2a esz=3
# SVE2 saturating multiply high (indexed)
SQDMULH_zzx_h 01000100 0. 1 ..... 111100 ..... ..... @rrx_3 esz=1
SQDMULH_zzx_s 01000100 10 1 ..... 111100 ..... ..... @rrx_2 esz=2
SQDMULH_zzx_d 01000100 11 1 ..... 111100 ..... ..... @rrx_1 esz=3
SQRDMULH_zzx_h 01000100 0. 1 ..... 111101 ..... ..... @rrx_3 esz=1
SQRDMULH_zzx_s 01000100 10 1 ..... 111101 ..... ..... @rrx_2 esz=2
SQRDMULH_zzx_d 01000100 11 1 ..... 111101 ..... ..... @rrx_1 esz=3
# SVE2 integer multiply (indexed)
MUL_zzx_h 01000100 0. 1 ..... 111110 ..... ..... @rrx_3 esz=1
MUL_zzx_s 01000100 10 1 ..... 111110 ..... ..... @rrx_2 esz=2
MUL_zzx_d 01000100 11 1 ..... 111110 ..... ..... @rrx_1 esz=3
# SVE floating-point complex add (predicated)
FCADD 01100100 esz:2 00000 rot:1 100 pg:3 rm:5 rd:5 \
@ -749,20 +934,19 @@ FCMLA_zzxz 01100100 11 1 index:1 rm:4 0001 rot:2 rn:5 rd:5 \
### SVE FP Multiply-Add Indexed Group
# SVE floating-point multiply-add (indexed)
FMLA_zzxz 01100100 0.1 .. rm:3 00000 sub:1 rn:5 rd:5 \
ra=%reg_movprfx index=%index3_22_19 esz=1
FMLA_zzxz 01100100 101 index:2 rm:3 00000 sub:1 rn:5 rd:5 \
ra=%reg_movprfx esz=2
FMLA_zzxz 01100100 111 index:1 rm:4 00000 sub:1 rn:5 rd:5 \
ra=%reg_movprfx esz=3
FMLA_zzxz 01100100 0. 1 ..... 000000 ..... ..... @rrxr_3 esz=1
FMLA_zzxz 01100100 10 1 ..... 000000 ..... ..... @rrxr_2 esz=2
FMLA_zzxz 01100100 11 1 ..... 000000 ..... ..... @rrxr_1 esz=3
FMLS_zzxz 01100100 0. 1 ..... 000001 ..... ..... @rrxr_3 esz=1
FMLS_zzxz 01100100 10 1 ..... 000001 ..... ..... @rrxr_2 esz=2
FMLS_zzxz 01100100 11 1 ..... 000001 ..... ..... @rrxr_1 esz=3
### SVE FP Multiply Indexed Group
# SVE floating-point multiply (indexed)
FMUL_zzx 01100100 0.1 .. rm:3 001000 rn:5 rd:5 \
index=%index3_22_19 esz=1
FMUL_zzx 01100100 101 index:2 rm:3 001000 rn:5 rd:5 esz=2
FMUL_zzx 01100100 111 index:1 rm:4 001000 rn:5 rd:5 esz=3
FMUL_zzx 01100100 0. 1 ..... 001000 ..... ..... @rrx_3 esz=1
FMUL_zzx 01100100 10 1 ..... 001000 ..... ..... @rrx_2 esz=2
FMUL_zzx 01100100 11 1 ..... 001000 ..... ..... @rrx_1 esz=3
### SVE FP Fast Reduction Group
@ -957,11 +1141,15 @@ LD_zpri 1010010 .. nreg:2 0.... 111 ... ..... ..... @rpri_load_msz
# SVE load and broadcast quadword (scalar plus scalar)
LD1RQ_zprr 1010010 .. 00 ..... 000 ... ..... ..... \
@rprr_load_msz nreg=0
LD1RO_zprr 1010010 .. 01 ..... 000 ... ..... ..... \
@rprr_load_msz nreg=0
# SVE load and broadcast quadword (scalar plus immediate)
# LD1RQB, LD1RQH, LD1RQS, LD1RQD
LD1RQ_zpri 1010010 .. 00 0.... 001 ... ..... ..... \
@rpri_load_msz nreg=0
LD1RO_zpri 1010010 .. 01 0.... 001 ... ..... ..... \
@rpri_load_msz nreg=0
# SVE 32-bit gather prefetch (scalar plus 32-bit scaled offsets)
PRF 1000010 00 -1 ----- 0-- --- ----- 0 ----
@ -1090,3 +1278,353 @@ ST1_zprz 1110010 .. 00 ..... 100 ... ..... ..... \
@rprr_scatter_store xs=0 esz=3 scale=0
ST1_zprz 1110010 .. 00 ..... 110 ... ..... ..... \
@rprr_scatter_store xs=1 esz=3 scale=0
#### SVE2 Support
### SVE2 Integer Multiply - Unpredicated
# SVE2 integer multiply vectors (unpredicated)
MUL_zzz 00000100 .. 1 ..... 0110 00 ..... ..... @rd_rn_rm
SMULH_zzz 00000100 .. 1 ..... 0110 10 ..... ..... @rd_rn_rm
UMULH_zzz 00000100 .. 1 ..... 0110 11 ..... ..... @rd_rn_rm
PMUL_zzz 00000100 00 1 ..... 0110 01 ..... ..... @rd_rn_rm_e0
# SVE2 signed saturating doubling multiply high (unpredicated)
SQDMULH_zzz 00000100 .. 1 ..... 0111 00 ..... ..... @rd_rn_rm
SQRDMULH_zzz 00000100 .. 1 ..... 0111 01 ..... ..... @rd_rn_rm
### SVE2 Integer - Predicated
SADALP_zpzz 01000100 .. 000 100 101 ... ..... ..... @rdm_pg_rn
UADALP_zpzz 01000100 .. 000 101 101 ... ..... ..... @rdm_pg_rn
### SVE2 integer unary operations (predicated)
URECPE 01000100 .. 000 000 101 ... ..... ..... @rd_pg_rn
URSQRTE 01000100 .. 000 001 101 ... ..... ..... @rd_pg_rn
SQABS 01000100 .. 001 000 101 ... ..... ..... @rd_pg_rn
SQNEG 01000100 .. 001 001 101 ... ..... ..... @rd_pg_rn
### SVE2 saturating/rounding bitwise shift left (predicated)
SRSHL 01000100 .. 000 010 100 ... ..... ..... @rdn_pg_rm
URSHL 01000100 .. 000 011 100 ... ..... ..... @rdn_pg_rm
SRSHL 01000100 .. 000 110 100 ... ..... ..... @rdm_pg_rn # SRSHLR
URSHL 01000100 .. 000 111 100 ... ..... ..... @rdm_pg_rn # URSHLR
SQSHL 01000100 .. 001 000 100 ... ..... ..... @rdn_pg_rm
UQSHL 01000100 .. 001 001 100 ... ..... ..... @rdn_pg_rm
SQSHL 01000100 .. 001 100 100 ... ..... ..... @rdm_pg_rn # SQSHLR
UQSHL 01000100 .. 001 101 100 ... ..... ..... @rdm_pg_rn # UQSHLR
SQRSHL 01000100 .. 001 010 100 ... ..... ..... @rdn_pg_rm
UQRSHL 01000100 .. 001 011 100 ... ..... ..... @rdn_pg_rm
SQRSHL 01000100 .. 001 110 100 ... ..... ..... @rdm_pg_rn # SQRSHLR
UQRSHL 01000100 .. 001 111 100 ... ..... ..... @rdm_pg_rn # UQRSHLR
### SVE2 integer halving add/subtract (predicated)
SHADD 01000100 .. 010 000 100 ... ..... ..... @rdn_pg_rm
UHADD 01000100 .. 010 001 100 ... ..... ..... @rdn_pg_rm
SHSUB 01000100 .. 010 010 100 ... ..... ..... @rdn_pg_rm
UHSUB 01000100 .. 010 011 100 ... ..... ..... @rdn_pg_rm
SRHADD 01000100 .. 010 100 100 ... ..... ..... @rdn_pg_rm
URHADD 01000100 .. 010 101 100 ... ..... ..... @rdn_pg_rm
SHSUB 01000100 .. 010 110 100 ... ..... ..... @rdm_pg_rn # SHSUBR
UHSUB 01000100 .. 010 111 100 ... ..... ..... @rdm_pg_rn # UHSUBR
### SVE2 integer pairwise arithmetic
ADDP 01000100 .. 010 001 101 ... ..... ..... @rdn_pg_rm
SMAXP 01000100 .. 010 100 101 ... ..... ..... @rdn_pg_rm
UMAXP 01000100 .. 010 101 101 ... ..... ..... @rdn_pg_rm
SMINP 01000100 .. 010 110 101 ... ..... ..... @rdn_pg_rm
UMINP 01000100 .. 010 111 101 ... ..... ..... @rdn_pg_rm
### SVE2 saturating add/subtract (predicated)
SQADD_zpzz 01000100 .. 011 000 100 ... ..... ..... @rdn_pg_rm
UQADD_zpzz 01000100 .. 011 001 100 ... ..... ..... @rdn_pg_rm
SQSUB_zpzz 01000100 .. 011 010 100 ... ..... ..... @rdn_pg_rm
UQSUB_zpzz 01000100 .. 011 011 100 ... ..... ..... @rdn_pg_rm
SUQADD 01000100 .. 011 100 100 ... ..... ..... @rdn_pg_rm
USQADD 01000100 .. 011 101 100 ... ..... ..... @rdn_pg_rm
SQSUB_zpzz 01000100 .. 011 110 100 ... ..... ..... @rdm_pg_rn # SQSUBR
UQSUB_zpzz 01000100 .. 011 111 100 ... ..... ..... @rdm_pg_rn # UQSUBR
#### SVE2 Widening Integer Arithmetic
## SVE2 integer add/subtract long
SADDLB 01000101 .. 0 ..... 00 0000 ..... ..... @rd_rn_rm
SADDLT 01000101 .. 0 ..... 00 0001 ..... ..... @rd_rn_rm
UADDLB 01000101 .. 0 ..... 00 0010 ..... ..... @rd_rn_rm
UADDLT 01000101 .. 0 ..... 00 0011 ..... ..... @rd_rn_rm
SSUBLB 01000101 .. 0 ..... 00 0100 ..... ..... @rd_rn_rm
SSUBLT 01000101 .. 0 ..... 00 0101 ..... ..... @rd_rn_rm
USUBLB 01000101 .. 0 ..... 00 0110 ..... ..... @rd_rn_rm
USUBLT 01000101 .. 0 ..... 00 0111 ..... ..... @rd_rn_rm
SABDLB 01000101 .. 0 ..... 00 1100 ..... ..... @rd_rn_rm
SABDLT 01000101 .. 0 ..... 00 1101 ..... ..... @rd_rn_rm
UABDLB 01000101 .. 0 ..... 00 1110 ..... ..... @rd_rn_rm
UABDLT 01000101 .. 0 ..... 00 1111 ..... ..... @rd_rn_rm
## SVE2 integer add/subtract interleaved long
SADDLBT 01000101 .. 0 ..... 1000 00 ..... ..... @rd_rn_rm
SSUBLBT 01000101 .. 0 ..... 1000 10 ..... ..... @rd_rn_rm
SSUBLTB 01000101 .. 0 ..... 1000 11 ..... ..... @rd_rn_rm
## SVE2 integer add/subtract wide
SADDWB 01000101 .. 0 ..... 010 000 ..... ..... @rd_rn_rm
SADDWT 01000101 .. 0 ..... 010 001 ..... ..... @rd_rn_rm
UADDWB 01000101 .. 0 ..... 010 010 ..... ..... @rd_rn_rm
UADDWT 01000101 .. 0 ..... 010 011 ..... ..... @rd_rn_rm
SSUBWB 01000101 .. 0 ..... 010 100 ..... ..... @rd_rn_rm
SSUBWT 01000101 .. 0 ..... 010 101 ..... ..... @rd_rn_rm
USUBWB 01000101 .. 0 ..... 010 110 ..... ..... @rd_rn_rm
USUBWT 01000101 .. 0 ..... 010 111 ..... ..... @rd_rn_rm
## SVE2 integer multiply long
SQDMULLB_zzz 01000101 .. 0 ..... 011 000 ..... ..... @rd_rn_rm
SQDMULLT_zzz 01000101 .. 0 ..... 011 001 ..... ..... @rd_rn_rm
PMULLB 01000101 .. 0 ..... 011 010 ..... ..... @rd_rn_rm
PMULLT 01000101 .. 0 ..... 011 011 ..... ..... @rd_rn_rm
SMULLB_zzz 01000101 .. 0 ..... 011 100 ..... ..... @rd_rn_rm
SMULLT_zzz 01000101 .. 0 ..... 011 101 ..... ..... @rd_rn_rm
UMULLB_zzz 01000101 .. 0 ..... 011 110 ..... ..... @rd_rn_rm
UMULLT_zzz 01000101 .. 0 ..... 011 111 ..... ..... @rd_rn_rm
## SVE2 bitwise shift left long
# Note bit23 == 0 is handled by esz > 0 in do_sve2_shll_tb.
SSHLLB 01000101 .. 0 ..... 1010 00 ..... ..... @rd_rn_tszimm_shl
SSHLLT 01000101 .. 0 ..... 1010 01 ..... ..... @rd_rn_tszimm_shl
USHLLB 01000101 .. 0 ..... 1010 10 ..... ..... @rd_rn_tszimm_shl
USHLLT 01000101 .. 0 ..... 1010 11 ..... ..... @rd_rn_tszimm_shl
## SVE2 bitwise exclusive-or interleaved
EORBT 01000101 .. 0 ..... 10010 0 ..... ..... @rd_rn_rm
EORTB 01000101 .. 0 ..... 10010 1 ..... ..... @rd_rn_rm
## SVE integer matrix multiply accumulate
SMMLA 01000101 00 0 ..... 10011 0 ..... ..... @rda_rn_rm_e0
USMMLA 01000101 10 0 ..... 10011 0 ..... ..... @rda_rn_rm_e0
UMMLA 01000101 11 0 ..... 10011 0 ..... ..... @rda_rn_rm_e0
## SVE2 bitwise permute
BEXT 01000101 .. 0 ..... 1011 00 ..... ..... @rd_rn_rm
BDEP 01000101 .. 0 ..... 1011 01 ..... ..... @rd_rn_rm
BGRP 01000101 .. 0 ..... 1011 10 ..... ..... @rd_rn_rm
#### SVE2 Accumulate
## SVE2 complex integer add
CADD_rot90 01000101 .. 00000 0 11011 0 ..... ..... @rdn_rm
CADD_rot270 01000101 .. 00000 0 11011 1 ..... ..... @rdn_rm
SQCADD_rot90 01000101 .. 00000 1 11011 0 ..... ..... @rdn_rm
SQCADD_rot270 01000101 .. 00000 1 11011 1 ..... ..... @rdn_rm
## SVE2 integer absolute difference and accumulate long
SABALB 01000101 .. 0 ..... 1100 00 ..... ..... @rda_rn_rm
SABALT 01000101 .. 0 ..... 1100 01 ..... ..... @rda_rn_rm
UABALB 01000101 .. 0 ..... 1100 10 ..... ..... @rda_rn_rm
UABALT 01000101 .. 0 ..... 1100 11 ..... ..... @rda_rn_rm
## SVE2 integer add/subtract long with carry
# ADC and SBC decoded via size in helper dispatch.
ADCLB 01000101 .. 0 ..... 11010 0 ..... ..... @rda_rn_rm
ADCLT 01000101 .. 0 ..... 11010 1 ..... ..... @rda_rn_rm
## SVE2 bitwise shift right and accumulate
# TODO: Use @rda and %reg_movprfx here.
SSRA 01000101 .. 0 ..... 1110 00 ..... ..... @rd_rn_tszimm_shr
USRA 01000101 .. 0 ..... 1110 01 ..... ..... @rd_rn_tszimm_shr
SRSRA 01000101 .. 0 ..... 1110 10 ..... ..... @rd_rn_tszimm_shr
URSRA 01000101 .. 0 ..... 1110 11 ..... ..... @rd_rn_tszimm_shr
## SVE2 bitwise shift and insert
SRI 01000101 .. 0 ..... 11110 0 ..... ..... @rd_rn_tszimm_shr
SLI 01000101 .. 0 ..... 11110 1 ..... ..... @rd_rn_tszimm_shl
## SVE2 integer absolute difference and accumulate
# TODO: Use @rda and %reg_movprfx here.
SABA 01000101 .. 0 ..... 11111 0 ..... ..... @rd_rn_rm
UABA 01000101 .. 0 ..... 11111 1 ..... ..... @rd_rn_rm
#### SVE2 Narrowing
## SVE2 saturating extract narrow
# Bits 23, 18-16 are zero, limited in the translator via esz < 3 & imm == 0.
SQXTNB 01000101 .. 1 ..... 010 000 ..... ..... @rd_rn_tszimm_shl
SQXTNT 01000101 .. 1 ..... 010 001 ..... ..... @rd_rn_tszimm_shl
UQXTNB 01000101 .. 1 ..... 010 010 ..... ..... @rd_rn_tszimm_shl
UQXTNT 01000101 .. 1 ..... 010 011 ..... ..... @rd_rn_tszimm_shl
SQXTUNB 01000101 .. 1 ..... 010 100 ..... ..... @rd_rn_tszimm_shl
SQXTUNT 01000101 .. 1 ..... 010 101 ..... ..... @rd_rn_tszimm_shl
## SVE2 bitwise shift right narrow
# Bit 23 == 0 is handled by esz > 0 in the translator.
SQSHRUNB 01000101 .. 1 ..... 00 0000 ..... ..... @rd_rn_tszimm_shr
SQSHRUNT 01000101 .. 1 ..... 00 0001 ..... ..... @rd_rn_tszimm_shr
SQRSHRUNB 01000101 .. 1 ..... 00 0010 ..... ..... @rd_rn_tszimm_shr
SQRSHRUNT 01000101 .. 1 ..... 00 0011 ..... ..... @rd_rn_tszimm_shr
SHRNB 01000101 .. 1 ..... 00 0100 ..... ..... @rd_rn_tszimm_shr
SHRNT 01000101 .. 1 ..... 00 0101 ..... ..... @rd_rn_tszimm_shr
RSHRNB 01000101 .. 1 ..... 00 0110 ..... ..... @rd_rn_tszimm_shr
RSHRNT 01000101 .. 1 ..... 00 0111 ..... ..... @rd_rn_tszimm_shr
SQSHRNB 01000101 .. 1 ..... 00 1000 ..... ..... @rd_rn_tszimm_shr
SQSHRNT 01000101 .. 1 ..... 00 1001 ..... ..... @rd_rn_tszimm_shr
SQRSHRNB 01000101 .. 1 ..... 00 1010 ..... ..... @rd_rn_tszimm_shr
SQRSHRNT 01000101 .. 1 ..... 00 1011 ..... ..... @rd_rn_tszimm_shr
UQSHRNB 01000101 .. 1 ..... 00 1100 ..... ..... @rd_rn_tszimm_shr
UQSHRNT 01000101 .. 1 ..... 00 1101 ..... ..... @rd_rn_tszimm_shr
UQRSHRNB 01000101 .. 1 ..... 00 1110 ..... ..... @rd_rn_tszimm_shr
UQRSHRNT 01000101 .. 1 ..... 00 1111 ..... ..... @rd_rn_tszimm_shr
## SVE2 integer add/subtract narrow high part
ADDHNB 01000101 .. 1 ..... 011 000 ..... ..... @rd_rn_rm
ADDHNT 01000101 .. 1 ..... 011 001 ..... ..... @rd_rn_rm
RADDHNB 01000101 .. 1 ..... 011 010 ..... ..... @rd_rn_rm
RADDHNT 01000101 .. 1 ..... 011 011 ..... ..... @rd_rn_rm
SUBHNB 01000101 .. 1 ..... 011 100 ..... ..... @rd_rn_rm
SUBHNT 01000101 .. 1 ..... 011 101 ..... ..... @rd_rn_rm
RSUBHNB 01000101 .. 1 ..... 011 110 ..... ..... @rd_rn_rm
RSUBHNT 01000101 .. 1 ..... 011 111 ..... ..... @rd_rn_rm
### SVE2 Character Match
MATCH 01000101 .. 1 ..... 100 ... ..... 0 .... @pd_pg_rn_rm
NMATCH 01000101 .. 1 ..... 100 ... ..... 1 .... @pd_pg_rn_rm
### SVE2 Histogram Computation
HISTCNT 01000101 .. 1 ..... 110 ... ..... ..... @rd_pg_rn_rm
HISTSEG 01000101 .. 1 ..... 101 000 ..... ..... @rd_rn_rm
## SVE2 floating-point pairwise operations
FADDP 01100100 .. 010 00 0 100 ... ..... ..... @rdn_pg_rm
FMAXNMP 01100100 .. 010 10 0 100 ... ..... ..... @rdn_pg_rm
FMINNMP 01100100 .. 010 10 1 100 ... ..... ..... @rdn_pg_rm
FMAXP 01100100 .. 010 11 0 100 ... ..... ..... @rdn_pg_rm
FMINP 01100100 .. 010 11 1 100 ... ..... ..... @rdn_pg_rm
#### SVE Integer Multiply-Add (unpredicated)
## SVE2 saturating multiply-add long
SQDMLALB_zzzw 01000100 .. 0 ..... 0110 00 ..... ..... @rda_rn_rm
SQDMLALT_zzzw 01000100 .. 0 ..... 0110 01 ..... ..... @rda_rn_rm
SQDMLSLB_zzzw 01000100 .. 0 ..... 0110 10 ..... ..... @rda_rn_rm
SQDMLSLT_zzzw 01000100 .. 0 ..... 0110 11 ..... ..... @rda_rn_rm
## SVE2 saturating multiply-add interleaved long
SQDMLALBT 01000100 .. 0 ..... 00001 0 ..... ..... @rda_rn_rm
SQDMLSLBT 01000100 .. 0 ..... 00001 1 ..... ..... @rda_rn_rm
## SVE2 saturating multiply-add high
SQRDMLAH_zzzz 01000100 .. 0 ..... 01110 0 ..... ..... @rda_rn_rm
SQRDMLSH_zzzz 01000100 .. 0 ..... 01110 1 ..... ..... @rda_rn_rm
## SVE2 integer multiply-add long
SMLALB_zzzw 01000100 .. 0 ..... 010 000 ..... ..... @rda_rn_rm
SMLALT_zzzw 01000100 .. 0 ..... 010 001 ..... ..... @rda_rn_rm
UMLALB_zzzw 01000100 .. 0 ..... 010 010 ..... ..... @rda_rn_rm
UMLALT_zzzw 01000100 .. 0 ..... 010 011 ..... ..... @rda_rn_rm
SMLSLB_zzzw 01000100 .. 0 ..... 010 100 ..... ..... @rda_rn_rm
SMLSLT_zzzw 01000100 .. 0 ..... 010 101 ..... ..... @rda_rn_rm
UMLSLB_zzzw 01000100 .. 0 ..... 010 110 ..... ..... @rda_rn_rm
UMLSLT_zzzw 01000100 .. 0 ..... 010 111 ..... ..... @rda_rn_rm
## SVE2 complex integer multiply-add
CMLA_zzzz 01000100 esz:2 0 rm:5 0010 rot:2 rn:5 rd:5 ra=%reg_movprfx
SQRDCMLAH_zzzz 01000100 esz:2 0 rm:5 0011 rot:2 rn:5 rd:5 ra=%reg_movprfx
## SVE mixed sign dot product
USDOT_zzzz 01000100 .. 0 ..... 011 110 ..... ..... @rda_rn_rm
### SVE2 floating point matrix multiply accumulate
FMMLA 01100100 .. 1 ..... 111001 ..... ..... @rda_rn_rm
### SVE2 Memory Gather Load Group
# SVE2 64-bit gather non-temporal load
# (scalar plus unpacked 32-bit unscaled offsets)
LDNT1_zprz 1100010 msz:2 00 rm:5 1 u:1 0 pg:3 rn:5 rd:5 \
&rprr_gather_load xs=0 esz=3 scale=0 ff=0
# SVE2 32-bit gather non-temporal load (scalar plus 32-bit unscaled offsets)
LDNT1_zprz 1000010 msz:2 00 rm:5 10 u:1 pg:3 rn:5 rd:5 \
&rprr_gather_load xs=0 esz=2 scale=0 ff=0
### SVE2 Memory Store Group
# SVE2 64-bit scatter non-temporal store (vector plus scalar)
STNT1_zprz 1110010 .. 00 ..... 001 ... ..... ..... \
@rprr_scatter_store xs=2 esz=3 scale=0
# SVE2 32-bit scatter non-temporal store (vector plus scalar)
STNT1_zprz 1110010 .. 10 ..... 001 ... ..... ..... \
@rprr_scatter_store xs=0 esz=2 scale=0
### SVE2 Crypto Extensions
# SVE2 crypto unary operations
# AESMC and AESIMC
AESMC 01000101 00 10000011100 decrypt:1 00000 rd:5
# SVE2 crypto destructive binary operations
AESE 01000101 00 10001 0 11100 0 ..... ..... @rdn_rm_e0
AESD 01000101 00 10001 0 11100 1 ..... ..... @rdn_rm_e0
SM4E 01000101 00 10001 1 11100 0 ..... ..... @rdn_rm_e0
# SVE2 crypto constructive binary operations
SM4EKEY 01000101 00 1 ..... 11110 0 ..... ..... @rd_rn_rm_e0
RAX1 01000101 00 1 ..... 11110 1 ..... ..... @rd_rn_rm_e0
### SVE2 floating-point convert precision odd elements
FCVTXNT_ds 01100100 00 0010 10 101 ... ..... ..... @rd_pg_rn_e0
FCVTX_ds 01100101 00 0010 10 101 ... ..... ..... @rd_pg_rn_e0
FCVTNT_sh 01100100 10 0010 00 101 ... ..... ..... @rd_pg_rn_e0
FCVTLT_hs 01100100 10 0010 01 101 ... ..... ..... @rd_pg_rn_e0
FCVTNT_ds 01100100 11 0010 10 101 ... ..... ..... @rd_pg_rn_e0
FCVTLT_sd 01100100 11 0010 11 101 ... ..... ..... @rd_pg_rn_e0
### SVE2 floating-point convert to integer
FLOGB 01100101 00 011 esz:2 0101 pg:3 rn:5 rd:5 &rpr_esz
### SVE2 floating-point multiply-add long (vectors)
FMLALB_zzzw 01100100 10 1 ..... 10 0 00 0 ..... ..... @rda_rn_rm_e0
FMLALT_zzzw 01100100 10 1 ..... 10 0 00 1 ..... ..... @rda_rn_rm_e0
FMLSLB_zzzw 01100100 10 1 ..... 10 1 00 0 ..... ..... @rda_rn_rm_e0
FMLSLT_zzzw 01100100 10 1 ..... 10 1 00 1 ..... ..... @rda_rn_rm_e0
### SVE2 floating-point multiply-add long (indexed)
FMLALB_zzxw 01100100 10 1 ..... 0100.0 ..... ..... @rrxr_3a esz=2
FMLALT_zzxw 01100100 10 1 ..... 0100.1 ..... ..... @rrxr_3a esz=2
FMLSLB_zzxw 01100100 10 1 ..... 0110.0 ..... ..... @rrxr_3a esz=2
FMLSLT_zzxw 01100100 10 1 ..... 0110.1 ..... ..... @rrxr_3a esz=2

2110
target/arm/sve_helper.c
View File

File diff suppressed because it is too large Load Diff

111
target/arm/translate-a64.c
View File

@ -683,6 +683,34 @@ static void gen_gvec_op3_qc(DisasContext *s, bool is_q, int rd, int rn,
tcg_temp_free_ptr(qc_ptr);
}
/* Expand a 4-operand operation using an out-of-line helper. */
static void gen_gvec_op4_ool(DisasContext *s, bool is_q, int rd, int rn,
int rm, int ra, int data, gen_helper_gvec_4 *fn)
{
tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd),
vec_full_reg_offset(s, rn),
vec_full_reg_offset(s, rm),
vec_full_reg_offset(s, ra),
is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
}
/*
* Expand a 4-operand + fpstatus pointer + simd data value operation using
* an out-of-line helper.
*/
static void gen_gvec_op4_fpst(DisasContext *s, bool is_q, int rd, int rn,
int rm, int ra, bool is_fp16, int data,
gen_helper_gvec_4_ptr *fn)
{
TCGv_ptr fpst = fpstatus_ptr(is_fp16 ? FPST_FPCR_F16 : FPST_FPCR);
tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd),
vec_full_reg_offset(s, rn),
vec_full_reg_offset(s, rm),
vec_full_reg_offset(s, ra), fpst,
is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
tcg_temp_free_ptr(fpst);
}
/* Set ZF and NF based on a 64 bit result. This is alas fiddlier
* than the 32 bit equivalent.
*/
@ -12147,6 +12175,22 @@ static void disas_simd_three_reg_same_extra(DisasContext *s, uint32_t insn)
}
feature = dc_isar_feature(aa64_dp, s);
break;
case 0x03: /* USDOT */
if (size != MO_32) {
unallocated_encoding(s);
return;
}
feature = dc_isar_feature(aa64_i8mm, s);
break;
case 0x04: /* SMMLA */
case 0x14: /* UMMLA */
case 0x05: /* USMMLA */
if (!is_q || size != MO_32) {
unallocated_encoding(s);
return;
}
feature = dc_isar_feature(aa64_i8mm, s);
break;
case 0x18: /* FCMLA, #0 */
case 0x19: /* FCMLA, #90 */
case 0x1a: /* FCMLA, #180 */
@ -12183,10 +12227,23 @@ static void disas_simd_three_reg_same_extra(DisasContext *s, uint32_t insn)
return;
case 0x2: /* SDOT / UDOT */
gen_gvec_op3_ool(s, is_q, rd, rn, rm, 0,
gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, 0,
u ? gen_helper_gvec_udot_b : gen_helper_gvec_sdot_b);
return;
case 0x3: /* USDOT */
gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, 0, gen_helper_gvec_usdot_b);
return;
case 0x04: /* SMMLA, UMMLA */
gen_gvec_op4_ool(s, 1, rd, rn, rm, rd, 0,
u ? gen_helper_gvec_ummla_b
: gen_helper_gvec_smmla_b);
return;
case 0x05: /* USMMLA */
gen_gvec_op4_ool(s, 1, rd, rn, rm, rd, 0, gen_helper_gvec_usmmla_b);
return;
case 0x8: /* FCMLA, #0 */
case 0x9: /* FCMLA, #90 */
case 0xa: /* FCMLA, #180 */
@ -12194,15 +12251,15 @@ static void disas_simd_three_reg_same_extra(DisasContext *s, uint32_t insn)
rot = extract32(opcode, 0, 2);
switch (size) {
case 1:
gen_gvec_op3_fpst(s, is_q, rd, rn, rm, true, rot,
gen_gvec_op4_fpst(s, is_q, rd, rn, rm, rd, true, rot,
gen_helper_gvec_fcmlah);
break;
case 2:
gen_gvec_op3_fpst(s, is_q, rd, rn, rm, false, rot,
gen_gvec_op4_fpst(s, is_q, rd, rn, rm, rd, false, rot,
gen_helper_gvec_fcmlas);
break;
case 3:
gen_gvec_op3_fpst(s, is_q, rd, rn, rm, false, rot,
gen_gvec_op4_fpst(s, is_q, rd, rn, rm, rd, false, rot,
gen_helper_gvec_fcmlad);
break;
default:
@ -13332,6 +13389,13 @@ static void disas_simd_indexed(DisasContext *s, uint32_t insn)
return;
}
break;
case 0x0f: /* SUDOT, USDOT */
if (is_scalar || (size & 1) || !dc_isar_feature(aa64_i8mm, s)) {
unallocated_encoding(s);
return;
}
size = MO_32;
break;
case 0x11: /* FCMLA #0 */
case 0x13: /* FCMLA #90 */
case 0x15: /* FCMLA #180 */
@ -13442,10 +13506,17 @@ static void disas_simd_indexed(DisasContext *s, uint32_t insn)
switch (16 * u + opcode) {
case 0x0e: /* SDOT */
case 0x1e: /* UDOT */
gen_gvec_op3_ool(s, is_q, rd, rn, rm, index,
gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
u ? gen_helper_gvec_udot_idx_b
: gen_helper_gvec_sdot_idx_b);
return;
case 0x0f: /* SUDOT, USDOT */
gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
extract32(insn, 23, 1)
? gen_helper_gvec_usdot_idx_b
: gen_helper_gvec_sudot_idx_b);
return;
case 0x11: /* FCMLA #0 */
case 0x13: /* FCMLA #90 */
case 0x15: /* FCMLA #180 */
@ -13453,9 +13524,10 @@ static void disas_simd_indexed(DisasContext *s, uint32_t insn)
{
int rot = extract32(insn, 13, 2);
int data = (index << 2) | rot;
tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd),
tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd),
vec_full_reg_offset(s, rn),
vec_full_reg_offset(s, rm), fpst,
vec_full_reg_offset(s, rm),
vec_full_reg_offset(s, rd), fpst,
is_q ? 16 : 8, vec_full_reg_size(s), data,
size == MO_64
? gen_helper_gvec_fcmlas_idx
@ -14349,8 +14421,6 @@ static void disas_crypto_xar(DisasContext *s, uint32_t insn)
int imm6 = extract32(insn, 10, 6);
int rn = extract32(insn, 5, 5);
int rd = extract32(insn, 0, 5);
TCGv_i64 tcg_op1, tcg_op2, tcg_res[2];
int pass;
if (!dc_isar_feature(aa64_sha3, s)) {
unallocated_encoding(s);
@ -14361,25 +14431,10 @@ static void disas_crypto_xar(DisasContext *s, uint32_t insn)
return;
}
tcg_op1 = tcg_temp_new_i64();
tcg_op2 = tcg_temp_new_i64();
tcg_res[0] = tcg_temp_new_i64();
tcg_res[1] = tcg_temp_new_i64();
for (pass = 0; pass < 2; pass++) {
read_vec_element(s, tcg_op1, rn, pass, MO_64);
read_vec_element(s, tcg_op2, rm, pass, MO_64);
tcg_gen_xor_i64(tcg_res[pass], tcg_op1, tcg_op2);
tcg_gen_rotri_i64(tcg_res[pass], tcg_res[pass], imm6);
}
write_vec_element(s, tcg_res[0], rd, 0, MO_64);
write_vec_element(s, tcg_res[1], rd, 1, MO_64);
tcg_temp_free_i64(tcg_op1);
tcg_temp_free_i64(tcg_op2);
tcg_temp_free_i64(tcg_res[0]);
tcg_temp_free_i64(tcg_res[1]);
gen_gvec_xar(MO_64, vec_full_reg_offset(s, rd),
vec_full_reg_offset(s, rn),
vec_full_reg_offset(s, rm), imm6, 16,
vec_full_reg_size(s));
}
/* Crypto three-reg imm2

3
target/arm/translate-a64.h
View File

@ -120,5 +120,8 @@ bool disas_sve(DisasContext *, uint32_t);
void gen_gvec_rax1(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz);
void gen_gvec_xar(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
uint32_t rm_ofs, int64_t shift,
uint32_t opr_sz, uint32_t max_sz);
#endif /* TARGET_ARM_TRANSLATE_A64_H */

231
target/arm/translate-neon.c
View File

@ -151,24 +151,20 @@ static void neon_store_element64(int reg, int ele, MemOp size, TCGv_i64 var)
}
}
static bool trans_VCMLA(DisasContext *s, arg_VCMLA *a)
static bool do_neon_ddda(DisasContext *s, int q, int vd, int vn, int vm,
int data, gen_helper_gvec_4 *fn_gvec)
{
int opr_sz;
TCGv_ptr fpst;
gen_helper_gvec_3_ptr *fn_gvec_ptr;
if (!dc_isar_feature(aa32_vcma, s)
|| (a->size == MO_16 && !dc_isar_feature(aa32_fp16_arith, s))) {
return false;
}
/* UNDEF accesses to D16-D31 if they don't exist. */
if (!dc_isar_feature(aa32_simd_r32, s) &&
((a->vd | a->vn | a->vm) & 0x10)) {
if (((vd | vn | vm) & 0x10) && !dc_isar_feature(aa32_simd_r32, s)) {
return false;
}
if ((a->vn | a->vm | a->vd) & a->q) {
/*
* UNDEF accesses to odd registers for each bit of Q.
* Q will be 0b111 for all Q-reg instructions, otherwise
* when we have mixed Q- and D-reg inputs.
*/
if (((vd & 1) * 4 | (vn & 1) * 2 | (vm & 1)) & q) {
return false;
}
@ -176,19 +172,65 @@ static bool trans_VCMLA(DisasContext *s, arg_VCMLA *a)
return true;
}
opr_sz = (1 + a->q) * 8;
fpst = fpstatus_ptr(a->size == MO_16 ? FPST_STD_F16 : FPST_STD);
fn_gvec_ptr = (a->size == MO_16) ?
gen_helper_gvec_fcmlah : gen_helper_gvec_fcmlas;
tcg_gen_gvec_3_ptr(vfp_reg_offset(1, a->vd),
vfp_reg_offset(1, a->vn),
vfp_reg_offset(1, a->vm),
fpst, opr_sz, opr_sz, a->rot,
fn_gvec_ptr);
int opr_sz = q ? 16 : 8;
tcg_gen_gvec_4_ool(vfp_reg_offset(1, vd),
vfp_reg_offset(1, vn),
vfp_reg_offset(1, vm),
vfp_reg_offset(1, vd),
opr_sz, opr_sz, data, fn_gvec);
return true;
}
static bool do_neon_ddda_fpst(DisasContext *s, int q, int vd, int vn, int vm,
int data, ARMFPStatusFlavour fp_flavour,
gen_helper_gvec_4_ptr *fn_gvec_ptr)
{
/* UNDEF accesses to D16-D31 if they don't exist. */
if (((vd | vn | vm) & 0x10) && !dc_isar_feature(aa32_simd_r32, s)) {
return false;
}
/*
* UNDEF accesses to odd registers for each bit of Q.
* Q will be 0b111 for all Q-reg instructions, otherwise
* when we have mixed Q- and D-reg inputs.
*/
if (((vd & 1) * 4 | (vn & 1) * 2 | (vm & 1)) & q) {
return false;
}
if (!vfp_access_check(s)) {
return true;
}
int opr_sz = q ? 16 : 8;
TCGv_ptr fpst = fpstatus_ptr(fp_flavour);
tcg_gen_gvec_4_ptr(vfp_reg_offset(1, vd),
vfp_reg_offset(1, vn),
vfp_reg_offset(1, vm),
vfp_reg_offset(1, vd),
fpst, opr_sz, opr_sz, data, fn_gvec_ptr);
tcg_temp_free_ptr(fpst);
return true;
}
static bool trans_VCMLA(DisasContext *s, arg_VCMLA *a)
{
if (!dc_isar_feature(aa32_vcma, s)) {
return false;
}
if (a->size == MO_16) {
if (!dc_isar_feature(aa32_fp16_arith, s)) {
return false;
}
return do_neon_ddda_fpst(s, a->q * 7, a->vd, a->vn, a->vm, a->rot,
FPST_STD_F16, gen_helper_gvec_fcmlah);
}
return do_neon_ddda_fpst(s, a->q * 7, a->vd, a->vn, a->vm, a->rot,
FPST_STD, gen_helper_gvec_fcmlas);
}
static bool trans_VCADD(DisasContext *s, arg_VCADD *a)
{
int opr_sz;
@ -227,36 +269,31 @@ static bool trans_VCADD(DisasContext *s, arg_VCADD *a)
return true;
}
static bool trans_VDOT(DisasContext *s, arg_VDOT *a)
static bool trans_VSDOT(DisasContext *s, arg_VSDOT *a)
{
int opr_sz;
gen_helper_gvec_3 *fn_gvec;
if (!dc_isar_feature(aa32_dp, s)) {
return false;
}
return do_neon_ddda(s, a->q * 7, a->vd, a->vn, a->vm, 0,
gen_helper_gvec_sdot_b);
}
/* UNDEF accesses to D16-D31 if they don't exist. */
if (!dc_isar_feature(aa32_simd_r32, s) &&
((a->vd | a->vn | a->vm) & 0x10)) {
static bool trans_VUDOT(DisasContext *s, arg_VUDOT *a)
{
if (!dc_isar_feature(aa32_dp, s)) {
return false;
}
return do_neon_ddda(s, a->q * 7, a->vd, a->vn, a->vm, 0,
gen_helper_gvec_udot_b);
}
if ((a->vn | a->vm | a->vd) & a->q) {
static bool trans_VUSDOT(DisasContext *s, arg_VUSDOT *a)
{
if (!dc_isar_feature(aa32_i8mm, s)) {
return false;
}
if (!vfp_access_check(s)) {
return true;
}
opr_sz = (1 + a->q) * 8;
fn_gvec = a->u ? gen_helper_gvec_udot_b : gen_helper_gvec_sdot_b;
tcg_gen_gvec_3_ool(vfp_reg_offset(1, a->vd),
vfp_reg_offset(1, a->vn),
vfp_reg_offset(1, a->vm),
opr_sz, opr_sz, 0, fn_gvec);
return true;
return do_neon_ddda(s, a->q * 7, a->vd, a->vn, a->vm, 0,
gen_helper_gvec_usdot_b);
}
static bool trans_VFML(DisasContext *s, arg_VFML *a)
@ -292,77 +329,56 @@ static bool trans_VFML(DisasContext *s, arg_VFML *a)
static bool trans_VCMLA_scalar(DisasContext *s, arg_VCMLA_scalar *a)
{
gen_helper_gvec_3_ptr *fn_gvec_ptr;
int opr_sz;
TCGv_ptr fpst;
int data = (a->index << 2) | a->rot;
if (!dc_isar_feature(aa32_vcma, s)) {
return false;
}
if (a->size == MO_16 && !dc_isar_feature(aa32_fp16_arith, s)) {
return false;
if (a->size == MO_16) {
if (!dc_isar_feature(aa32_fp16_arith, s)) {
return false;
}
return do_neon_ddda_fpst(s, a->q * 6, a->vd, a->vn, a->vm, data,
FPST_STD_F16, gen_helper_gvec_fcmlah_idx);
}
/* UNDEF accesses to D16-D31 if they don't exist. */
if (!dc_isar_feature(aa32_simd_r32, s) &&
((a->vd | a->vn | a->vm) & 0x10)) {
return false;
}
if ((a->vd | a->vn) & a->q) {
return false;
}
if (!vfp_access_check(s)) {
return true;
}
fn_gvec_ptr = (a->size == MO_16) ?
gen_helper_gvec_fcmlah_idx : gen_helper_gvec_fcmlas_idx;
opr_sz = (1 + a->q) * 8;
fpst = fpstatus_ptr(a->size == MO_16 ? FPST_STD_F16 : FPST_STD);
tcg_gen_gvec_3_ptr(vfp_reg_offset(1, a->vd),
vfp_reg_offset(1, a->vn),
vfp_reg_offset(1, a->vm),
fpst, opr_sz, opr_sz,
(a->index << 2) | a->rot, fn_gvec_ptr);
tcg_temp_free_ptr(fpst);
return true;
return do_neon_ddda_fpst(s, a->q * 6, a->vd, a->vn, a->vm, data,
FPST_STD, gen_helper_gvec_fcmlas_idx);
}
static bool trans_VDOT_scalar(DisasContext *s, arg_VDOT_scalar *a)
static bool trans_VSDOT_scalar(DisasContext *s, arg_VSDOT_scalar *a)
{
gen_helper_gvec_3 *fn_gvec;
int opr_sz;
TCGv_ptr fpst;
if (!dc_isar_feature(aa32_dp, s)) {
return false;
}
return do_neon_ddda(s, a->q * 6, a->vd, a->vn, a->vm, a->index,
gen_helper_gvec_sdot_idx_b);
}
/* UNDEF accesses to D16-D31 if they don't exist. */
if (!dc_isar_feature(aa32_simd_r32, s) &&
((a->vd | a->vn) & 0x10)) {
static bool trans_VUDOT_scalar(DisasContext *s, arg_VUDOT_scalar *a)
{
if (!dc_isar_feature(aa32_dp, s)) {
return false;
}
return do_neon_ddda(s, a->q * 6, a->vd, a->vn, a->vm, a->index,
gen_helper_gvec_udot_idx_b);
}
if ((a->vd | a->vn) & a->q) {
static bool trans_VUSDOT_scalar(DisasContext *s, arg_VUSDOT_scalar *a)
{
if (!dc_isar_feature(aa32_i8mm, s)) {
return false;
}
return do_neon_ddda(s, a->q * 6, a->vd, a->vn, a->vm, a->index,
gen_helper_gvec_usdot_idx_b);
}
if (!vfp_access_check(s)) {
return true;
static bool trans_VSUDOT_scalar(DisasContext *s, arg_VSUDOT_scalar *a)
{
if (!dc_isar_feature(aa32_i8mm, s)) {
return false;
}
fn_gvec = a->u ? gen_helper_gvec_udot_idx_b : gen_helper_gvec_sdot_idx_b;
opr_sz = (1 + a->q) * 8;
fpst = fpstatus_ptr(FPST_STD);
tcg_gen_gvec_3_ool(vfp_reg_offset(1, a->vd),
vfp_reg_offset(1, a->vn),
vfp_reg_offset(1, a->rm),
opr_sz, opr_sz, a->index, fn_gvec);
tcg_temp_free_ptr(fpst);
return true;
return do_neon_ddda(s, a->q * 6, a->vd, a->vn, a->vm, a->index,
gen_helper_gvec_sudot_idx_b);
}
static bool trans_VFML_scalar(DisasContext *s, arg_VFML_scalar *a)
@ -4020,3 +4036,30 @@ static bool trans_VTRN(DisasContext *s, arg_2misc *a)
tcg_temp_free_i32(tmp2);
return true;
}
static bool trans_VSMMLA(DisasContext *s, arg_VSMMLA *a)
{
if (!dc_isar_feature(aa32_i8mm, s)) {
return false;
}
return do_neon_ddda(s, 7, a->vd, a->vn, a->vm, 0,
gen_helper_gvec_smmla_b);
}
static bool trans_VUMMLA(DisasContext *s, arg_VUMMLA *a)
{
if (!dc_isar_feature(aa32_i8mm, s)) {
return false;
}
return do_neon_ddda(s, 7, a->vd, a->vn, a->vm, 0,
gen_helper_gvec_ummla_b);
}
static bool trans_VUSMMLA(DisasContext *s, arg_VUSMMLA *a)
{
if (!dc_isar_feature(aa32_i8mm, s)) {
return false;
}
return do_neon_ddda(s, 7, a->vd, a->vn, a->vm, 0,
gen_helper_gvec_usmmla_b);
}

3200
target/arm/translate-sve.c
View File

File diff suppressed because it is too large Load Diff

887
target/arm/vec_helper.c
View File

File diff suppressed because it is too large Load Diff

167
target/arm/vec_internal.h
View File

@ -20,6 +20,30 @@
#ifndef TARGET_ARM_VEC_INTERNALS_H
#define TARGET_ARM_VEC_INTERNALS_H
/*
* Note that vector data is stored in host-endian 64-bit chunks,
* so addressing units smaller than that needs a host-endian fixup.
*
* The H<N> macros are used when indexing an array of elements of size N.
*
* The H1_<N> macros are used when performing byte arithmetic and then
* casting the final pointer to a type of size N.
*/
#ifdef HOST_WORDS_BIGENDIAN
#define H1(x) ((x) ^ 7)
#define H1_2(x) ((x) ^ 6)
#define H1_4(x) ((x) ^ 4)
#define H2(x) ((x) ^ 3)
#define H4(x) ((x) ^ 1)
#else
#define H1(x) (x)
#define H1_2(x) (x)
#define H1_4(x) (x)
#define H2(x) (x)
#define H4(x) (x)
#endif
static inline void clear_tail(void *vd, uintptr_t opr_sz, uintptr_t max_sz)
{
uint64_t *d = vd + opr_sz;
@ -30,4 +54,147 @@ static inline void clear_tail(void *vd, uintptr_t opr_sz, uintptr_t max_sz)
}
}
static inline int32_t do_sqrshl_bhs(int32_t src, int32_t shift, int bits,
bool round, uint32_t *sat)
{
if (shift <= -bits) {
/* Rounding the sign bit always produces 0. */
if (round) {
return 0;
}
return src >> 31;
} else if (shift < 0) {
if (round) {
src >>= -shift - 1;
return (src >> 1) + (src & 1);
}
return src >> -shift;
} else if (shift < bits) {
int32_t val = src << shift;
if (bits == 32) {
if (!sat || val >> shift == src) {
return val;
}
} else {
int32_t extval = sextract32(val, 0, bits);
if (!sat || val == extval) {
return extval;
}
}
} else if (!sat || src == 0) {
return 0;
}
*sat = 1;
return (1u << (bits - 1)) - (src >= 0);
}
static inline uint32_t do_uqrshl_bhs(uint32_t src, int32_t shift, int bits,
bool round, uint32_t *sat)
{
if (shift <= -(bits + round)) {
return 0;
} else if (shift < 0) {
if (round) {
src >>= -shift - 1;
return (src >> 1) + (src & 1);
}
return src >> -shift;
} else if (shift < bits) {
uint32_t val = src << shift;
if (bits == 32) {
if (!sat || val >> shift == src) {
return val;
}
} else {
uint32_t extval = extract32(val, 0, bits);
if (!sat || val == extval) {
return extval;
}
}
} else if (!sat || src == 0) {
return 0;
}
*sat = 1;
return MAKE_64BIT_MASK(0, bits);
}
static inline int32_t do_suqrshl_bhs(int32_t src, int32_t shift, int bits,
bool round, uint32_t *sat)
{
if (sat && src < 0) {
*sat = 1;
return 0;
}
return do_uqrshl_bhs(src, shift, bits, round, sat);
}
static inline int64_t do_sqrshl_d(int64_t src, int64_t shift,
bool round, uint32_t *sat)
{
if (shift <= -64) {
/* Rounding the sign bit always produces 0. */
if (round) {
return 0;
}
return src >> 63;
} else if (shift < 0) {
if (round) {
src >>= -shift - 1;
return (src >> 1) + (src & 1);
}
return src >> -shift;
} else if (shift < 64) {
int64_t val = src << shift;
if (!sat || val >> shift == src) {
return val;
}
} else if (!sat || src == 0) {
return 0;
}
*sat = 1;
return src < 0 ? INT64_MIN : INT64_MAX;
}
static inline uint64_t do_uqrshl_d(uint64_t src, int64_t shift,
bool round, uint32_t *sat)
{
if (shift <= -(64 + round)) {
return 0;
} else if (shift < 0) {
if (round) {
src >>= -shift - 1;
return (src >> 1) + (src & 1);
}
return src >> -shift;
} else if (shift < 64) {
uint64_t val = src << shift;
if (!sat || val >> shift == src) {
return val;
}
} else if (!sat || src == 0) {
return 0;
}
*sat = 1;
return UINT64_MAX;
}
static inline int64_t do_suqrshl_d(int64_t src, int64_t shift,
bool round, uint32_t *sat)
{
if (sat && src < 0) {
*sat = 1;
return 0;
}
return do_uqrshl_d(src, shift, round, sat);
}
int8_t do_sqrdmlah_b(int8_t, int8_t, int8_t, bool, bool);
int16_t do_sqrdmlah_h(int16_t, int16_t, int16_t, bool, bool, uint32_t *);
int32_t do_sqrdmlah_s(int32_t, int32_t, int32_t, bool, bool, uint32_t *);
int64_t do_sqrdmlah_d(int64_t, int64_t, int64_t, bool, bool);
#endif /* TARGET_ARM_VEC_INTERNALS_H */