RPCS3/llvm-mirror
1
0
Fork 0
mirror of https://github.com/RPCS3/llvm-mirror.git synced 2024-12-11 13:37:07 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[globalisel][irtranslator] Add support for atomicrmw and (strong) cmpxchg

Browse Source 
Summary:
This patch adds support for the atomicrmw instructions and the strong
cmpxchg instruction to the IRTranslator.

I've left out weak cmpxchg because LangRef.rst isn't entirely clear on what
difference it makes to the backend. As far as I can tell from the code, it
only matters to AtomicExpandPass which is run at the LLVM-IR level.

Reviewers: ab, t.p.northover, qcolombet, rovka, aditya_nandakumar, volkan, javed.absar

Reviewed By: qcolombet

Subscribers: kristof.beyls, javed.absar, igorb, llvm-commits

Differential Revision: https://reviews.llvm.org/D40092

llvm-svn: 336589
This commit is contained in:
Daniel Sanders 2018-07-09 19:33:40 +00:00
parent cd23bf1411
commit ed6e598662
5 changed files with 701 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
include/llvm/CodeGen/GlobalISel/IRTranslator.h
View File

@ -399,6 +399,9 @@ private:
bool translateShuffleVector(const User &U, MachineIRBuilder &MIRBuilder);
bool translateAtomicCmpXchg(const User &U, MachineIRBuilder &MIRBuilder);
bool translateAtomicRMW(const User &U, MachineIRBuilder &MIRBuilder);
// Stubs to keep the compiler happy while we implement the rest of the
// translation.
bool translateResume(const User &U, MachineIRBuilder &MIRBuilder) {
@ -416,12 +419,6 @@ private:
bool translateFence(const User &U, MachineIRBuilder &MIRBuilder) {
return false;
}
bool translateAtomicCmpXchg(const User &U, MachineIRBuilder &MIRBuilder) {
return false;
}
bool translateAtomicRMW(const User &U, MachineIRBuilder &MIRBuilder) {
return false;
}
bool translateAddrSpaceCast(const User &U, MachineIRBuilder &MIRBuilder) {
return translateCast(TargetOpcode::G_ADDRSPACE_CAST, U, MIRBuilder);
}

210
include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
View File

@ -717,7 +717,28 @@ public:
MachineInstrBuilder buildExtractVectorElement(unsigned Res, unsigned Val,
unsigned Idx);
/// Build and insert `OldValRes = G_ATOMIC_CMPXCHG Addr, CmpVal, NewVal,
/// Build and insert `OldValRes<def>, SuccessRes<def> =
/// G_ATOMIC_CMPXCHG_WITH_SUCCESS Addr, CmpVal, NewVal, MMO`.
///
/// Atomically replace the value at \p Addr with \p NewVal if it is currently
/// \p CmpVal otherwise leaves it unchanged. Puts the original value from \p
/// Addr in \p Res, along with an s1 indicating whether it was replaced.
///
/// \pre setBasicBlock or setMI must have been called.
/// \pre \p OldValRes must be a generic virtual register of scalar type.
/// \pre \p SuccessRes must be a generic virtual register of scalar type. It
/// will be assigned 0 on failure and 1 on success.
/// \pre \p Addr must be a generic virtual register with pointer type.
/// \pre \p OldValRes, \p CmpVal, and \p NewVal must be generic virtual
/// registers of the same type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
MachineInstrBuilder
buildAtomicCmpXchgWithSuccess(unsigned OldValRes, unsigned SuccessRes,
unsigned Addr, unsigned CmpVal, unsigned NewVal,
MachineMemOperand &MMO);
/// Build and insert `OldValRes<def> = G_ATOMIC_CMPXCHG Addr, CmpVal, NewVal,
/// MMO`.
///
/// Atomically replace the value at \p Addr with \p NewVal if it is currently
@ -734,6 +755,193 @@ public:
MachineInstrBuilder buildAtomicCmpXchg(unsigned OldValRes, unsigned Addr,
unsigned CmpVal, unsigned NewVal,
MachineMemOperand &MMO);
/// Build and insert `OldValRes<def> = G_ATOMICRMW_<Opcode> Addr, Val, MMO`.
///
/// Atomically read-modify-update the value at \p Addr with \p Val. Puts the
/// original value from \p Addr in \p OldValRes. The modification is
/// determined by the opcode.
///
/// \pre setBasicBlock or setMI must have been called.
/// \pre \p OldValRes must be a generic virtual register.
/// \pre \p Addr must be a generic virtual register with pointer type.
/// \pre \p OldValRes, and \p Val must be generic virtual registers of the
/// same type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
MachineInstrBuilder buildAtomicRMW(unsigned Opcode, unsigned OldValRes,
unsigned Addr, unsigned Val,
MachineMemOperand &MMO);
/// Build and insert `OldValRes<def> = G_ATOMICRMW_XCHG Addr, Val, MMO`.
///
/// Atomically replace the value at \p Addr with \p Val. Puts the original
/// value from \p Addr in \p OldValRes.
///
/// \pre setBasicBlock or setMI must have been called.
/// \pre \p OldValRes must be a generic virtual register.
/// \pre \p Addr must be a generic virtual register with pointer type.
/// \pre \p OldValRes, and \p Val must be generic virtual registers of the
/// same type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
MachineInstrBuilder buildAtomicRMWXchg(unsigned OldValRes, unsigned Addr,
unsigned Val, MachineMemOperand &MMO);
/// Build and insert `OldValRes<def> = G_ATOMICRMW_ADD Addr, Val, MMO`.
///
/// Atomically replace the value at \p Addr with the addition of \p Val and
/// the original value. Puts the original value from \p Addr in \p OldValRes.
///
/// \pre setBasicBlock or setMI must have been called.
/// \pre \p OldValRes must be a generic virtual register.
/// \pre \p Addr must be a generic virtual register with pointer type.
/// \pre \p OldValRes, and \p Val must be generic virtual registers of the
/// same type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
MachineInstrBuilder buildAtomicRMWAdd(unsigned OldValRes, unsigned Addr,
unsigned Val, MachineMemOperand &MMO);
/// Build and insert `OldValRes<def> = G_ATOMICRMW_SUB Addr, Val, MMO`.
///
/// Atomically replace the value at \p Addr with the subtraction of \p Val and
/// the original value. Puts the original value from \p Addr in \p OldValRes.
///
/// \pre setBasicBlock or setMI must have been called.
/// \pre \p OldValRes must be a generic virtual register.
/// \pre \p Addr must be a generic virtual register with pointer type.
/// \pre \p OldValRes, and \p Val must be generic virtual registers of the
/// same type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
MachineInstrBuilder buildAtomicRMWSub(unsigned OldValRes, unsigned Addr,
unsigned Val, MachineMemOperand &MMO);
/// Build and insert `OldValRes<def> = G_ATOMICRMW_AND Addr, Val, MMO`.
///
/// Atomically replace the value at \p Addr with the bitwise and of \p Val and
/// the original value. Puts the original value from \p Addr in \p OldValRes.
///
/// \pre setBasicBlock or setMI must have been called.
/// \pre \p OldValRes must be a generic virtual register.
/// \pre \p Addr must be a generic virtual register with pointer type.
/// \pre \p OldValRes, and \p Val must be generic virtual registers of the
/// same type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
MachineInstrBuilder buildAtomicRMWAnd(unsigned OldValRes, unsigned Addr,
unsigned Val, MachineMemOperand &MMO);
/// Build and insert `OldValRes<def> = G_ATOMICRMW_NAND Addr, Val, MMO`.
///
/// Atomically replace the value at \p Addr with the bitwise nand of \p Val
/// and the original value. Puts the original value from \p Addr in \p
/// OldValRes.
///
/// \pre setBasicBlock or setMI must have been called.
/// \pre \p OldValRes must be a generic virtual register.
/// \pre \p Addr must be a generic virtual register with pointer type.
/// \pre \p OldValRes, and \p Val must be generic virtual registers of the
/// same type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
MachineInstrBuilder buildAtomicRMWNand(unsigned OldValRes, unsigned Addr,
unsigned Val, MachineMemOperand &MMO);
/// Build and insert `OldValRes<def> = G_ATOMICRMW_OR Addr, Val, MMO`.
///
/// Atomically replace the value at \p Addr with the bitwise or of \p Val and
/// the original value. Puts the original value from \p Addr in \p OldValRes.
///
/// \pre setBasicBlock or setMI must have been called.
/// \pre \p OldValRes must be a generic virtual register.
/// \pre \p Addr must be a generic virtual register with pointer type.
/// \pre \p OldValRes, and \p Val must be generic virtual registers of the
/// same type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
MachineInstrBuilder buildAtomicRMWOr(unsigned OldValRes, unsigned Addr,
unsigned Val, MachineMemOperand &MMO);
/// Build and insert `OldValRes<def> = G_ATOMICRMW_XOR Addr, Val, MMO`.
///
/// Atomically replace the value at \p Addr with the bitwise xor of \p Val and
/// the original value. Puts the original value from \p Addr in \p OldValRes.
///
/// \pre setBasicBlock or setMI must have been called.
/// \pre \p OldValRes must be a generic virtual register.
/// \pre \p Addr must be a generic virtual register with pointer type.
/// \pre \p OldValRes, and \p Val must be generic virtual registers of the
/// same type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
MachineInstrBuilder buildAtomicRMWXor(unsigned OldValRes, unsigned Addr,
unsigned Val, MachineMemOperand &MMO);
/// Build and insert `OldValRes<def> = G_ATOMICRMW_MAX Addr, Val, MMO`.
///
/// Atomically replace the value at \p Addr with the signed maximum of \p
/// Val and the original value. Puts the original value from \p Addr in \p
/// OldValRes.
///
/// \pre setBasicBlock or setMI must have been called.
/// \pre \p OldValRes must be a generic virtual register.
/// \pre \p Addr must be a generic virtual register with pointer type.
/// \pre \p OldValRes, and \p Val must be generic virtual registers of the
/// same type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
MachineInstrBuilder buildAtomicRMWMax(unsigned OldValRes, unsigned Addr,
unsigned Val, MachineMemOperand &MMO);
/// Build and insert `OldValRes<def> = G_ATOMICRMW_MIN Addr, Val, MMO`.
///
/// Atomically replace the value at \p Addr with the signed minimum of \p
/// Val and the original value. Puts the original value from \p Addr in \p
/// OldValRes.
///
/// \pre setBasicBlock or setMI must have been called.
/// \pre \p OldValRes must be a generic virtual register.
/// \pre \p Addr must be a generic virtual register with pointer type.
/// \pre \p OldValRes, and \p Val must be generic virtual registers of the
/// same type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
MachineInstrBuilder buildAtomicRMWMin(unsigned OldValRes, unsigned Addr,
unsigned Val, MachineMemOperand &MMO);
/// Build and insert `OldValRes<def> = G_ATOMICRMW_UMAX Addr, Val, MMO`.
///
/// Atomically replace the value at \p Addr with the unsigned maximum of \p
/// Val and the original value. Puts the original value from \p Addr in \p
/// OldValRes.
///
/// \pre setBasicBlock or setMI must have been called.
/// \pre \p OldValRes must be a generic virtual register.
/// \pre \p Addr must be a generic virtual register with pointer type.
/// \pre \p OldValRes, and \p Val must be generic virtual registers of the
/// same type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
MachineInstrBuilder buildAtomicRMWUmax(unsigned OldValRes, unsigned Addr,
unsigned Val, MachineMemOperand &MMO);
/// Build and insert `OldValRes<def> = G_ATOMICRMW_UMIN Addr, Val, MMO`.
///
/// Atomically replace the value at \p Addr with the unsigned minimum of \p
/// Val and the original value. Puts the original value from \p Addr in \p
/// OldValRes.
///
/// \pre setBasicBlock or setMI must have been called.
/// \pre \p OldValRes must be a generic virtual register.
/// \pre \p Addr must be a generic virtual register with pointer type.
/// \pre \p OldValRes, and \p Val must be generic virtual registers of the
/// same type.
///
/// \return a MachineInstrBuilder for the newly created instruction.
MachineInstrBuilder buildAtomicRMWUmin(unsigned OldValRes, unsigned Addr,
unsigned Val, MachineMemOperand &MMO);
};
/// A CRTP class that contains methods for building instructions that can

108
lib/CodeGen/GlobalISel/IRTranslator.cpp
View File

@ -228,6 +228,20 @@ unsigned IRTranslator::getMemOpAlignment(const Instruction &I) {
} else if (const LoadInst *LI = dyn_cast<LoadInst>(&I)) {
Alignment = LI->getAlignment();
ValTy = LI->getType();
} else if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(&I)) {
// TODO(PR27168): This instruction has no alignment attribute, but unlike
// the default alignment for load/store, the default here is to assume
// it has NATURAL alignment, not DataLayout-specified alignment.
const DataLayout &DL = AI->getModule()->getDataLayout();
Alignment = DL.getTypeStoreSize(AI->getCompareOperand()->getType());
ValTy = AI->getCompareOperand()->getType();
} else if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(&I)) {
// TODO(PR27168): This instruction has no alignment attribute, but unlike
// the default alignment for load/store, the default here is to assume
// it has NATURAL alignment, not DataLayout-specified alignment.
const DataLayout &DL = AI->getModule()->getDataLayout();
Alignment = DL.getTypeStoreSize(AI->getValOperand()->getType());
ValTy = AI->getType();
} else {
OptimizationRemarkMissed R("gisel-irtranslator", "", &I);
R << "unable to translate memop: " << ore::NV("Opcode", &I);
@ -1285,6 +1299,100 @@ bool IRTranslator::translatePHI(const User &U, MachineIRBuilder &MIRBuilder) {
return true;
}
bool IRTranslator::translateAtomicCmpXchg(const User &U,
MachineIRBuilder &MIRBuilder) {
const AtomicCmpXchgInst &I = cast<AtomicCmpXchgInst>(U);
if (I.isWeak())
return false;
auto Flags = I.isVolatile() ? MachineMemOperand::MOVolatile
: MachineMemOperand::MONone;
Flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
Type *ResType = I.getType();
Type *ValType = ResType->Type::getStructElementType(0);
auto Res = getOrCreateVRegs(I);
unsigned OldValRes = Res[0];
unsigned SuccessRes = Res[1];
unsigned Addr = getOrCreateVReg(*I.getPointerOperand());
unsigned Cmp = getOrCreateVReg(*I.getCompareOperand());
unsigned NewVal = getOrCreateVReg(*I.getNewValOperand());
MIRBuilder.buildAtomicCmpXchgWithSuccess(
OldValRes, SuccessRes, Addr, Cmp, NewVal,
*MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
Flags, DL->getTypeStoreSize(ValType),
getMemOpAlignment(I), AAMDNodes(), nullptr,
I.getSyncScopeID(), I.getSuccessOrdering(),
I.getFailureOrdering()));
return true;
}
bool IRTranslator::translateAtomicRMW(const User &U,
MachineIRBuilder &MIRBuilder) {
const AtomicRMWInst &I = cast<AtomicRMWInst>(U);
auto Flags = I.isVolatile() ? MachineMemOperand::MOVolatile
: MachineMemOperand::MONone;
Flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
Type *ResType = I.getType();
unsigned Res = getOrCreateVReg(I);
unsigned Addr = getOrCreateVReg(*I.getPointerOperand());
unsigned Val = getOrCreateVReg(*I.getValOperand());
unsigned Opcode = 0;
switch (I.getOperation()) {
default:
llvm_unreachable("Unknown atomicrmw op");
return false;
case AtomicRMWInst::Xchg:
Opcode = TargetOpcode::G_ATOMICRMW_XCHG;
break;
case AtomicRMWInst::Add:
Opcode = TargetOpcode::G_ATOMICRMW_ADD;
break;
case AtomicRMWInst::Sub:
Opcode = TargetOpcode::G_ATOMICRMW_SUB;
break;
case AtomicRMWInst::And:
Opcode = TargetOpcode::G_ATOMICRMW_AND;
break;
case AtomicRMWInst::Nand:
Opcode = TargetOpcode::G_ATOMICRMW_NAND;
break;
case AtomicRMWInst::Or:
Opcode = TargetOpcode::G_ATOMICRMW_OR;
break;
case AtomicRMWInst::Xor:
Opcode = TargetOpcode::G_ATOMICRMW_XOR;
break;
case AtomicRMWInst::Max:
Opcode = TargetOpcode::G_ATOMICRMW_MAX;
break;
case AtomicRMWInst::Min:
Opcode = TargetOpcode::G_ATOMICRMW_MIN;
break;
case AtomicRMWInst::UMax:
Opcode = TargetOpcode::G_ATOMICRMW_UMAX;
break;
case AtomicRMWInst::UMin:
Opcode = TargetOpcode::G_ATOMICRMW_UMIN;
break;
}
MIRBuilder.buildAtomicRMW(
Opcode, Res, Addr, Val,
*MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
Flags, DL->getTypeStoreSize(ResType),
getMemOpAlignment(I), AAMDNodes(), nullptr,
I.getSyncScopeID(), I.getOrdering()));
return true;
}
void IRTranslator::finishPendingPhis() {
for (auto &Phi : PendingPHIs) {
const PHINode *PI = Phi.first;

117
lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
View File

@ -669,6 +669,33 @@ MachineIRBuilderBase::buildExtractVectorElement(unsigned Res, unsigned Val,
.addUse(Idx);
}
MachineInstrBuilder MachineIRBuilderBase::buildAtomicCmpXchgWithSuccess(
unsigned OldValRes, unsigned SuccessRes, unsigned Addr, unsigned CmpVal,
unsigned NewVal, MachineMemOperand &MMO) {
#ifndef NDEBUG
LLT OldValResTy = getMRI()->getType(OldValRes);
LLT SuccessResTy = getMRI()->getType(SuccessRes);
LLT AddrTy = getMRI()->getType(Addr);
LLT CmpValTy = getMRI()->getType(CmpVal);
LLT NewValTy = getMRI()->getType(NewVal);
assert(OldValResTy.isScalar() && "invalid operand type");
assert(SuccessResTy.isScalar() && "invalid operand type");
assert(AddrTy.isPointer() && "invalid operand type");
assert(CmpValTy.isValid() && "invalid operand type");
assert(NewValTy.isValid() && "invalid operand type");
assert(OldValResTy == CmpValTy && "type mismatch");
assert(OldValResTy == NewValTy && "type mismatch");
#endif
return buildInstr(TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS)
.addDef(OldValRes)
.addDef(SuccessRes)
.addUse(Addr)
.addUse(CmpVal)
.addUse(NewVal)
.addMemOperand(&MMO);
}
MachineInstrBuilder
MachineIRBuilderBase::buildAtomicCmpXchg(unsigned OldValRes, unsigned Addr,
unsigned CmpVal, unsigned NewVal,
@ -694,6 +721,94 @@ MachineIRBuilderBase::buildAtomicCmpXchg(unsigned OldValRes, unsigned Addr,
.addMemOperand(&MMO);
}
MachineInstrBuilder
MachineIRBuilderBase::buildAtomicRMW(unsigned Opcode, unsigned OldValRes,
unsigned Addr, unsigned Val,
MachineMemOperand &MMO) {
#ifndef NDEBUG
LLT OldValResTy = getMRI()->getType(OldValRes);
LLT AddrTy = getMRI()->getType(Addr);
LLT ValTy = getMRI()->getType(Val);
assert(OldValResTy.isScalar() && "invalid operand type");
assert(AddrTy.isPointer() && "invalid operand type");
assert(ValTy.isValid() && "invalid operand type");
assert(OldValResTy == ValTy && "type mismatch");
#endif
return buildInstr(Opcode)
.addDef(OldValRes)
.addUse(Addr)
.addUse(Val)
.addMemOperand(&MMO);
}
MachineInstrBuilder
MachineIRBuilderBase::buildAtomicRMWXchg(unsigned OldValRes, unsigned Addr,
unsigned Val, MachineMemOperand &MMO) {
return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_XCHG, OldValRes, Addr, Val,
MMO);
}
MachineInstrBuilder
MachineIRBuilderBase::buildAtomicRMWAdd(unsigned OldValRes, unsigned Addr,
unsigned Val, MachineMemOperand &MMO) {
return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_ADD, OldValRes, Addr, Val,
MMO);
}
MachineInstrBuilder
MachineIRBuilderBase::buildAtomicRMWSub(unsigned OldValRes, unsigned Addr,
unsigned Val, MachineMemOperand &MMO) {
return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_SUB, OldValRes, Addr, Val,
MMO);
}
MachineInstrBuilder
MachineIRBuilderBase::buildAtomicRMWAnd(unsigned OldValRes, unsigned Addr,
unsigned Val, MachineMemOperand &MMO) {
return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_AND, OldValRes, Addr, Val,
MMO);
}
MachineInstrBuilder
MachineIRBuilderBase::buildAtomicRMWNand(unsigned OldValRes, unsigned Addr,
unsigned Val, MachineMemOperand &MMO) {
return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_NAND, OldValRes, Addr, Val,
MMO);
}
MachineInstrBuilder
MachineIRBuilderBase::buildAtomicRMWOr(unsigned OldValRes, unsigned Addr,
unsigned Val, MachineMemOperand &MMO) {
return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_OR, OldValRes, Addr, Val,
MMO);
}
MachineInstrBuilder
MachineIRBuilderBase::buildAtomicRMWXor(unsigned OldValRes, unsigned Addr,
unsigned Val, MachineMemOperand &MMO) {
return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_XOR, OldValRes, Addr, Val,
MMO);
}
MachineInstrBuilder
MachineIRBuilderBase::buildAtomicRMWMax(unsigned OldValRes, unsigned Addr,
unsigned Val, MachineMemOperand &MMO) {
return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_MAX, OldValRes, Addr, Val,
MMO);
}
MachineInstrBuilder
MachineIRBuilderBase::buildAtomicRMWMin(unsigned OldValRes, unsigned Addr,
unsigned Val, MachineMemOperand &MMO) {
return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_MIN, OldValRes, Addr, Val,
MMO);
}
MachineInstrBuilder
MachineIRBuilderBase::buildAtomicRMWUmax(unsigned OldValRes, unsigned Addr,
unsigned Val, MachineMemOperand &MMO) {
return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_UMAX, OldValRes, Addr, Val,
MMO);
}
MachineInstrBuilder
MachineIRBuilderBase::buildAtomicRMWUmin(unsigned OldValRes, unsigned Addr,
unsigned Val, MachineMemOperand &MMO) {
return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_UMIN, OldValRes, Addr, Val,
MMO);
}
void MachineIRBuilderBase::validateTruncExt(unsigned Dst, unsigned Src,
bool IsExtend) {
#ifndef NDEBUG
@ -701,7 +816,7 @@ void MachineIRBuilderBase::validateTruncExt(unsigned Dst, unsigned Src,
LLT DstTy = getMRI()->getType(Dst);
if (DstTy.isVector()) {
assert(SrcTy.isVector() && "mismatched cast between vecot and non-vector");
assert(SrcTy.isVector() && "mismatched cast between vector and non-vector");
assert(SrcTy.getNumElements() == DstTy.getNumElements() &&
"different number of elements in a trunc/ext");
} else

265
test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
View File

@ -1882,3 +1882,268 @@ define i1 @return_i1_zext() {
; CHECK: RET_ReallyLR implicit $w0
ret i1 true
}
; Try one cmpxchg
define i32 @test_atomic_cmpxchg_1(i32* %addr) {
; CHECK-LABEL: name: test_atomic_cmpxchg_1
; CHECK: bb.1.entry:
; CHECK-NEXT: successors: %bb.{{[^)]+}}
; CHECK-NEXT: liveins: $x0
; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
; CHECK-NEXT: [[OLDVAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
; CHECK-NEXT: [[NEWVAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
; CHECK: bb.2.repeat:
; CHECK-NEXT: successors: %bb.3({{[^)]+}}), %bb.2({{[^)]+}})
; CHECK: [[OLDVALRES:%[0-9]+]]:_(s32), [[SUCCESS:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[ADDR]](p0), [[OLDVAL]], [[NEWVAL]] :: (load store monotonic monotonic 4 on %ir.addr)
; CHECK-NEXT: G_BRCOND [[SUCCESS]](s1), %bb.3
; CHECK-NEXT: G_BR %bb.2
; CHECK: bb.3.done:
entry:
br label %repeat
repeat:
%val_success = cmpxchg i32* %addr, i32 0, i32 1 monotonic monotonic
%value_loaded = extractvalue { i32, i1 } %val_success, 0
%success = extractvalue { i32, i1 } %val_success, 1
br i1 %success, label %done, label %repeat
done:
ret i32 %value_loaded
}
; Try one cmpxchg with a small type and high atomic ordering.
define i16 @test_atomic_cmpxchg_2(i16* %addr) {
; CHECK-LABEL: name: test_atomic_cmpxchg_2
; CHECK: bb.1.entry:
; CHECK-NEXT: successors: %bb.2({{[^)]+}})
; CHECK-NEXT: liveins: $x0
; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
; CHECK-NEXT: [[OLDVAL:%[0-9]+]]:_(s16) = G_CONSTANT i16 0
; CHECK-NEXT: [[NEWVAL:%[0-9]+]]:_(s16) = G_CONSTANT i16 1
; CHECK: bb.2.repeat:
; CHECK-NEXT: successors: %bb.3({{[^)]+}}), %bb.2({{[^)]+}})
; CHECK: [[OLDVALRES:%[0-9]+]]:_(s16), [[SUCCESS:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[ADDR]](p0), [[OLDVAL]], [[NEWVAL]] :: (load store seq_cst seq_cst 2 on %ir.addr)
; CHECK-NEXT: G_BRCOND [[SUCCESS]](s1), %bb.3
; CHECK-NEXT: G_BR %bb.2
; CHECK: bb.3.done:
entry:
br label %repeat
repeat:
%val_success = cmpxchg i16* %addr, i16 0, i16 1 seq_cst seq_cst
%value_loaded = extractvalue { i16, i1 } %val_success, 0
%success = extractvalue { i16, i1 } %val_success, 1
br i1 %success, label %done, label %repeat
done:
ret i16 %value_loaded
}
; Try one cmpxchg where the success order and failure order differ.
define i64 @test_atomic_cmpxchg_3(i64* %addr) {
; CHECK-LABEL: name: test_atomic_cmpxchg_3
; CHECK: bb.1.entry:
; CHECK-NEXT: successors: %bb.2({{[^)]+}})
; CHECK-NEXT: liveins: $x0
; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
; CHECK-NEXT: [[OLDVAL:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
; CHECK-NEXT: [[NEWVAL:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
; CHECK: bb.2.repeat:
; CHECK-NEXT: successors: %bb.3({{[^)]+}}), %bb.2({{[^)]+}})
; CHECK: [[OLDVALRES:%[0-9]+]]:_(s64), [[SUCCESS:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[ADDR]](p0), [[OLDVAL]], [[NEWVAL]] :: (load store seq_cst acquire 8 on %ir.addr)
; CHECK-NEXT: G_BRCOND [[SUCCESS]](s1), %bb.3
; CHECK-NEXT: G_BR %bb.2
; CHECK: bb.3.done:
entry:
br label %repeat
repeat:
%val_success = cmpxchg i64* %addr, i64 0, i64 1 seq_cst acquire
%value_loaded = extractvalue { i64, i1 } %val_success, 0
%success = extractvalue { i64, i1 } %val_success, 1
br i1 %success, label %done, label %repeat
done:
ret i64 %value_loaded
}
; Try a monotonic atomicrmw xchg
; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
define i32 @test_atomicrmw_xchg(i256* %addr) {
; CHECK-LABEL: name: test_atomicrmw_xchg
; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
; CHECK-NEXT: liveins: $x0
; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_XCHG [[ADDR]](p0), [[VAL]] :: (load store monotonic 32 on %ir.addr)
; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
%oldval = atomicrmw xchg i256* %addr, i256 1 monotonic
; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
; test so work around it by truncating to i32 for now.
%oldval.trunc = trunc i256 %oldval to i32
ret i32 %oldval.trunc
}
; Try an acquire atomicrmw add
; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
define i32 @test_atomicrmw_add(i256* %addr) {
; CHECK-LABEL: name: test_atomicrmw_add
; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
; CHECK-NEXT: liveins: $x0
; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_ADD [[ADDR]](p0), [[VAL]] :: (load store acquire 32 on %ir.addr)
; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
%oldval = atomicrmw add i256* %addr, i256 1 acquire
; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
; test so work around it by truncating to i32 for now.
%oldval.trunc = trunc i256 %oldval to i32
ret i32 %oldval.trunc
}
; Try a release atomicrmw sub
; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
define i32 @test_atomicrmw_sub(i256* %addr) {
; CHECK-LABEL: name: test_atomicrmw_sub
; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
; CHECK-NEXT: liveins: $x0
; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_SUB [[ADDR]](p0), [[VAL]] :: (load store release 32 on %ir.addr)
; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
%oldval = atomicrmw sub i256* %addr, i256 1 release
; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
; test so work around it by truncating to i32 for now.
%oldval.trunc = trunc i256 %oldval to i32
ret i32 %oldval.trunc
}
; Try an acq_rel atomicrmw and
; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
define i32 @test_atomicrmw_and(i256* %addr) {
; CHECK-LABEL: name: test_atomicrmw_and
; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
; CHECK-NEXT: liveins: $x0
; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_AND [[ADDR]](p0), [[VAL]] :: (load store acq_rel 32 on %ir.addr)
; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
%oldval = atomicrmw and i256* %addr, i256 1 acq_rel
; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
; test so work around it by truncating to i32 for now.
%oldval.trunc = trunc i256 %oldval to i32
ret i32 %oldval.trunc
}
; Try an seq_cst atomicrmw nand
; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
define i32 @test_atomicrmw_nand(i256* %addr) {
; CHECK-LABEL: name: test_atomicrmw_nand
; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
; CHECK-NEXT: liveins: $x0
; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_NAND [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr)
; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
%oldval = atomicrmw nand i256* %addr, i256 1 seq_cst
; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
; test so work around it by truncating to i32 for now.
%oldval.trunc = trunc i256 %oldval to i32
ret i32 %oldval.trunc
}
; Try an seq_cst atomicrmw or
; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
define i32 @test_atomicrmw_or(i256* %addr) {
; CHECK-LABEL: name: test_atomicrmw_or
; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
; CHECK-NEXT: liveins: $x0
; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_OR [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr)
; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
%oldval = atomicrmw or i256* %addr, i256 1 seq_cst
; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
; test so work around it by truncating to i32 for now.
%oldval.trunc = trunc i256 %oldval to i32
ret i32 %oldval.trunc
}
; Try an seq_cst atomicrmw xor
; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
define i32 @test_atomicrmw_xor(i256* %addr) {
; CHECK-LABEL: name: test_atomicrmw_xor
; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
; CHECK-NEXT: liveins: $x0
; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_XOR [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr)
; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
%oldval = atomicrmw xor i256* %addr, i256 1 seq_cst
; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
; test so work around it by truncating to i32 for now.
%oldval.trunc = trunc i256 %oldval to i32
ret i32 %oldval.trunc
}
; Try an seq_cst atomicrmw min
; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
define i32 @test_atomicrmw_min(i256* %addr) {
; CHECK-LABEL: name: test_atomicrmw_min
; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
; CHECK-NEXT: liveins: $x0
; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_MIN [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr)
; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
%oldval = atomicrmw min i256* %addr, i256 1 seq_cst
; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
; test so work around it by truncating to i32 for now.
%oldval.trunc = trunc i256 %oldval to i32
ret i32 %oldval.trunc
}
; Try an seq_cst atomicrmw max
; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
define i32 @test_atomicrmw_max(i256* %addr) {
; CHECK-LABEL: name: test_atomicrmw_max
; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
; CHECK-NEXT: liveins: $x0
; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_MAX [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr)
; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
%oldval = atomicrmw max i256* %addr, i256 1 seq_cst
; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
; test so work around it by truncating to i32 for now.
%oldval.trunc = trunc i256 %oldval to i32
ret i32 %oldval.trunc
}
; Try an seq_cst atomicrmw unsigned min
; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
define i32 @test_atomicrmw_umin(i256* %addr) {
; CHECK-LABEL: name: test_atomicrmw_umin
; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
; CHECK-NEXT: liveins: $x0
; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_UMIN [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr)
; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
%oldval = atomicrmw umin i256* %addr, i256 1 seq_cst
; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
; test so work around it by truncating to i32 for now.
%oldval.trunc = trunc i256 %oldval to i32
ret i32 %oldval.trunc
}
; Try an seq_cst atomicrmw unsigned max
; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
define i32 @test_atomicrmw_umax(i256* %addr) {
; CHECK-LABEL: name: test_atomicrmw_umax
; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
; CHECK-NEXT: liveins: $x0
; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_UMAX [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr)
; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
%oldval = atomicrmw umax i256* %addr, i256 1 seq_cst
; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
; test so work around it by truncating to i32 for now.
%oldval.trunc = trunc i256 %oldval to i32
ret i32 %oldval.trunc
}