Magic-Mirror/llvm-capstone
1
0
Fork 0
mirror of https://github.com/capstone-engine/llvm-capstone.git synced 2024-11-29 16:41:27 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Add Alignment argument to IRBuilder CreateAtomicRMW and CreateAtomicCmpXchg.

Browse Source 
And then push those change throughout LLVM.

Keep the old signature in Clang's CGBuilder for now -- that will be
updated in a follow-on patch (D97224).

The MLIR LLVM-IR dialect is not updated to support the new alignment
attribute, but preserves its existing behavior.

Differential Revision: https://reviews.llvm.org/D97223
This commit is contained in:
James Y Knight 2021-02-08 23:07:12 -05:00
parent 52c781f6f1
commit 24539f1ef2
10 changed files with 131 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
clang/lib/CodeGen/CGBuilder.h
View File

@ -132,6 +132,28 @@ public:
return CreateAlignedStore(getInt1(Value), Addr, CharUnits::One());
}
// Temporarily use old signature; clang will be updated to an Address overload
// in a subsequent patch.
llvm::AtomicCmpXchgInst *
CreateAtomicCmpXchg(llvm::Value *Ptr, llvm::Value *Cmp, llvm::Value *New,
llvm::AtomicOrdering SuccessOrdering,
llvm::AtomicOrdering FailureOrdering,
llvm::SyncScope::ID SSID = llvm::SyncScope::System) {
return CGBuilderBaseTy::CreateAtomicCmpXchg(
Ptr, Cmp, New, llvm::MaybeAlign(), SuccessOrdering, FailureOrdering,
SSID);
}
// Temporarily use old signature; clang will be updated to an Address overload
// in a subsequent patch.
llvm::AtomicRMWInst *
CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, llvm::Value *Ptr,
llvm::Value *Val, llvm::AtomicOrdering Ordering,
llvm::SyncScope::ID SSID = llvm::SyncScope::System) {
return CGBuilderBaseTy::CreateAtomicRMW(Op, Ptr, Val, llvm::MaybeAlign(),
Ordering, SSID);
}
using CGBuilderBaseTy::CreateBitCast;
Address CreateBitCast(Address Addr, llvm::Type *Ty,
const llvm::Twine &Name = "") {

4
llvm/include/llvm/CodeGen/AtomicExpandUtils.h
View File

@ -22,8 +22,8 @@ class Value;
/// (the builder, %addr, %loaded, %new_val, ordering,
/// /* OUT */ %success, /* OUT */ %new_loaded)
using CreateCmpXchgInstFun =
function_ref<void(IRBuilder<> &, Value *, Value *, Value *, AtomicOrdering,
Value *&, Value *&)>;
function_ref<void(IRBuilder<> &, Value *, Value *, Value *, Align,
AtomicOrdering, Value *&, Value *&)>;
/// Expand an atomic RMW instruction into a loop utilizing
/// cmpxchg. You'll want to make sure your target machine likes cmpxchg

31
llvm/include/llvm/IR/IRBuilder.h
View File

@ -1704,21 +1704,30 @@ public:
return Insert(new FenceInst(Context, Ordering, SSID), Name);
}
AtomicCmpXchgInst *CreateAtomicCmpXchg(
Value *Ptr, Value *Cmp, Value *New, AtomicOrdering SuccessOrdering,
AtomicOrdering FailureOrdering, SyncScope::ID SSID = SyncScope::System) {
const DataLayout &DL = BB->getModule()->getDataLayout();
Align Alignment(DL.getTypeStoreSize(New->getType()));
return Insert(new AtomicCmpXchgInst(
Ptr, Cmp, New, Alignment, SuccessOrdering, FailureOrdering, SSID));
AtomicCmpXchgInst *
CreateAtomicCmpXchg(Value *Ptr, Value *Cmp, Value *New, MaybeAlign Align,
AtomicOrdering SuccessOrdering,
AtomicOrdering FailureOrdering,
SyncScope::ID SSID = SyncScope::System) {
if (!Align) {
const DataLayout &DL = BB->getModule()->getDataLayout();
Align = llvm::Align(DL.getTypeStoreSize(New->getType()));
}
return Insert(new AtomicCmpXchgInst(Ptr, Cmp, New, *Align, SuccessOrdering,
FailureOrdering, SSID));
}
AtomicRMWInst *CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val,
AtomicRMWInst *CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr,
Value *Val, MaybeAlign Align,
AtomicOrdering Ordering,
SyncScope::ID SSID = SyncScope::System) {
const DataLayout &DL = BB->getModule()->getDataLayout();
Align Alignment(DL.getTypeStoreSize(Val->getType()));
return Insert(new AtomicRMWInst(Op, Ptr, Val, Alignment, Ordering, SSID));
if (!Align) {
const DataLayout &DL = BB->getModule()->getDataLayout();
Align = llvm::Align(DL.getTypeStoreSize(Val->getType()));
}
return Insert(new AtomicRMWInst(Op, Ptr, Val, *Align, Ordering, SSID));
}
Value *CreateGEP(Value *Ptr, ArrayRef<Value *> IdxList,

128
llvm/lib/CodeGen/AtomicExpandPass.cpp
View File

@ -80,10 +80,11 @@ namespace {
bool tryExpandAtomicRMW(AtomicRMWInst *AI);
Value *
insertRMWLLSCLoop(IRBuilder<> &Builder, Type *ResultTy, Value *Addr,
AtomicOrdering MemOpOrder,
Align AddrAlign, AtomicOrdering MemOpOrder,
function_ref<Value *(IRBuilder<> &, Value *)> PerformOp);
void expandAtomicOpToLLSC(
Instruction *I, Type *ResultTy, Value *Addr, AtomicOrdering MemOpOrder,
Instruction *I, Type *ResultTy, Value *Addr, Align AddrAlign,
AtomicOrdering MemOpOrder,
function_ref<Value *(IRBuilder<> &, Value *)> PerformOp);
void expandPartwordAtomicRMW(
AtomicRMWInst *I,
@ -95,7 +96,7 @@ namespace {
AtomicCmpXchgInst *convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI);
static Value *insertRMWCmpXchgLoop(
IRBuilder<> &Builder, Type *ResultType, Value *Addr,
IRBuilder<> &Builder, Type *ResultType, Value *Addr, Align AddrAlign,
AtomicOrdering MemOpOrder,
function_ref<Value *(IRBuilder<> &, Value *)> PerformOp,
CreateCmpXchgInstFun CreateCmpXchg);
@ -368,7 +369,8 @@ bool AtomicExpand::tryExpandAtomicLoad(LoadInst *LI) {
return false;
case TargetLoweringBase::AtomicExpansionKind::LLSC:
expandAtomicOpToLLSC(
LI, LI->getType(), LI->getPointerOperand(), LI->getOrdering(),
LI, LI->getType(), LI->getPointerOperand(), LI->getAlign(),
LI->getOrdering(),
[](IRBuilder<> &Builder, Value *Loaded) { return Loaded; });
return true;
case TargetLoweringBase::AtomicExpansionKind::LLOnly:
@ -407,7 +409,7 @@ bool AtomicExpand::expandAtomicLoadToCmpXchg(LoadInst *LI) {
Constant *DummyVal = Constant::getNullValue(Ty);
Value *Pair = Builder.CreateAtomicCmpXchg(
Addr, DummyVal, DummyVal, Order,
Addr, DummyVal, DummyVal, LI->getAlign(), Order,
AtomicCmpXchgInst::getStrongestFailureOrdering(Order));
Value *Loaded = Builder.CreateExtractValue(Pair, 0, "loaded");
@ -454,9 +456,9 @@ bool AtomicExpand::expandAtomicStore(StoreInst *SI) {
// It is the responsibility of the target to only signal expansion via
// shouldExpandAtomicRMW in cases where this is required and possible.
IRBuilder<> Builder(SI);
AtomicRMWInst *AI =
Builder.CreateAtomicRMW(AtomicRMWInst::Xchg, SI->getPointerOperand(),
SI->getValueOperand(), SI->getOrdering());
AtomicRMWInst *AI = Builder.CreateAtomicRMW(
AtomicRMWInst::Xchg, SI->getPointerOperand(), SI->getValueOperand(),
SI->getAlign(), SI->getOrdering());
SI->eraseFromParent();
// Now we have an appropriate swap instruction, lower it as usual.
@ -464,9 +466,9 @@ bool AtomicExpand::expandAtomicStore(StoreInst *SI) {
}
static void createCmpXchgInstFun(IRBuilder<> &Builder, Value *Addr,
Value *Loaded, Value *NewVal,
AtomicOrdering MemOpOrder,
Value *&Success, Value *&NewLoaded) {
Value *Loaded, Value *NewVal, Align AddrAlign,
AtomicOrdering MemOpOrder, Value *&Success,
Value *&NewLoaded) {
Type *OrigTy = NewVal->getType();
// This code can go away when cmpxchg supports FP types.
@ -479,8 +481,8 @@ static void createCmpXchgInstFun(IRBuilder<> &Builder, Value *Addr,
Loaded = Builder.CreateBitCast(Loaded, IntTy);
}
Value* Pair = Builder.CreateAtomicCmpXchg(
Addr, Loaded, NewVal, MemOpOrder,
Value *Pair = Builder.CreateAtomicCmpXchg(
Addr, Loaded, NewVal, AddrAlign, MemOpOrder,
AtomicCmpXchgInst::getStrongestFailureOrdering(MemOpOrder));
Success = Builder.CreateExtractValue(Pair, 1, "success");
NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded");
@ -546,7 +548,7 @@ bool AtomicExpand::tryExpandAtomicRMW(AtomicRMWInst *AI) {
AI->getValOperand());
};
expandAtomicOpToLLSC(AI, AI->getType(), AI->getPointerOperand(),
AI->getOrdering(), PerformOp);
AI->getAlign(), AI->getOrdering(), PerformOp);
}
return true;
}
@ -581,6 +583,7 @@ struct PartwordMaskValues {
Type *WordType = nullptr;
Type *ValueType = nullptr;
Value *AlignedAddr = nullptr;
Align AlignedAddrAlignment;
// The remaining fields can be null.
Value *ShiftAmt = nullptr;
Value *Mask = nullptr;
@ -603,6 +606,7 @@ raw_ostream &operator<<(raw_ostream &O, const PartwordMaskValues &PMV) {
PrintObj(PMV.ValueType);
O << " AlignedAddr: ";
PrintObj(PMV.AlignedAddr);
O << " AlignedAddrAlignment: " << PMV.AlignedAddrAlignment.value() << '\n';
O << " ShiftAmt: ";
PrintObj(PMV.ShiftAmt);
O << " Mask: ";
@ -633,6 +637,7 @@ raw_ostream &operator<<(raw_ostream &O, const PartwordMaskValues &PMV) {
/// Inv_Mask: The inverse of Mask.
static PartwordMaskValues createMaskInstrs(IRBuilder<> &Builder, Instruction *I,
Type *ValueType, Value *Addr,
Align AddrAlign,
unsigned MinWordSize) {
PartwordMaskValues PMV;
@ -646,6 +651,7 @@ static PartwordMaskValues createMaskInstrs(IRBuilder<> &Builder, Instruction *I,
: ValueType;
if (PMV.ValueType == PMV.WordType) {
PMV.AlignedAddr = Addr;
PMV.AlignedAddrAlignment = AddrAlign;
return PMV;
}
@ -654,10 +660,12 @@ static PartwordMaskValues createMaskInstrs(IRBuilder<> &Builder, Instruction *I,
Type *WordPtrType =
PMV.WordType->getPointerTo(Addr->getType()->getPointerAddressSpace());
// TODO: we could skip some of this if AddrAlign >= MinWordSize.
Value *AddrInt = Builder.CreatePtrToInt(Addr, DL.getIntPtrType(Ctx));
PMV.AlignedAddr = Builder.CreateIntToPtr(
Builder.CreateAnd(AddrInt, ~(uint64_t)(MinWordSize - 1)), WordPtrType,
"AlignedAddr");
PMV.AlignedAddrAlignment = Align(MinWordSize);
Value *PtrLSB = Builder.CreateAnd(AddrInt, MinWordSize - 1, "PtrLSB");
if (DL.isLittleEndian()) {
@ -765,7 +773,7 @@ void AtomicExpand::expandPartwordAtomicRMW(
PartwordMaskValues PMV =
createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
TLI->getMinCmpXchgSizeInBits() / 8);
AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
Value *ValOperand_Shifted =
Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType),
@ -778,13 +786,14 @@ void AtomicExpand::expandPartwordAtomicRMW(
Value *OldResult;
if (ExpansionKind == TargetLoweringBase::AtomicExpansionKind::CmpXChg) {
OldResult =
insertRMWCmpXchgLoop(Builder, PMV.WordType, PMV.AlignedAddr, MemOpOrder,
PerformPartwordOp, createCmpXchgInstFun);
OldResult = insertRMWCmpXchgLoop(Builder, PMV.WordType, PMV.AlignedAddr,
PMV.AlignedAddrAlignment, MemOpOrder,
PerformPartwordOp, createCmpXchgInstFun);
} else {
assert(ExpansionKind == TargetLoweringBase::AtomicExpansionKind::LLSC);
OldResult = insertRMWLLSCLoop(Builder, PMV.WordType, PMV.AlignedAddr,
MemOpOrder, PerformPartwordOp);
PMV.AlignedAddrAlignment, MemOpOrder,
PerformPartwordOp);
}
Value *FinalOldResult = extractMaskedValue(Builder, OldResult, PMV);
@ -803,7 +812,7 @@ AtomicRMWInst *AtomicExpand::widenPartwordAtomicRMW(AtomicRMWInst *AI) {
PartwordMaskValues PMV =
createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
TLI->getMinCmpXchgSizeInBits() / 8);
AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
Value *ValOperand_Shifted =
Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType),
@ -817,8 +826,9 @@ AtomicRMWInst *AtomicExpand::widenPartwordAtomicRMW(AtomicRMWInst *AI) {
else
NewOperand = ValOperand_Shifted;
AtomicRMWInst *NewAI = Builder.CreateAtomicRMW(Op, PMV.AlignedAddr,
NewOperand, AI->getOrdering());
AtomicRMWInst *NewAI =
Builder.CreateAtomicRMW(Op, PMV.AlignedAddr, NewOperand,
PMV.AlignedAddrAlignment, AI->getOrdering());
Value *FinalOldResult = extractMaskedValue(Builder, NewAI, PMV);
AI->replaceAllUsesWith(FinalOldResult);
@ -871,8 +881,6 @@ bool AtomicExpand::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {
IRBuilder<> Builder(CI);
LLVMContext &Ctx = Builder.getContext();
const int WordSize = TLI->getMinCmpXchgSizeInBits() / 8;
BasicBlock *EndBB =
BB->splitBasicBlock(CI->getIterator(), "partword.cmpxchg.end");
auto FailureBB =
@ -884,8 +892,9 @@ bool AtomicExpand::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {
std::prev(BB->end())->eraseFromParent();
Builder.SetInsertPoint(BB);
PartwordMaskValues PMV = createMaskInstrs(
Builder, CI, CI->getCompareOperand()->getType(), Addr, WordSize);
PartwordMaskValues PMV =
createMaskInstrs(Builder, CI, CI->getCompareOperand()->getType(), Addr,
CI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
// Shift the incoming values over, into the right location in the word.
Value *NewVal_Shifted =
@ -909,8 +918,8 @@ bool AtomicExpand::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {
Value *FullWord_NewVal = Builder.CreateOr(Loaded_MaskOut, NewVal_Shifted);
Value *FullWord_Cmp = Builder.CreateOr(Loaded_MaskOut, Cmp_Shifted);
AtomicCmpXchgInst *NewCI = Builder.CreateAtomicCmpXchg(
PMV.AlignedAddr, FullWord_Cmp, FullWord_NewVal, CI->getSuccessOrdering(),
CI->getFailureOrdering(), CI->getSyncScopeID());
PMV.AlignedAddr, FullWord_Cmp, FullWord_NewVal, PMV.AlignedAddrAlignment,
CI->getSuccessOrdering(), CI->getFailureOrdering(), CI->getSyncScopeID());
NewCI->setVolatile(CI->isVolatile());
// When we're building a strong cmpxchg, we need a loop, so you
// might think we could use a weak cmpxchg inside. But, using strong
@ -953,11 +962,12 @@ bool AtomicExpand::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {
}
void AtomicExpand::expandAtomicOpToLLSC(
Instruction *I, Type *ResultType, Value *Addr, AtomicOrdering MemOpOrder,
Instruction *I, Type *ResultType, Value *Addr, Align AddrAlign,
AtomicOrdering MemOpOrder,
function_ref<Value *(IRBuilder<> &, Value *)> PerformOp) {
IRBuilder<> Builder(I);
Value *Loaded =
insertRMWLLSCLoop(Builder, ResultType, Addr, MemOpOrder, PerformOp);
Value *Loaded = insertRMWLLSCLoop(Builder, ResultType, Addr, AddrAlign,
MemOpOrder, PerformOp);
I->replaceAllUsesWith(Loaded);
I->eraseFromParent();
@ -968,7 +978,7 @@ void AtomicExpand::expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI) {
PartwordMaskValues PMV =
createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
TLI->getMinCmpXchgSizeInBits() / 8);
AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
// The value operand must be sign-extended for signed min/max so that the
// target's signed comparison instructions can be used. Otherwise, just
@ -994,7 +1004,7 @@ void AtomicExpand::expandAtomicCmpXchgToMaskedIntrinsic(AtomicCmpXchgInst *CI) {
PartwordMaskValues PMV = createMaskInstrs(
Builder, CI, CI->getCompareOperand()->getType(), CI->getPointerOperand(),
TLI->getMinCmpXchgSizeInBits() / 8);
CI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
Value *CmpVal_Shifted = Builder.CreateShl(
Builder.CreateZExt(CI->getCompareOperand(), PMV.WordType), PMV.ShiftAmt,
@ -1017,13 +1027,16 @@ void AtomicExpand::expandAtomicCmpXchgToMaskedIntrinsic(AtomicCmpXchgInst *CI) {
}
Value *AtomicExpand::insertRMWLLSCLoop(
IRBuilder<> &Builder, Type *ResultTy, Value *Addr,
IRBuilder<> &Builder, Type *ResultTy, Value *Addr, Align AddrAlign,
AtomicOrdering MemOpOrder,
function_ref<Value *(IRBuilder<> &, Value *)> PerformOp) {
LLVMContext &Ctx = Builder.getContext();
BasicBlock *BB = Builder.GetInsertBlock();
Function *F = BB->getParent();
assert(AddrAlign >= ResultTy->getPrimitiveSizeInBits() / 8 &&
"Expected at least natural alignment at this point.");
// Given: atomicrmw some_op iN* %addr, iN %incr ordering
//
// The standard expansion we produce is:
@ -1082,11 +1095,9 @@ AtomicCmpXchgInst *AtomicExpand::convertCmpXchgToIntegerType(AtomicCmpXchgInst *
Value *NewCmp = Builder.CreatePtrToInt(CI->getCompareOperand(), NewTy);
Value *NewNewVal = Builder.CreatePtrToInt(CI->getNewValOperand(), NewTy);
auto *NewCI = Builder.CreateAtomicCmpXchg(NewAddr, NewCmp, NewNewVal,
CI->getSuccessOrdering(),
CI->getFailureOrdering(),
CI->getSyncScopeID());
auto *NewCI = Builder.CreateAtomicCmpXchg(
NewAddr, NewCmp, NewNewVal, CI->getAlign(), CI->getSuccessOrdering(),
CI->getFailureOrdering(), CI->getSyncScopeID());
NewCI->setVolatile(CI->isVolatile());
NewCI->setWeak(CI->isWeak());
LLVM_DEBUG(dbgs() << "Replaced " << *CI << " with " << *NewCI << "\n");
@ -1211,7 +1222,7 @@ bool AtomicExpand::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
PartwordMaskValues PMV =
createMaskInstrs(Builder, CI, CI->getCompareOperand()->getType(), Addr,
TLI->getMinCmpXchgSizeInBits() / 8);
CI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
Builder.CreateBr(StartBB);
// Start the main loop block now that we've taken care of the preliminaries.
@ -1379,7 +1390,7 @@ bool AtomicExpand::simplifyIdempotentRMW(AtomicRMWInst* RMWI) {
}
Value *AtomicExpand::insertRMWCmpXchgLoop(
IRBuilder<> &Builder, Type *ResultTy, Value *Addr,
IRBuilder<> &Builder, Type *ResultTy, Value *Addr, Align AddrAlign,
AtomicOrdering MemOpOrder,
function_ref<Value *(IRBuilder<> &, Value *)> PerformOp,
CreateCmpXchgInstFun CreateCmpXchg) {
@ -1411,9 +1422,7 @@ Value *AtomicExpand::insertRMWCmpXchgLoop(
// the branch entirely.
std::prev(BB->end())->eraseFromParent();
Builder.SetInsertPoint(BB);
LoadInst *InitLoaded = Builder.CreateLoad(ResultTy, Addr);
// Atomics require at least natural alignment.
InitLoaded->setAlignment(Align(ResultTy->getPrimitiveSizeInBits() / 8));
LoadInst *InitLoaded = Builder.CreateAlignedLoad(ResultTy, Addr, AddrAlign);
Builder.CreateBr(LoopBB);
// Start the main loop block now that we've taken care of the preliminaries.
@ -1426,7 +1435,7 @@ Value *AtomicExpand::insertRMWCmpXchgLoop(
Value *NewLoaded = nullptr;
Value *Success = nullptr;
CreateCmpXchg(Builder, Addr, Loaded, NewVal,
CreateCmpXchg(Builder, Addr, Loaded, NewVal, AddrAlign,
MemOpOrder == AtomicOrdering::Unordered
? AtomicOrdering::Monotonic
: MemOpOrder,
@ -1466,7 +1475,8 @@ bool llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
CreateCmpXchgInstFun CreateCmpXchg) {
IRBuilder<> Builder(AI);
Value *Loaded = AtomicExpand::insertRMWCmpXchgLoop(
Builder, AI->getType(), AI->getPointerOperand(), AI->getOrdering(),
Builder, AI->getType(), AI->getPointerOperand(), AI->getAlign(),
AI->getOrdering(),
[&](IRBuilder<> &Builder, Value *Loaded) {
return performAtomicOp(AI->getOperation(), Builder, Loaded,
AI->getValOperand());
@ -1614,20 +1624,20 @@ void AtomicExpand::expandAtomicRMWToLibcall(AtomicRMWInst *I) {
// libcalls (add/sub/etc) and we needed a generic. So, expand to a
// CAS libcall, via a CAS loop, instead.
if (!Success) {
expandAtomicRMWToCmpXchg(I, [this](IRBuilder<> &Builder, Value *Addr,
Value *Loaded, Value *NewVal,
AtomicOrdering MemOpOrder,
Value *&Success, Value *&NewLoaded) {
// Create the CAS instruction normally...
AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg(
Addr, Loaded, NewVal, MemOpOrder,
AtomicCmpXchgInst::getStrongestFailureOrdering(MemOpOrder));
Success = Builder.CreateExtractValue(Pair, 1, "success");
NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded");
expandAtomicRMWToCmpXchg(
I, [this](IRBuilder<> &Builder, Value *Addr, Value *Loaded,
Value *NewVal, Align Alignment, AtomicOrdering MemOpOrder,
Value *&Success, Value *&NewLoaded) {
// Create the CAS instruction normally...
AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg(
Addr, Loaded, NewVal, Alignment, MemOpOrder,
AtomicCmpXchgInst::getStrongestFailureOrdering(MemOpOrder));
Success = Builder.CreateExtractValue(Pair, 1, "success");
NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded");
// ...and then expand the CAS into a libcall.
expandAtomicCASToLibcall(Pair);
});
// ...and then expand the CAS into a libcall.
expandAtomicCASToLibcall(Pair);
});
}
}

2
llvm/lib/IR/AutoUpgrade.cpp
View File

@ -3594,7 +3594,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
Name.startswith("atomic.load.add.f64.p"))) {
Value *Ptr = CI->getArgOperand(0);
Value *Val = CI->getArgOperand(1);
Rep = Builder.CreateAtomicRMW(AtomicRMWInst::FAdd, Ptr, Val,
Rep = Builder.CreateAtomicRMW(AtomicRMWInst::FAdd, Ptr, Val, MaybeAlign(),
AtomicOrdering::SequentiallyConsistent);
} else if (IsNVVM && (Name == "max.i" || Name == "max.ll" ||
Name == "max.ui" || Name == "max.ull")) {

16
llvm/lib/IR/Core.cpp
View File

@ -3977,9 +3977,10 @@ LLVMValueRef LLVMBuildAtomicRMW(LLVMBuilderRef B,LLVMAtomicRMWBinOp op,
LLVMAtomicOrdering ordering,
LLVMBool singleThread) {
AtomicRMWInst::BinOp intop = mapFromLLVMRMWBinOp(op);
return wrap(unwrap(B)->CreateAtomicRMW(intop, unwrap(PTR), unwrap(Val),
mapFromLLVMOrdering(ordering), singleThread ? SyncScope::SingleThread
: SyncScope::System));
return wrap(unwrap(B)->CreateAtomicRMW(
intop, unwrap(PTR), unwrap(Val), MaybeAlign(),
mapFromLLVMOrdering(ordering),
singleThread ? SyncScope::SingleThread : SyncScope::System));
}
LLVMValueRef LLVMBuildAtomicCmpXchg(LLVMBuilderRef B, LLVMValueRef Ptr,
@ -3988,10 +3989,11 @@ LLVMValueRef LLVMBuildAtomicCmpXchg(LLVMBuilderRef B, LLVMValueRef Ptr,
LLVMAtomicOrdering FailureOrdering,
LLVMBool singleThread) {
return wrap(unwrap(B)->CreateAtomicCmpXchg(unwrap(Ptr), unwrap(Cmp),
unwrap(New), mapFromLLVMOrdering(SuccessOrdering),
mapFromLLVMOrdering(FailureOrdering),
singleThread ? SyncScope::SingleThread : SyncScope::System));
return wrap(unwrap(B)->CreateAtomicCmpXchg(
unwrap(Ptr), unwrap(Cmp), unwrap(New), MaybeAlign(),
mapFromLLVMOrdering(SuccessOrdering),
mapFromLLVMOrdering(FailureOrdering),
singleThread ? SyncScope::SingleThread : SyncScope::System));
}
unsigned LLVMGetNumMaskElements(LLVMValueRef SVInst) {

2
llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp
View File

@ -965,7 +965,7 @@ bool GCOVProfiler::emitProfileNotes(
Counters->getValueType(), Counters, 0, I);
if (Options.Atomic) {
Builder.CreateAtomicRMW(AtomicRMWInst::Add, V, Builder.getInt64(1),
AtomicOrdering::Monotonic);
MaybeAlign(), AtomicOrdering::Monotonic);
} else {
Value *Count =
Builder.CreateLoad(Builder.getInt64Ty(), V, "gcov_ctr");

2
llvm/lib/Transforms/Instrumentation/InstrOrderFile.cpp
View File

@ -144,7 +144,7 @@ public:
// Fill up UpdateOrderFileBB: grab the index, update the buffer!
Value *IdxVal = updateB.CreateAtomicRMW(
AtomicRMWInst::Add, BufferIdx, ConstantInt::get(Int32Ty, 1),
AtomicOrdering::SequentiallyConsistent);
MaybeAlign(), AtomicOrdering::SequentiallyConsistent);
// We need to wrap around the index to fit it inside the buffer.
Value *WrappedIdx = updateB.CreateAnd(
IdxVal, ConstantInt::get(Int32Ty, INSTR_ORDER_FILE_BUFFER_MASK));

3
llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
View File

@ -205,6 +205,7 @@ public:
// automic update currently can only be promoted across the current
// loop, not the whole loop nest.
Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue,
MaybeAlign(),
AtomicOrdering::SequentiallyConsistent);
else {
LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted");
@ -702,7 +703,7 @@ void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
if (Options.Atomic || AtomicCounterUpdateAll ||
(Index == 0 && AtomicFirstCounter)) {
Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(),
AtomicOrdering::Monotonic);
MaybeAlign(), AtomicOrdering::Monotonic);
} else {
Value *IncStep = Inc->getStep();
Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount");

5
mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
View File

@ -1526,6 +1526,7 @@ def AtomicOrdering : I64EnumAttr<
def LLVM_AtomicRMWType : AnyTypeOf<[LLVM_AnyFloat, LLVM_AnyInteger]>;
// FIXME: Need to add alignment attribute to MLIR atomicrmw operation.
def LLVM_AtomicRMWOp : LLVM_Op<"atomicrmw"> {
let arguments = (ins AtomicBinOp:$bin_op,
LLVM_PointerTo<LLVM_AtomicRMWType>:$ptr,
@ -1533,6 +1534,7 @@ def LLVM_AtomicRMWOp : LLVM_Op<"atomicrmw"> {
let results = (outs LLVM_AtomicRMWType:$res);
let llvmBuilder = [{
$res = builder.CreateAtomicRMW(getLLVMAtomicBinOp($bin_op), $ptr, $val,
llvm::MaybeAlign(),
getLLVMAtomicOrdering($ordering));
}];
let parser = [{ return parseAtomicRMWOp(parser, result); }];
@ -1553,6 +1555,7 @@ def LLVM_AtomicCmpXchgResultType : Type<And<[
"an LLVM struct type with any integer or pointer followed by a single-bit "
"integer">;
// FIXME: Need to add alignment attribute to MLIR cmpxchg operation.
def LLVM_AtomicCmpXchgOp : LLVM_Op<"cmpxchg"> {
let arguments = (ins LLVM_PointerTo<LLVM_AtomicCmpXchgType>:$ptr,
LLVM_AtomicCmpXchgType:$cmp, LLVM_AtomicCmpXchgType:$val,
@ -1560,7 +1563,7 @@ def LLVM_AtomicCmpXchgOp : LLVM_Op<"cmpxchg"> {
AtomicOrdering:$failure_ordering);
let results = (outs LLVM_AtomicCmpXchgResultType:$res);
let llvmBuilder = [{
$res = builder.CreateAtomicCmpXchg($ptr, $cmp, $val,
$res = builder.CreateAtomicCmpXchg($ptr, $cmp, $val, llvm::MaybeAlign(),
getLLVMAtomicOrdering($success_ordering),
getLLVMAtomicOrdering($failure_ordering));
}];