From 2aff7f39199f214ee606e4e18a8eba178ffe0d94 Mon Sep 17 00:00:00 2001 From: Krzysztof Drewniak Date: Thu, 4 Jan 2024 09:33:09 -0600 Subject: [PATCH] [mlir][LLVM] Add !invariant.load metadata support to llvm.load (#76754) Add support for !invariant.load metadata (by way of a unit attribute) to the MLIR representation of llvm.load. --- mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td | 9 ++++++++- mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 6 +++--- mlir/test/Dialect/LLVMIR/roundtrip.mlir | 7 +++++++ mlir/test/Target/LLVMIR/Import/instructions.ll | 13 +++++++++++++ mlir/test/Target/LLVMIR/llvmir.mlir | 11 +++++++++++ 5 files changed, 42 insertions(+), 4 deletions(-) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td index 9e65898154bd..2376f1f0e2dc 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td @@ -330,6 +330,7 @@ def LLVM_LoadOp : LLVM_MemAccessOpBase<"load", OptionalAttr:$alignment, UnitAttr:$volatile_, UnitAttr:$nontemporal, + UnitAttr:$invariant, DefaultValuedAttr< AtomicOrdering, "AtomicOrdering::not_atomic">:$ordering, OptionalAttr:$syncscope); @@ -364,11 +365,16 @@ def LLVM_LoadOp : LLVM_MemAccessOpBase<"load", let assemblyFormat = [{ (`volatile` $volatile_^)? $addr (`atomic` (`syncscope` `(` $syncscope^ `)`)? $ordering^)? + (`invariant` $invariant^)? attr-dict `:` qualified(type($addr)) `->` type($res) }]; string llvmBuilder = [{ auto *inst = builder.CreateLoad($_resultType, $addr, $volatile_); $res = inst; + if ($invariant) { + llvm::MDNode *metadata = llvm::MDNode::get(inst->getContext(), std::nullopt); + inst->setMetadata(llvm::LLVMContext::MD_invariant_load, metadata); + } }] # setOrderingCode # setSyncScopeCode # setAlignmentCode @@ -381,13 +387,14 @@ def LLVM_LoadOp : LLVM_MemAccessOpBase<"load", $res = $_builder.create($_location, $_resultType, $addr, alignment, loadInst->isVolatile(), loadInst->hasMetadata(llvm::LLVMContext::MD_nontemporal), + loadInst->hasMetadata(llvm::LLVMContext::MD_invariant_load), convertAtomicOrderingFromLLVM(loadInst->getOrdering()), getLLVMSyncScope(loadInst)); }]; let builders = [ OpBuilder<(ins "Type":$type, "Value":$addr, CArg<"unsigned", "0">:$alignment, CArg<"bool", "false">:$isVolatile, - CArg<"bool", "false">:$isNonTemporal, + CArg<"bool", "false">:$isNonTemporal, CArg<"bool", "false">:$isInvariant, CArg<"AtomicOrdering", "AtomicOrdering::not_atomic">:$ordering, CArg<"StringRef", "StringRef()">:$syncscope)> ]; diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp index 64388a9a0181..c2336f44b33e 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -822,11 +822,11 @@ LogicalResult LoadOp::verify() { void LoadOp::build(OpBuilder &builder, OperationState &state, Type type, Value addr, unsigned alignment, bool isVolatile, - bool isNonTemporal, AtomicOrdering ordering, - StringRef syncscope) { + bool isNonTemporal, bool isInvariant, + AtomicOrdering ordering, StringRef syncscope) { build(builder, state, type, addr, alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile, - isNonTemporal, ordering, + isNonTemporal, isInvariant, ordering, syncscope.empty() ? nullptr : builder.getStringAttr(syncscope), /*access_groups=*/nullptr, /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, diff --git a/mlir/test/Dialect/LLVMIR/roundtrip.mlir b/mlir/test/Dialect/LLVMIR/roundtrip.mlir index 49f34785ebad..1958dd56bab7 100644 --- a/mlir/test/Dialect/LLVMIR/roundtrip.mlir +++ b/mlir/test/Dialect/LLVMIR/roundtrip.mlir @@ -401,6 +401,13 @@ func.func @cmpxchg(%ptr : !llvm.ptr, %cmp : i32, %new : i32) { llvm.return } +// CHECK-LABEL: @invariant_load +func.func @invariant_load(%ptr : !llvm.ptr) -> i32 { + // CHECK: llvm.load %{{.+}} invariant {alignment = 4 : i64} : !llvm.ptr -> i32 + %0 = llvm.load %ptr invariant {alignment = 4 : i64} : !llvm.ptr -> i32 + func.return %0 : i32 +} + llvm.mlir.global external constant @_ZTIi() : !llvm.ptr llvm.func @bar(!llvm.ptr, !llvm.ptr, !llvm.ptr) llvm.func @__gxx_personality_v0(...) -> i32 diff --git a/mlir/test/Target/LLVMIR/Import/instructions.ll b/mlir/test/Target/LLVMIR/Import/instructions.ll index 036efad0d099..005aafb20a51 100644 --- a/mlir/test/Target/LLVMIR/Import/instructions.ll +++ b/mlir/test/Target/LLVMIR/Import/instructions.ll @@ -370,6 +370,19 @@ define void @load_store(ptr %ptr) { ; // ----- +; CHECK-LABEL: @invariant_load +; CHECK-SAME: %[[PTR:[a-zA-Z0-9]+]] +define float @invariant_load(ptr %ptr) { + ; CHECK: %[[V:[0-9]+]] = llvm.load %[[PTR]] invariant {alignment = 4 : i64} : !llvm.ptr -> f32 + %1 = load float, ptr %ptr, align 4, !invariant.load !0 + ; CHECK: llvm.return %[[V]] + ret float %1 +} + +!0 = !{} + +; // ----- + ; CHECK-LABEL: @atomic_load_store ; CHECK-SAME: %[[PTR:[a-zA-Z0-9]+]] define void @atomic_load_store(ptr %ptr) { diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir index 13e61b6ce10b..4a036b0497ff 100644 --- a/mlir/test/Target/LLVMIR/llvmir.mlir +++ b/mlir/test/Target/LLVMIR/llvmir.mlir @@ -1911,6 +1911,17 @@ llvm.func @nontemporal_store_and_load() { // ----- +// Check that invariantLoad attribute is exported as metadata node. +llvm.func @nontemporal_store_and_load(%ptr : !llvm.ptr) -> i32 { + // CHECK: !invariant.load ![[NODE:[0-9]+]] + %1 = llvm.load %ptr invariant : !llvm.ptr -> i32 + llvm.return %1 : i32 +} + +// CHECK: ![[NODE]] = !{} + +// ----- + llvm.func @atomic_store_and_load(%ptr : !llvm.ptr) { // CHECK: load atomic // CHECK-SAME: acquire, align 4