From 02f640672e2875c4e7578366bb2e22582548d7c1 Mon Sep 17 00:00:00 2001 From: Yang Keao Date: Wed, 1 Jun 2022 13:09:58 -0500 Subject: [PATCH] [Polly] Migrate -polly-mse to the new pass manager. This patch implements the `MaximalStaticExpansion` and its printer in NPM. Reviewed By: Meinersbur Differential Revision: https://reviews.llvm.org/D125870 --- polly/include/polly/LinkAllPasses.h | 2 +- polly/include/polly/MaximalStaticExpansion.h | 42 + polly/lib/Support/PollyPasses.def | 2 + polly/lib/Support/RegisterPasses.cpp | 3 +- .../lib/Transform/MaximalStaticExpansion.cpp | 779 ++++++++++-------- .../load_after_store_same_statement.ll | 2 + .../read_from_original.ll | 2 + .../MaximalStaticExpansion/too_many_writes.ll | 2 + .../working_deps_between_inners.ll | 1 + .../working_deps_between_inners_phi.ll | 2 + .../working_expansion.ll | 1 + ...sion_multiple_dependences_per_statement.ll | 1 + ...sion_multiple_instruction_per_statement.ll | 1 + .../working_phi_expansion.ll | 2 + .../working_phi_two_scalars.ll | 2 + .../working_value_expansion.ll | 1 + 16 files changed, 487 insertions(+), 358 deletions(-) create mode 100644 polly/include/polly/MaximalStaticExpansion.h diff --git a/polly/include/polly/LinkAllPasses.h b/polly/include/polly/LinkAllPasses.h index 3fced85f527b..71a74e684576 100644 --- a/polly/include/polly/LinkAllPasses.h +++ b/polly/include/polly/LinkAllPasses.h @@ -162,7 +162,7 @@ void initializeManagedMemoryRewritePassPass(llvm::PassRegistry &); #endif void initializeIslScheduleOptimizerWrapperPassPass(llvm::PassRegistry &); void initializeIslScheduleOptimizerPrinterLegacyPassPass(llvm::PassRegistry &); -void initializeMaximalStaticExpanderPass(llvm::PassRegistry &); +void initializeMaximalStaticExpanderWrapperPassPass(llvm::PassRegistry &); void initializePollyCanonicalizePass(llvm::PassRegistry &); void initializeFlattenSchedulePass(llvm::PassRegistry &); void initializeFlattenSchedulePrinterLegacyPassPass(llvm::PassRegistry &); diff --git a/polly/include/polly/MaximalStaticExpansion.h b/polly/include/polly/MaximalStaticExpansion.h new file mode 100644 index 000000000000..88827b270088 --- /dev/null +++ b/polly/include/polly/MaximalStaticExpansion.h @@ -0,0 +1,42 @@ +//===- polly/MaximalStaticExpansion.h - expand memory access -*- C++ -*-======// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This pass fully expand the memory accesses of a Scop to get rid of +// dependencies. +// +//===----------------------------------------------------------------------===// + +#ifndef POLLY_MAXIMALSTATICEXPANSION_H +#define POLLY_MAXIMALSTATICEXPANSION_H + +#include "polly/ScopPass.h" +#include "llvm/IR/PassManager.h" + +namespace polly { + +class MaximalStaticExpansionPass + : public llvm::PassInfoMixin { +public: + llvm::PreservedAnalyses run(Scop &, ScopAnalysisManager &, + ScopStandardAnalysisResults &, SPMUpdater &); +}; + +struct MaximalStaticExpansionPrinterPass + : llvm::PassInfoMixin { + MaximalStaticExpansionPrinterPass(raw_ostream &OS) : OS(OS) {} + + PreservedAnalyses run(Scop &S, ScopAnalysisManager &, + ScopStandardAnalysisResults &SAR, SPMUpdater &); + +private: + llvm::raw_ostream &OS; +}; + +} // namespace polly + +#endif /* POLLY_MAXIMALSTATICEXPANSION_H */ diff --git a/polly/lib/Support/PollyPasses.def b/polly/lib/Support/PollyPasses.def index 9eb667b98cff..e068f31fdb70 100644 --- a/polly/lib/Support/PollyPasses.def +++ b/polly/lib/Support/PollyPasses.def @@ -43,4 +43,6 @@ SCOP_PASS("polly-prune-unprofitable", PruneUnprofitablePass()) SCOP_PASS("polly-opt-isl", IslScheduleOptimizerPass()) SCOP_PASS("print", IslScheduleOptimizerPrinterPass(llvm::outs())) SCOP_PASS("polly-dce", DeadCodeElimPass()) +SCOP_PASS("polly-mse", MaximalStaticExpansionPass()) +SCOP_PASS("print", MaximalStaticExpansionPrinterPass(llvm::outs())) #undef SCOP_PASS diff --git a/polly/lib/Support/RegisterPasses.cpp b/polly/lib/Support/RegisterPasses.cpp index e5fbb280db8c..573393b6e465 100644 --- a/polly/lib/Support/RegisterPasses.cpp +++ b/polly/lib/Support/RegisterPasses.cpp @@ -30,6 +30,7 @@ #include "polly/ForwardOpTree.h" #include "polly/JSONExporter.h" #include "polly/LinkAllPasses.h" +#include "polly/MaximalStaticExpansion.h" #include "polly/PolyhedralInfo.h" #include "polly/PruneUnprofitable.h" #include "polly/ScheduleOptimizer.h" @@ -263,7 +264,7 @@ void initializePollyPasses(llvm::PassRegistry &Registry) { initializeJSONExporterPass(Registry); initializeJSONImporterPass(Registry); initializeJSONImporterPrinterLegacyPassPass(Registry); - initializeMaximalStaticExpanderPass(Registry); + initializeMaximalStaticExpanderWrapperPassPass(Registry); initializeIslAstInfoWrapperPassPass(Registry); initializeIslAstInfoPrinterLegacyPassPass(Registry); initializeIslScheduleOptimizerWrapperPassPass(Registry); diff --git a/polly/lib/Transform/MaximalStaticExpansion.cpp b/polly/lib/Transform/MaximalStaticExpansion.cpp index c1baa2080b46..dd1bae11d657 100644 --- a/polly/lib/Transform/MaximalStaticExpansion.cpp +++ b/polly/lib/Transform/MaximalStaticExpansion.cpp @@ -11,6 +11,7 @@ // //===----------------------------------------------------------------------===// +#include "polly/MaximalStaticExpansion.h" #include "polly/DependenceInfo.h" #include "polly/LinkAllPasses.h" #include "polly/ScopInfo.h" @@ -34,13 +35,13 @@ using namespace polly; namespace { -class MaximalStaticExpander final : public ScopPass { +class MaximalStaticExpanderWrapperPass final : public ScopPass { public: static char ID; - explicit MaximalStaticExpander() : ScopPass(ID) {} + explicit MaximalStaticExpanderWrapperPass() : ScopPass(ID) {} - ~MaximalStaticExpander() override = default; + ~MaximalStaticExpanderWrapperPass() override = default; /// Expand the accesses of the SCoP. /// @@ -55,63 +56,7 @@ public: /// Register all analyses and transformations required. void getAnalysisUsage(AnalysisUsage &AU) const override; - -private: - /// OptimizationRemarkEmitter object for displaying diagnostic remarks. - OptimizationRemarkEmitter *ORE; - - /// Emit remark - void emitRemark(StringRef Msg, Instruction *Inst); - - /// Return true if the SAI in parameter is expandable. - /// - /// @param SAI the SAI that need to be checked. - /// @param Writes A set that will contains all the write accesses. - /// @param Reads A set that will contains all the read accesses. - /// @param S The SCop in which the SAI is in. - /// @param Dependences The RAW dependences of the SCop. - bool isExpandable(const ScopArrayInfo *SAI, - SmallPtrSetImpl &Writes, - SmallPtrSetImpl &Reads, Scop &S, - const isl::union_map &Dependences); - - /// Expand the MemoryAccess according to its domain. - /// - /// @param S The SCop in which the memory access appears in. - /// @param MA The memory access that need to be expanded. - ScopArrayInfo *expandAccess(Scop &S, MemoryAccess *MA); - - /// Filter the dependences to have only one related to current memory access. - /// - /// @param S The SCop in which the memory access appears in. - /// @param MapDependences The dependences to filter. - /// @param MA The memory access that need to be expanded. - isl::union_map filterDependences(Scop &S, - const isl::union_map &MapDependences, - MemoryAccess *MA); - - /// Expand the MemoryAccess according to Dependences and already expanded - /// MemoryAccesses. - /// - /// @param The SCop in which the memory access appears in. - /// @param The memory access that need to be expanded. - /// @param Dependences The RAW dependences of the SCop. - /// @param ExpandedSAI The expanded SAI created during write expansion. - /// @param Reverse if true, the Dependences union_map is reversed before - /// intersection. - void mapAccess(Scop &S, SmallPtrSetImpl &Accesses, - const isl::union_map &Dependences, ScopArrayInfo *ExpandedSAI, - bool Reverse); - - /// Expand PHI memory accesses. - /// - /// @param The SCop in which the memory access appears in. - /// @param The ScopArrayInfo representing the PHI accesses to expand. - /// @param Dependences The RAW dependences of the SCop. - void expandPhi(Scop &S, const ScopArrayInfo *SAI, - const isl::union_map &Dependences); }; -} // namespace #ifndef NDEBUG /// Whether a dimension of a set is bounded (lower and upper) by a constant, @@ -128,360 +73,482 @@ static bool isDimBoundedByConstant(isl::set Set, unsigned dim) { } #endif -char MaximalStaticExpander::ID = 0; +class MaximalStaticExpansionImpl { + OptimizationRemarkEmitter &ORE; + Scop &S; + isl::union_map &Dependences; -isl::union_map MaximalStaticExpander::filterDependences( - Scop &S, const isl::union_map &Dependences, MemoryAccess *MA) { - auto SAI = MA->getLatestScopArrayInfo(); - - auto AccessDomainSet = MA->getAccessRelation().domain(); - auto AccessDomainId = AccessDomainSet.get_tuple_id(); - - isl::union_map MapDependences = isl::union_map::empty(S.getIslCtx()); - - for (isl::map Map : Dependences.get_map_list()) { - // Filter out Statement to Statement dependences. - if (!Map.can_curry()) - continue; - - // Intersect with the relevant SAI. - auto TmpMapDomainId = - Map.get_space().domain().unwrap().range().get_tuple_id(isl::dim::set); - - ScopArrayInfo *UserSAI = - static_cast(TmpMapDomainId.get_user()); - - if (SAI != UserSAI) - continue; - - // Get the correct S1[] -> S2[] dependence. - auto NewMap = Map.factor_domain(); - auto NewMapDomainId = NewMap.domain().get_tuple_id(); - - if (AccessDomainId.get() != NewMapDomainId.get()) - continue; - - // Add the corresponding map to MapDependences. - MapDependences = MapDependences.unite(NewMap); + /// Emit remark + void emitRemark(StringRef Msg, Instruction *Inst) { + ORE.emit(OptimizationRemarkAnalysis(DEBUG_TYPE, "ExpansionRejection", Inst) + << Msg); } - return MapDependences; -} + /// Filter the dependences to have only one related to current memory access. + /// + /// @param S The SCop in which the memory access appears in. + /// @param MapDependences The dependences to filter. + /// @param MA The memory access that need to be expanded. + isl::union_map filterDependences(const isl::union_map &Dependences, + MemoryAccess *MA) { + auto SAI = MA->getLatestScopArrayInfo(); -bool MaximalStaticExpander::isExpandable( - const ScopArrayInfo *SAI, SmallPtrSetImpl &Writes, - SmallPtrSetImpl &Reads, Scop &S, - const isl::union_map &Dependences) { - if (SAI->isValueKind()) { - Writes.insert(S.getValueDef(SAI)); - for (auto MA : S.getValueUses(SAI)) - Reads.insert(MA); - return true; - } else if (SAI->isPHIKind()) { - auto Read = S.getPHIRead(SAI); + auto AccessDomainSet = MA->getAccessRelation().domain(); + auto AccessDomainId = AccessDomainSet.get_tuple_id(); - auto StmtDomain = isl::union_set(Read->getStatement()->getDomain()); + isl::union_map MapDependences = isl::union_map::empty(S.getIslCtx()); - auto Writes = S.getPHIIncomings(SAI); + for (isl::map Map : Dependences.get_map_list()) { + // Filter out Statement to Statement dependences. + if (!Map.can_curry()) + continue; - // Get the domain where all the writes are writing to. - auto WriteDomain = isl::union_set::empty(S.getIslCtx()); + // Intersect with the relevant SAI. + auto TmpMapDomainId = + Map.get_space().domain().unwrap().range().get_tuple_id(isl::dim::set); - for (auto Write : Writes) { - auto MapDeps = filterDependences(S, Dependences, Write); - for (isl::map Map : MapDeps.get_map_list()) - WriteDomain = WriteDomain.unite(Map.range()); + ScopArrayInfo *UserSAI = + static_cast(TmpMapDomainId.get_user()); + + if (SAI != UserSAI) + continue; + + // Get the correct S1[] -> S2[] dependence. + auto NewMap = Map.factor_domain(); + auto NewMapDomainId = NewMap.domain().get_tuple_id(); + + if (AccessDomainId.get() != NewMapDomainId.get()) + continue; + + // Add the corresponding map to MapDependences. + MapDependences = MapDependences.unite(NewMap); } - // For now, read from original scalar is not possible. - if (!StmtDomain.is_equal(WriteDomain)) { - emitRemark(SAI->getName() + " read from its original value.", - Read->getAccessInstruction()); + return MapDependences; + } + + /// Return true if the SAI in parameter is expandable. + /// + /// @param SAI the SAI that need to be checked. + /// @param Writes A set that will contains all the write accesses. + /// @param Reads A set that will contains all the read accesses. + /// @param S The SCop in which the SAI is in. + /// @param Dependences The RAW dependences of the SCop. + bool isExpandable(const ScopArrayInfo *SAI, + SmallPtrSetImpl &Writes, + SmallPtrSetImpl &Reads, Scop &S) { + if (SAI->isValueKind()) { + Writes.insert(S.getValueDef(SAI)); + for (auto MA : S.getValueUses(SAI)) + Reads.insert(MA); + return true; + } else if (SAI->isPHIKind()) { + auto Read = S.getPHIRead(SAI); + + auto StmtDomain = isl::union_set(Read->getStatement()->getDomain()); + + auto Writes = S.getPHIIncomings(SAI); + + // Get the domain where all the writes are writing to. + auto WriteDomain = isl::union_set::empty(S.getIslCtx()); + + for (auto Write : Writes) { + auto MapDeps = filterDependences(Dependences, Write); + for (isl::map Map : MapDeps.get_map_list()) + WriteDomain = WriteDomain.unite(Map.range()); + } + + // For now, read from original scalar is not possible. + if (!StmtDomain.is_equal(WriteDomain)) { + emitRemark(SAI->getName() + " read from its original value.", + Read->getAccessInstruction()); + return false; + } + + return true; + } else if (SAI->isExitPHIKind()) { + // For now, we are not able to expand ExitPhi. + emitRemark(SAI->getName() + " is a ExitPhi node.", + S.getEnteringBlock()->getFirstNonPHI()); + return false; + } + + int NumberWrites = 0; + for (ScopStmt &Stmt : S) { + auto StmtReads = isl::union_map::empty(S.getIslCtx()); + auto StmtWrites = isl::union_map::empty(S.getIslCtx()); + + for (MemoryAccess *MA : Stmt) { + // Check if the current MemoryAccess involved the current SAI. + if (SAI != MA->getLatestScopArrayInfo()) + continue; + + // For now, we are not able to expand array where read come after write + // (to the same location) in a same statement. + auto AccRel = isl::union_map(MA->getAccessRelation()); + if (MA->isRead()) { + // Reject load after store to same location. + if (!StmtWrites.is_disjoint(AccRel)) { + emitRemark(SAI->getName() + " has read after write to the same " + "element in same statement. The " + "dependences found during analysis may " + "be wrong because Polly is not able to " + "handle such case for now.", + MA->getAccessInstruction()); + return false; + } + + StmtReads = StmtReads.unite(AccRel); + } else { + StmtWrites = StmtWrites.unite(AccRel); + } + + // For now, we are not able to expand MayWrite. + if (MA->isMayWrite()) { + emitRemark(SAI->getName() + " has a maywrite access.", + MA->getAccessInstruction()); + return false; + } + + // For now, we are not able to expand SAI with more than one write. + if (MA->isMustWrite()) { + Writes.insert(MA); + NumberWrites++; + if (NumberWrites > 1) { + emitRemark(SAI->getName() + " has more than 1 write access.", + MA->getAccessInstruction()); + return false; + } + } + + // Check if it is possible to expand this read. + if (MA->isRead()) { + // Get the domain of the current ScopStmt. + auto StmtDomain = Stmt.getDomain(); + + // Get the domain of the future Read access. + auto ReadDomainSet = MA->getAccessRelation().domain(); + auto ReadDomain = isl::union_set(ReadDomainSet); + + // Get the dependences relevant for this MA + auto MapDependences = filterDependences(Dependences.reverse(), MA); + unsigned NumberElementMap = isl_union_map_n_map(MapDependences.get()); + + if (NumberElementMap == 0) { + emitRemark("The expansion of " + SAI->getName() + + " would lead to a read from the original array.", + MA->getAccessInstruction()); + return false; + } + + auto DepsDomain = MapDependences.domain(); + + // If there are multiple maps in the Deps, we cannot handle this case + // for now. + if (NumberElementMap != 1) { + emitRemark(SAI->getName() + + " has too many dependences to be handle for now.", + MA->getAccessInstruction()); + return false; + } + + auto DepsDomainSet = isl::set(DepsDomain); + + // For now, read from the original array is not possible. + if (!StmtDomain.is_subset(DepsDomainSet)) { + emitRemark("The expansion of " + SAI->getName() + + " would lead to a read from the original array.", + MA->getAccessInstruction()); + return false; + } + + Reads.insert(MA); + } + } + } + + // No need to expand SAI with no write. + if (NumberWrites == 0) { + emitRemark(SAI->getName() + " has 0 write access.", + S.getEnteringBlock()->getFirstNonPHI()); return false; } return true; - } else if (SAI->isExitPHIKind()) { - // For now, we are not able to expand ExitPhi. - emitRemark(SAI->getName() + " is a ExitPhi node.", - S.getEnteringBlock()->getFirstNonPHI()); - return false; } - int NumberWrites = 0; - for (ScopStmt &Stmt : S) { - auto StmtReads = isl::union_map::empty(S.getIslCtx()); - auto StmtWrites = isl::union_map::empty(S.getIslCtx()); + /// Expand the MemoryAccess according to Dependences and already expanded + /// MemoryAccesses. + /// + /// @param The SCop in which the memory access appears in. + /// @param The memory access that need to be expanded. + /// @param Dependences The RAW dependences of the SCop. + /// @param ExpandedSAI The expanded SAI created during write expansion. + /// @param Reverse if true, the Dependences union_map is reversed before + /// intersection. + void mapAccess(SmallPtrSetImpl &Accesses, + const isl::union_map &Dependences, ScopArrayInfo *ExpandedSAI, + bool Reverse) { + for (auto MA : Accesses) { + // Get the current AM. + auto CurrentAccessMap = MA->getAccessRelation(); - for (MemoryAccess *MA : Stmt) { - // Check if the current MemoryAccess involved the current SAI. - if (SAI != MA->getLatestScopArrayInfo()) + // Get RAW dependences for the current WA. + auto DomainSet = MA->getAccessRelation().domain(); + auto Domain = isl::union_set(DomainSet); + + // Get the dependences relevant for this MA. + isl::union_map MapDependences = + filterDependences(Reverse ? Dependences.reverse() : Dependences, MA); + + // If no dependences, no need to modify anything. + if (MapDependences.is_empty()) + return; + + assert(isl_union_map_n_map(MapDependences.get()) == 1 && + "There are more than one RAW dependencies in the union map."); + auto NewAccessMap = isl::map::from_union_map(MapDependences); + + auto Id = ExpandedSAI->getBasePtrId(); + + // Replace the out tuple id with the one of the access array. + NewAccessMap = NewAccessMap.set_tuple_id(isl::dim::out, Id); + + // Set the new access relation. + MA->setNewAccessRelation(NewAccessMap); + } + } + + /// Expand the MemoryAccess according to its domain. + /// + /// @param S The SCop in which the memory access appears in. + /// @param MA The memory access that need to be expanded. + ScopArrayInfo *expandAccess(MemoryAccess *MA) { + // Get the current AM. + auto CurrentAccessMap = MA->getAccessRelation(); + + unsigned in_dimensions = + unsignedFromIslSize(CurrentAccessMap.domain_tuple_dim()); + + // Get domain from the current AM. + auto Domain = CurrentAccessMap.domain(); + + // Create a new AM from the domain. + auto NewAccessMap = isl::map::from_domain(Domain); + + // Add dimensions to the new AM according to the current in_dim. + NewAccessMap = NewAccessMap.add_dims(isl::dim::out, in_dimensions); + + // Create the string representing the name of the new SAI. + // One new SAI for each statement so that each write go to a different + // memory cell. + auto CurrentStmtDomain = MA->getStatement()->getDomain(); + auto CurrentStmtName = CurrentStmtDomain.get_tuple_name(); + auto CurrentOutId = CurrentAccessMap.get_tuple_id(isl::dim::out); + std::string CurrentOutIdString = + MA->getScopArrayInfo()->getName() + "_" + CurrentStmtName + "_expanded"; + + // Set the tuple id for the out dimension. + NewAccessMap = NewAccessMap.set_tuple_id(isl::dim::out, CurrentOutId); + + // Create the size vector. + std::vector Sizes; + for (unsigned i = 0; i < in_dimensions; i++) { + assert(isDimBoundedByConstant(CurrentStmtDomain, i) && + "Domain boundary are not constant."); + auto UpperBound = getConstant(CurrentStmtDomain.dim_max(i), true, false); + assert(!UpperBound.is_null() && UpperBound.is_pos() && + !UpperBound.is_nan() && + "The upper bound is not a positive integer."); + assert(UpperBound.le(isl::val(CurrentAccessMap.ctx(), + std::numeric_limits::max() - 1)) && + "The upper bound overflow a int."); + Sizes.push_back(UpperBound.get_num_si() + 1); + } + + // Get the ElementType of the current SAI. + auto ElementType = MA->getLatestScopArrayInfo()->getElementType(); + + // Create (or get if already existing) the new expanded SAI. + auto ExpandedSAI = + S.createScopArrayInfo(ElementType, CurrentOutIdString, Sizes); + ExpandedSAI->setIsOnHeap(true); + + // Get the out Id of the expanded Array. + auto NewOutId = ExpandedSAI->getBasePtrId(); + + // Set the out id of the new AM to the new SAI id. + NewAccessMap = NewAccessMap.set_tuple_id(isl::dim::out, NewOutId); + + // Add constraints to linked output with input id. + auto SpaceMap = NewAccessMap.get_space(); + auto ConstraintBasicMap = isl::basic_map::equal( + SpaceMap, unsignedFromIslSize(SpaceMap.dim(isl::dim::in))); + NewAccessMap = isl::map(ConstraintBasicMap); + + // Set the new access relation map. + MA->setNewAccessRelation(NewAccessMap); + + return ExpandedSAI; + } + + /// Expand PHI memory accesses. + /// + /// @param The SCop in which the memory access appears in. + /// @param The ScopArrayInfo representing the PHI accesses to expand. + /// @param Dependences The RAW dependences of the SCop. + void expandPhi(Scop &S, const ScopArrayInfo *SAI, + const isl::union_map &Dependences) { + SmallPtrSet Writes; + for (auto MA : S.getPHIIncomings(SAI)) + Writes.insert(MA); + auto Read = S.getPHIRead(SAI); + auto ExpandedSAI = expandAccess(Read); + + mapAccess(Writes, Dependences, ExpandedSAI, false); + } + +public: + MaximalStaticExpansionImpl(Scop &S, isl::union_map &Dependences, + OptimizationRemarkEmitter &ORE) + : S(S), ORE(ORE), Dependences(Dependences) {} + + /// Expand the accesses of the SCoP + /// + /// @param S The SCoP that must be expanded + /// @param D The dependencies information of SCoP + void expand() { + SmallVector CurrentSAI(S.arrays().begin(), + S.arrays().end()); + for (auto SAI : CurrentSAI) { + SmallPtrSet AllWrites; + SmallPtrSet AllReads; + if (!isExpandable(SAI, AllWrites, AllReads, S)) continue; - // For now, we are not able to expand array where read come after write - // (to the same location) in a same statement. - auto AccRel = isl::union_map(MA->getAccessRelation()); - if (MA->isRead()) { - // Reject load after store to same location. - if (!StmtWrites.is_disjoint(AccRel)) { - emitRemark(SAI->getName() + " has read after write to the same " - "element in same statement. The " - "dependences found during analysis may " - "be wrong because Polly is not able to " - "handle such case for now.", - MA->getAccessInstruction()); - return false; - } + if (SAI->isValueKind() || SAI->isArrayKind()) { + assert(AllWrites.size() == 1 || SAI->isValueKind()); - StmtReads = StmtReads.unite(AccRel); - } else { - StmtWrites = StmtWrites.unite(AccRel); - } + auto TheWrite = *(AllWrites.begin()); + ScopArrayInfo *ExpandedArray = expandAccess(TheWrite); - // For now, we are not able to expand MayWrite. - if (MA->isMayWrite()) { - emitRemark(SAI->getName() + " has a maywrite access.", - MA->getAccessInstruction()); - return false; - } - - // For now, we are not able to expand SAI with more than one write. - if (MA->isMustWrite()) { - Writes.insert(MA); - NumberWrites++; - if (NumberWrites > 1) { - emitRemark(SAI->getName() + " has more than 1 write access.", - MA->getAccessInstruction()); - return false; - } - } - - // Check if it is possible to expand this read. - if (MA->isRead()) { - // Get the domain of the current ScopStmt. - auto StmtDomain = Stmt.getDomain(); - - // Get the domain of the future Read access. - auto ReadDomainSet = MA->getAccessRelation().domain(); - auto ReadDomain = isl::union_set(ReadDomainSet); - - // Get the dependences relevant for this MA - auto MapDependences = filterDependences(S, Dependences.reverse(), MA); - unsigned NumberElementMap = isl_union_map_n_map(MapDependences.get()); - - if (NumberElementMap == 0) { - emitRemark("The expansion of " + SAI->getName() + - " would lead to a read from the original array.", - MA->getAccessInstruction()); - return false; - } - - auto DepsDomain = MapDependences.domain(); - - // If there are multiple maps in the Deps, we cannot handle this case - // for now. - if (NumberElementMap != 1) { - emitRemark(SAI->getName() + - " has too many dependences to be handle for now.", - MA->getAccessInstruction()); - return false; - } - - auto DepsDomainSet = isl::set(DepsDomain); - - // For now, read from the original array is not possible. - if (!StmtDomain.is_subset(DepsDomainSet)) { - emitRemark("The expansion of " + SAI->getName() + - " would lead to a read from the original array.", - MA->getAccessInstruction()); - return false; - } - - Reads.insert(MA); + mapAccess(AllReads, Dependences, ExpandedArray, true); + } else if (SAI->isPHIKind()) { + expandPhi(S, SAI, Dependences); } } } - // No need to expand SAI with no write. - if (NumberWrites == 0) { - emitRemark(SAI->getName() + " has 0 write access.", - S.getEnteringBlock()->getFirstNonPHI()); - return false; + /// Dump the internal information about a performed MSE to @p OS. + void print(llvm::raw_ostream &OS) { + OS << "After arrays {\n"; + + for (auto &Array : S.arrays()) + Array->print(OS); + + OS << "}\n"; + + OS << "After accesses {\n"; + for (auto &Stmt : S) { + OS.indent(4) << Stmt.getBaseName() << "{\n"; + for (auto *MA : Stmt) + MA->print(OS); + OS.indent(4) << "}\n"; + } + OS << "}\n"; + } +}; + +static std::unique_ptr +runMaximalStaticExpansion(Scop &S, OptimizationRemarkEmitter &ORE, + const Dependences &D) { + auto Dependences = D.getDependences(Dependences::TYPE_RAW); + + std::unique_ptr Impl = + std::make_unique(S, Dependences, ORE); + + Impl->expand(); + return Impl; +} + +static PreservedAnalyses runMSEUsingNPM(Scop &S, ScopAnalysisManager &SAM, + ScopStandardAnalysisResults &SAR, + raw_ostream *OS) { + OptimizationRemarkEmitter ORE(&S.getFunction()); + + auto &DI = SAM.getResult(S, SAR); + auto &D = DI.getDependences(Dependences::AL_Reference); + + std::unique_ptr Impl = + runMaximalStaticExpansion(S, ORE, D); + + if (OS) { + *OS << "Printing analysis 'Polly - Maximal static expansion of SCoP' for " + "region: '" + << S.getName() << "' in function '" << S.getFunction().getName() + << "':\n"; + + if (Impl) { + *OS << "MSE result:\n"; + Impl->print(*OS); + } } - return true; + return PreservedAnalyses::all(); } -void MaximalStaticExpander::mapAccess(Scop &S, - SmallPtrSetImpl &Accesses, - const isl::union_map &Dependences, - ScopArrayInfo *ExpandedSAI, - bool Reverse) { - for (auto MA : Accesses) { - // Get the current AM. - auto CurrentAccessMap = MA->getAccessRelation(); +} // namespace - // Get RAW dependences for the current WA. - auto DomainSet = MA->getAccessRelation().domain(); - auto Domain = isl::union_set(DomainSet); - - // Get the dependences relevant for this MA. - isl::union_map MapDependences = - filterDependences(S, Reverse ? Dependences.reverse() : Dependences, MA); - - // If no dependences, no need to modify anything. - if (MapDependences.is_empty()) - return; - - assert(isl_union_map_n_map(MapDependences.get()) == 1 && - "There are more than one RAW dependencies in the union map."); - auto NewAccessMap = isl::map::from_union_map(MapDependences); - - auto Id = ExpandedSAI->getBasePtrId(); - - // Replace the out tuple id with the one of the access array. - NewAccessMap = NewAccessMap.set_tuple_id(isl::dim::out, Id); - - // Set the new access relation. - MA->setNewAccessRelation(NewAccessMap); - } +PreservedAnalyses +MaximalStaticExpansionPass::run(Scop &S, ScopAnalysisManager &SAM, + ScopStandardAnalysisResults &SAR, + SPMUpdater &) { + return runMSEUsingNPM(S, SAM, SAR, nullptr); } -ScopArrayInfo *MaximalStaticExpander::expandAccess(Scop &S, MemoryAccess *MA) { - // Get the current AM. - auto CurrentAccessMap = MA->getAccessRelation(); - - unsigned in_dimensions = - unsignedFromIslSize(CurrentAccessMap.domain_tuple_dim()); - - // Get domain from the current AM. - auto Domain = CurrentAccessMap.domain(); - - // Create a new AM from the domain. - auto NewAccessMap = isl::map::from_domain(Domain); - - // Add dimensions to the new AM according to the current in_dim. - NewAccessMap = NewAccessMap.add_dims(isl::dim::out, in_dimensions); - - // Create the string representing the name of the new SAI. - // One new SAI for each statement so that each write go to a different memory - // cell. - auto CurrentStmtDomain = MA->getStatement()->getDomain(); - auto CurrentStmtName = CurrentStmtDomain.get_tuple_name(); - auto CurrentOutId = CurrentAccessMap.get_tuple_id(isl::dim::out); - std::string CurrentOutIdString = - MA->getScopArrayInfo()->getName() + "_" + CurrentStmtName + "_expanded"; - - // Set the tuple id for the out dimension. - NewAccessMap = NewAccessMap.set_tuple_id(isl::dim::out, CurrentOutId); - - // Create the size vector. - std::vector Sizes; - for (unsigned i = 0; i < in_dimensions; i++) { - assert(isDimBoundedByConstant(CurrentStmtDomain, i) && - "Domain boundary are not constant."); - auto UpperBound = getConstant(CurrentStmtDomain.dim_max(i), true, false); - assert(!UpperBound.is_null() && UpperBound.is_pos() && - !UpperBound.is_nan() && - "The upper bound is not a positive integer."); - assert(UpperBound.le(isl::val(CurrentAccessMap.ctx(), - std::numeric_limits::max() - 1)) && - "The upper bound overflow a int."); - Sizes.push_back(UpperBound.get_num_si() + 1); - } - - // Get the ElementType of the current SAI. - auto ElementType = MA->getLatestScopArrayInfo()->getElementType(); - - // Create (or get if already existing) the new expanded SAI. - auto ExpandedSAI = - S.createScopArrayInfo(ElementType, CurrentOutIdString, Sizes); - ExpandedSAI->setIsOnHeap(true); - - // Get the out Id of the expanded Array. - auto NewOutId = ExpandedSAI->getBasePtrId(); - - // Set the out id of the new AM to the new SAI id. - NewAccessMap = NewAccessMap.set_tuple_id(isl::dim::out, NewOutId); - - // Add constraints to linked output with input id. - auto SpaceMap = NewAccessMap.get_space(); - auto ConstraintBasicMap = isl::basic_map::equal( - SpaceMap, unsignedFromIslSize(SpaceMap.dim(isl::dim::in))); - NewAccessMap = isl::map(ConstraintBasicMap); - - // Set the new access relation map. - MA->setNewAccessRelation(NewAccessMap); - - return ExpandedSAI; +PreservedAnalyses +MaximalStaticExpansionPrinterPass::run(Scop &S, ScopAnalysisManager &SAM, + ScopStandardAnalysisResults &SAR, + SPMUpdater &) { + return runMSEUsingNPM(S, SAM, SAR, &OS); } -void MaximalStaticExpander::expandPhi(Scop &S, const ScopArrayInfo *SAI, - const isl::union_map &Dependences) { - SmallPtrSet Writes; - for (auto MA : S.getPHIIncomings(SAI)) - Writes.insert(MA); - auto Read = S.getPHIRead(SAI); - auto ExpandedSAI = expandAccess(S, Read); +char MaximalStaticExpanderWrapperPass::ID = 0; - mapAccess(S, Writes, Dependences, ExpandedSAI, false); -} - -void MaximalStaticExpander::emitRemark(StringRef Msg, Instruction *Inst) { - ORE->emit(OptimizationRemarkAnalysis(DEBUG_TYPE, "ExpansionRejection", Inst) - << Msg); -} - -bool MaximalStaticExpander::runOnScop(Scop &S) { +bool MaximalStaticExpanderWrapperPass::runOnScop(Scop &S) { // Get the ORE from OptimizationRemarkEmitterWrapperPass. - ORE = &(getAnalysis().getORE()); + OptimizationRemarkEmitter *ORE = + &getAnalysis().getORE(); // Get the RAW Dependences. auto &DI = getAnalysis(); auto &D = DI.getDependences(Dependences::AL_Reference); - isl::union_map Dependences = D.getDependences(Dependences::TYPE_RAW); - SmallVector CurrentSAI(S.arrays().begin(), - S.arrays().end()); - - for (auto SAI : CurrentSAI) { - SmallPtrSet AllWrites; - SmallPtrSet AllReads; - if (!isExpandable(SAI, AllWrites, AllReads, S, Dependences)) - continue; - - if (SAI->isValueKind() || SAI->isArrayKind()) { - assert(AllWrites.size() == 1 || SAI->isValueKind()); - - auto TheWrite = *(AllWrites.begin()); - ScopArrayInfo *ExpandedArray = expandAccess(S, TheWrite); - - mapAccess(S, AllReads, Dependences, ExpandedArray, true); - } else if (SAI->isPHIKind()) { - expandPhi(S, SAI, Dependences); - } - } + std::unique_ptr Impl = + runMaximalStaticExpansion(S, *ORE, D); return false; } -void MaximalStaticExpander::printScop(raw_ostream &OS, Scop &S) const { +void MaximalStaticExpanderWrapperPass::printScop(raw_ostream &OS, + Scop &S) const { S.print(OS, false); } -void MaximalStaticExpander::getAnalysisUsage(AnalysisUsage &AU) const { +void MaximalStaticExpanderWrapperPass::getAnalysisUsage( + AnalysisUsage &AU) const { ScopPass::getAnalysisUsage(AU); AU.addRequired(); AU.addRequired(); } Pass *polly::createMaximalStaticExpansionPass() { - return new MaximalStaticExpander(); + return new MaximalStaticExpanderWrapperPass(); } -INITIALIZE_PASS_BEGIN(MaximalStaticExpander, "polly-mse", +INITIALIZE_PASS_BEGIN(MaximalStaticExpanderWrapperPass, "polly-mse", "Polly - Maximal static expansion of SCoP", false, false); INITIALIZE_PASS_DEPENDENCY(DependenceInfo); INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass); -INITIALIZE_PASS_END(MaximalStaticExpander, "polly-mse", +INITIALIZE_PASS_END(MaximalStaticExpanderWrapperPass, "polly-mse", "Polly - Maximal static expansion of SCoP", false, false) diff --git a/polly/test/MaximalStaticExpansion/load_after_store_same_statement.ll b/polly/test/MaximalStaticExpansion/load_after_store_same_statement.ll index 036f39e5911b..26ed404fd4f6 100644 --- a/polly/test/MaximalStaticExpansion/load_after_store_same_statement.ll +++ b/polly/test/MaximalStaticExpansion/load_after_store_same_statement.ll @@ -1,5 +1,7 @@ ; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s +; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print)" -disable-output < %s | FileCheck %s ; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-mse -polly-print-scops -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1| FileCheck %s --check-prefix=MSE +; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE ; ; Verify that the expansion of an array with load after store in a same statement is not done. ; diff --git a/polly/test/MaximalStaticExpansion/read_from_original.ll b/polly/test/MaximalStaticExpansion/read_from_original.ll index fa32a3d3e8d2..95fd37bc251f 100644 --- a/polly/test/MaximalStaticExpansion/read_from_original.ll +++ b/polly/test/MaximalStaticExpansion/read_from_original.ll @@ -1,5 +1,7 @@ ; RUN: opt %loadPolly -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s +; RUN: opt %loadNPMPolly "-passes=scop(print)" -disable-output < %s | FileCheck %s ; RUN: opt %loadPolly -polly-mse -polly-print-scops -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1| FileCheck %s --check-prefix=MSE +; RUN: opt %loadNPMPolly "-passes=scop(print)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE ; ; Verify that Polly detects problems and does not expand the array ; diff --git a/polly/test/MaximalStaticExpansion/too_many_writes.ll b/polly/test/MaximalStaticExpansion/too_many_writes.ll index f4af7de03a8c..594f2a235dab 100644 --- a/polly/test/MaximalStaticExpansion/too_many_writes.ll +++ b/polly/test/MaximalStaticExpansion/too_many_writes.ll @@ -1,5 +1,7 @@ ; RUN: opt %loadPolly -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s +; RUN: opt %loadNPMPolly "-passes=scop(print)" -disable-output < %s | FileCheck %s ; RUN: opt %loadPolly -polly-mse -polly-print-scops -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE +; RUN: opt %loadNPMPolly "-passes=scop(print)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE ; ; Verify that Polly detects problems and does not expand the array ; diff --git a/polly/test/MaximalStaticExpansion/working_deps_between_inners.ll b/polly/test/MaximalStaticExpansion/working_deps_between_inners.ll index 7d72a4a6ab88..f754982317fb 100644 --- a/polly/test/MaximalStaticExpansion/working_deps_between_inners.ll +++ b/polly/test/MaximalStaticExpansion/working_deps_between_inners.ll @@ -1,4 +1,5 @@ ; RUN: opt %loadPolly -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s +; RUN: opt %loadNPMPolly "-passes=scop(print)" -disable-output < %s | FileCheck %s ; ; Verify that the accesses are correctly expanded for MemoryKind::Array ; diff --git a/polly/test/MaximalStaticExpansion/working_deps_between_inners_phi.ll b/polly/test/MaximalStaticExpansion/working_deps_between_inners_phi.ll index e405fdeb9f3b..288148811fbc 100644 --- a/polly/test/MaximalStaticExpansion/working_deps_between_inners_phi.ll +++ b/polly/test/MaximalStaticExpansion/working_deps_between_inners_phi.ll @@ -1,5 +1,7 @@ ; RUN: opt %loadPolly -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s +; RUN: opt %loadNPMPolly "-passes=scop(print)" -disable-output < %s | FileCheck %s ; RUN: opt %loadPolly -polly-mse -polly-print-scops -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE +; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE ; ; Verify that the accesses are correctly expanded for MemoryKind::Array and MemoryKind::PHI. ; tmp_06_phi is not expanded because it need copy in. diff --git a/polly/test/MaximalStaticExpansion/working_expansion.ll b/polly/test/MaximalStaticExpansion/working_expansion.ll index a420f88f10d5..269c892cfe22 100644 --- a/polly/test/MaximalStaticExpansion/working_expansion.ll +++ b/polly/test/MaximalStaticExpansion/working_expansion.ll @@ -1,4 +1,5 @@ ; RUN: opt %loadPolly -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s +; RUN: opt %loadNPMPolly "-passes=scop(print)" -disable-output < %s | FileCheck %s ; ; Verify that the accesses are correctly expanded for MemoryKind::Array ; diff --git a/polly/test/MaximalStaticExpansion/working_expansion_multiple_dependences_per_statement.ll b/polly/test/MaximalStaticExpansion/working_expansion_multiple_dependences_per_statement.ll index e58b7464dad0..3f425a79cbed 100644 --- a/polly/test/MaximalStaticExpansion/working_expansion_multiple_dependences_per_statement.ll +++ b/polly/test/MaximalStaticExpansion/working_expansion_multiple_dependences_per_statement.ll @@ -1,4 +1,5 @@ ; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s +; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print)" -disable-output < %s | FileCheck %s ; ; Verify that the accesses are correctly expanded ; diff --git a/polly/test/MaximalStaticExpansion/working_expansion_multiple_instruction_per_statement.ll b/polly/test/MaximalStaticExpansion/working_expansion_multiple_instruction_per_statement.ll index f39b38a3bfcc..62edfa626e13 100644 --- a/polly/test/MaximalStaticExpansion/working_expansion_multiple_instruction_per_statement.ll +++ b/polly/test/MaximalStaticExpansion/working_expansion_multiple_instruction_per_statement.ll @@ -1,4 +1,5 @@ ; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s +; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print)" -disable-output < %s | FileCheck %s ; ; Verify that the accesses are correctly expanded ; diff --git a/polly/test/MaximalStaticExpansion/working_phi_expansion.ll b/polly/test/MaximalStaticExpansion/working_phi_expansion.ll index 813258e85dd1..666d8c2bc4fe 100644 --- a/polly/test/MaximalStaticExpansion/working_phi_expansion.ll +++ b/polly/test/MaximalStaticExpansion/working_phi_expansion.ll @@ -1,5 +1,7 @@ ; RUN: opt %loadPolly -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s +; RUN: opt %loadNPMPolly "-passes=scop(print)" -disable-output < %s | FileCheck %s ; RUN: opt %loadPolly -polly-mse -polly-print-scops -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE +; RUN: opt %loadNPMPolly "-passes=scop(print)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE ; ; Verify that the accesses are correctly expanded for MemoryKind::PHI ; tmp_04 is not expanded because it need copy-in. diff --git a/polly/test/MaximalStaticExpansion/working_phi_two_scalars.ll b/polly/test/MaximalStaticExpansion/working_phi_two_scalars.ll index 445035adda10..55c8466590c9 100644 --- a/polly/test/MaximalStaticExpansion/working_phi_two_scalars.ll +++ b/polly/test/MaximalStaticExpansion/working_phi_two_scalars.ll @@ -1,5 +1,7 @@ ; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s +; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print)" -disable-output < %s | FileCheck %s ; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-mse -polly-print-scops -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE +; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE ; ; Verify that the accesses are correctly expanded for MemoryKind::PHI ; tmp_05 and tmp2_06 are not expanded because they need copy-in. diff --git a/polly/test/MaximalStaticExpansion/working_value_expansion.ll b/polly/test/MaximalStaticExpansion/working_value_expansion.ll index f7e9bf6a1755..ca910b2052e0 100644 --- a/polly/test/MaximalStaticExpansion/working_value_expansion.ll +++ b/polly/test/MaximalStaticExpansion/working_value_expansion.ll @@ -1,4 +1,5 @@ ; RUN: opt %loadPolly -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s +; RUN: opt %loadNPMPolly "-passes=scop(print)" -disable-output < %s | FileCheck %s ; ; Verify that the accesses are correctly expanded for MemoryKind::Value ;