[Polly] Add support for -polly-position=early with the NPM.

This required support for the canonicalization passes, inlcuding
porting RewriteByReferenceParams to the NPM.

For some reason, the legacy pass pipeline with -polly-position=early did
not run the CodePreparation pass. This was fixed as well.

polly/include/polly/Canonicalization.h

@@ -9,6 +9,8 @@
 #ifndef POLLY_CANONICALIZATION_H
 #define POLLY_CANONICALIZATION_H
 
+#include "llvm/Passes/PassBuilder.h"
+
 namespace llvm {
 namespace legacy {
 class PassManagerBase;
@@ -25,6 +27,11 @@ namespace polly {
 /// of Polly. The set of optimization passes scheduled here is probably not yet
 /// optimal. TODO: Optimize the set of canonicalization passes.
 void registerCanonicalicationPasses(llvm::legacy::PassManagerBase &PM);
+
+llvm::FunctionPassManager
+buildCanonicalicationPassesForNPM(llvm::ModulePassManager &MPM,
+                                  llvm::PassBuilder::OptimizationLevel Level);
+
 } // namespace polly
 
 #endif

polly/include/polly/LinkAllPasses.h

@@ -42,7 +42,7 @@ llvm::Pass *createPolyhedralInfoPass();
 llvm::Pass *createScopDetectionWrapperPassPass();
 llvm::Pass *createScopInfoRegionPassPass();
 llvm::Pass *createScopInfoWrapperPassPass();
-llvm::Pass *createRewriteByrefParamsPass();
+llvm::Pass *createRewriteByrefParamsWrapperPass();
 llvm::Pass *createIslAstInfoWrapperPassPass();
 llvm::Pass *createCodeGenerationPass();
 #ifdef GPU_CODEGEN
@@ -87,6 +87,7 @@ struct PollyForcePassLinking {
     polly::createScopInfoRegionPassPass();
     polly::createPollyCanonicalizePass();
     polly::createPolyhedralInfoPass();
+    polly::createRewriteByrefParamsWrapperPass();
     polly::createIslAstInfoWrapperPassPass();
     polly::createCodeGenerationPass();
 #ifdef GPU_CODEGEN
@@ -114,7 +115,7 @@ void initializeJSONExporterPass(llvm::PassRegistry &);
 void initializeJSONImporterPass(llvm::PassRegistry &);
 void initializeIslAstInfoWrapperPassPass(llvm::PassRegistry &);
 void initializeCodeGenerationPass(llvm::PassRegistry &);
-void initializeRewriteByrefParamsPass(llvm::PassRegistry &);
+void initializeRewriteByrefParamsWrapperPassPass(llvm::PassRegistry &);
 #ifdef GPU_CODEGEN
 void initializePPCGCodeGenerationPass(llvm::PassRegistry &);
 void initializeManagedMemoryRewritePassPass(llvm::PassRegistry &);

polly/include/polly/RewriteByReferenceParameters.h

@@ -0,0 +1,38 @@
+//===- RewriteByReferenceParameters.h -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef POLLY_REWRITEBYREFERENCEPARAMETERS_H
+#define POLLY_REWRITEBYREFERENCEPARAMETERS_H
+
+#include "polly/ScopPass.h"
+
+namespace llvm {
+class PassRegistry;
+class Pass;
+class raw_ostream;
+} // namespace llvm
+
+namespace polly {
+llvm::Pass *createRewriteByrefParamsWrapperPass();
+
+struct RewriteByrefParamsPass : llvm::PassInfoMixin<RewriteByrefParamsPass> {
+  RewriteByrefParamsPass() {}
+
+  llvm::PreservedAnalyses run(llvm::Function &F,
+    llvm::FunctionAnalysisManager &FAM);
+};
+
+} // namespace polly
+
+namespace llvm {
+void initializeRewriteByrefParamsWrapperPassPass(llvm::PassRegistry &);
+} // namespace llvm
+
+#endif /* POLLY_REWRITEBYREFERENCEPARAMETERS_H */

polly/lib/Support/RegisterPasses.cpp

@@ -263,7 +263,7 @@ void initializePollyPasses(PassRegistry &Registry) {
   initializeScopInlinerPass(Registry);
   initializeScopInfoRegionPassPass(Registry);
   initializeScopInfoWrapperPassPass(Registry);
-  initializeRewriteByrefParamsPass(Registry);
+  initializeRewriteByrefParamsWrapperPassPass(Registry);
   initializeCodegenCleanupPass(Registry);
   initializeFlattenSchedulePass(Registry);
   initializeForwardOpTreeWrapperPassPass(Registry);
@@ -429,6 +429,7 @@ registerPollyEarlyAsPossiblePasses(const llvm::PassManagerBuilder &Builder,
     return;
 
   registerCanonicalicationPasses(PM);
+  PM.add(polly::createCodePreparationPass());
   registerPollyPasses(PM, EnableForOpt);
 }
 
@@ -466,13 +467,17 @@ registerPollyScalarOptimizerLatePasses(const llvm::PassManagerBuilder &Builder,
     PM.add(createCodegenCleanupPass());
 }
 
-static void buildDefaultPollyPipeline(FunctionPassManager &PM,
-                                      PassBuilder::OptimizationLevel Level) {
-  bool EnableForOpt =
-      shouldEnablePollyForOptimization() && Level.isOptimizingForSpeed();
-  if (!shouldEnablePollyForDiagnostic() && !EnableForOpt)
-    return;
-
+/// Add the pass sequence required for Polly to the New Pass Manager.
+///
+/// @param PM           The pass manager itself.
+/// @param Level        The optimization level. Used for the cleanup of Polly's
+///                     output.
+/// @param EnableForOpt Whether to add Polly IR transformations. If False, only
+///                     the analysis passes are added, skipping Polly itself.
+///                     The IR may still be modified.
+static void buildCommonPollyPipeline(FunctionPassManager &PM,
+                                     PassBuilder::OptimizationLevel Level,
+                                     bool EnableForOpt) {
   PassBuilder PB;
   ScopPassManager SPM;
 
@@ -583,6 +588,43 @@ static void buildDefaultPollyPipeline(FunctionPassManager &PM,
     PM.addPass(llvm::CFGPrinterPass());
 }
 
+static void buildEarlyPollyPipeline(ModulePassManager &MPM,
+                                    PassBuilder::OptimizationLevel Level) {
+  bool EnableForOpt =
+      shouldEnablePollyForOptimization() && Level.isOptimizingForSpeed();
+  if (!shouldEnablePollyForDiagnostic() && !EnableForOpt)
+    return;
+
+  FunctionPassManager FPM = buildCanonicalicationPassesForNPM(MPM, Level);
+
+  if (DumpBefore)
+    report_fatal_error("Option -polly-dump-before not supported with NPM",
+                       false);
+  if (!DumpBeforeFile.empty())
+    report_fatal_error("Option -polly-dump-before-file not supported with NPM",
+                       false);
+
+  buildCommonPollyPipeline(FPM, Level, EnableForOpt);
+  MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+}
+
+static void buildLatePollyPipeline(FunctionPassManager &PM,
+                                   PassBuilder::OptimizationLevel Level) {
+  bool EnableForOpt =
+      shouldEnablePollyForOptimization() && Level.isOptimizingForSpeed();
+  if (!shouldEnablePollyForDiagnostic() && !EnableForOpt)
+    return;
+
+  if (DumpBefore)
+    report_fatal_error("Option -polly-dump-before not supported with NPM",
+                       false);
+  if (!DumpBeforeFile.empty())
+    report_fatal_error("Option -polly-dump-before-file not supported with NPM",
+                       false);
+
+  buildCommonPollyPipeline(PM, Level, EnableForOpt);
+}
+
 /// Register Polly to be available as an optimizer
 ///
 ///
@@ -776,9 +818,20 @@ void registerPollyPasses(PassBuilder &PB) {
         return parseTopLevelPipeline(MPM, PIC, Pipeline);
       });
 
-  if (PassPosition != POSITION_BEFORE_VECTORIZER)
-    report_fatal_error("Option -polly-position not supported with NPM", false);
-  PB.registerVectorizerStartEPCallback(buildDefaultPollyPipeline);
+  switch (PassPosition) {
+  case POSITION_EARLY:
+    PB.registerPipelineStartEPCallback(buildEarlyPollyPipeline);
+    break;
+  case POSITION_AFTER_LOOPOPT:
+    report_fatal_error(
+        "Option -polly-position=after-loopopt not supported with NPM", false);
+    break;
+  case POSITION_BEFORE_VECTORIZER:
+    PB.registerVectorizerStartEPCallback(buildLatePollyPipeline);
+    break;
+  default:
+    llvm_unreachable("Unknown -polly-position option");
+  }
 }
 } // namespace polly
 

polly/lib/Transform/Canonicalization.cpp

@@ -15,11 +15,22 @@
 #include "polly/Canonicalization.h"
 #include "polly/LinkAllPasses.h"
 #include "polly/Options.h"
+#include "polly/RewriteByReferenceParameters.h"
+#include "llvm/Analysis/GlobalsModRef.h"
+#include "llvm/Analysis/ProfileSummaryInfo.h"
 #include "llvm/IR/LegacyPassManager.h"
 #include "llvm/Transforms/IPO.h"
+#include "llvm/Transforms/IPO/FunctionAttrs.h"
 #include "llvm/Transforms/InstCombine/InstCombine.h"
 #include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Scalar/EarlyCSE.h"
+#include "llvm/Transforms/Scalar/IndVarSimplify.h"
+#include "llvm/Transforms/Scalar/LoopRotation.h"
+#include "llvm/Transforms/Scalar/Reassociate.h"
+#include "llvm/Transforms/Scalar/SimplifyCFG.h"
+#include "llvm/Transforms/Scalar/TailRecursionElimination.h"
 #include "llvm/Transforms/Utils.h"
+#include "llvm/Transforms/Utils/Mem2Reg.h"
 
 using namespace llvm;
 using namespace polly;
@@ -31,7 +42,7 @@ static cl::opt<bool>
 
 void polly::registerCanonicalicationPasses(llvm::legacy::PassManagerBase &PM) {
   bool UseMemSSA = true;
-  PM.add(polly::createRewriteByrefParamsPass());
+  PM.add(polly::createRewriteByrefParamsWrapperPass());
   PM.add(llvm::createPromoteMemoryToRegisterPass());
   PM.add(llvm::createEarlyCSEPass(UseMemSSA));
   PM.add(llvm::createInstructionCombiningPass());
@@ -49,7 +60,77 @@ void polly::registerCanonicalicationPasses(llvm::legacy::PassManagerBase &PM) {
   }
   PM.add(llvm::createInstructionCombiningPass());
   PM.add(llvm::createIndVarSimplifyPass());
-  PM.add(polly::createCodePreparationPass());
+}
+
+/// Adapted from llvm::PassBuilder::buildInlinerPipeline
+static ModuleInlinerWrapperPass
+buildInlinePasses(llvm::PassBuilder::OptimizationLevel Level) {
+  InlineParams IP = getInlineParams(200);
+  ModuleInlinerWrapperPass MIWP(IP);
+
+  // Require the GlobalsAA analysis for the module so we can query it within
+  // the CGSCC pipeline.
+  MIWP.addModulePass(RequireAnalysisPass<GlobalsAA, Module>());
+  // Invalidate AAManager so it can be recreated and pick up the newly available
+  // GlobalsAA.
+  MIWP.addModulePass(
+      createModuleToFunctionPassAdaptor(InvalidateAnalysisPass<AAManager>()));
+
+  // Require the ProfileSummaryAnalysis for the module so we can query it within
+  // the inliner pass.
+  MIWP.addModulePass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
+
+  // Now begin the main postorder CGSCC pipeline.
+  // FIXME: The current CGSCC pipeline has its origins in the legacy pass
+  // manager and trying to emulate its precise behavior. Much of this doesn't
+  // make a lot of sense and we should revisit the core CGSCC structure.
+  CGSCCPassManager &MainCGPipeline = MIWP.getPM();
+
+  // Now deduce any function attributes based in the current code.
+  MainCGPipeline.addPass(PostOrderFunctionAttrsPass());
+
+  return MIWP;
+}
+
+FunctionPassManager polly::buildCanonicalicationPassesForNPM(
+    llvm::ModulePassManager &MPM, llvm::PassBuilder::OptimizationLevel Level) {
+  FunctionPassManager FPM;
+
+  bool UseMemSSA = true;
+  FPM.addPass(RewriteByrefParamsPass());
+  FPM.addPass(PromotePass());
+  FPM.addPass(EarlyCSEPass(UseMemSSA));
+  FPM.addPass(InstCombinePass());
+  FPM.addPass(SimplifyCFGPass());
+  FPM.addPass(TailCallElimPass());
+  FPM.addPass(SimplifyCFGPass());
+  FPM.addPass(ReassociatePass());
+  {
+    LoopPassManager LPM;
+    LPM.addPass(LoopRotatePass(Level != PassBuilder::OptimizationLevel::Oz));
+    FPM.addPass(createFunctionToLoopPassAdaptor<LoopPassManager>(
+        std::move(LPM), /*UseMemorySSA=*/false,
+        /*UseBlockFrequencyInfo=*/false));
+  }
+  if (PollyInliner) {
+    MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+    MPM.addPass(buildInlinePasses(Level));
+    FPM = FunctionPassManager();
+
+    FPM.addPass(PromotePass());
+    FPM.addPass(SimplifyCFGPass());
+    FPM.addPass(InstCombinePass());
+  }
+  FPM.addPass(InstCombinePass());
+  {
+    LoopPassManager LPM;
+    LPM.addPass(IndVarSimplifyPass());
+    FPM.addPass(createFunctionToLoopPassAdaptor<LoopPassManager>(
+        std::move(LPM), /*UseMemorySSA=*/false,
+        /*UseBlockFrequencyInfo=*/true));
+  }
+
+  return FPM;
 }
 
 namespace {

polly/lib/Transform/RewriteByReferenceParameters.cpp

@@ -13,6 +13,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "polly/RewriteByReferenceParameters.h"
 #include "polly/LinkAllPasses.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
@@ -23,76 +24,89 @@
 using namespace llvm;
 
 namespace {
+static void tryRewriteInstruction(Instruction &Inst) {
+  BasicBlock *Entry = &Inst.getParent()->getParent()->getEntryBlock();
 
-class RewriteByrefParams : public FunctionPass {
+  auto *Call = dyn_cast<CallInst>(&Inst);
+
+  if (!Call)
+    return;
+
+  llvm::Function *F = Call->getCalledFunction();
+
+  if (!F)
+    return;
+
+  // We currently match for a very specific function. In case this proves
+  // useful, we can make this code dependent on readonly metadata.
+  if (!F->hasName() || F->getName() != "_gfortran_transfer_integer_write")
+    return;
+
+  auto *BitCast = dyn_cast<BitCastInst>(Call->getOperand(1));
+
+  if (!BitCast)
+    return;
+
+  auto *Alloca = dyn_cast<AllocaInst>(BitCast->getOperand(0));
+
+  if (!Alloca)
+    return;
+
+  std::string InstName = Alloca->getName().str();
+
+  auto NewAlloca =
+      new AllocaInst(Alloca->getAllocatedType(), 0,
+                     "polly_byref_alloca_" + InstName, &*Entry->begin());
+
+  auto *LoadedVal = new LoadInst(Alloca->getAllocatedType(), Alloca,
+                                 "polly_byref_load_" + InstName, &Inst);
+
+  new StoreInst(LoadedVal, NewAlloca, &Inst);
+  auto *NewBitCast = new BitCastInst(NewAlloca, BitCast->getType(),
+                                     "polly_byref_cast_" + InstName, &Inst);
+  Call->setOperand(1, NewBitCast);
+}
+
+static void runRewriteByrefParams(Function &F) {
+  for (BasicBlock &BB : F)
+    for (Instruction &Inst : BB)
+      tryRewriteInstruction(Inst);
+}
+
+class RewriteByrefParamsWrapperPass : public FunctionPass {
 private:
-  RewriteByrefParams(const RewriteByrefParams &) = delete;
-  const RewriteByrefParams &operator=(const RewriteByrefParams &) = delete;
+  RewriteByrefParamsWrapperPass(const RewriteByrefParamsWrapperPass &) = delete;
+  const RewriteByrefParamsWrapperPass &
+  operator=(const RewriteByrefParamsWrapperPass &) = delete;
 
 public:
   static char ID;
-  explicit RewriteByrefParams() : FunctionPass(ID) {}
+  explicit RewriteByrefParamsWrapperPass() : FunctionPass(ID) {}
 
   virtual void getAnalysisUsage(AnalysisUsage &AU) const override {}
 
-  void tryRewriteInstruction(Instruction &Inst) {
-    BasicBlock *Entry = &Inst.getParent()->getParent()->getEntryBlock();
-
-    auto *Call = dyn_cast<CallInst>(&Inst);
-
-    if (!Call)
-      return;
-
-    llvm::Function *F = Call->getCalledFunction();
-
-    if (!F)
-      return;
-
-    // We currently match for a very specific function. In case this proves
-    // useful, we can make this code dependent on readonly metadata.
-    if (!F->hasName() || F->getName() != "_gfortran_transfer_integer_write")
-      return;
-
-    auto *BitCast = dyn_cast<BitCastInst>(Call->getOperand(1));
-
-    if (!BitCast)
-      return;
-
-    auto *Alloca = dyn_cast<AllocaInst>(BitCast->getOperand(0));
-
-    if (!Alloca)
-      return;
-
-    std::string InstName = Alloca->getName().str();
-
-    auto NewAlloca =
-        new AllocaInst(Alloca->getAllocatedType(), 0,
-                       "polly_byref_alloca_" + InstName, &*Entry->begin());
-
-    auto *LoadedVal = new LoadInst(Alloca->getAllocatedType(), Alloca,
-                                   "polly_byref_load_" + InstName, &Inst);
-
-    new StoreInst(LoadedVal, NewAlloca, &Inst);
-    auto *NewBitCast = new BitCastInst(NewAlloca, BitCast->getType(),
-                                       "polly_byref_cast_" + InstName, &Inst);
-    Call->setOperand(1, NewBitCast);
-  }
-
   virtual bool runOnFunction(Function &F) override {
-    for (BasicBlock &BB : F)
-      for (Instruction &Inst : BB)
-        tryRewriteInstruction(Inst);
-
+    runRewriteByrefParams(F);
     return true;
   }
 };
 
-char RewriteByrefParams::ID;
+char RewriteByrefParamsWrapperPass::ID;
 } // anonymous namespace
 
-Pass *polly::createRewriteByrefParamsPass() { return new RewriteByrefParams(); }
+Pass *polly::createRewriteByrefParamsWrapperPass() {
+  return new RewriteByrefParamsWrapperPass();
+}
 
-INITIALIZE_PASS_BEGIN(RewriteByrefParams, "polly-rewrite-byref-params",
+llvm::PreservedAnalyses
+polly ::RewriteByrefParamsPass::run(llvm::Function &F,
+                                    llvm::FunctionAnalysisManager &FAM) {
+  runRewriteByrefParams(F);
+  return PreservedAnalyses::none();
+}
+
+INITIALIZE_PASS_BEGIN(RewriteByrefParamsWrapperPass,
+                      "polly-rewrite-byref-params",
                       "Polly - Rewrite by reference parameters", false, false)
-INITIALIZE_PASS_END(RewriteByrefParams, "polly-rewrite-byref-params",
+INITIALIZE_PASS_END(RewriteByrefParamsWrapperPass, "polly-rewrite-byref-params",
                     "Polly - Rewrite by reference parameters", false, false)

polly/test/Support/pipelineposition.ll

@@ -0,0 +1,91 @@
+; Legacy pass manager
+; RUN: opt %loadPolly -O3 -enable-new-pm=0 -polly -polly-position=early                    -disable-output -debug-only=polly-scops < %s 2>&1 | FileCheck %s --check-prefix=NOINLINE
+; RUN: opt %loadPolly -O3 -enable-new-pm=0 -polly -polly-position=early -polly-run-inliner -disable-output -debug-only=polly-scops < %s 2>&1 | FileCheck %s --check-prefix=INLINED1
+; RUN: opt %loadPolly -O3 -enable-new-pm=0 -polly -polly-position=after-loopopt            -disable-output -debug-only=polly-scops < %s 2>&1 | FileCheck %s --check-prefix=INLINED2
+; RUN: opt %loadPolly -O3 -enable-new-pm=0 -polly -polly-position=before-vectorizer        -disable-output -debug-only=polly-scops < %s 2>&1 | FileCheck %s --check-prefix=INLINED3
+;
+; New pass manager
+; RUN: opt %loadPolly -O3 -enable-new-pm=1 -polly -polly-position=early                    -disable-output -debug-only=polly-scops < %s 2>&1 | FileCheck %s --check-prefix=NOINLINE
+; RUN: opt %loadPolly -O3 -enable-new-pm=1 -polly -polly-position=early -polly-run-inliner -disable-output -debug-only=polly-scops < %s 2>&1 | FileCheck %s --check-prefix=INLINED1
+; RUN: opt %loadPolly -O3 -enable-new-pm=1 -polly -polly-position=before-vectorizer        -disable-output -debug-only=polly-scops < %s 2>&1 | FileCheck %s --check-prefix=INLINED3
+;
+; REQUIRES: asserts
+;
+; void callee(int n, double A[], int i) {
+;   for (int j = 0; j < n; j += 1)
+;     A[i+j] = 42.0;
+; }
+;
+; void caller(int n, double A[]) {
+;   for (int i = 0; i < n; i += 1)
+;     callee(n, A, i);
+; }
+
+
+define internal void @callee(i32 %n, double* noalias nonnull %A, i32 %i) {
+entry:
+  br label %for
+
+for:
+  %j = phi i32 [0, %entry], [%j.inc, %inc]
+  %j.cmp = icmp slt i32 %j, %n
+  br i1 %j.cmp, label %body, label %exit
+
+    body:
+      %idx = add i32 %i, %j
+      %arrayidx = getelementptr inbounds double, double* %A, i32 %idx
+      store double 42.0, double* %arrayidx
+      br label %inc
+
+inc:
+  %j.inc = add nuw nsw i32 %j, 1
+  br label %for
+
+exit:
+  br label %return
+
+return:
+  ret void
+}
+
+
+define void @caller(i32 %n, double* noalias nonnull %A) {
+entry:
+  br label %for
+
+for:
+  %i = phi i32 [0, %entry], [%j.inc, %inc]
+  %i.cmp = icmp slt i32 %i, %n
+  br i1 %i.cmp, label %body, label %exit
+
+    body:
+      call void @callee(i32 %n, double* %A, i32 %i)
+      br label %inc
+
+inc:
+  %j.inc = add nuw nsw i32 %i, 1
+  br label %for
+
+exit:
+  br label %return
+
+return:
+  ret void
+}
+
+
+; NOINLINE-LABEL: Function: callee
+; NOINLINE:       Schedule :=
+; NOINLINE-NEXT:    [n, i] -> { Stmt_body[i0] -> [i0] };
+
+; INLINED1-LABEL: Function: caller
+; INLINED1:       Schedule :=
+; INLINED1-NEXT:    [n] -> { Stmt_body_i[i0, i1] -> [i0, i1] };
+
+; INLINED2-LABEL: Function: caller
+; INLINED2:       Schedule :=
+; INLINED2-NEXT:    [n] -> { Stmt_polly_loop_header_i_us_us[i0, i1] -> [i0, 1, i1] };
+
+; INLINED3-LABEL: Function: caller
+; INLINED3:       Schedule :=
+; INLINED3-NEXT:    [n] -> { Stmt_body_i_us[i0, i1] -> [i0, i1] };