From d7c1427953188a4ff366758f9d993f46cd80affe Mon Sep 17 00:00:00 2001 From: Arthur Eubanks Date: Mon, 14 Nov 2022 09:33:38 -0800 Subject: [PATCH] Revert "[LegacyPM] Remove cl::opts controlling optimization pass manager passes" This reverts commit 7ec05fec7115a910b2e172de794adc462388c25e. Breaks bots, e.g. https://lab.llvm.org/buildbot#builders/217/builds/15008 --- llvm/lib/Passes/PassBuilderPipelines.cpp | 121 ++------- .../lib/Transforms/IPO/PassManagerBuilder.cpp | 257 +++++++++++++++++- 2 files changed, 280 insertions(+), 98 deletions(-) diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp index f978b2cefe9e..b32d53d8dfaf 100644 --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -134,11 +134,11 @@ static cl::opt UseInlineAdvisor( "enable-ml-inliner", cl::init(InliningAdvisorMode::Default), cl::Hidden, cl::desc("Enable ML policy for inliner. Currently trained for -Oz only"), cl::values(clEnumValN(InliningAdvisorMode::Default, "default", - "Heuristics-based inliner version"), + "Heuristics-based inliner version."), clEnumValN(InliningAdvisorMode::Development, "development", - "Use development mode (runtime-loadable model)"), + "Use development mode (runtime-loadable model)."), clEnumValN(InliningAdvisorMode::Release, "release", - "Use release mode (AOT-compiled model)"))); + "Use release mode (AOT-compiled model)."))); static cl::opt EnableSyntheticCounts( "enable-npm-synthetic-counts", cl::Hidden, @@ -161,7 +161,7 @@ static cl::opt EnableModuleInliner("enable-module-inliner", static cl::opt PerformMandatoryInliningsFirst( "mandatory-inlining-first", cl::init(true), cl::Hidden, cl::desc("Perform mandatory inlinings module-wide, before performing " - "inlining")); + "inlining.")); static cl::opt EnableO3NonTrivialUnswitching( "enable-npm-O3-nontrivial-unswitch", cl::init(true), cl::Hidden, @@ -190,99 +190,6 @@ static cl::opt EnableGlobalAnalyses( "enable-global-analyses", cl::init(true), cl::Hidden, cl::desc("Enable inter-procedural analyses")); -static cl::opt - RunPartialInlining("enable-partial-inlining", cl::init(false), cl::Hidden, - cl::desc("Run Partial inlinining pass")); - -static cl::opt ExtraVectorizerPasses( - "extra-vectorizer-passes", cl::init(false), cl::Hidden, - cl::desc("Run cleanup optimization passes after vectorization")); - -static cl::opt RunNewGVN("enable-newgvn", cl::init(false), cl::Hidden, - cl::desc("Run the NewGVN pass")); - -static cl::opt EnableLoopInterchange( - "enable-loopinterchange", cl::init(false), cl::Hidden, - cl::desc("Enable the experimental LoopInterchange Pass")); - -static cl::opt EnableUnrollAndJam("enable-unroll-and-jam", - cl::init(false), cl::Hidden, - cl::desc("Enable Unroll And Jam Pass")); - -static cl::opt EnableLoopFlatten("enable-loop-flatten", cl::init(false), - cl::Hidden, - cl::desc("Enable the LoopFlatten Pass")); - -static cl::opt - EnableDFAJumpThreading("enable-dfa-jump-thread", - cl::desc("Enable DFA jump threading"), - cl::init(false), cl::Hidden); - -static cl::opt - EnableHotColdSplit("hot-cold-split", - cl::desc("Enable hot-cold splitting pass")); - -static cl::opt EnableIROutliner("ir-outliner", cl::init(false), - cl::Hidden, - cl::desc("Enable ir outliner pass")); - -static cl::opt - DisablePreInliner("disable-preinline", cl::init(false), cl::Hidden, - cl::desc("Disable pre-instrumentation inliner")); - -static cl::opt PreInlineThreshold( - "preinline-threshold", cl::Hidden, cl::init(75), - cl::desc("Control the amount of inlining in pre-instrumentation inliner " - "(default = 75)")); - -static cl::opt - EnableGVNHoist("enable-gvn-hoist", - cl::desc("Enable the GVN hoisting pass (default = off)")); - -static cl::opt - EnableGVNSink("enable-gvn-sink", - cl::desc("Enable the GVN sinking pass (default = off)")); - -// This option is used in simplifying testing SampleFDO optimizations for -// profile loading. -static cl::opt - EnableCHR("enable-chr", cl::init(true), cl::Hidden, - cl::desc("Enable control height reduction optimization (CHR)")); - -static cl::opt FlattenedProfileUsed( - "flattened-profile-used", cl::init(false), cl::Hidden, - cl::desc("Indicate the sample profile being used is flattened, i.e., " - "no inline hierachy exists in the profile")); - -static cl::opt EnableOrderFileInstrumentation( - "enable-order-file-instrumentation", cl::init(false), cl::Hidden, - cl::desc("Enable order file instrumentation (default = off)")); - -static cl::opt - EnableMatrix("enable-matrix", cl::init(false), cl::Hidden, - cl::desc("Enable lowering of the matrix intrinsics")); - -static cl::opt EnableConstraintElimination( - "enable-constraint-elimination", cl::init(false), cl::Hidden, - cl::desc( - "Enable pass to eliminate conditions based on linear constraints")); - -static cl::opt EnableFunctionSpecialization( - "enable-function-specialization", cl::init(false), cl::Hidden, - cl::desc("Enable Function Specialization pass")); - -static cl::opt AttributorRun( - "attributor-enable", cl::Hidden, cl::init(AttributorRunOption::NONE), - cl::desc("Enable the attributor inter-procedural deduction pass"), - cl::values(clEnumValN(AttributorRunOption::ALL, "all", - "enable all attributor runs"), - clEnumValN(AttributorRunOption::MODULE, "module", - "enable module-wide attributor runs"), - clEnumValN(AttributorRunOption::CGSCC, "cgscc", - "enable call graph SCC attributor runs"), - clEnumValN(AttributorRunOption::NONE, "none", - "disable attributor runs"))); - PipelineTuningOptions::PipelineTuningOptions() { LoopInterleaving = true; LoopVectorization = true; @@ -299,7 +206,27 @@ PipelineTuningOptions::PipelineTuningOptions() { namespace llvm { extern cl::opt MaxDevirtIterations; +extern cl::opt EnableConstraintElimination; +extern cl::opt EnableFunctionSpecialization; +extern cl::opt EnableGVNHoist; +extern cl::opt EnableGVNSink; +extern cl::opt EnableHotColdSplit; +extern cl::opt EnableIROutliner; +extern cl::opt EnableOrderFileInstrumentation; +extern cl::opt EnableCHR; +extern cl::opt EnableLoopInterchange; +extern cl::opt EnableUnrollAndJam; +extern cl::opt EnableLoopFlatten; +extern cl::opt EnableDFAJumpThreading; +extern cl::opt RunNewGVN; +extern cl::opt RunPartialInlining; +extern cl::opt ExtraVectorizerPasses; +extern cl::opt FlattenedProfileUsed; +extern cl::opt AttributorRun; extern cl::opt EnableKnowledgeRetention; +extern cl::opt EnableMatrix; +extern cl::opt DisablePreInliner; +extern cl::opt PreInlineThreshold; } // namespace llvm void PassBuilder::invokePeepholeEPCallbacks(FunctionPassManager &FPM, diff --git a/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp b/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp index a81a3fb8d891..52e4fa401183 100644 --- a/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp +++ b/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp @@ -43,6 +43,129 @@ using namespace llvm; +namespace llvm { +cl::opt RunPartialInlining("enable-partial-inlining", cl::Hidden, + cl::desc("Run Partial inlinining pass")); + +static cl::opt +UseGVNAfterVectorization("use-gvn-after-vectorization", + cl::init(false), cl::Hidden, + cl::desc("Run GVN instead of Early CSE after vectorization passes")); + +cl::opt ExtraVectorizerPasses( + "extra-vectorizer-passes", cl::init(false), cl::Hidden, + cl::desc("Run cleanup optimization passes after vectorization.")); + +static cl::opt +RunLoopRerolling("reroll-loops", cl::Hidden, + cl::desc("Run the loop rerolling pass")); + +cl::opt RunNewGVN("enable-newgvn", cl::init(false), cl::Hidden, + cl::desc("Run the NewGVN pass")); + +// Experimental option to use CFL-AA +static cl::opt<::CFLAAType> + UseCFLAA("use-cfl-aa", cl::init(::CFLAAType::None), cl::Hidden, + cl::desc("Enable the new, experimental CFL alias analysis"), + cl::values(clEnumValN(::CFLAAType::None, "none", "Disable CFL-AA"), + clEnumValN(::CFLAAType::Steensgaard, "steens", + "Enable unification-based CFL-AA"), + clEnumValN(::CFLAAType::Andersen, "anders", + "Enable inclusion-based CFL-AA"), + clEnumValN(::CFLAAType::Both, "both", + "Enable both variants of CFL-AA"))); + +cl::opt EnableLoopInterchange( + "enable-loopinterchange", cl::init(false), cl::Hidden, + cl::desc("Enable the experimental LoopInterchange Pass")); + +cl::opt EnableUnrollAndJam("enable-unroll-and-jam", cl::init(false), + cl::Hidden, + cl::desc("Enable Unroll And Jam Pass")); + +cl::opt EnableLoopFlatten("enable-loop-flatten", cl::init(false), + cl::Hidden, + cl::desc("Enable the LoopFlatten Pass")); + +cl::opt EnableDFAJumpThreading("enable-dfa-jump-thread", + cl::desc("Enable DFA jump threading."), + cl::init(false), cl::Hidden); + +cl::opt EnableHotColdSplit("hot-cold-split", + cl::desc("Enable hot-cold splitting pass")); + +cl::opt EnableIROutliner("ir-outliner", cl::init(false), cl::Hidden, + cl::desc("Enable ir outliner pass")); + +static cl::opt UseLoopVersioningLICM( + "enable-loop-versioning-licm", cl::init(false), cl::Hidden, + cl::desc("Enable the experimental Loop Versioning LICM pass")); + +cl::opt + DisablePreInliner("disable-preinline", cl::init(false), cl::Hidden, + cl::desc("Disable pre-instrumentation inliner")); + +cl::opt PreInlineThreshold( + "preinline-threshold", cl::Hidden, cl::init(75), + cl::desc("Control the amount of inlining in pre-instrumentation inliner " + "(default = 75)")); + +cl::opt + EnableGVNHoist("enable-gvn-hoist", + cl::desc("Enable the GVN hoisting pass (default = off)")); + +static cl::opt + DisableLibCallsShrinkWrap("disable-libcalls-shrinkwrap", cl::init(false), + cl::Hidden, + cl::desc("Disable shrink-wrap library calls")); + +cl::opt + EnableGVNSink("enable-gvn-sink", + cl::desc("Enable the GVN sinking pass (default = off)")); + +// This option is used in simplifying testing SampleFDO optimizations for +// profile loading. +cl::opt + EnableCHR("enable-chr", cl::init(true), cl::Hidden, + cl::desc("Enable control height reduction optimization (CHR)")); + +cl::opt FlattenedProfileUsed( + "flattened-profile-used", cl::init(false), cl::Hidden, + cl::desc("Indicate the sample profile being used is flattened, i.e., " + "no inline hierachy exists in the profile. ")); + +cl::opt EnableOrderFileInstrumentation( + "enable-order-file-instrumentation", cl::init(false), cl::Hidden, + cl::desc("Enable order file instrumentation (default = off)")); + +cl::opt EnableMatrix( + "enable-matrix", cl::init(false), cl::Hidden, + cl::desc("Enable lowering of the matrix intrinsics")); + +cl::opt EnableConstraintElimination( + "enable-constraint-elimination", cl::init(false), cl::Hidden, + cl::desc( + "Enable pass to eliminate conditions based on linear constraints.")); + +cl::opt EnableFunctionSpecialization( + "enable-function-specialization", cl::init(false), cl::Hidden, + cl::desc("Enable Function Specialization pass")); + +cl::opt AttributorRun( + "attributor-enable", cl::Hidden, cl::init(AttributorRunOption::NONE), + cl::desc("Enable the attributor inter-procedural deduction pass."), + cl::values(clEnumValN(AttributorRunOption::ALL, "all", + "enable all attributor runs"), + clEnumValN(AttributorRunOption::MODULE, "module", + "enable module-wide attributor runs"), + clEnumValN(AttributorRunOption::CGSCC, "cgscc", + "enable call graph SCC attributor runs"), + clEnumValN(AttributorRunOption::NONE, "none", + "disable attributor runs"))); + +extern cl::opt EnableKnowledgeRetention; +} // namespace llvm + PassManagerBuilder::PassManagerBuilder() { OptLevel = 2; SizeLevel = 0; @@ -52,6 +175,8 @@ PassManagerBuilder::PassManagerBuilder() { SLPVectorize = false; LoopVectorize = true; LoopsInterleaved = true; + RerollLoops = RunLoopRerolling; + NewGVN = RunNewGVN; LicmMssaOptCap = SetLicmMssaOptCap; LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap; DisableGVNLoadPRE = false; @@ -128,6 +253,21 @@ void PassManagerBuilder::addExtensionsToPM(ExtensionPointTy ETy, void PassManagerBuilder::addInitialAliasAnalysisPasses( legacy::PassManagerBase &PM) const { + switch (UseCFLAA) { + case ::CFLAAType::Steensgaard: + PM.add(createCFLSteensAAWrapperPass()); + break; + case ::CFLAAType::Andersen: + PM.add(createCFLAndersAAWrapperPass()); + break; + case ::CFLAAType::Both: + PM.add(createCFLSteensAAWrapperPass()); + PM.add(createCFLAndersAAWrapperPass()); + break; + default: + break; + } + // Add TypeBasedAliasAnalysis before BasicAliasAnalysis so that // BasicAliasAnalysis wins if they disagree. This is intended to help // support "obvious" type-punning idioms. @@ -143,6 +283,13 @@ void PassManagerBuilder::populateFunctionPassManager( if (LibraryInfo) FPM.add(new TargetLibraryInfoWrapperPass(*LibraryInfo)); + // The backends do not handle matrix intrinsics currently. + // Make sure they are also lowered in O0. + // FIXME: A lightweight version of the pass should run in the backend + // pipeline on demand. + if (EnableMatrix && OptLevel == 0) + FPM.add(createLowerMatrixIntrinsicsMinimalPass()); + if (OptLevel == 0) return; addInitialAliasAnalysisPasses(FPM); @@ -162,6 +309,21 @@ void PassManagerBuilder::addFunctionSimplificationPasses( assert(OptLevel >= 1 && "Calling function optimizer with no optimization level!"); MPM.add(createSROAPass()); MPM.add(createEarlyCSEPass(true /* Enable mem-ssa. */)); // Catch trivial redundancies + if (EnableKnowledgeRetention) + MPM.add(createAssumeSimplifyPass()); + + if (OptLevel > 1) { + if (EnableGVNHoist) + MPM.add(createGVNHoistPass()); + if (EnableGVNSink) { + MPM.add(createGVNSinkPass()); + MPM.add(createCFGSimplificationPass( + SimplifyCFGOptions().convertSwitchRangeToICmp(true))); + } + } + + if (EnableConstraintElimination) + MPM.add(createConstraintEliminationPass()); if (OptLevel > 1) { // Speculative execution if the target has divergent branches; otherwise nop. @@ -177,7 +339,7 @@ void PassManagerBuilder::addFunctionSimplificationPasses( if (OptLevel > 2) MPM.add(createAggressiveInstCombinerPass()); MPM.add(createInstructionCombiningPass()); - if (SizeLevel == 0) + if (SizeLevel == 0 && !DisableLibCallsShrinkWrap) MPM.add(createLibCallsShrinkWrapPass()); addExtensionsToPM(EP_Peephole, MPM); @@ -189,6 +351,11 @@ void PassManagerBuilder::addFunctionSimplificationPasses( true))); // Merge & remove BBs MPM.add(createReassociatePass()); // Reassociate expressions + // The matrix extension can introduce large vector operations early, which can + // benefit from running vector-combine early on. + if (EnableMatrix) + MPM.add(createVectorCombinePass()); + // Begin the loop pass pipeline. // The simple loop unswitch pass relies on separate cleanup passes. Schedule @@ -218,11 +385,18 @@ void PassManagerBuilder::addFunctionSimplificationPasses( SimplifyCFGOptions().convertSwitchRangeToICmp(true))); MPM.add(createInstructionCombiningPass()); // We resume loop passes creating a second loop pipeline here. + if (EnableLoopFlatten) { + MPM.add(createLoopFlattenPass()); // Flatten loops + MPM.add(createLoopSimplifyCFGPass()); + } MPM.add(createLoopIdiomPass()); // Recognize idioms like memset. MPM.add(createIndVarSimplifyPass()); // Canonicalize indvars addExtensionsToPM(EP_LateLoopOptimizations, MPM); MPM.add(createLoopDeletionPass()); // Delete dead loops + if (EnableLoopInterchange) + MPM.add(createLoopInterchangePass()); // Interchange loops + // Unroll small loops and perform peeling. MPM.add(createSimpleLoopUnrollPass(OptLevel, DisableUnrollLoops, ForgetAllSCEVInLoopUnroll)); @@ -239,6 +413,9 @@ void PassManagerBuilder::addFunctionSimplificationPasses( } MPM.add(createSCCPPass()); // Constant prop with SCCP + if (EnableConstraintElimination) + MPM.add(createConstraintEliminationPass()); + // Delete dead bit computations (instcombine runs after to fold away the dead // computations, and then ADCE will run later to exploit any new DCE // opportunities that creates). @@ -249,6 +426,9 @@ void PassManagerBuilder::addFunctionSimplificationPasses( MPM.add(createInstructionCombiningPass()); addExtensionsToPM(EP_Peephole, MPM); if (OptLevel > 1) { + if (EnableDFAJumpThreading && SizeLevel == 0) + MPM.add(createDFAJumpThreadingPass()); + MPM.add(createJumpThreadingPass()); // Thread jumps MPM.add(createCorrelatedValuePropagationPass()); } @@ -264,6 +444,9 @@ void PassManagerBuilder::addFunctionSimplificationPasses( addExtensionsToPM(EP_ScalarOptimizerLate, MPM); + if (RerollLoops) + MPM.add(createLoopRerollPass()); + // Merge & remove BBs and sink & hoist common instructions. MPM.add(createCFGSimplificationPass( SimplifyCFGOptions().hoistCommonInsts(true).sinkCommonInsts(true))); @@ -285,6 +468,9 @@ void PassManagerBuilder::addVectorPasses(legacy::PassManagerBase &PM, // FIXME: It would be really good to use a loop-integrated instruction // combiner for cleanup here so that the unrolling and LICM can be pipelined // across the loop nests. + // We do UnrollAndJam in a separate LPM to ensure it happens before unroll + if (EnableUnrollAndJam && !DisableUnrollLoops) + PM.add(createLoopUnrollAndJamPass(OptLevel)); PM.add(createLoopUnrollPass(OptLevel, DisableUnrollLoops, ForgetAllSCEVInLoopUnroll)); PM.add(createWarnMissedTransformationsPass()); @@ -298,6 +484,24 @@ void PassManagerBuilder::addVectorPasses(legacy::PassManagerBase &PM, // Cleanup after the loop optimization passes. PM.add(createInstructionCombiningPass()); + if (OptLevel > 1 && ExtraVectorizerPasses) { + // At higher optimization levels, try to clean up any runtime overlap and + // alignment checks inserted by the vectorizer. We want to track correlated + // runtime checks for two inner loops in the same outer loop, fold any + // common computations, hoist loop-invariant aspects out of any outer loop, + // and unswitch the runtime checks if possible. Once hoisted, we may have + // dead (or speculatable) control flows or more combining opportunities. + PM.add(createEarlyCSEPass()); + PM.add(createCorrelatedValuePropagationPass()); + PM.add(createInstructionCombiningPass()); + PM.add(createLICMPass(LicmMssaOptCap, LicmMssaNoAccForPromotionCap, + /*AllowSpeculation=*/true)); + PM.add(createSimpleLoopUnswitchLegacyPass()); + PM.add(createCFGSimplificationPass( + SimplifyCFGOptions().convertSwitchRangeToICmp(true))); + PM.add(createInstructionCombiningPass()); + } + // Now that we've formed fast to execute loop structures, we do further // optimizations. These are run afterward as they might block doing complex // analyses and transforms such as what are needed for loop vectorization. @@ -324,6 +528,8 @@ void PassManagerBuilder::addVectorPasses(legacy::PassManagerBase &PM, // Optimize parallel scalar instruction chains into SIMD instructions. if (SLPVectorize) { PM.add(createSLPVectorizerPass()); + if (OptLevel > 1 && ExtraVectorizerPasses) + PM.add(createEarlyCSEPass()); } // Enhance/cleanup vector code. @@ -333,6 +539,13 @@ void PassManagerBuilder::addVectorPasses(legacy::PassManagerBase &PM, addExtensionsToPM(EP_Peephole, PM); PM.add(createInstructionCombiningPass()); + if (EnableUnrollAndJam && !DisableUnrollLoops) { + // Unroll and Jam. We do this before unroll but need to be in a separate + // loop pass manager in order for the outer loop to be processed by + // unroll and jam before the inner loop is unrolled. + PM.add(createLoopUnrollAndJamPass(OptLevel)); + } + // Unroll small loops PM.add(createLoopUnrollPass(OptLevel, DisableUnrollLoops, ForgetAllSCEVInLoopUnroll)); @@ -400,11 +613,19 @@ void PassManagerBuilder::populateModulePassManager( // Infer attributes about declarations if possible. MPM.add(createInferFunctionAttrsLegacyPass()); + // Infer attributes on declarations, call sites, arguments, etc. + if (AttributorRun & AttributorRunOption::MODULE) + MPM.add(createAttributorLegacyPass()); + addExtensionsToPM(EP_ModuleOptimizerEarly, MPM); if (OptLevel > 2) MPM.add(createCallSiteSplittingPass()); + // Propage constant function arguments by specializing the functions. + if (OptLevel > 2 && EnableFunctionSpecialization) + MPM.add(createFunctionSpecializationPass()); + MPM.add(createIPSCCPPass()); // IP SCCP MPM.add(createCalledValuePropagationPass()); @@ -433,6 +654,10 @@ void PassManagerBuilder::populateModulePassManager( RunInliner = true; } + // Infer attributes on declarations, call sites, arguments, etc. for an SCC. + if (AttributorRun & AttributorRunOption::CGSCC) + MPM.add(createAttributorCGSCCLegacyPass()); + // Try to perform OpenMP specific optimizations. This is a (quick!) no-op if // there are no OpenMP runtime calls present in the module. if (OptLevel > 1) @@ -448,6 +673,9 @@ void PassManagerBuilder::populateModulePassManager( // we must insert a no-op module pass to reset the pass manager. MPM.add(createBarrierNoopPass()); + if (RunPartialInlining) + MPM.add(createPartialInliningPass()); + if (OptLevel > 1) // Remove avail extern fns and globals definitions if we aren't // compiling an object file for later LTO. For LTO we want to preserve @@ -473,6 +701,17 @@ void PassManagerBuilder::populateModulePassManager( MPM.add(createGlobalDCEPass()); } + // Scheduling LoopVersioningLICM when inlining is over, because after that + // we may see more accurate aliasing. Reason to run this late is that too + // early versioning may prevent further inlining due to increase of code + // size. By placing it just after inlining other optimizations which runs + // later might get benefit of no-alias assumption in clone loop. + if (UseLoopVersioningLICM) { + MPM.add(createLoopVersioningLICMPass()); // Do LoopVersioningLICM + MPM.add(createLICMPass(LicmMssaOptCap, LicmMssaNoAccForPromotionCap, + /*AllowSpeculation=*/true)); + } + // We add a fresh GlobalsModRef run at this point. This is particularly // useful as the above will have inlined, DCE'ed, and function-attr // propagated everything. We should at this point have a reasonably minimal @@ -493,6 +732,14 @@ void PassManagerBuilder::populateModulePassManager( MPM.add(createFloat2IntPass()); MPM.add(createLowerConstantIntrinsicsPass()); + if (EnableMatrix) { + MPM.add(createLowerMatrixIntrinsicsPass()); + // CSE the pointer arithmetic of the column vectors. This allows alias + // analysis to establish no-aliasing between loads and stores of different + // columns of the same matrix. + MPM.add(createEarlyCSEPass(false)); + } + addExtensionsToPM(EP_VectorizerStart, MPM); // Re-rotate loops in all our loop nests. These may have fallout out of @@ -518,6 +765,14 @@ void PassManagerBuilder::populateModulePassManager( MPM.add(createConstantMergePass()); // Merge dup global constants } + // See comment in the new PM for justification of scheduling splitting at + // this stage (\ref buildModuleSimplificationPipeline). + if (EnableHotColdSplit) + MPM.add(createHotColdSplittingPass()); + + if (EnableIROutliner) + MPM.add(createIROutlinerPass()); + if (MergeFunctions) MPM.add(createMergeFunctionsPass());