38fe128d7c1e6d01358f2a7f589f13eb3d3d0bd8
[lldb.git] / llvm / lib / Passes / PassBuilder.cpp
1 //===- Parsing, selection, and construction of pass pipelines -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 ///
10 /// This file provides the implementation of the PassBuilder based on our
11 /// static pass registry as well as related functionality. It also provides
12 /// helpers to aid in analyzing, debugging, and testing passes and pass
13 /// pipelines.
14 ///
15 //===----------------------------------------------------------------------===//
16
17 #include "llvm/Passes/PassBuilder.h"
18 #include "llvm/ADT/StringSwitch.h"
19 #include "llvm/Analysis/AliasAnalysis.h"
20 #include "llvm/Analysis/AliasAnalysisEvaluator.h"
21 #include "llvm/Analysis/AliasSetTracker.h"
22 #include "llvm/Analysis/AssumptionCache.h"
23 #include "llvm/Analysis/BasicAliasAnalysis.h"
24 #include "llvm/Analysis/BlockFrequencyInfo.h"
25 #include "llvm/Analysis/BranchProbabilityInfo.h"
26 #include "llvm/Analysis/CFGPrinter.h"
27 #include "llvm/Analysis/CFLAndersAliasAnalysis.h"
28 #include "llvm/Analysis/CFLSteensAliasAnalysis.h"
29 #include "llvm/Analysis/CGSCCPassManager.h"
30 #include "llvm/Analysis/CallGraph.h"
31 #include "llvm/Analysis/DDG.h"
32 #include "llvm/Analysis/Delinearization.h"
33 #include "llvm/Analysis/DemandedBits.h"
34 #include "llvm/Analysis/DependenceAnalysis.h"
35 #include "llvm/Analysis/DominanceFrontier.h"
36 #include "llvm/Analysis/FunctionPropertiesAnalysis.h"
37 #include "llvm/Analysis/GlobalsModRef.h"
38 #include "llvm/Analysis/IRSimilarityIdentifier.h"
39 #include "llvm/Analysis/IVUsers.h"
40 #include "llvm/Analysis/InlineAdvisor.h"
41 #include "llvm/Analysis/InlineSizeEstimatorAnalysis.h"
42 #include "llvm/Analysis/InstCount.h"
43 #include "llvm/Analysis/LazyCallGraph.h"
44 #include "llvm/Analysis/LazyValueInfo.h"
45 #include "llvm/Analysis/Lint.h"
46 #include "llvm/Analysis/LoopAccessAnalysis.h"
47 #include "llvm/Analysis/LoopCacheAnalysis.h"
48 #include "llvm/Analysis/LoopInfo.h"
49 #include "llvm/Analysis/LoopNestAnalysis.h"
50 #include "llvm/Analysis/MemoryDependenceAnalysis.h"
51 #include "llvm/Analysis/MemorySSA.h"
52 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
53 #include "llvm/Analysis/ObjCARCAliasAnalysis.h"
54 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
55 #include "llvm/Analysis/PhiValues.h"
56 #include "llvm/Analysis/PostDominators.h"
57 #include "llvm/Analysis/ProfileSummaryInfo.h"
58 #include "llvm/Analysis/RegionInfo.h"
59 #include "llvm/Analysis/ScalarEvolution.h"
60 #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
61 #include "llvm/Analysis/ScopedNoAliasAA.h"
62 #include "llvm/Analysis/StackLifetime.h"
63 #include "llvm/Analysis/StackSafetyAnalysis.h"
64 #include "llvm/Analysis/TargetLibraryInfo.h"
65 #include "llvm/Analysis/TargetTransformInfo.h"
66 #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
67 #include "llvm/IR/Dominators.h"
68 #include "llvm/IR/IRPrintingPasses.h"
69 #include "llvm/IR/PassManager.h"
70 #include "llvm/IR/SafepointIRVerifier.h"
71 #include "llvm/IR/Verifier.h"
72 #include "llvm/Support/CommandLine.h"
73 #include "llvm/Support/Debug.h"
74 #include "llvm/Support/FormatVariadic.h"
75 #include "llvm/Support/Regex.h"
76 #include "llvm/Target/TargetMachine.h"
77 #include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
78 #include "llvm/Transforms/Coroutines/CoroCleanup.h"
79 #include "llvm/Transforms/Coroutines/CoroEarly.h"
80 #include "llvm/Transforms/Coroutines/CoroElide.h"
81 #include "llvm/Transforms/Coroutines/CoroSplit.h"
82 #include "llvm/Transforms/HelloNew/HelloWorld.h"
83 #include "llvm/Transforms/IPO/AlwaysInliner.h"
84 #include "llvm/Transforms/IPO/ArgumentPromotion.h"
85 #include "llvm/Transforms/IPO/Attributor.h"
86 #include "llvm/Transforms/IPO/CalledValuePropagation.h"
87 #include "llvm/Transforms/IPO/ConstantMerge.h"
88 #include "llvm/Transforms/IPO/CrossDSOCFI.h"
89 #include "llvm/Transforms/IPO/DeadArgumentElimination.h"
90 #include "llvm/Transforms/IPO/ElimAvailExtern.h"
91 #include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
92 #include "llvm/Transforms/IPO/FunctionAttrs.h"
93 #include "llvm/Transforms/IPO/FunctionImport.h"
94 #include "llvm/Transforms/IPO/GlobalDCE.h"
95 #include "llvm/Transforms/IPO/GlobalOpt.h"
96 #include "llvm/Transforms/IPO/GlobalSplit.h"
97 #include "llvm/Transforms/IPO/HotColdSplitting.h"
98 #include "llvm/Transforms/IPO/InferFunctionAttrs.h"
99 #include "llvm/Transforms/IPO/Inliner.h"
100 #include "llvm/Transforms/IPO/Internalize.h"
101 #include "llvm/Transforms/IPO/LowerTypeTests.h"
102 #include "llvm/Transforms/IPO/MergeFunctions.h"
103 #include "llvm/Transforms/IPO/OpenMPOpt.h"
104 #include "llvm/Transforms/IPO/PartialInlining.h"
105 #include "llvm/Transforms/IPO/SCCP.h"
106 #include "llvm/Transforms/IPO/SampleProfile.h"
107 #include "llvm/Transforms/IPO/StripDeadPrototypes.h"
108 #include "llvm/Transforms/IPO/StripSymbols.h"
109 #include "llvm/Transforms/IPO/SyntheticCountsPropagation.h"
110 #include "llvm/Transforms/IPO/WholeProgramDevirt.h"
111 #include "llvm/Transforms/InstCombine/InstCombine.h"
112 #include "llvm/Transforms/Instrumentation.h"
113 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
114 #include "llvm/Transforms/Instrumentation/BoundsChecking.h"
115 #include "llvm/Transforms/Instrumentation/CGProfile.h"
116 #include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
117 #include "llvm/Transforms/Instrumentation/DataFlowSanitizer.h"
118 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
119 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
120 #include "llvm/Transforms/Instrumentation/InstrOrderFile.h"
121 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
122 #include "llvm/Transforms/Instrumentation/MemProfiler.h"
123 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
124 #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
125 #include "llvm/Transforms/Instrumentation/PoisonChecking.h"
126 #include "llvm/Transforms/Instrumentation/SanitizerCoverage.h"
127 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
128 #include "llvm/Transforms/ObjCARC.h"
129 #include "llvm/Transforms/Scalar/ADCE.h"
130 #include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
131 #include "llvm/Transforms/Scalar/BDCE.h"
132 #include "llvm/Transforms/Scalar/CallSiteSplitting.h"
133 #include "llvm/Transforms/Scalar/ConstantHoisting.h"
134 #include "llvm/Transforms/Scalar/ConstraintElimination.h"
135 #include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h"
136 #include "llvm/Transforms/Scalar/DCE.h"
137 #include "llvm/Transforms/Scalar/DeadStoreElimination.h"
138 #include "llvm/Transforms/Scalar/DivRemPairs.h"
139 #include "llvm/Transforms/Scalar/EarlyCSE.h"
140 #include "llvm/Transforms/Scalar/Float2Int.h"
141 #include "llvm/Transforms/Scalar/GVN.h"
142 #include "llvm/Transforms/Scalar/GuardWidening.h"
143 #include "llvm/Transforms/Scalar/IVUsersPrinter.h"
144 #include "llvm/Transforms/Scalar/IndVarSimplify.h"
145 #include "llvm/Transforms/Scalar/InductiveRangeCheckElimination.h"
146 #include "llvm/Transforms/Scalar/InstSimplifyPass.h"
147 #include "llvm/Transforms/Scalar/JumpThreading.h"
148 #include "llvm/Transforms/Scalar/LICM.h"
149 #include "llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h"
150 #include "llvm/Transforms/Scalar/LoopDataPrefetch.h"
151 #include "llvm/Transforms/Scalar/LoopDeletion.h"
152 #include "llvm/Transforms/Scalar/LoopDistribute.h"
153 #include "llvm/Transforms/Scalar/LoopFlatten.h"
154 #include "llvm/Transforms/Scalar/LoopFuse.h"
155 #include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
156 #include "llvm/Transforms/Scalar/LoopInstSimplify.h"
157 #include "llvm/Transforms/Scalar/LoopLoadElimination.h"
158 #include "llvm/Transforms/Scalar/LoopPassManager.h"
159 #include "llvm/Transforms/Scalar/LoopPredication.h"
160 #include "llvm/Transforms/Scalar/LoopReroll.h"
161 #include "llvm/Transforms/Scalar/LoopRotation.h"
162 #include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"
163 #include "llvm/Transforms/Scalar/LoopSink.h"
164 #include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
165 #include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
166 #include "llvm/Transforms/Scalar/LoopUnrollPass.h"
167 #include "llvm/Transforms/Scalar/LowerAtomic.h"
168 #include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
169 #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
170 #include "llvm/Transforms/Scalar/LowerGuardIntrinsic.h"
171 #include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h"
172 #include "llvm/Transforms/Scalar/LowerWidenableCondition.h"
173 #include "llvm/Transforms/Scalar/MakeGuardsExplicit.h"
174 #include "llvm/Transforms/Scalar/MemCpyOptimizer.h"
175 #include "llvm/Transforms/Scalar/MergeICmps.h"
176 #include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h"
177 #include "llvm/Transforms/Scalar/NaryReassociate.h"
178 #include "llvm/Transforms/Scalar/NewGVN.h"
179 #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
180 #include "llvm/Transforms/Scalar/Reassociate.h"
181 #include "llvm/Transforms/Scalar/RewriteStatepointsForGC.h"
182 #include "llvm/Transforms/Scalar/SCCP.h"
183 #include "llvm/Transforms/Scalar/SROA.h"
184 #include "llvm/Transforms/Scalar/Scalarizer.h"
185 #include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
186 #include "llvm/Transforms/Scalar/SimplifyCFG.h"
187 #include "llvm/Transforms/Scalar/Sink.h"
188 #include "llvm/Transforms/Scalar/SpeculateAroundPHIs.h"
189 #include "llvm/Transforms/Scalar/SpeculativeExecution.h"
190 #include "llvm/Transforms/Scalar/TailRecursionElimination.h"
191 #include "llvm/Transforms/Scalar/WarnMissedTransforms.h"
192 #include "llvm/Transforms/Utils/AddDiscriminators.h"
193 #include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
194 #include "llvm/Transforms/Utils/BreakCriticalEdges.h"
195 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
196 #include "llvm/Transforms/Utils/CanonicalizeFreezeInLoops.h"
197 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
198 #include "llvm/Transforms/Utils/InjectTLIMappings.h"
199 #include "llvm/Transforms/Utils/LCSSA.h"
200 #include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
201 #include "llvm/Transforms/Utils/LoopSimplify.h"
202 #include "llvm/Transforms/Utils/LoopVersioning.h"
203 #include "llvm/Transforms/Utils/LowerInvoke.h"
204 #include "llvm/Transforms/Utils/LowerSwitch.h"
205 #include "llvm/Transforms/Utils/Mem2Reg.h"
206 #include "llvm/Transforms/Utils/MetaRenamer.h"
207 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
208 #include "llvm/Transforms/Utils/SymbolRewriter.h"
209 #include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
210 #include "llvm/Transforms/Vectorize/LoopVectorize.h"
211 #include "llvm/Transforms/Vectorize/SLPVectorizer.h"
212 #include "llvm/Transforms/Vectorize/VectorCombine.h"
213
214 using namespace llvm;
215
216 static cl::opt<unsigned> MaxDevirtIterations("pm-max-devirt-iterations",
217                                              cl::ReallyHidden, cl::init(4));
218 static cl::opt<bool>
219     RunPartialInlining("enable-npm-partial-inlining", cl::init(false),
220                        cl::Hidden, cl::ZeroOrMore,
221                        cl::desc("Run Partial inlinining pass"));
222
223 static cl::opt<int> PreInlineThreshold(
224     "npm-preinline-threshold", cl::Hidden, cl::init(75), cl::ZeroOrMore,
225     cl::desc("Control the amount of inlining in pre-instrumentation inliner "
226              "(default = 75)"));
227
228 static cl::opt<bool>
229     RunNewGVN("enable-npm-newgvn", cl::init(false),
230               cl::Hidden, cl::ZeroOrMore,
231               cl::desc("Run NewGVN instead of GVN"));
232
233 static cl::opt<bool> EnableGVNHoist(
234     "enable-npm-gvn-hoist", cl::init(false), cl::Hidden,
235     cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
236
237 static cl::opt<InliningAdvisorMode> UseInlineAdvisor(
238     "enable-ml-inliner", cl::init(InliningAdvisorMode::Default), cl::Hidden,
239     cl::desc("Enable ML policy for inliner. Currently trained for -Oz only"),
240     cl::values(clEnumValN(InliningAdvisorMode::Default, "default",
241                           "Heuristics-based inliner version."),
242                clEnumValN(InliningAdvisorMode::Development, "development",
243                           "Use development mode (runtime-loadable model)."),
244                clEnumValN(InliningAdvisorMode::Release, "release",
245                           "Use release mode (AOT-compiled model).")));
246
247 static cl::opt<bool> EnableGVNSink(
248     "enable-npm-gvn-sink", cl::init(false), cl::Hidden,
249     cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
250
251 static cl::opt<bool> EnableUnrollAndJam(
252     "enable-npm-unroll-and-jam", cl::init(false), cl::Hidden,
253     cl::desc("Enable the Unroll and Jam pass for the new PM (default = off)"));
254
255 static cl::opt<bool> EnableLoopFlatten(
256     "enable-npm-loop-flatten", cl::init(false), cl::Hidden,
257     cl::desc("Enable the Loop flattening pass for the new PM (default = off)"));
258
259 static cl::opt<bool> EnableSyntheticCounts(
260     "enable-npm-synthetic-counts", cl::init(false), cl::Hidden, cl::ZeroOrMore,
261     cl::desc("Run synthetic function entry count generation "
262              "pass"));
263
264 static const Regex DefaultAliasRegex(
265     "^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$");
266
267 // This option is used in simplifying testing SampleFDO optimizations for
268 // profile loading.
269 static cl::opt<bool>
270     EnableCHR("enable-chr-npm", cl::init(true), cl::Hidden,
271               cl::desc("Enable control height reduction optimization (CHR)"));
272
273 /// Flag to enable inline deferral during PGO.
274 static cl::opt<bool>
275     EnablePGOInlineDeferral("enable-npm-pgo-inline-deferral", cl::init(true),
276                             cl::Hidden,
277                             cl::desc("Enable inline deferral during PGO"));
278
279 static cl::opt<bool> EnableMemProfiler("enable-mem-prof", cl::init(false),
280                                        cl::Hidden, cl::ZeroOrMore,
281                                        cl::desc("Enable memory profiler"));
282
283 PipelineTuningOptions::PipelineTuningOptions() {
284   LoopInterleaving = true;
285   LoopVectorization = true;
286   SLPVectorization = false;
287   LoopUnrolling = true;
288   ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll;
289   Coroutines = false;
290   LicmMssaOptCap = SetLicmMssaOptCap;
291   LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap;
292   CallGraphProfile = true;
293 }
294
295 extern cl::opt<bool> EnableConstraintElimination;
296 extern cl::opt<bool> EnableHotColdSplit;
297 extern cl::opt<bool> EnableOrderFileInstrumentation;
298
299 extern cl::opt<bool> FlattenedProfileUsed;
300
301 extern cl::opt<AttributorRunOption> AttributorRun;
302 extern cl::opt<bool> EnableKnowledgeRetention;
303
304 extern cl::opt<bool> EnableMatrix;
305
306 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O0 = {
307     /*SpeedLevel*/ 0,
308     /*SizeLevel*/ 0};
309 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O1 = {
310     /*SpeedLevel*/ 1,
311     /*SizeLevel*/ 0};
312 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O2 = {
313     /*SpeedLevel*/ 2,
314     /*SizeLevel*/ 0};
315 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O3 = {
316     /*SpeedLevel*/ 3,
317     /*SizeLevel*/ 0};
318 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::Os = {
319     /*SpeedLevel*/ 2,
320     /*SizeLevel*/ 1};
321 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::Oz = {
322     /*SpeedLevel*/ 2,
323     /*SizeLevel*/ 2};
324
325 namespace {
326
327 // The following passes/analyses have custom names, otherwise their name will
328 // include `(anonymous namespace)`. These are special since they are only for
329 // testing purposes and don't live in a header file.
330
331 /// No-op module pass which does nothing.
332 struct NoOpModulePass : PassInfoMixin<NoOpModulePass> {
333   PreservedAnalyses run(Module &M, ModuleAnalysisManager &) {
334     return PreservedAnalyses::all();
335   }
336
337   static StringRef name() { return "NoOpModulePass"; }
338 };
339
340 /// No-op module analysis.
341 class NoOpModuleAnalysis : public AnalysisInfoMixin<NoOpModuleAnalysis> {
342   friend AnalysisInfoMixin<NoOpModuleAnalysis>;
343   static AnalysisKey Key;
344
345 public:
346   struct Result {};
347   Result run(Module &, ModuleAnalysisManager &) { return Result(); }
348   static StringRef name() { return "NoOpModuleAnalysis"; }
349 };
350
351 /// No-op CGSCC pass which does nothing.
352 struct NoOpCGSCCPass : PassInfoMixin<NoOpCGSCCPass> {
353   PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &,
354                         LazyCallGraph &, CGSCCUpdateResult &UR) {
355     return PreservedAnalyses::all();
356   }
357   static StringRef name() { return "NoOpCGSCCPass"; }
358 };
359
360 /// No-op CGSCC analysis.
361 class NoOpCGSCCAnalysis : public AnalysisInfoMixin<NoOpCGSCCAnalysis> {
362   friend AnalysisInfoMixin<NoOpCGSCCAnalysis>;
363   static AnalysisKey Key;
364
365 public:
366   struct Result {};
367   Result run(LazyCallGraph::SCC &, CGSCCAnalysisManager &, LazyCallGraph &G) {
368     return Result();
369   }
370   static StringRef name() { return "NoOpCGSCCAnalysis"; }
371 };
372
373 /// No-op function pass which does nothing.
374 struct NoOpFunctionPass : PassInfoMixin<NoOpFunctionPass> {
375   PreservedAnalyses run(Function &F, FunctionAnalysisManager &) {
376     return PreservedAnalyses::all();
377   }
378   static StringRef name() { return "NoOpFunctionPass"; }
379 };
380
381 /// No-op function analysis.
382 class NoOpFunctionAnalysis : public AnalysisInfoMixin<NoOpFunctionAnalysis> {
383   friend AnalysisInfoMixin<NoOpFunctionAnalysis>;
384   static AnalysisKey Key;
385
386 public:
387   struct Result {};
388   Result run(Function &, FunctionAnalysisManager &) { return Result(); }
389   static StringRef name() { return "NoOpFunctionAnalysis"; }
390 };
391
392 /// No-op loop pass which does nothing.
393 struct NoOpLoopPass : PassInfoMixin<NoOpLoopPass> {
394   PreservedAnalyses run(Loop &L, LoopAnalysisManager &,
395                         LoopStandardAnalysisResults &, LPMUpdater &) {
396     return PreservedAnalyses::all();
397   }
398   static StringRef name() { return "NoOpLoopPass"; }
399 };
400
401 /// No-op loop analysis.
402 class NoOpLoopAnalysis : public AnalysisInfoMixin<NoOpLoopAnalysis> {
403   friend AnalysisInfoMixin<NoOpLoopAnalysis>;
404   static AnalysisKey Key;
405
406 public:
407   struct Result {};
408   Result run(Loop &, LoopAnalysisManager &, LoopStandardAnalysisResults &) {
409     return Result();
410   }
411   static StringRef name() { return "NoOpLoopAnalysis"; }
412 };
413
414 AnalysisKey NoOpModuleAnalysis::Key;
415 AnalysisKey NoOpCGSCCAnalysis::Key;
416 AnalysisKey NoOpFunctionAnalysis::Key;
417 AnalysisKey NoOpLoopAnalysis::Key;
418
419 } // namespace
420
421 void PassBuilder::invokePeepholeEPCallbacks(
422     FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
423   for (auto &C : PeepholeEPCallbacks)
424     C(FPM, Level);
425 }
426
427 void PassBuilder::registerModuleAnalyses(ModuleAnalysisManager &MAM) {
428 #define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
429   MAM.registerPass([&] { return CREATE_PASS; });
430 #include "PassRegistry.def"
431
432   for (auto &C : ModuleAnalysisRegistrationCallbacks)
433     C(MAM);
434 }
435
436 void PassBuilder::registerCGSCCAnalyses(CGSCCAnalysisManager &CGAM) {
437 #define CGSCC_ANALYSIS(NAME, CREATE_PASS)                                      \
438   CGAM.registerPass([&] { return CREATE_PASS; });
439 #include "PassRegistry.def"
440
441   for (auto &C : CGSCCAnalysisRegistrationCallbacks)
442     C(CGAM);
443 }
444
445 void PassBuilder::registerFunctionAnalyses(FunctionAnalysisManager &FAM) {
446 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
447   FAM.registerPass([&] { return CREATE_PASS; });
448 #include "PassRegistry.def"
449
450   for (auto &C : FunctionAnalysisRegistrationCallbacks)
451     C(FAM);
452 }
453
454 void PassBuilder::registerLoopAnalyses(LoopAnalysisManager &LAM) {
455 #define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
456   LAM.registerPass([&] { return CREATE_PASS; });
457 #include "PassRegistry.def"
458
459   for (auto &C : LoopAnalysisRegistrationCallbacks)
460     C(LAM);
461 }
462
463 // TODO: Investigate the cost/benefit of tail call elimination on debugging.
464 FunctionPassManager PassBuilder::buildO1FunctionSimplificationPipeline(
465     OptimizationLevel Level, ThinLTOPhase Phase, bool DebugLogging) {
466
467   FunctionPassManager FPM(DebugLogging);
468
469   // Form SSA out of local memory accesses after breaking apart aggregates into
470   // scalars.
471   FPM.addPass(SROA());
472
473   // Catch trivial redundancies
474   FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
475
476   // Hoisting of scalars and load expressions.
477   FPM.addPass(SimplifyCFGPass());
478   FPM.addPass(InstCombinePass());
479
480   FPM.addPass(LibCallsShrinkWrapPass());
481
482   invokePeepholeEPCallbacks(FPM, Level);
483
484   FPM.addPass(SimplifyCFGPass());
485
486   // Form canonically associated expression trees, and simplify the trees using
487   // basic mathematical properties. For example, this will form (nearly)
488   // minimal multiplication trees.
489   FPM.addPass(ReassociatePass());
490
491   // Add the primary loop simplification pipeline.
492   // FIXME: Currently this is split into two loop pass pipelines because we run
493   // some function passes in between them. These can and should be removed
494   // and/or replaced by scheduling the loop pass equivalents in the correct
495   // positions. But those equivalent passes aren't powerful enough yet.
496   // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
497   // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
498   // fully replace `SimplifyCFGPass`, and the closest to the other we have is
499   // `LoopInstSimplify`.
500   LoopPassManager LPM1(DebugLogging), LPM2(DebugLogging);
501
502   // Simplify the loop body. We do this initially to clean up after other loop
503   // passes run, either when iterating on a loop or on inner loops with
504   // implications on the outer loop.
505   LPM1.addPass(LoopInstSimplifyPass());
506   LPM1.addPass(LoopSimplifyCFGPass());
507
508   LPM1.addPass(LoopRotatePass(/* Disable header duplication */ true));
509   // TODO: Investigate promotion cap for O1.
510   LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
511   LPM1.addPass(SimpleLoopUnswitchPass());
512   LPM2.addPass(IndVarSimplifyPass());
513   LPM2.addPass(LoopIdiomRecognizePass());
514
515   for (auto &C : LateLoopOptimizationsEPCallbacks)
516     C(LPM2, Level);
517
518   LPM2.addPass(LoopDeletionPass());
519   if (EnableLoopFlatten)
520     LPM2.addPass(LoopFlattenPass());
521   // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
522   // because it changes IR to makes profile annotation in back compile
523   // inaccurate. The normal unroller doesn't pay attention to forced full unroll
524   // attributes so we need to make sure and allow the full unroll pass to pay
525   // attention to it.
526   if (Phase != ThinLTOPhase::PreLink || !PGOOpt ||
527       PGOOpt->Action != PGOOptions::SampleUse)
528     LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
529                                     /* OnlyWhenForced= */ !PTO.LoopUnrolling,
530                                     PTO.ForgetAllSCEVInLoopUnroll));
531
532   for (auto &C : LoopOptimizerEndEPCallbacks)
533     C(LPM2, Level);
534
535   // We provide the opt remark emitter pass for LICM to use. We only need to do
536   // this once as it is immutable.
537   FPM.addPass(
538       RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
539   FPM.addPass(createFunctionToLoopPassAdaptor(
540       std::move(LPM1), EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true,
541       DebugLogging));
542   FPM.addPass(SimplifyCFGPass());
543   FPM.addPass(InstCombinePass());
544   // The loop passes in LPM2 (LoopFullUnrollPass) do not preserve MemorySSA.
545   // *All* loop passes must preserve it, in order to be able to use it.
546   FPM.addPass(createFunctionToLoopPassAdaptor(
547       std::move(LPM2), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false,
548       DebugLogging));
549
550   // Delete small array after loop unroll.
551   FPM.addPass(SROA());
552
553   // Specially optimize memory movement as it doesn't look like dataflow in SSA.
554   FPM.addPass(MemCpyOptPass());
555
556   // Sparse conditional constant propagation.
557   // FIXME: It isn't clear why we do this *after* loop passes rather than
558   // before...
559   FPM.addPass(SCCPPass());
560
561   // Delete dead bit computations (instcombine runs after to fold away the dead
562   // computations, and then ADCE will run later to exploit any new DCE
563   // opportunities that creates).
564   FPM.addPass(BDCEPass());
565
566   // Run instcombine after redundancy and dead bit elimination to exploit
567   // opportunities opened up by them.
568   FPM.addPass(InstCombinePass());
569   invokePeepholeEPCallbacks(FPM, Level);
570
571   if (PTO.Coroutines)
572     FPM.addPass(CoroElidePass());
573
574   for (auto &C : ScalarOptimizerLateEPCallbacks)
575     C(FPM, Level);
576
577   // Finally, do an expensive DCE pass to catch all the dead code exposed by
578   // the simplifications and basic cleanup after all the simplifications.
579   // TODO: Investigate if this is too expensive.
580   FPM.addPass(ADCEPass());
581   FPM.addPass(SimplifyCFGPass());
582   FPM.addPass(InstCombinePass());
583   invokePeepholeEPCallbacks(FPM, Level);
584
585   return FPM;
586 }
587
588 FunctionPassManager
589 PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
590                                                  ThinLTOPhase Phase,
591                                                  bool DebugLogging) {
592   assert(Level != OptimizationLevel::O0 && "Must request optimizations!");
593
594   // The O1 pipeline has a separate pipeline creation function to simplify
595   // construction readability.
596   if (Level.getSpeedupLevel() == 1)
597     return buildO1FunctionSimplificationPipeline(Level, Phase, DebugLogging);
598
599   FunctionPassManager FPM(DebugLogging);
600
601   // Form SSA out of local memory accesses after breaking apart aggregates into
602   // scalars.
603   FPM.addPass(SROA());
604
605   // Catch trivial redundancies
606   FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
607   if (EnableKnowledgeRetention)
608     FPM.addPass(AssumeSimplifyPass());
609
610   // Hoisting of scalars and load expressions.
611   if (EnableGVNHoist)
612     FPM.addPass(GVNHoistPass());
613
614   // Global value numbering based sinking.
615   if (EnableGVNSink) {
616     FPM.addPass(GVNSinkPass());
617     FPM.addPass(SimplifyCFGPass());
618   }
619
620   if (EnableConstraintElimination)
621     FPM.addPass(ConstraintEliminationPass());
622
623   // Speculative execution if the target has divergent branches; otherwise nop.
624   FPM.addPass(SpeculativeExecutionPass(/* OnlyIfDivergentTarget =*/true));
625
626   // Optimize based on known information about branches, and cleanup afterward.
627   FPM.addPass(JumpThreadingPass());
628   FPM.addPass(CorrelatedValuePropagationPass());
629
630   FPM.addPass(SimplifyCFGPass());
631   if (Level == OptimizationLevel::O3)
632     FPM.addPass(AggressiveInstCombinePass());
633   FPM.addPass(InstCombinePass());
634
635   if (!Level.isOptimizingForSize())
636     FPM.addPass(LibCallsShrinkWrapPass());
637
638   invokePeepholeEPCallbacks(FPM, Level);
639
640   // For PGO use pipeline, try to optimize memory intrinsics such as memcpy
641   // using the size value profile. Don't perform this when optimizing for size.
642   if (PGOOpt && PGOOpt->Action == PGOOptions::IRUse &&
643       !Level.isOptimizingForSize())
644     FPM.addPass(PGOMemOPSizeOpt());
645
646   FPM.addPass(TailCallElimPass());
647   FPM.addPass(SimplifyCFGPass());
648
649   // Form canonically associated expression trees, and simplify the trees using
650   // basic mathematical properties. For example, this will form (nearly)
651   // minimal multiplication trees.
652   FPM.addPass(ReassociatePass());
653
654   // Add the primary loop simplification pipeline.
655   // FIXME: Currently this is split into two loop pass pipelines because we run
656   // some function passes in between them. These can and should be removed
657   // and/or replaced by scheduling the loop pass equivalents in the correct
658   // positions. But those equivalent passes aren't powerful enough yet.
659   // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
660   // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
661   // fully replace `SimplifyCFGPass`, and the closest to the other we have is
662   // `LoopInstSimplify`.
663   LoopPassManager LPM1(DebugLogging), LPM2(DebugLogging);
664
665   // Simplify the loop body. We do this initially to clean up after other loop
666   // passes run, either when iterating on a loop or on inner loops with
667   // implications on the outer loop.
668   LPM1.addPass(LoopInstSimplifyPass());
669   LPM1.addPass(LoopSimplifyCFGPass());
670
671   // Rotate Loop - disable header duplication at -Oz
672   LPM1.addPass(LoopRotatePass(Level != OptimizationLevel::Oz));
673   // TODO: Investigate promotion cap for O1.
674   LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
675   LPM1.addPass(SimpleLoopUnswitchPass());
676   LPM2.addPass(IndVarSimplifyPass());
677   LPM2.addPass(LoopIdiomRecognizePass());
678
679   for (auto &C : LateLoopOptimizationsEPCallbacks)
680     C(LPM2, Level);
681
682   LPM2.addPass(LoopDeletionPass());
683   // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
684   // because it changes IR to makes profile annotation in back compile
685   // inaccurate. The normal unroller doesn't pay attention to forced full unroll
686   // attributes so we need to make sure and allow the full unroll pass to pay
687   // attention to it.
688   if (Phase != ThinLTOPhase::PreLink || !PGOOpt ||
689       PGOOpt->Action != PGOOptions::SampleUse)
690     LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
691                                     /* OnlyWhenForced= */ !PTO.LoopUnrolling,
692                                     PTO.ForgetAllSCEVInLoopUnroll));
693
694   for (auto &C : LoopOptimizerEndEPCallbacks)
695     C(LPM2, Level);
696
697   // We provide the opt remark emitter pass for LICM to use. We only need to do
698   // this once as it is immutable.
699   FPM.addPass(
700       RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
701   FPM.addPass(createFunctionToLoopPassAdaptor(
702       std::move(LPM1), EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true,
703       DebugLogging));
704   FPM.addPass(SimplifyCFGPass());
705   FPM.addPass(InstCombinePass());
706   // The loop passes in LPM2 (IndVarSimplifyPass, LoopIdiomRecognizePass,
707   // LoopDeletionPass and LoopFullUnrollPass) do not preserve MemorySSA.
708   // *All* loop passes must preserve it, in order to be able to use it.
709   FPM.addPass(createFunctionToLoopPassAdaptor(
710       std::move(LPM2), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false,
711       DebugLogging));
712
713   // Delete small array after loop unroll.
714   FPM.addPass(SROA());
715
716   // Eliminate redundancies.
717   FPM.addPass(MergedLoadStoreMotionPass());
718   if (RunNewGVN)
719     FPM.addPass(NewGVNPass());
720   else
721     FPM.addPass(GVN());
722
723   // Specially optimize memory movement as it doesn't look like dataflow in SSA.
724   FPM.addPass(MemCpyOptPass());
725
726   // Sparse conditional constant propagation.
727   // FIXME: It isn't clear why we do this *after* loop passes rather than
728   // before...
729   FPM.addPass(SCCPPass());
730
731   // Delete dead bit computations (instcombine runs after to fold away the dead
732   // computations, and then ADCE will run later to exploit any new DCE
733   // opportunities that creates).
734   FPM.addPass(BDCEPass());
735
736   // Run instcombine after redundancy and dead bit elimination to exploit
737   // opportunities opened up by them.
738   FPM.addPass(InstCombinePass());
739   invokePeepholeEPCallbacks(FPM, Level);
740
741   // Re-consider control flow based optimizations after redundancy elimination,
742   // redo DCE, etc.
743   FPM.addPass(JumpThreadingPass());
744   FPM.addPass(CorrelatedValuePropagationPass());
745   FPM.addPass(DSEPass());
746   FPM.addPass(createFunctionToLoopPassAdaptor(
747       LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
748       EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true, DebugLogging));
749
750   if (PTO.Coroutines)
751     FPM.addPass(CoroElidePass());
752
753   for (auto &C : ScalarOptimizerLateEPCallbacks)
754     C(FPM, Level);
755
756   // Finally, do an expensive DCE pass to catch all the dead code exposed by
757   // the simplifications and basic cleanup after all the simplifications.
758   // TODO: Investigate if this is too expensive.
759   FPM.addPass(ADCEPass());
760   FPM.addPass(SimplifyCFGPass());
761   FPM.addPass(InstCombinePass());
762   invokePeepholeEPCallbacks(FPM, Level);
763
764   if (EnableCHR && Level == OptimizationLevel::O3 && PGOOpt &&
765       (PGOOpt->Action == PGOOptions::IRUse ||
766        PGOOpt->Action == PGOOptions::SampleUse))
767     FPM.addPass(ControlHeightReductionPass());
768
769   return FPM;
770 }
771
772 void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging,
773                                     PassBuilder::OptimizationLevel Level,
774                                     bool RunProfileGen, bool IsCS,
775                                     std::string ProfileFile,
776                                     std::string ProfileRemappingFile) {
777   assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!");
778   // Generally running simplification passes and the inliner with an high
779   // threshold results in smaller executables, but there may be cases where
780   // the size grows, so let's be conservative here and skip this simplification
781   // at -Os/Oz. We will not do this  inline for context sensistive PGO (when
782   // IsCS is true).
783   if (!Level.isOptimizingForSize() && !IsCS) {
784     InlineParams IP;
785
786     IP.DefaultThreshold = PreInlineThreshold;
787
788     // FIXME: The hint threshold has the same value used by the regular inliner.
789     // This should probably be lowered after performance testing.
790     // FIXME: this comment is cargo culted from the old pass manager, revisit).
791     IP.HintThreshold = 325;
792     ModuleInlinerWrapperPass MIWP(IP, DebugLogging);
793     CGSCCPassManager &CGPipeline = MIWP.getPM();
794
795     FunctionPassManager FPM;
796     FPM.addPass(SROA());
797     FPM.addPass(EarlyCSEPass());    // Catch trivial redundancies.
798     FPM.addPass(SimplifyCFGPass()); // Merge & remove basic blocks.
799     FPM.addPass(InstCombinePass()); // Combine silly sequences.
800     invokePeepholeEPCallbacks(FPM, Level);
801
802     CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
803
804     MPM.addPass(std::move(MIWP));
805
806     // Delete anything that is now dead to make sure that we don't instrument
807     // dead code. Instrumentation can end up keeping dead code around and
808     // dramatically increase code size.
809     MPM.addPass(GlobalDCEPass());
810   }
811
812   if (!RunProfileGen) {
813     assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
814     MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
815     // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
816     // RequireAnalysisPass for PSI before subsequent non-module passes.
817     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
818     return;
819   }
820
821   // Perform PGO instrumentation.
822   MPM.addPass(PGOInstrumentationGen(IsCS));
823
824   FunctionPassManager FPM;
825   FPM.addPass(createFunctionToLoopPassAdaptor(
826       LoopRotatePass(), EnableMSSALoopDependency,
827       /*UseBlockFrequencyInfo=*/false, DebugLogging));
828   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
829
830   // Add the profile lowering pass.
831   InstrProfOptions Options;
832   if (!ProfileFile.empty())
833     Options.InstrProfileOutput = ProfileFile;
834   // Do counter promotion at Level greater than O0.
835   Options.DoCounterPromotion = true;
836   Options.UseBFIInPromotion = IsCS;
837   MPM.addPass(InstrProfiling(Options, IsCS));
838 }
839
840 void PassBuilder::addPGOInstrPassesForO0(ModulePassManager &MPM,
841                                          bool DebugLogging, bool RunProfileGen,
842                                          bool IsCS, std::string ProfileFile,
843                                          std::string ProfileRemappingFile) {
844   if (!RunProfileGen) {
845     assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
846     MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
847     // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
848     // RequireAnalysisPass for PSI before subsequent non-module passes.
849     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
850     return;
851   }
852
853   // Perform PGO instrumentation.
854   MPM.addPass(PGOInstrumentationGen(IsCS));
855   // Add the profile lowering pass.
856   InstrProfOptions Options;
857   if (!ProfileFile.empty())
858     Options.InstrProfileOutput = ProfileFile;
859   // Do not do counter promotion at O0.
860   Options.DoCounterPromotion = false;
861   Options.UseBFIInPromotion = IsCS;
862   MPM.addPass(InstrProfiling(Options, IsCS));
863 }
864
865 static InlineParams
866 getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level) {
867   return getInlineParams(Level.getSpeedupLevel(), Level.getSizeLevel());
868 }
869
870 ModuleInlinerWrapperPass
871 PassBuilder::buildInlinerPipeline(OptimizationLevel Level, ThinLTOPhase Phase,
872                                   bool DebugLogging) {
873   InlineParams IP = getInlineParamsFromOptLevel(Level);
874   if (Phase == PassBuilder::ThinLTOPhase::PreLink && PGOOpt &&
875       PGOOpt->Action == PGOOptions::SampleUse)
876     IP.HotCallSiteThreshold = 0;
877
878   if (PGOOpt)
879     IP.EnableDeferral = EnablePGOInlineDeferral;
880
881   ModuleInlinerWrapperPass MIWP(IP, DebugLogging, UseInlineAdvisor,
882                                 MaxDevirtIterations);
883
884   // Require the GlobalsAA analysis for the module so we can query it within
885   // the CGSCC pipeline.
886   MIWP.addRequiredModuleAnalysis<GlobalsAA>();
887
888   // Require the ProfileSummaryAnalysis for the module so we can query it within
889   // the inliner pass.
890   MIWP.addRequiredModuleAnalysis<ProfileSummaryAnalysis>();
891
892   // Now begin the main postorder CGSCC pipeline.
893   // FIXME: The current CGSCC pipeline has its origins in the legacy pass
894   // manager and trying to emulate its precise behavior. Much of this doesn't
895   // make a lot of sense and we should revisit the core CGSCC structure.
896   CGSCCPassManager &MainCGPipeline = MIWP.getPM();
897
898   // Note: historically, the PruneEH pass was run first to deduce nounwind and
899   // generally clean up exception handling overhead. It isn't clear this is
900   // valuable as the inliner doesn't currently care whether it is inlining an
901   // invoke or a call.
902
903   if (AttributorRun & AttributorRunOption::CGSCC)
904     MainCGPipeline.addPass(AttributorCGSCCPass());
905
906   if (PTO.Coroutines)
907     MainCGPipeline.addPass(CoroSplitPass(Level != OptimizationLevel::O0));
908
909   // Now deduce any function attributes based in the current code.
910   MainCGPipeline.addPass(PostOrderFunctionAttrsPass());
911
912   // When at O3 add argument promotion to the pass pipeline.
913   // FIXME: It isn't at all clear why this should be limited to O3.
914   if (Level == OptimizationLevel::O3)
915     MainCGPipeline.addPass(ArgumentPromotionPass());
916
917   // Try to perform OpenMP specific optimizations. This is a (quick!) no-op if
918   // there are no OpenMP runtime calls present in the module.
919   if (Level == OptimizationLevel::O2 || Level == OptimizationLevel::O3)
920     MainCGPipeline.addPass(OpenMPOptPass());
921
922   // Lastly, add the core function simplification pipeline nested inside the
923   // CGSCC walk.
924   MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor(
925       buildFunctionSimplificationPipeline(Level, Phase, DebugLogging)));
926
927   for (auto &C : CGSCCOptimizerLateEPCallbacks)
928     C(MainCGPipeline, Level);
929
930   return MIWP;
931 }
932
933 ModulePassManager PassBuilder::buildModuleSimplificationPipeline(
934     OptimizationLevel Level, ThinLTOPhase Phase, bool DebugLogging) {
935   ModulePassManager MPM(DebugLogging);
936
937   bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse);
938
939   // In ThinLTO mode, when flattened profile is used, all the available
940   // profile information will be annotated in PreLink phase so there is
941   // no need to load the profile again in PostLink.
942   bool LoadSampleProfile =
943       HasSampleProfile &&
944       !(FlattenedProfileUsed && Phase == ThinLTOPhase::PostLink);
945
946   // During the ThinLTO backend phase we perform early indirect call promotion
947   // here, before globalopt. Otherwise imported available_externally functions
948   // look unreferenced and are removed. If we are going to load the sample
949   // profile then defer until later.
950   // TODO: See if we can move later and consolidate with the location where
951   // we perform ICP when we are loading a sample profile.
952   // TODO: We pass HasSampleProfile (whether there was a sample profile file
953   // passed to the compile) to the SamplePGO flag of ICP. This is used to
954   // determine whether the new direct calls are annotated with prof metadata.
955   // Ideally this should be determined from whether the IR is annotated with
956   // sample profile, and not whether the a sample profile was provided on the
957   // command line. E.g. for flattened profiles where we will not be reloading
958   // the sample profile in the ThinLTO backend, we ideally shouldn't have to
959   // provide the sample profile file.
960   if (Phase == ThinLTOPhase::PostLink && !LoadSampleProfile)
961     MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile));
962
963   // Do basic inference of function attributes from known properties of system
964   // libraries and other oracles.
965   MPM.addPass(InferFunctionAttrsPass());
966
967   // Create an early function pass manager to cleanup the output of the
968   // frontend.
969   FunctionPassManager EarlyFPM(DebugLogging);
970   EarlyFPM.addPass(SimplifyCFGPass());
971   EarlyFPM.addPass(SROA());
972   EarlyFPM.addPass(EarlyCSEPass());
973   EarlyFPM.addPass(LowerExpectIntrinsicPass());
974   if (PTO.Coroutines)
975     EarlyFPM.addPass(CoroEarlyPass());
976   if (Level == OptimizationLevel::O3)
977     EarlyFPM.addPass(CallSiteSplittingPass());
978
979   // In SamplePGO ThinLTO backend, we need instcombine before profile annotation
980   // to convert bitcast to direct calls so that they can be inlined during the
981   // profile annotation prepration step.
982   // More details about SamplePGO design can be found in:
983   // https://research.google.com/pubs/pub45290.html
984   // FIXME: revisit how SampleProfileLoad/Inliner/ICP is structured.
985   if (LoadSampleProfile)
986     EarlyFPM.addPass(InstCombinePass());
987   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM)));
988
989   if (LoadSampleProfile) {
990     // Annotate sample profile right after early FPM to ensure freshness of
991     // the debug info.
992     MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
993                                         PGOOpt->ProfileRemappingFile,
994                                         Phase == ThinLTOPhase::PreLink));
995     // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
996     // RequireAnalysisPass for PSI before subsequent non-module passes.
997     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
998     // Do not invoke ICP in the ThinLTOPrelink phase as it makes it hard
999     // for the profile annotation to be accurate in the ThinLTO backend.
1000     if (Phase != ThinLTOPhase::PreLink)
1001       // We perform early indirect call promotion here, before globalopt.
1002       // This is important for the ThinLTO backend phase because otherwise
1003       // imported available_externally functions look unreferenced and are
1004       // removed.
1005       MPM.addPass(PGOIndirectCallPromotion(Phase == ThinLTOPhase::PostLink,
1006                                            true /* SamplePGO */));
1007   }
1008
1009   if (AttributorRun & AttributorRunOption::MODULE)
1010     MPM.addPass(AttributorPass());
1011
1012   // Lower type metadata and the type.test intrinsic in the ThinLTO
1013   // post link pipeline after ICP. This is to enable usage of the type
1014   // tests in ICP sequences.
1015   if (Phase == ThinLTOPhase::PostLink)
1016     MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1017
1018   // Interprocedural constant propagation now that basic cleanup has occurred
1019   // and prior to optimizing globals.
1020   // FIXME: This position in the pipeline hasn't been carefully considered in
1021   // years, it should be re-analyzed.
1022   MPM.addPass(IPSCCPPass());
1023
1024   // Attach metadata to indirect call sites indicating the set of functions
1025   // they may target at run-time. This should follow IPSCCP.
1026   MPM.addPass(CalledValuePropagationPass());
1027
1028   // Optimize globals to try and fold them into constants.
1029   MPM.addPass(GlobalOptPass());
1030
1031   // Promote any localized globals to SSA registers.
1032   // FIXME: Should this instead by a run of SROA?
1033   // FIXME: We should probably run instcombine and simplify-cfg afterward to
1034   // delete control flows that are dead once globals have been folded to
1035   // constants.
1036   MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
1037
1038   // Remove any dead arguments exposed by cleanups and constant folding
1039   // globals.
1040   MPM.addPass(DeadArgumentEliminationPass());
1041
1042   // Create a small function pass pipeline to cleanup after all the global
1043   // optimizations.
1044   FunctionPassManager GlobalCleanupPM(DebugLogging);
1045   GlobalCleanupPM.addPass(InstCombinePass());
1046   invokePeepholeEPCallbacks(GlobalCleanupPM, Level);
1047
1048   GlobalCleanupPM.addPass(SimplifyCFGPass());
1049   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM)));
1050
1051   // Add all the requested passes for instrumentation PGO, if requested.
1052   if (PGOOpt && Phase != ThinLTOPhase::PostLink &&
1053       (PGOOpt->Action == PGOOptions::IRInstr ||
1054        PGOOpt->Action == PGOOptions::IRUse)) {
1055     addPGOInstrPasses(MPM, DebugLogging, Level,
1056                       /* RunProfileGen */ PGOOpt->Action == PGOOptions::IRInstr,
1057                       /* IsCS */ false, PGOOpt->ProfileFile,
1058                       PGOOpt->ProfileRemappingFile);
1059     MPM.addPass(PGOIndirectCallPromotion(false, false));
1060   }
1061   if (PGOOpt && Phase != ThinLTOPhase::PostLink &&
1062       PGOOpt->CSAction == PGOOptions::CSIRInstr)
1063     MPM.addPass(PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile));
1064
1065   // Synthesize function entry counts for non-PGO compilation.
1066   if (EnableSyntheticCounts && !PGOOpt)
1067     MPM.addPass(SyntheticCountsPropagation());
1068
1069   MPM.addPass(buildInlinerPipeline(Level, Phase, DebugLogging));
1070
1071   if (EnableMemProfiler && Phase != ThinLTOPhase::PreLink) {
1072     MPM.addPass(createModuleToFunctionPassAdaptor(MemProfilerPass()));
1073     MPM.addPass(ModuleMemProfilerPass());
1074   }
1075
1076   return MPM;
1077 }
1078
1079 ModulePassManager PassBuilder::buildModuleOptimizationPipeline(
1080     OptimizationLevel Level, bool DebugLogging, bool LTOPreLink) {
1081   ModulePassManager MPM(DebugLogging);
1082
1083   // Optimize globals now that the module is fully simplified.
1084   MPM.addPass(GlobalOptPass());
1085   MPM.addPass(GlobalDCEPass());
1086
1087   // Run partial inlining pass to partially inline functions that have
1088   // large bodies.
1089   if (RunPartialInlining)
1090     MPM.addPass(PartialInlinerPass());
1091
1092   // Remove avail extern fns and globals definitions since we aren't compiling
1093   // an object file for later LTO. For LTO we want to preserve these so they
1094   // are eligible for inlining at link-time. Note if they are unreferenced they
1095   // will be removed by GlobalDCE later, so this only impacts referenced
1096   // available externally globals. Eventually they will be suppressed during
1097   // codegen, but eliminating here enables more opportunity for GlobalDCE as it
1098   // may make globals referenced by available external functions dead and saves
1099   // running remaining passes on the eliminated functions. These should be
1100   // preserved during prelinking for link-time inlining decisions.
1101   if (!LTOPreLink)
1102     MPM.addPass(EliminateAvailableExternallyPass());
1103
1104   if (EnableOrderFileInstrumentation)
1105     MPM.addPass(InstrOrderFilePass());
1106
1107   // Do RPO function attribute inference across the module to forward-propagate
1108   // attributes where applicable.
1109   // FIXME: Is this really an optimization rather than a canonicalization?
1110   MPM.addPass(ReversePostOrderFunctionAttrsPass());
1111
1112   // Do a post inline PGO instrumentation and use pass. This is a context
1113   // sensitive PGO pass. We don't want to do this in LTOPreLink phrase as
1114   // cross-module inline has not been done yet. The context sensitive
1115   // instrumentation is after all the inlines are done.
1116   if (!LTOPreLink && PGOOpt) {
1117     if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1118       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ true,
1119                         /* IsCS */ true, PGOOpt->CSProfileGenFile,
1120                         PGOOpt->ProfileRemappingFile);
1121     else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1122       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ false,
1123                         /* IsCS */ true, PGOOpt->ProfileFile,
1124                         PGOOpt->ProfileRemappingFile);
1125   }
1126
1127   // Re-require GloblasAA here prior to function passes. This is particularly
1128   // useful as the above will have inlined, DCE'ed, and function-attr
1129   // propagated everything. We should at this point have a reasonably minimal
1130   // and richly annotated call graph. By computing aliasing and mod/ref
1131   // information for all local globals here, the late loop passes and notably
1132   // the vectorizer will be able to use them to help recognize vectorizable
1133   // memory operations.
1134   MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
1135
1136   FunctionPassManager OptimizePM(DebugLogging);
1137   OptimizePM.addPass(Float2IntPass());
1138   OptimizePM.addPass(LowerConstantIntrinsicsPass());
1139
1140   if (EnableMatrix) {
1141     OptimizePM.addPass(LowerMatrixIntrinsicsPass());
1142     OptimizePM.addPass(EarlyCSEPass());
1143   }
1144
1145   // FIXME: We need to run some loop optimizations to re-rotate loops after
1146   // simplify-cfg and others undo their rotation.
1147
1148   // Optimize the loop execution. These passes operate on entire loop nests
1149   // rather than on each loop in an inside-out manner, and so they are actually
1150   // function passes.
1151
1152   for (auto &C : VectorizerStartEPCallbacks)
1153     C(OptimizePM, Level);
1154
1155   // First rotate loops that may have been un-rotated by prior passes.
1156   OptimizePM.addPass(createFunctionToLoopPassAdaptor(
1157       LoopRotatePass(), EnableMSSALoopDependency,
1158       /*UseBlockFrequencyInfo=*/false, DebugLogging));
1159
1160   // Distribute loops to allow partial vectorization.  I.e. isolate dependences
1161   // into separate loop that would otherwise inhibit vectorization.  This is
1162   // currently only performed for loops marked with the metadata
1163   // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
1164   OptimizePM.addPass(LoopDistributePass());
1165
1166   // Populates the VFABI attribute with the scalar-to-vector mappings
1167   // from the TargetLibraryInfo.
1168   OptimizePM.addPass(InjectTLIMappings());
1169
1170   // Now run the core loop vectorizer.
1171   OptimizePM.addPass(LoopVectorizePass(
1172       LoopVectorizeOptions(!PTO.LoopInterleaving, !PTO.LoopVectorization)));
1173
1174   // Eliminate loads by forwarding stores from the previous iteration to loads
1175   // of the current iteration.
1176   OptimizePM.addPass(LoopLoadEliminationPass());
1177
1178   // Cleanup after the loop optimization passes.
1179   OptimizePM.addPass(InstCombinePass());
1180
1181   // Now that we've formed fast to execute loop structures, we do further
1182   // optimizations. These are run afterward as they might block doing complex
1183   // analyses and transforms such as what are needed for loop vectorization.
1184
1185   // Cleanup after loop vectorization, etc. Simplification passes like CVP and
1186   // GVN, loop transforms, and others have already run, so it's now better to
1187   // convert to more optimized IR using more aggressive simplify CFG options.
1188   // The extra sinking transform can create larger basic blocks, so do this
1189   // before SLP vectorization.
1190   // FIXME: study whether hoisting and/or sinking of common instructions should
1191   //        be delayed until after SLP vectorizer.
1192   OptimizePM.addPass(SimplifyCFGPass(SimplifyCFGOptions()
1193                                          .forwardSwitchCondToPhi(true)
1194                                          .convertSwitchToLookupTable(true)
1195                                          .needCanonicalLoops(false)
1196                                          .hoistCommonInsts(true)
1197                                          .sinkCommonInsts(true)));
1198
1199   // Optimize parallel scalar instruction chains into SIMD instructions.
1200   if (PTO.SLPVectorization)
1201     OptimizePM.addPass(SLPVectorizerPass());
1202
1203   // Enhance/cleanup vector code.
1204   OptimizePM.addPass(VectorCombinePass());
1205   OptimizePM.addPass(InstCombinePass());
1206
1207   // Unroll small loops to hide loop backedge latency and saturate any parallel
1208   // execution resources of an out-of-order processor. We also then need to
1209   // clean up redundancies and loop invariant code.
1210   // FIXME: It would be really good to use a loop-integrated instruction
1211   // combiner for cleanup here so that the unrolling and LICM can be pipelined
1212   // across the loop nests.
1213   // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
1214   if (EnableUnrollAndJam && PTO.LoopUnrolling) {
1215     OptimizePM.addPass(LoopUnrollAndJamPass(Level.getSpeedupLevel()));
1216   }
1217   OptimizePM.addPass(LoopUnrollPass(LoopUnrollOptions(
1218       Level.getSpeedupLevel(), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
1219       PTO.ForgetAllSCEVInLoopUnroll)));
1220   OptimizePM.addPass(WarnMissedTransformationsPass());
1221   OptimizePM.addPass(InstCombinePass());
1222   OptimizePM.addPass(RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
1223   OptimizePM.addPass(createFunctionToLoopPassAdaptor(
1224       LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
1225       EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true, DebugLogging));
1226
1227   // Now that we've vectorized and unrolled loops, we may have more refined
1228   // alignment information, try to re-derive it here.
1229   OptimizePM.addPass(AlignmentFromAssumptionsPass());
1230
1231   // Split out cold code. Splitting is done late to avoid hiding context from
1232   // other optimizations and inadvertently regressing performance. The tradeoff
1233   // is that this has a higher code size cost than splitting early.
1234   if (EnableHotColdSplit && !LTOPreLink)
1235     MPM.addPass(HotColdSplittingPass());
1236
1237   // LoopSink pass sinks instructions hoisted by LICM, which serves as a
1238   // canonicalization pass that enables other optimizations. As a result,
1239   // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
1240   // result too early.
1241   OptimizePM.addPass(LoopSinkPass());
1242
1243   // And finally clean up LCSSA form before generating code.
1244   OptimizePM.addPass(InstSimplifyPass());
1245
1246   // This hoists/decomposes div/rem ops. It should run after other sink/hoist
1247   // passes to avoid re-sinking, but before SimplifyCFG because it can allow
1248   // flattening of blocks.
1249   OptimizePM.addPass(DivRemPairsPass());
1250
1251   // LoopSink (and other loop passes since the last simplifyCFG) might have
1252   // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
1253   OptimizePM.addPass(SimplifyCFGPass());
1254
1255   // Optimize PHIs by speculating around them when profitable. Note that this
1256   // pass needs to be run after any PRE or similar pass as it is essentially
1257   // inserting redundancies into the program. This even includes SimplifyCFG.
1258   OptimizePM.addPass(SpeculateAroundPHIsPass());
1259
1260   if (PTO.Coroutines)
1261     OptimizePM.addPass(CoroCleanupPass());
1262
1263   // Add the core optimizing pipeline.
1264   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM)));
1265
1266   for (auto &C : OptimizerLastEPCallbacks)
1267     C(MPM, Level);
1268
1269   if (PTO.CallGraphProfile)
1270     MPM.addPass(CGProfilePass());
1271
1272   // Now we need to do some global optimization transforms.
1273   // FIXME: It would seem like these should come first in the optimization
1274   // pipeline and maybe be the bottom of the canonicalization pipeline? Weird
1275   // ordering here.
1276   MPM.addPass(GlobalDCEPass());
1277   MPM.addPass(ConstantMergePass());
1278
1279   return MPM;
1280 }
1281
1282 ModulePassManager
1283 PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level,
1284                                            bool DebugLogging, bool LTOPreLink) {
1285   assert(Level != OptimizationLevel::O0 &&
1286          "Must request optimizations for the default pipeline!");
1287
1288   ModulePassManager MPM(DebugLogging);
1289
1290   // Force any function attributes we want the rest of the pipeline to observe.
1291   MPM.addPass(ForceFunctionAttrsPass());
1292
1293   // Apply module pipeline start EP callback.
1294   for (auto &C : PipelineStartEPCallbacks)
1295     C(MPM);
1296
1297   if (PGOOpt && PGOOpt->SamplePGOSupport)
1298     MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
1299
1300   // Add the core simplification pipeline.
1301   MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::None,
1302                                                 DebugLogging));
1303
1304   // Now add the optimization pipeline.
1305   MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging, LTOPreLink));
1306
1307   return MPM;
1308 }
1309
1310 ModulePassManager
1311 PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level,
1312                                                 bool DebugLogging) {
1313   assert(Level != OptimizationLevel::O0 &&
1314          "Must request optimizations for the default pipeline!");
1315
1316   ModulePassManager MPM(DebugLogging);
1317
1318   // Force any function attributes we want the rest of the pipeline to observe.
1319   MPM.addPass(ForceFunctionAttrsPass());
1320
1321   if (PGOOpt && PGOOpt->SamplePGOSupport)
1322     MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
1323
1324   // Apply module pipeline start EP callback.
1325   for (auto &C : PipelineStartEPCallbacks)
1326     C(MPM);
1327
1328   // If we are planning to perform ThinLTO later, we don't bloat the code with
1329   // unrolling/vectorization/... now. Just simplify the module as much as we
1330   // can.
1331   MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PreLink,
1332                                                 DebugLogging));
1333
1334   // Run partial inlining pass to partially inline functions that have
1335   // large bodies.
1336   // FIXME: It isn't clear whether this is really the right place to run this
1337   // in ThinLTO. Because there is another canonicalization and simplification
1338   // phase that will run after the thin link, running this here ends up with
1339   // less information than will be available later and it may grow functions in
1340   // ways that aren't beneficial.
1341   if (RunPartialInlining)
1342     MPM.addPass(PartialInlinerPass());
1343
1344   // Reduce the size of the IR as much as possible.
1345   MPM.addPass(GlobalOptPass());
1346
1347   // Module simplification splits coroutines, but does not fully clean up
1348   // coroutine intrinsics. To ensure ThinLTO optimization passes don't trip up
1349   // on these, we schedule the cleanup here.
1350   if (PTO.Coroutines)
1351     MPM.addPass(createModuleToFunctionPassAdaptor(CoroCleanupPass()));
1352
1353   return MPM;
1354 }
1355
1356 ModulePassManager PassBuilder::buildThinLTODefaultPipeline(
1357     OptimizationLevel Level, bool DebugLogging,
1358     const ModuleSummaryIndex *ImportSummary) {
1359   ModulePassManager MPM(DebugLogging);
1360
1361   if (ImportSummary) {
1362     // These passes import type identifier resolutions for whole-program
1363     // devirtualization and CFI. They must run early because other passes may
1364     // disturb the specific instruction patterns that these passes look for,
1365     // creating dependencies on resolutions that may not appear in the summary.
1366     //
1367     // For example, GVN may transform the pattern assume(type.test) appearing in
1368     // two basic blocks into assume(phi(type.test, type.test)), which would
1369     // transform a dependency on a WPD resolution into a dependency on a type
1370     // identifier resolution for CFI.
1371     //
1372     // Also, WPD has access to more precise information than ICP and can
1373     // devirtualize more effectively, so it should operate on the IR first.
1374     //
1375     // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1376     // metadata and intrinsics.
1377     MPM.addPass(WholeProgramDevirtPass(nullptr, ImportSummary));
1378     MPM.addPass(LowerTypeTestsPass(nullptr, ImportSummary));
1379   }
1380
1381   if (Level == OptimizationLevel::O0)
1382     return MPM;
1383
1384   // Force any function attributes we want the rest of the pipeline to observe.
1385   MPM.addPass(ForceFunctionAttrsPass());
1386
1387   // Add the core simplification pipeline.
1388   MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PostLink,
1389                                                 DebugLogging));
1390
1391   // Now add the optimization pipeline.
1392   MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging));
1393
1394   return MPM;
1395 }
1396
1397 ModulePassManager
1398 PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level,
1399                                             bool DebugLogging) {
1400   assert(Level != OptimizationLevel::O0 &&
1401          "Must request optimizations for the default pipeline!");
1402   // FIXME: We should use a customized pre-link pipeline!
1403   return buildPerModuleDefaultPipeline(Level, DebugLogging,
1404                                        /* LTOPreLink */ true);
1405 }
1406
1407 ModulePassManager
1408 PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, bool DebugLogging,
1409                                      ModuleSummaryIndex *ExportSummary) {
1410   ModulePassManager MPM(DebugLogging);
1411
1412   if (Level == OptimizationLevel::O0) {
1413     // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1414     // metadata and intrinsics.
1415     MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1416     MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1417     // Run a second time to clean up any type tests left behind by WPD for use
1418     // in ICP.
1419     MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1420     return MPM;
1421   }
1422
1423   if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
1424     // Load sample profile before running the LTO optimization pipeline.
1425     MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
1426                                         PGOOpt->ProfileRemappingFile,
1427                                         false /* ThinLTOPhase::PreLink */));
1428     // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1429     // RequireAnalysisPass for PSI before subsequent non-module passes.
1430     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
1431   }
1432
1433   // Remove unused virtual tables to improve the quality of code generated by
1434   // whole-program devirtualization and bitset lowering.
1435   MPM.addPass(GlobalDCEPass());
1436
1437   // Force any function attributes we want the rest of the pipeline to observe.
1438   MPM.addPass(ForceFunctionAttrsPass());
1439
1440   // Do basic inference of function attributes from known properties of system
1441   // libraries and other oracles.
1442   MPM.addPass(InferFunctionAttrsPass());
1443
1444   if (Level.getSpeedupLevel() > 1) {
1445     FunctionPassManager EarlyFPM(DebugLogging);
1446     EarlyFPM.addPass(CallSiteSplittingPass());
1447     MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM)));
1448
1449     // Indirect call promotion. This should promote all the targets that are
1450     // left by the earlier promotion pass that promotes intra-module targets.
1451     // This two-step promotion is to save the compile time. For LTO, it should
1452     // produce the same result as if we only do promotion here.
1453     MPM.addPass(PGOIndirectCallPromotion(
1454         true /* InLTO */, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
1455     // Propagate constants at call sites into the functions they call.  This
1456     // opens opportunities for globalopt (and inlining) by substituting function
1457     // pointers passed as arguments to direct uses of functions.
1458     MPM.addPass(IPSCCPPass());
1459
1460     // Attach metadata to indirect call sites indicating the set of functions
1461     // they may target at run-time. This should follow IPSCCP.
1462     MPM.addPass(CalledValuePropagationPass());
1463   }
1464
1465   // Now deduce any function attributes based in the current code.
1466   MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1467               PostOrderFunctionAttrsPass()));
1468
1469   // Do RPO function attribute inference across the module to forward-propagate
1470   // attributes where applicable.
1471   // FIXME: Is this really an optimization rather than a canonicalization?
1472   MPM.addPass(ReversePostOrderFunctionAttrsPass());
1473
1474   // Use in-range annotations on GEP indices to split globals where beneficial.
1475   MPM.addPass(GlobalSplitPass());
1476
1477   // Run whole program optimization of virtual call when the list of callees
1478   // is fixed.
1479   MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1480
1481   // Stop here at -O1.
1482   if (Level == OptimizationLevel::O1) {
1483     // The LowerTypeTestsPass needs to run to lower type metadata and the
1484     // type.test intrinsics. The pass does nothing if CFI is disabled.
1485     MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1486     // Run a second time to clean up any type tests left behind by WPD for use
1487     // in ICP (which is performed earlier than this in the regular LTO
1488     // pipeline).
1489     MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1490     return MPM;
1491   }
1492
1493   // Optimize globals to try and fold them into constants.
1494   MPM.addPass(GlobalOptPass());
1495
1496   // Promote any localized globals to SSA registers.
1497   MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
1498
1499   // Linking modules together can lead to duplicate global constant, only
1500   // keep one copy of each constant.
1501   MPM.addPass(ConstantMergePass());
1502
1503   // Remove unused arguments from functions.
1504   MPM.addPass(DeadArgumentEliminationPass());
1505
1506   // Reduce the code after globalopt and ipsccp.  Both can open up significant
1507   // simplification opportunities, and both can propagate functions through
1508   // function pointers.  When this happens, we often have to resolve varargs
1509   // calls, etc, so let instcombine do this.
1510   FunctionPassManager PeepholeFPM(DebugLogging);
1511   if (Level == OptimizationLevel::O3)
1512     PeepholeFPM.addPass(AggressiveInstCombinePass());
1513   PeepholeFPM.addPass(InstCombinePass());
1514   invokePeepholeEPCallbacks(PeepholeFPM, Level);
1515
1516   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM)));
1517
1518   // Note: historically, the PruneEH pass was run first to deduce nounwind and
1519   // generally clean up exception handling overhead. It isn't clear this is
1520   // valuable as the inliner doesn't currently care whether it is inlining an
1521   // invoke or a call.
1522   // Run the inliner now.
1523   MPM.addPass(ModuleInlinerWrapperPass(getInlineParamsFromOptLevel(Level),
1524                                        DebugLogging));
1525
1526   // Optimize globals again after we ran the inliner.
1527   MPM.addPass(GlobalOptPass());
1528
1529   // Garbage collect dead functions.
1530   // FIXME: Add ArgumentPromotion pass after once it's ported.
1531   MPM.addPass(GlobalDCEPass());
1532
1533   FunctionPassManager FPM(DebugLogging);
1534   // The IPO Passes may leave cruft around. Clean up after them.
1535   FPM.addPass(InstCombinePass());
1536   invokePeepholeEPCallbacks(FPM, Level);
1537
1538   FPM.addPass(JumpThreadingPass(/*InsertFreezeWhenUnfoldingSelect*/ true));
1539
1540   // Do a post inline PGO instrumentation and use pass. This is a context
1541   // sensitive PGO pass.
1542   if (PGOOpt) {
1543     if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1544       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ true,
1545                         /* IsCS */ true, PGOOpt->CSProfileGenFile,
1546                         PGOOpt->ProfileRemappingFile);
1547     else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1548       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ false,
1549                         /* IsCS */ true, PGOOpt->ProfileFile,
1550                         PGOOpt->ProfileRemappingFile);
1551   }
1552
1553   // Break up allocas
1554   FPM.addPass(SROA());
1555
1556   // LTO provides additional opportunities for tailcall elimination due to
1557   // link-time inlining, and visibility of nocapture attribute.
1558   FPM.addPass(TailCallElimPass());
1559
1560   // Run a few AA driver optimizations here and now to cleanup the code.
1561   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
1562
1563   MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1564               PostOrderFunctionAttrsPass()));
1565   // FIXME: here we run IP alias analysis in the legacy PM.
1566
1567   FunctionPassManager MainFPM;
1568
1569   // FIXME: once we fix LoopPass Manager, add LICM here.
1570   // FIXME: once we provide support for enabling MLSM, add it here.
1571   if (RunNewGVN)
1572     MainFPM.addPass(NewGVNPass());
1573   else
1574     MainFPM.addPass(GVN());
1575
1576   // Remove dead memcpy()'s.
1577   MainFPM.addPass(MemCpyOptPass());
1578
1579   // Nuke dead stores.
1580   MainFPM.addPass(DSEPass());
1581
1582   // FIXME: at this point, we run a bunch of loop passes:
1583   // indVarSimplify, loopDeletion, loopInterchange, loopUnroll,
1584   // loopVectorize. Enable them once the remaining issue with LPM
1585   // are sorted out.
1586
1587   MainFPM.addPass(InstCombinePass());
1588   MainFPM.addPass(SimplifyCFGPass());
1589   MainFPM.addPass(SCCPPass());
1590   MainFPM.addPass(InstCombinePass());
1591   MainFPM.addPass(BDCEPass());
1592
1593   // FIXME: We may want to run SLPVectorizer here.
1594   // After vectorization, assume intrinsics may tell us more
1595   // about pointer alignments.
1596 #if 0
1597   MainFPM.add(AlignmentFromAssumptionsPass());
1598 #endif
1599
1600   // FIXME: Conditionally run LoadCombine here, after it's ported
1601   // (in case we still have this pass, given its questionable usefulness).
1602
1603   MainFPM.addPass(InstCombinePass());
1604   invokePeepholeEPCallbacks(MainFPM, Level);
1605   MainFPM.addPass(JumpThreadingPass(/*InsertFreezeWhenUnfoldingSelect*/ true));
1606   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM)));
1607
1608   // Create a function that performs CFI checks for cross-DSO calls with
1609   // targets in the current module.
1610   MPM.addPass(CrossDSOCFIPass());
1611
1612   // Lower type metadata and the type.test intrinsic. This pass supports
1613   // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs
1614   // to be run at link time if CFI is enabled. This pass does nothing if
1615   // CFI is disabled.
1616   MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1617   // Run a second time to clean up any type tests left behind by WPD for use
1618   // in ICP (which is performed earlier than this in the regular LTO pipeline).
1619   MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1620
1621   // Enable splitting late in the FullLTO post-link pipeline. This is done in
1622   // the same stage in the old pass manager (\ref addLateLTOOptimizationPasses).
1623   if (EnableHotColdSplit)
1624     MPM.addPass(HotColdSplittingPass());
1625
1626   // Add late LTO optimization passes.
1627   // Delete basic blocks, which optimization passes may have killed.
1628   MPM.addPass(createModuleToFunctionPassAdaptor(SimplifyCFGPass()));
1629
1630   // Drop bodies of available eternally objects to improve GlobalDCE.
1631   MPM.addPass(EliminateAvailableExternallyPass());
1632
1633   // Now that we have optimized the program, discard unreachable functions.
1634   MPM.addPass(GlobalDCEPass());
1635
1636   // FIXME: Maybe enable MergeFuncs conditionally after it's ported.
1637   return MPM;
1638 }
1639
1640 AAManager PassBuilder::buildDefaultAAPipeline() {
1641   AAManager AA;
1642
1643   // The order in which these are registered determines their priority when
1644   // being queried.
1645
1646   // First we register the basic alias analysis that provides the majority of
1647   // per-function local AA logic. This is a stateless, on-demand local set of
1648   // AA techniques.
1649   AA.registerFunctionAnalysis<BasicAA>();
1650
1651   // Next we query fast, specialized alias analyses that wrap IR-embedded
1652   // information about aliasing.
1653   AA.registerFunctionAnalysis<ScopedNoAliasAA>();
1654   AA.registerFunctionAnalysis<TypeBasedAA>();
1655
1656   // Add support for querying global aliasing information when available.
1657   // Because the `AAManager` is a function analysis and `GlobalsAA` is a module
1658   // analysis, all that the `AAManager` can do is query for any *cached*
1659   // results from `GlobalsAA` through a readonly proxy.
1660   AA.registerModuleAnalysis<GlobalsAA>();
1661
1662   return AA;
1663 }
1664
1665 static Optional<int> parseRepeatPassName(StringRef Name) {
1666   if (!Name.consume_front("repeat<") || !Name.consume_back(">"))
1667     return None;
1668   int Count;
1669   if (Name.getAsInteger(0, Count) || Count <= 0)
1670     return None;
1671   return Count;
1672 }
1673
1674 static Optional<int> parseDevirtPassName(StringRef Name) {
1675   if (!Name.consume_front("devirt<") || !Name.consume_back(">"))
1676     return None;
1677   int Count;
1678   if (Name.getAsInteger(0, Count) || Count <= 0)
1679     return None;
1680   return Count;
1681 }
1682
1683 static bool checkParametrizedPassName(StringRef Name, StringRef PassName) {
1684   if (!Name.consume_front(PassName))
1685     return false;
1686   // normal pass name w/o parameters == default parameters
1687   if (Name.empty())
1688     return true;
1689   return Name.startswith("<") && Name.endswith(">");
1690 }
1691
1692 namespace {
1693
1694 /// This performs customized parsing of pass name with parameters.
1695 ///
1696 /// We do not need parametrization of passes in textual pipeline very often,
1697 /// yet on a rare occasion ability to specify parameters right there can be
1698 /// useful.
1699 ///
1700 /// \p Name - parameterized specification of a pass from a textual pipeline
1701 /// is a string in a form of :
1702 ///      PassName '<' parameter-list '>'
1703 ///
1704 /// Parameter list is being parsed by the parser callable argument, \p Parser,
1705 /// It takes a string-ref of parameters and returns either StringError or a
1706 /// parameter list in a form of a custom parameters type, all wrapped into
1707 /// Expected<> template class.
1708 ///
1709 template <typename ParametersParseCallableT>
1710 auto parsePassParameters(ParametersParseCallableT &&Parser, StringRef Name,
1711                          StringRef PassName) -> decltype(Parser(StringRef{})) {
1712   using ParametersT = typename decltype(Parser(StringRef{}))::value_type;
1713
1714   StringRef Params = Name;
1715   if (!Params.consume_front(PassName)) {
1716     assert(false &&
1717            "unable to strip pass name from parametrized pass specification");
1718   }
1719   if (Params.empty())
1720     return ParametersT{};
1721   if (!Params.consume_front("<") || !Params.consume_back(">")) {
1722     assert(false && "invalid format for parametrized pass name");
1723   }
1724
1725   Expected<ParametersT> Result = Parser(Params);
1726   assert((Result || Result.template errorIsA<StringError>()) &&
1727          "Pass parameter parser can only return StringErrors.");
1728   return Result;
1729 }
1730
1731 /// Parser of parameters for LoopUnroll pass.
1732 Expected<LoopUnrollOptions> parseLoopUnrollOptions(StringRef Params) {
1733   LoopUnrollOptions UnrollOpts;
1734   while (!Params.empty()) {
1735     StringRef ParamName;
1736     std::tie(ParamName, Params) = Params.split(';');
1737     int OptLevel = StringSwitch<int>(ParamName)
1738                        .Case("O0", 0)
1739                        .Case("O1", 1)
1740                        .Case("O2", 2)
1741                        .Case("O3", 3)
1742                        .Default(-1);
1743     if (OptLevel >= 0) {
1744       UnrollOpts.setOptLevel(OptLevel);
1745       continue;
1746     }
1747     if (ParamName.consume_front("full-unroll-max=")) {
1748       int Count;
1749       if (ParamName.getAsInteger(0, Count))
1750         return make_error<StringError>(
1751             formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
1752             inconvertibleErrorCode());
1753       UnrollOpts.setFullUnrollMaxCount(Count);
1754       continue;
1755     }
1756
1757     bool Enable = !ParamName.consume_front("no-");
1758     if (ParamName == "partial") {
1759       UnrollOpts.setPartial(Enable);
1760     } else if (ParamName == "peeling") {
1761       UnrollOpts.setPeeling(Enable);
1762     } else if (ParamName == "profile-peeling") {
1763       UnrollOpts.setProfileBasedPeeling(Enable);
1764     } else if (ParamName == "runtime") {
1765       UnrollOpts.setRuntime(Enable);
1766     } else if (ParamName == "upperbound") {
1767       UnrollOpts.setUpperBound(Enable);
1768     } else {
1769       return make_error<StringError>(
1770           formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
1771           inconvertibleErrorCode());
1772     }
1773   }
1774   return UnrollOpts;
1775 }
1776
1777 Expected<MemorySanitizerOptions> parseMSanPassOptions(StringRef Params) {
1778   MemorySanitizerOptions Result;
1779   while (!Params.empty()) {
1780     StringRef ParamName;
1781     std::tie(ParamName, Params) = Params.split(';');
1782
1783     if (ParamName == "recover") {
1784       Result.Recover = true;
1785     } else if (ParamName == "kernel") {
1786       Result.Kernel = true;
1787     } else if (ParamName.consume_front("track-origins=")) {
1788       if (ParamName.getAsInteger(0, Result.TrackOrigins))
1789         return make_error<StringError>(
1790             formatv("invalid argument to MemorySanitizer pass track-origins "
1791                     "parameter: '{0}' ",
1792                     ParamName)
1793                 .str(),
1794             inconvertibleErrorCode());
1795     } else {
1796       return make_error<StringError>(
1797           formatv("invalid MemorySanitizer pass parameter '{0}' ", ParamName)
1798               .str(),
1799           inconvertibleErrorCode());
1800     }
1801   }
1802   return Result;
1803 }
1804
1805 /// Parser of parameters for SimplifyCFG pass.
1806 Expected<SimplifyCFGOptions> parseSimplifyCFGOptions(StringRef Params) {
1807   SimplifyCFGOptions Result;
1808   while (!Params.empty()) {
1809     StringRef ParamName;
1810     std::tie(ParamName, Params) = Params.split(';');
1811
1812     bool Enable = !ParamName.consume_front("no-");
1813     if (ParamName == "forward-switch-cond") {
1814       Result.forwardSwitchCondToPhi(Enable);
1815     } else if (ParamName == "switch-to-lookup") {
1816       Result.convertSwitchToLookupTable(Enable);
1817     } else if (ParamName == "keep-loops") {
1818       Result.needCanonicalLoops(Enable);
1819     } else if (ParamName == "hoist-common-insts") {
1820       Result.hoistCommonInsts(Enable);
1821     } else if (ParamName == "sink-common-insts") {
1822       Result.sinkCommonInsts(Enable);
1823     } else if (Enable && ParamName.consume_front("bonus-inst-threshold=")) {
1824       APInt BonusInstThreshold;
1825       if (ParamName.getAsInteger(0, BonusInstThreshold))
1826         return make_error<StringError>(
1827             formatv("invalid argument to SimplifyCFG pass bonus-threshold "
1828                     "parameter: '{0}' ",
1829                     ParamName).str(),
1830             inconvertibleErrorCode());
1831       Result.bonusInstThreshold(BonusInstThreshold.getSExtValue());
1832     } else {
1833       return make_error<StringError>(
1834           formatv("invalid SimplifyCFG pass parameter '{0}' ", ParamName).str(),
1835           inconvertibleErrorCode());
1836     }
1837   }
1838   return Result;
1839 }
1840
1841 /// Parser of parameters for LoopVectorize pass.
1842 Expected<LoopVectorizeOptions> parseLoopVectorizeOptions(StringRef Params) {
1843   LoopVectorizeOptions Opts;
1844   while (!Params.empty()) {
1845     StringRef ParamName;
1846     std::tie(ParamName, Params) = Params.split(';');
1847
1848     bool Enable = !ParamName.consume_front("no-");
1849     if (ParamName == "interleave-forced-only") {
1850       Opts.setInterleaveOnlyWhenForced(Enable);
1851     } else if (ParamName == "vectorize-forced-only") {
1852       Opts.setVectorizeOnlyWhenForced(Enable);
1853     } else {
1854       return make_error<StringError>(
1855           formatv("invalid LoopVectorize parameter '{0}' ", ParamName).str(),
1856           inconvertibleErrorCode());
1857     }
1858   }
1859   return Opts;
1860 }
1861
1862 Expected<bool> parseLoopUnswitchOptions(StringRef Params) {
1863   bool Result = false;
1864   while (!Params.empty()) {
1865     StringRef ParamName;
1866     std::tie(ParamName, Params) = Params.split(';');
1867
1868     bool Enable = !ParamName.consume_front("no-");
1869     if (ParamName == "nontrivial") {
1870       Result = Enable;
1871     } else {
1872       return make_error<StringError>(
1873           formatv("invalid LoopUnswitch pass parameter '{0}' ", ParamName)
1874               .str(),
1875           inconvertibleErrorCode());
1876     }
1877   }
1878   return Result;
1879 }
1880
1881 Expected<bool> parseMergedLoadStoreMotionOptions(StringRef Params) {
1882   bool Result = false;
1883   while (!Params.empty()) {
1884     StringRef ParamName;
1885     std::tie(ParamName, Params) = Params.split(';');
1886
1887     bool Enable = !ParamName.consume_front("no-");
1888     if (ParamName == "split-footer-bb") {
1889       Result = Enable;
1890     } else {
1891       return make_error<StringError>(
1892           formatv("invalid MergedLoadStoreMotion pass parameter '{0}' ",
1893                   ParamName)
1894               .str(),
1895           inconvertibleErrorCode());
1896     }
1897   }
1898   return Result;
1899 }
1900
1901 Expected<GVNOptions> parseGVNOptions(StringRef Params) {
1902   GVNOptions Result;
1903   while (!Params.empty()) {
1904     StringRef ParamName;
1905     std::tie(ParamName, Params) = Params.split(';');
1906
1907     bool Enable = !ParamName.consume_front("no-");
1908     if (ParamName == "pre") {
1909       Result.setPRE(Enable);
1910     } else if (ParamName == "load-pre") {
1911       Result.setLoadPRE(Enable);
1912     } else if (ParamName == "memdep") {
1913       Result.setMemDep(Enable);
1914     } else {
1915       return make_error<StringError>(
1916           formatv("invalid GVN pass parameter '{0}' ", ParamName).str(),
1917           inconvertibleErrorCode());
1918     }
1919   }
1920   return Result;
1921 }
1922
1923 Expected<StackLifetime::LivenessType>
1924 parseStackLifetimeOptions(StringRef Params) {
1925   StackLifetime::LivenessType Result = StackLifetime::LivenessType::May;
1926   while (!Params.empty()) {
1927     StringRef ParamName;
1928     std::tie(ParamName, Params) = Params.split(';');
1929
1930     if (ParamName == "may") {
1931       Result = StackLifetime::LivenessType::May;
1932     } else if (ParamName == "must") {
1933       Result = StackLifetime::LivenessType::Must;
1934     } else {
1935       return make_error<StringError>(
1936           formatv("invalid StackLifetime parameter '{0}' ", ParamName).str(),
1937           inconvertibleErrorCode());
1938     }
1939   }
1940   return Result;
1941 }
1942
1943 } // namespace
1944
1945 /// Tests whether a pass name starts with a valid prefix for a default pipeline
1946 /// alias.
1947 static bool startsWithDefaultPipelineAliasPrefix(StringRef Name) {
1948   return Name.startswith("default") || Name.startswith("thinlto") ||
1949          Name.startswith("lto");
1950 }
1951
1952 /// Tests whether registered callbacks will accept a given pass name.
1953 ///
1954 /// When parsing a pipeline text, the type of the outermost pipeline may be
1955 /// omitted, in which case the type is automatically determined from the first
1956 /// pass name in the text. This may be a name that is handled through one of the
1957 /// callbacks. We check this through the oridinary parsing callbacks by setting
1958 /// up a dummy PassManager in order to not force the client to also handle this
1959 /// type of query.
1960 template <typename PassManagerT, typename CallbacksT>
1961 static bool callbacksAcceptPassName(StringRef Name, CallbacksT &Callbacks) {
1962   if (!Callbacks.empty()) {
1963     PassManagerT DummyPM;
1964     for (auto &CB : Callbacks)
1965       if (CB(Name, DummyPM, {}))
1966         return true;
1967   }
1968   return false;
1969 }
1970
1971 template <typename CallbacksT>
1972 static bool isModulePassName(StringRef Name, CallbacksT &Callbacks) {
1973   // Manually handle aliases for pre-configured pipeline fragments.
1974   if (startsWithDefaultPipelineAliasPrefix(Name))
1975     return DefaultAliasRegex.match(Name);
1976
1977   // Explicitly handle pass manager names.
1978   if (Name == "module")
1979     return true;
1980   if (Name == "cgscc")
1981     return true;
1982   if (Name == "function")
1983     return true;
1984
1985   // Explicitly handle custom-parsed pass names.
1986   if (parseRepeatPassName(Name))
1987     return true;
1988
1989 #define MODULE_PASS(NAME, CREATE_PASS)                                         \
1990   if (Name == NAME)                                                            \
1991     return true;
1992 #define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
1993   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
1994     return true;
1995 #include "PassRegistry.def"
1996
1997   return callbacksAcceptPassName<ModulePassManager>(Name, Callbacks);
1998 }
1999
2000 template <typename CallbacksT>
2001 static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks) {
2002   // Explicitly handle pass manager names.
2003   if (Name == "cgscc")
2004     return true;
2005   if (Name == "function")
2006     return true;
2007
2008   // Explicitly handle custom-parsed pass names.
2009   if (parseRepeatPassName(Name))
2010     return true;
2011   if (parseDevirtPassName(Name))
2012     return true;
2013
2014 #define CGSCC_PASS(NAME, CREATE_PASS)                                          \
2015   if (Name == NAME)                                                            \
2016     return true;
2017 #define CGSCC_ANALYSIS(NAME, CREATE_PASS)                                      \
2018   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
2019     return true;
2020 #include "PassRegistry.def"
2021
2022   return callbacksAcceptPassName<CGSCCPassManager>(Name, Callbacks);
2023 }
2024
2025 template <typename CallbacksT>
2026 static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks) {
2027   // Explicitly handle pass manager names.
2028   if (Name == "function")
2029     return true;
2030   if (Name == "loop" || Name == "loop-mssa")
2031     return true;
2032
2033   // Explicitly handle custom-parsed pass names.
2034   if (parseRepeatPassName(Name))
2035     return true;
2036
2037 #define FUNCTION_PASS(NAME, CREATE_PASS)                                       \
2038   if (Name == NAME)                                                            \
2039     return true;
2040 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                   \
2041   if (checkParametrizedPassName(Name, NAME))                                   \
2042     return true;
2043 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
2044   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
2045     return true;
2046 #include "PassRegistry.def"
2047
2048   return callbacksAcceptPassName<FunctionPassManager>(Name, Callbacks);
2049 }
2050
2051 template <typename CallbacksT>
2052 static bool isLoopPassName(StringRef Name, CallbacksT &Callbacks) {
2053   // Explicitly handle pass manager names.
2054   if (Name == "loop" || Name == "loop-mssa")
2055     return true;
2056
2057   // Explicitly handle custom-parsed pass names.
2058   if (parseRepeatPassName(Name))
2059     return true;
2060
2061 #define LOOP_PASS(NAME, CREATE_PASS)                                           \
2062   if (Name == NAME)                                                            \
2063     return true;
2064 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                       \
2065   if (checkParametrizedPassName(Name, NAME))                                   \
2066     return true;
2067 #define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
2068   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
2069     return true;
2070 #include "PassRegistry.def"
2071
2072   return callbacksAcceptPassName<LoopPassManager>(Name, Callbacks);
2073 }
2074
2075 Optional<std::vector<PassBuilder::PipelineElement>>
2076 PassBuilder::parsePipelineText(StringRef Text) {
2077   std::vector<PipelineElement> ResultPipeline;
2078
2079   SmallVector<std::vector<PipelineElement> *, 4> PipelineStack = {
2080       &ResultPipeline};
2081   for (;;) {
2082     std::vector<PipelineElement> &Pipeline = *PipelineStack.back();
2083     size_t Pos = Text.find_first_of(",()");
2084     Pipeline.push_back({Text.substr(0, Pos), {}});
2085
2086     // If we have a single terminating name, we're done.
2087     if (Pos == Text.npos)
2088       break;
2089
2090     char Sep = Text[Pos];
2091     Text = Text.substr(Pos + 1);
2092     if (Sep == ',')
2093       // Just a name ending in a comma, continue.
2094       continue;
2095
2096     if (Sep == '(') {
2097       // Push the inner pipeline onto the stack to continue processing.
2098       PipelineStack.push_back(&Pipeline.back().InnerPipeline);
2099       continue;
2100     }
2101
2102     assert(Sep == ')' && "Bogus separator!");
2103     // When handling the close parenthesis, we greedily consume them to avoid
2104     // empty strings in the pipeline.
2105     do {
2106       // If we try to pop the outer pipeline we have unbalanced parentheses.
2107       if (PipelineStack.size() == 1)
2108         return None;
2109
2110       PipelineStack.pop_back();
2111     } while (Text.consume_front(")"));
2112
2113     // Check if we've finished parsing.
2114     if (Text.empty())
2115       break;
2116
2117     // Otherwise, the end of an inner pipeline always has to be followed by
2118     // a comma, and then we can continue.
2119     if (!Text.consume_front(","))
2120       return None;
2121   }
2122
2123   if (PipelineStack.size() > 1)
2124     // Unbalanced paretheses.
2125     return None;
2126
2127   assert(PipelineStack.back() == &ResultPipeline &&
2128          "Wrong pipeline at the bottom of the stack!");
2129   return {std::move(ResultPipeline)};
2130 }
2131
2132 Error PassBuilder::parseModulePass(ModulePassManager &MPM,
2133                                    const PipelineElement &E,
2134                                    bool VerifyEachPass, bool DebugLogging) {
2135   auto &Name = E.Name;
2136   auto &InnerPipeline = E.InnerPipeline;
2137
2138   // First handle complex passes like the pass managers which carry pipelines.
2139   if (!InnerPipeline.empty()) {
2140     if (Name == "module") {
2141       ModulePassManager NestedMPM(DebugLogging);
2142       if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline,
2143                                              VerifyEachPass, DebugLogging))
2144         return Err;
2145       MPM.addPass(std::move(NestedMPM));
2146       return Error::success();
2147     }
2148     if (Name == "cgscc") {
2149       CGSCCPassManager CGPM(DebugLogging);
2150       if (auto Err = parseCGSCCPassPipeline(CGPM, InnerPipeline, VerifyEachPass,
2151                                             DebugLogging))
2152         return Err;
2153       MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
2154       return Error::success();
2155     }
2156     if (Name == "function") {
2157       FunctionPassManager FPM(DebugLogging);
2158       if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline,
2159                                                VerifyEachPass, DebugLogging))
2160         return Err;
2161       MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
2162       return Error::success();
2163     }
2164     if (auto Count = parseRepeatPassName(Name)) {
2165       ModulePassManager NestedMPM(DebugLogging);
2166       if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline,
2167                                              VerifyEachPass, DebugLogging))
2168         return Err;
2169       MPM.addPass(createRepeatedPass(*Count, std::move(NestedMPM)));
2170       return Error::success();
2171     }
2172
2173     for (auto &C : ModulePipelineParsingCallbacks)
2174       if (C(Name, MPM, InnerPipeline))
2175         return Error::success();
2176
2177     // Normal passes can't have pipelines.
2178     return make_error<StringError>(
2179         formatv("invalid use of '{0}' pass as module pipeline", Name).str(),
2180         inconvertibleErrorCode());
2181     ;
2182   }
2183
2184   // Manually handle aliases for pre-configured pipeline fragments.
2185   if (startsWithDefaultPipelineAliasPrefix(Name)) {
2186     SmallVector<StringRef, 3> Matches;
2187     if (!DefaultAliasRegex.match(Name, &Matches))
2188       return make_error<StringError>(
2189           formatv("unknown default pipeline alias '{0}'", Name).str(),
2190           inconvertibleErrorCode());
2191
2192     assert(Matches.size() == 3 && "Must capture two matched strings!");
2193
2194     OptimizationLevel L = StringSwitch<OptimizationLevel>(Matches[2])
2195                               .Case("O0", OptimizationLevel::O0)
2196                               .Case("O1", OptimizationLevel::O1)
2197                               .Case("O2", OptimizationLevel::O2)
2198                               .Case("O3", OptimizationLevel::O3)
2199                               .Case("Os", OptimizationLevel::Os)
2200                               .Case("Oz", OptimizationLevel::Oz);
2201     if (L == OptimizationLevel::O0) {
2202       // Add instrumentation PGO passes -- at O0 we can still do PGO.
2203       if (PGOOpt && Matches[1] != "thinlto" &&
2204           (PGOOpt->Action == PGOOptions::IRInstr ||
2205            PGOOpt->Action == PGOOptions::IRUse))
2206         addPGOInstrPassesForO0(
2207             MPM, DebugLogging,
2208             /* RunProfileGen */ (PGOOpt->Action == PGOOptions::IRInstr),
2209             /* IsCS */ false, PGOOpt->ProfileFile,
2210             PGOOpt->ProfileRemappingFile);
2211
2212       // For IR that makes use of coroutines intrinsics, coroutine passes must
2213       // be run, even at -O0.
2214       if (PTO.Coroutines) {
2215         MPM.addPass(createModuleToFunctionPassAdaptor(CoroEarlyPass()));
2216
2217         CGSCCPassManager CGPM(DebugLogging);
2218         CGPM.addPass(CoroSplitPass());
2219         CGPM.addPass(createCGSCCToFunctionPassAdaptor(CoroElidePass()));
2220         MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
2221
2222         MPM.addPass(createModuleToFunctionPassAdaptor(CoroCleanupPass()));
2223       }
2224
2225       // Do nothing else at all!
2226       return Error::success();
2227     }
2228
2229     // This is consistent with old pass manager invoked via opt, but
2230     // inconsistent with clang. Clang doesn't enable loop vectorization
2231     // but does enable slp vectorization at Oz.
2232     PTO.LoopVectorization =
2233         L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz;
2234     PTO.SLPVectorization =
2235         L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz;
2236
2237     if (Matches[1] == "default") {
2238       MPM.addPass(buildPerModuleDefaultPipeline(L, DebugLogging));
2239     } else if (Matches[1] == "thinlto-pre-link") {
2240       MPM.addPass(buildThinLTOPreLinkDefaultPipeline(L, DebugLogging));
2241     } else if (Matches[1] == "thinlto") {
2242       MPM.addPass(buildThinLTODefaultPipeline(L, DebugLogging, nullptr));
2243     } else if (Matches[1] == "lto-pre-link") {
2244       MPM.addPass(buildLTOPreLinkDefaultPipeline(L, DebugLogging));
2245     } else {
2246       assert(Matches[1] == "lto" && "Not one of the matched options!");
2247       MPM.addPass(buildLTODefaultPipeline(L, DebugLogging, nullptr));
2248     }
2249     return Error::success();
2250   }
2251
2252   // Finally expand the basic registered passes from the .inc file.
2253 #define MODULE_PASS(NAME, CREATE_PASS)                                         \
2254   if (Name == NAME) {                                                          \
2255     MPM.addPass(CREATE_PASS);                                                  \
2256     return Error::success();                                                   \
2257   }
2258 #define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
2259   if (Name == "require<" NAME ">") {                                           \
2260     MPM.addPass(                                                               \
2261         RequireAnalysisPass<                                                   \
2262             std::remove_reference<decltype(CREATE_PASS)>::type, Module>());    \
2263     return Error::success();                                                   \
2264   }                                                                            \
2265   if (Name == "invalidate<" NAME ">") {                                        \
2266     MPM.addPass(InvalidateAnalysisPass<                                        \
2267                 std::remove_reference<decltype(CREATE_PASS)>::type>());        \
2268     return Error::success();                                                   \
2269   }
2270 #define CGSCC_PASS(NAME, CREATE_PASS)                                          \
2271   if (Name == NAME) {                                                          \
2272     MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(CREATE_PASS));         \
2273     return Error::success();                                                   \
2274   }
2275 #define FUNCTION_PASS(NAME, CREATE_PASS)                                       \
2276   if (Name == NAME) {                                                          \
2277     MPM.addPass(createModuleToFunctionPassAdaptor(CREATE_PASS));               \
2278     return Error::success();                                                   \
2279   }
2280 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                   \
2281   if (checkParametrizedPassName(Name, NAME)) {                                 \
2282     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
2283     if (!Params)                                                               \
2284       return Params.takeError();                                               \
2285     MPM.addPass(createModuleToFunctionPassAdaptor(CREATE_PASS(Params.get()))); \
2286     return Error::success();                                                   \
2287   }
2288 #define LOOP_PASS(NAME, CREATE_PASS)                                           \
2289   if (Name == NAME) {                                                          \
2290     MPM.addPass(                                                               \
2291         createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor(     \
2292             CREATE_PASS, false, false, DebugLogging)));                        \
2293     return Error::success();                                                   \
2294   }
2295 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                       \
2296   if (checkParametrizedPassName(Name, NAME)) {                                 \
2297     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
2298     if (!Params)                                                               \
2299       return Params.takeError();                                               \
2300     MPM.addPass(                                                               \
2301         createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor(     \
2302             CREATE_PASS(Params.get()), false, false, DebugLogging)));          \
2303     return Error::success();                                                   \
2304   }
2305 #include "PassRegistry.def"
2306
2307   for (auto &C : ModulePipelineParsingCallbacks)
2308     if (C(Name, MPM, InnerPipeline))
2309       return Error::success();
2310   return make_error<StringError>(
2311       formatv("unknown module pass '{0}'", Name).str(),
2312       inconvertibleErrorCode());
2313 }
2314
2315 Error PassBuilder::parseCGSCCPass(CGSCCPassManager &CGPM,
2316                                   const PipelineElement &E, bool VerifyEachPass,
2317                                   bool DebugLogging) {
2318   auto &Name = E.Name;
2319   auto &InnerPipeline = E.InnerPipeline;
2320
2321   // First handle complex passes like the pass managers which carry pipelines.
2322   if (!InnerPipeline.empty()) {
2323     if (Name == "cgscc") {
2324       CGSCCPassManager NestedCGPM(DebugLogging);
2325       if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline,
2326                                             VerifyEachPass, DebugLogging))
2327         return Err;
2328       // Add the nested pass manager with the appropriate adaptor.
2329       CGPM.addPass(std::move(NestedCGPM));
2330       return Error::success();
2331     }
2332     if (Name == "function") {
2333       FunctionPassManager FPM(DebugLogging);
2334       if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline,
2335                                                VerifyEachPass, DebugLogging))
2336         return Err;
2337       // Add the nested pass manager with the appropriate adaptor.
2338       CGPM.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
2339       return Error::success();
2340     }
2341     if (auto Count = parseRepeatPassName(Name)) {
2342       CGSCCPassManager NestedCGPM(DebugLogging);
2343       if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline,
2344                                             VerifyEachPass, DebugLogging))
2345         return Err;
2346       CGPM.addPass(createRepeatedPass(*Count, std::move(NestedCGPM)));
2347       return Error::success();
2348     }
2349     if (auto MaxRepetitions = parseDevirtPassName(Name)) {
2350       CGSCCPassManager NestedCGPM(DebugLogging);
2351       if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline,
2352                                             VerifyEachPass, DebugLogging))
2353         return Err;
2354       CGPM.addPass(
2355           createDevirtSCCRepeatedPass(std::move(NestedCGPM), *MaxRepetitions));
2356       return Error::success();
2357     }
2358
2359     for (auto &C : CGSCCPipelineParsingCallbacks)
2360       if (C(Name, CGPM, InnerPipeline))
2361         return Error::success();
2362
2363     // Normal passes can't have pipelines.
2364     return make_error<StringError>(
2365         formatv("invalid use of '{0}' pass as cgscc pipeline", Name).str(),
2366         inconvertibleErrorCode());
2367   }
2368
2369 // Now expand the basic registered passes from the .inc file.
2370 #define CGSCC_PASS(NAME, CREATE_PASS)                                          \
2371   if (Name == NAME) {                                                          \
2372     CGPM.addPass(CREATE_PASS);                                                 \
2373     return Error::success();                                                   \
2374   }
2375 #define CGSCC_ANALYSIS(NAME, CREATE_PASS)                                      \
2376   if (Name == "require<" NAME ">") {                                           \
2377     CGPM.addPass(RequireAnalysisPass<                                          \
2378                  std::remove_reference<decltype(CREATE_PASS)>::type,           \
2379                  LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &,    \
2380                  CGSCCUpdateResult &>());                                      \
2381     return Error::success();                                                   \
2382   }                                                                            \
2383   if (Name == "invalidate<" NAME ">") {                                        \
2384     CGPM.addPass(InvalidateAnalysisPass<                                       \
2385                  std::remove_reference<decltype(CREATE_PASS)>::type>());       \
2386     return Error::success();                                                   \
2387   }
2388 #define FUNCTION_PASS(NAME, CREATE_PASS)                                       \
2389   if (Name == NAME) {                                                          \
2390     CGPM.addPass(createCGSCCToFunctionPassAdaptor(CREATE_PASS));               \
2391     return Error::success();                                                   \
2392   }
2393 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                   \
2394   if (checkParametrizedPassName(Name, NAME)) {                                 \
2395     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
2396     if (!Params)                                                               \
2397       return Params.takeError();                                               \
2398     CGPM.addPass(createCGSCCToFunctionPassAdaptor(CREATE_PASS(Params.get()))); \
2399     return Error::success();                                                   \
2400   }
2401 #define LOOP_PASS(NAME, CREATE_PASS)                                           \
2402   if (Name == NAME) {                                                          \
2403     CGPM.addPass(                                                              \
2404         createCGSCCToFunctionPassAdaptor(createFunctionToLoopPassAdaptor(      \
2405             CREATE_PASS, false, false, DebugLogging)));                        \
2406     return Error::success();                                                   \
2407   }
2408 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                       \
2409   if (checkParametrizedPassName(Name, NAME)) {                                 \
2410     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
2411     if (!Params)                                                               \
2412       return Params.takeError();                                               \
2413     CGPM.addPass(                                                              \
2414         createCGSCCToFunctionPassAdaptor(createFunctionToLoopPassAdaptor(      \
2415             CREATE_PASS(Params.get()), false, false, DebugLogging)));          \
2416     return Error::success();                                                   \
2417   }
2418 #include "PassRegistry.def"
2419
2420   for (auto &C : CGSCCPipelineParsingCallbacks)
2421     if (C(Name, CGPM, InnerPipeline))
2422       return Error::success();
2423   return make_error<StringError>(
2424       formatv("unknown cgscc pass '{0}'", Name).str(),
2425       inconvertibleErrorCode());
2426 }
2427
2428 Error PassBuilder::parseFunctionPass(FunctionPassManager &FPM,
2429                                      const PipelineElement &E,
2430                                      bool VerifyEachPass, bool DebugLogging) {
2431   auto &Name = E.Name;
2432   auto &InnerPipeline = E.InnerPipeline;
2433
2434   // First handle complex passes like the pass managers which carry pipelines.
2435   if (!InnerPipeline.empty()) {
2436     if (Name == "function") {
2437       FunctionPassManager NestedFPM(DebugLogging);
2438       if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline,
2439                                                VerifyEachPass, DebugLogging))
2440         return Err;
2441       // Add the nested pass manager with the appropriate adaptor.
2442       FPM.addPass(std::move(NestedFPM));
2443       return Error::success();
2444     }
2445     if (Name == "loop" || Name == "loop-mssa") {
2446       LoopPassManager LPM(DebugLogging);
2447       if (auto Err = parseLoopPassPipeline(LPM, InnerPipeline, VerifyEachPass,
2448                                            DebugLogging))
2449         return Err;
2450       // Add the nested pass manager with the appropriate adaptor.
2451       bool UseMemorySSA = (Name == "loop-mssa");
2452       bool UseBFI =
2453           std::any_of(InnerPipeline.begin(), InnerPipeline.end(),
2454                       [](auto Pipeline) { return Pipeline.Name == "licm"; });
2455       FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), UseMemorySSA,
2456                                                   UseBFI, DebugLogging));
2457       return Error::success();
2458     }
2459     if (auto Count = parseRepeatPassName(Name)) {
2460       FunctionPassManager NestedFPM(DebugLogging);
2461       if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline,
2462                                                VerifyEachPass, DebugLogging))
2463         return Err;
2464       FPM.addPass(createRepeatedPass(*Count, std::move(NestedFPM)));
2465       return Error::success();
2466     }
2467
2468     for (auto &C : FunctionPipelineParsingCallbacks)
2469       if (C(Name, FPM, InnerPipeline))
2470         return Error::success();
2471
2472     // Normal passes can't have pipelines.
2473     return make_error<StringError>(
2474         formatv("invalid use of '{0}' pass as function pipeline", Name).str(),
2475         inconvertibleErrorCode());
2476   }
2477
2478 // Now expand the basic registered passes from the .inc file.
2479 #define FUNCTION_PASS(NAME, CREATE_PASS)                                       \
2480   if (Name == NAME) {                                                          \
2481     FPM.addPass(CREATE_PASS);                                                  \
2482     return Error::success();                                                   \
2483   }
2484 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                   \
2485   if (checkParametrizedPassName(Name, NAME)) {                                 \
2486     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
2487     if (!Params)                                                               \
2488       return Params.takeError();                                               \
2489     FPM.addPass(CREATE_PASS(Params.get()));                                    \
2490     return Error::success();                                                   \
2491   }
2492 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
2493   if (Name == "require<" NAME ">") {                                           \
2494     FPM.addPass(                                                               \
2495         RequireAnalysisPass<                                                   \
2496             std::remove_reference<decltype(CREATE_PASS)>::type, Function>());  \
2497     return Error::success();                                                   \
2498   }                                                                            \
2499   if (Name == "invalidate<" NAME ">") {                                        \
2500     FPM.addPass(InvalidateAnalysisPass<                                        \
2501                 std::remove_reference<decltype(CREATE_PASS)>::type>());        \
2502     return Error::success();                                                   \
2503   }
2504 // FIXME: UseMemorySSA is set to false. Maybe we could do things like:
2505 //        bool UseMemorySSA = !("canon-freeze" || "loop-predication" ||
2506 //                              "guard-widening");
2507 //        The risk is that it may become obsolete if we're not careful.
2508 #define LOOP_PASS(NAME, CREATE_PASS)                                           \
2509   if (Name == NAME) {                                                          \
2510     FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS, false, false,     \
2511                                                 DebugLogging));                \
2512     return Error::success();                                                   \
2513   }
2514 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                       \
2515   if (checkParametrizedPassName(Name, NAME)) {                                 \
2516     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
2517     if (!Params)                                                               \
2518       return Params.takeError();                                               \
2519     FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS(Params.get()),     \
2520                                                 false, false, DebugLogging));  \
2521     return Error::success();                                                   \
2522   }
2523 #include "PassRegistry.def"
2524
2525   for (auto &C : FunctionPipelineParsingCallbacks)
2526     if (C(Name, FPM, InnerPipeline))
2527       return Error::success();
2528   return make_error<StringError>(
2529       formatv("unknown function pass '{0}'", Name).str(),
2530       inconvertibleErrorCode());
2531 }
2532
2533 Error PassBuilder::parseLoopPass(LoopPassManager &LPM, const PipelineElement &E,
2534                                  bool VerifyEachPass, bool DebugLogging) {
2535   StringRef Name = E.Name;
2536   auto &InnerPipeline = E.InnerPipeline;
2537
2538   // First handle complex passes like the pass managers which carry pipelines.
2539   if (!InnerPipeline.empty()) {
2540     if (Name == "loop") {
2541       LoopPassManager NestedLPM(DebugLogging);
2542       if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline,
2543                                            VerifyEachPass, DebugLogging))
2544         return Err;
2545       // Add the nested pass manager with the appropriate adaptor.
2546       LPM.addPass(std::move(NestedLPM));
2547       return Error::success();
2548     }
2549     if (auto Count = parseRepeatPassName(Name)) {
2550       LoopPassManager NestedLPM(DebugLogging);
2551       if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline,
2552                                            VerifyEachPass, DebugLogging))
2553         return Err;
2554       LPM.addPass(createRepeatedPass(*Count, std::move(NestedLPM)));
2555       return Error::success();
2556     }
2557
2558     for (auto &C : LoopPipelineParsingCallbacks)
2559       if (C(Name, LPM, InnerPipeline))
2560         return Error::success();
2561
2562     // Normal passes can't have pipelines.
2563     return make_error<StringError>(
2564         formatv("invalid use of '{0}' pass as loop pipeline", Name).str(),
2565         inconvertibleErrorCode());
2566   }
2567
2568 // Now expand the basic registered passes from the .inc file.
2569 #define LOOP_PASS(NAME, CREATE_PASS)                                           \
2570   if (Name == NAME) {                                                          \
2571     LPM.addPass(CREATE_PASS);                                                  \
2572     return Error::success();                                                   \
2573   }
2574 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                       \
2575   if (checkParametrizedPassName(Name, NAME)) {                                 \
2576     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
2577     if (!Params)                                                               \
2578       return Params.takeError();                                               \
2579     LPM.addPass(CREATE_PASS(Params.get()));                                    \
2580     return Error::success();                                                   \
2581   }
2582 #define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
2583   if (Name == "require<" NAME ">") {                                           \
2584     LPM.addPass(RequireAnalysisPass<                                           \
2585                 std::remove_reference<decltype(CREATE_PASS)>::type, Loop,      \
2586                 LoopAnalysisManager, LoopStandardAnalysisResults &,            \
2587                 LPMUpdater &>());                                              \
2588     return Error::success();                                                   \
2589   }                                                                            \
2590   if (Name == "invalidate<" NAME ">") {                                        \
2591     LPM.addPass(InvalidateAnalysisPass<                                        \
2592                 std::remove_reference<decltype(CREATE_PASS)>::type>());        \
2593     return Error::success();                                                   \
2594   }
2595 #include "PassRegistry.def"
2596
2597   for (auto &C : LoopPipelineParsingCallbacks)
2598     if (C(Name, LPM, InnerPipeline))
2599       return Error::success();
2600   return make_error<StringError>(formatv("unknown loop pass '{0}'", Name).str(),
2601                                  inconvertibleErrorCode());
2602 }
2603
2604 bool PassBuilder::parseAAPassName(AAManager &AA, StringRef Name) {
2605 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS)                               \
2606   if (Name == NAME) {                                                          \
2607     AA.registerModuleAnalysis<                                                 \
2608         std::remove_reference<decltype(CREATE_PASS)>::type>();                 \
2609     return true;                                                               \
2610   }
2611 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS)                             \
2612   if (Name == NAME) {                                                          \
2613     AA.registerFunctionAnalysis<                                               \
2614         std::remove_reference<decltype(CREATE_PASS)>::type>();                 \
2615     return true;                                                               \
2616   }
2617 #include "PassRegistry.def"
2618
2619   for (auto &C : AAParsingCallbacks)
2620     if (C(Name, AA))
2621       return true;
2622   return false;
2623 }
2624
2625 Error PassBuilder::parseLoopPassPipeline(LoopPassManager &LPM,
2626                                          ArrayRef<PipelineElement> Pipeline,
2627                                          bool VerifyEachPass,
2628                                          bool DebugLogging) {
2629   for (const auto &Element : Pipeline) {
2630     if (auto Err = parseLoopPass(LPM, Element, VerifyEachPass, DebugLogging))
2631       return Err;
2632     // FIXME: No verifier support for Loop passes!
2633   }
2634   return Error::success();
2635 }
2636
2637 Error PassBuilder::parseFunctionPassPipeline(FunctionPassManager &FPM,
2638                                              ArrayRef<PipelineElement> Pipeline,
2639                                              bool VerifyEachPass,
2640                                              bool DebugLogging) {
2641   for (const auto &Element : Pipeline) {
2642     if (auto Err =
2643             parseFunctionPass(FPM, Element, VerifyEachPass, DebugLogging))
2644       return Err;
2645     if (VerifyEachPass)
2646       FPM.addPass(VerifierPass());
2647   }
2648   return Error::success();
2649 }
2650
2651 Error PassBuilder::parseCGSCCPassPipeline(CGSCCPassManager &CGPM,
2652                                           ArrayRef<PipelineElement> Pipeline,
2653                                           bool VerifyEachPass,
2654                                           bool DebugLogging) {
2655   for (const auto &Element : Pipeline) {
2656     if (auto Err = parseCGSCCPass(CGPM, Element, VerifyEachPass, DebugLogging))
2657       return Err;
2658     // FIXME: No verifier support for CGSCC passes!
2659   }
2660   return Error::success();
2661 }
2662
2663 void PassBuilder::crossRegisterProxies(LoopAnalysisManager &LAM,
2664                                        FunctionAnalysisManager &FAM,
2665                                        CGSCCAnalysisManager &CGAM,
2666                                        ModuleAnalysisManager &MAM) {
2667   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
2668   MAM.registerPass([&] { return CGSCCAnalysisManagerModuleProxy(CGAM); });
2669   CGAM.registerPass([&] { return ModuleAnalysisManagerCGSCCProxy(MAM); });
2670   FAM.registerPass([&] { return CGSCCAnalysisManagerFunctionProxy(CGAM); });
2671   FAM.registerPass([&] { return ModuleAnalysisManagerFunctionProxy(MAM); });
2672   FAM.registerPass([&] { return LoopAnalysisManagerFunctionProxy(LAM); });
2673   LAM.registerPass([&] { return FunctionAnalysisManagerLoopProxy(FAM); });
2674 }
2675
2676 Error PassBuilder::parseModulePassPipeline(ModulePassManager &MPM,
2677                                            ArrayRef<PipelineElement> Pipeline,
2678                                            bool VerifyEachPass,
2679                                            bool DebugLogging) {
2680   for (const auto &Element : Pipeline) {
2681     if (auto Err = parseModulePass(MPM, Element, VerifyEachPass, DebugLogging))
2682       return Err;
2683     if (VerifyEachPass)
2684       MPM.addPass(VerifierPass());
2685   }
2686   return Error::success();
2687 }
2688
2689 // Primary pass pipeline description parsing routine for a \c ModulePassManager
2690 // FIXME: Should this routine accept a TargetMachine or require the caller to
2691 // pre-populate the analysis managers with target-specific stuff?
2692 Error PassBuilder::parsePassPipeline(ModulePassManager &MPM,
2693                                      StringRef PipelineText,
2694                                      bool VerifyEachPass, bool DebugLogging) {
2695   auto Pipeline = parsePipelineText(PipelineText);
2696   if (!Pipeline || Pipeline->empty())
2697     return make_error<StringError>(
2698         formatv("invalid pipeline '{0}'", PipelineText).str(),
2699         inconvertibleErrorCode());
2700
2701   // If the first name isn't at the module layer, wrap the pipeline up
2702   // automatically.
2703   StringRef FirstName = Pipeline->front().Name;
2704
2705   if (!isModulePassName(FirstName, ModulePipelineParsingCallbacks)) {
2706     if (isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks)) {
2707       Pipeline = {{"cgscc", std::move(*Pipeline)}};
2708     } else if (isFunctionPassName(FirstName,
2709                                   FunctionPipelineParsingCallbacks)) {
2710       Pipeline = {{"function", std::move(*Pipeline)}};
2711     } else if (isLoopPassName(FirstName, LoopPipelineParsingCallbacks)) {
2712       Pipeline = {{"function", {{"loop", std::move(*Pipeline)}}}};
2713     } else {
2714       for (auto &C : TopLevelPipelineParsingCallbacks)
2715         if (C(MPM, *Pipeline, VerifyEachPass, DebugLogging))
2716           return Error::success();
2717
2718       // Unknown pass or pipeline name!
2719       auto &InnerPipeline = Pipeline->front().InnerPipeline;
2720       return make_error<StringError>(
2721           formatv("unknown {0} name '{1}'",
2722                   (InnerPipeline.empty() ? "pass" : "pipeline"), FirstName)
2723               .str(),
2724           inconvertibleErrorCode());
2725     }
2726   }
2727
2728   if (auto Err =
2729           parseModulePassPipeline(MPM, *Pipeline, VerifyEachPass, DebugLogging))
2730     return Err;
2731   return Error::success();
2732 }
2733
2734 // Primary pass pipeline description parsing routine for a \c CGSCCPassManager
2735 Error PassBuilder::parsePassPipeline(CGSCCPassManager &CGPM,
2736                                      StringRef PipelineText,
2737                                      bool VerifyEachPass, bool DebugLogging) {
2738   auto Pipeline = parsePipelineText(PipelineText);
2739   if (!Pipeline || Pipeline->empty())
2740     return make_error<StringError>(
2741         formatv("invalid pipeline '{0}'", PipelineText).str(),
2742         inconvertibleErrorCode());
2743
2744   StringRef FirstName = Pipeline->front().Name;
2745   if (!isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks))
2746     return make_error<StringError>(
2747         formatv("unknown cgscc pass '{0}' in pipeline '{1}'", FirstName,
2748                 PipelineText)
2749             .str(),
2750         inconvertibleErrorCode());
2751
2752   if (auto Err =
2753           parseCGSCCPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging))
2754     return Err;
2755   return Error::success();
2756 }
2757
2758 // Primary pass pipeline description parsing routine for a \c
2759 // FunctionPassManager
2760 Error PassBuilder::parsePassPipeline(FunctionPassManager &FPM,
2761                                      StringRef PipelineText,
2762                                      bool VerifyEachPass, bool DebugLogging) {
2763   auto Pipeline = parsePipelineText(PipelineText);
2764   if (!Pipeline || Pipeline->empty())
2765     return make_error<StringError>(
2766         formatv("invalid pipeline '{0}'", PipelineText).str(),
2767         inconvertibleErrorCode());
2768
2769   StringRef FirstName = Pipeline->front().Name;
2770   if (!isFunctionPassName(FirstName, FunctionPipelineParsingCallbacks))
2771     return make_error<StringError>(
2772         formatv("unknown function pass '{0}' in pipeline '{1}'", FirstName,
2773                 PipelineText)
2774             .str(),
2775         inconvertibleErrorCode());
2776
2777   if (auto Err = parseFunctionPassPipeline(FPM, *Pipeline, VerifyEachPass,
2778                                            DebugLogging))
2779     return Err;
2780   return Error::success();
2781 }
2782
2783 // Primary pass pipeline description parsing routine for a \c LoopPassManager
2784 Error PassBuilder::parsePassPipeline(LoopPassManager &CGPM,
2785                                      StringRef PipelineText,
2786                                      bool VerifyEachPass, bool DebugLogging) {
2787   auto Pipeline = parsePipelineText(PipelineText);
2788   if (!Pipeline || Pipeline->empty())
2789     return make_error<StringError>(
2790         formatv("invalid pipeline '{0}'", PipelineText).str(),
2791         inconvertibleErrorCode());
2792
2793   if (auto Err =
2794           parseLoopPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging))
2795     return Err;
2796
2797   return Error::success();
2798 }
2799
2800 Error PassBuilder::parseAAPipeline(AAManager &AA, StringRef PipelineText) {
2801   // If the pipeline just consists of the word 'default' just replace the AA
2802   // manager with our default one.
2803   if (PipelineText == "default") {
2804     AA = buildDefaultAAPipeline();
2805     return Error::success();
2806   }
2807
2808   while (!PipelineText.empty()) {
2809     StringRef Name;
2810     std::tie(Name, PipelineText) = PipelineText.split(',');
2811     if (!parseAAPassName(AA, Name))
2812       return make_error<StringError>(
2813           formatv("unknown alias analysis name '{0}'", Name).str(),
2814           inconvertibleErrorCode());
2815   }
2816
2817   return Error::success();
2818 }
2819
2820 bool PassBuilder::isAAPassName(StringRef PassName) {
2821 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS)                               \
2822   if (PassName == NAME)                                                        \
2823     return true;
2824 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS)                             \
2825   if (PassName == NAME)                                                        \
2826     return true;
2827 #include "PassRegistry.def"
2828   return false;
2829 }
2830
2831 bool PassBuilder::isAnalysisPassName(StringRef PassName) {
2832 #define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
2833   if (PassName == NAME)                                                        \
2834     return true;
2835 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
2836   if (PassName == NAME)                                                        \
2837     return true;
2838 #define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
2839   if (PassName == NAME)                                                        \
2840     return true;
2841 #define CGSSC_ANALYSIS(NAME, CREATE_PASS)                                      \
2842   if (PassName == NAME)                                                        \
2843     return true;
2844 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS)                               \
2845   if (PassName == NAME)                                                        \
2846     return true;
2847 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS)                             \
2848   if (PassName == NAME)                                                        \
2849     return true;
2850 #include "PassRegistry.def"
2851   return false;
2852 }
2853
2854 void PassBuilder::registerParseTopLevelPipelineCallback(
2855     const std::function<bool(ModulePassManager &, ArrayRef<PipelineElement>,
2856                              bool VerifyEachPass, bool DebugLogging)> &C) {
2857   TopLevelPipelineParsingCallbacks.push_back(C);
2858 }