[NPM] Port strip nonlinetable debuginfo pass to the new pass manager
[lldb.git] / llvm / lib / Passes / PassBuilder.cpp
1 //===- Parsing, selection, and construction of pass pipelines -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 ///
10 /// This file provides the implementation of the PassBuilder based on our
11 /// static pass registry as well as related functionality. It also provides
12 /// helpers to aid in analyzing, debugging, and testing passes and pass
13 /// pipelines.
14 ///
15 //===----------------------------------------------------------------------===//
16
17 #include "llvm/Passes/PassBuilder.h"
18 #include "llvm/ADT/StringSwitch.h"
19 #include "llvm/Analysis/AliasAnalysis.h"
20 #include "llvm/Analysis/AliasAnalysisEvaluator.h"
21 #include "llvm/Analysis/AliasSetTracker.h"
22 #include "llvm/Analysis/AssumptionCache.h"
23 #include "llvm/Analysis/BasicAliasAnalysis.h"
24 #include "llvm/Analysis/BlockFrequencyInfo.h"
25 #include "llvm/Analysis/BranchProbabilityInfo.h"
26 #include "llvm/Analysis/CFGPrinter.h"
27 #include "llvm/Analysis/CFLAndersAliasAnalysis.h"
28 #include "llvm/Analysis/CFLSteensAliasAnalysis.h"
29 #include "llvm/Analysis/CGSCCPassManager.h"
30 #include "llvm/Analysis/CallGraph.h"
31 #include "llvm/Analysis/DDG.h"
32 #include "llvm/Analysis/Delinearization.h"
33 #include "llvm/Analysis/DemandedBits.h"
34 #include "llvm/Analysis/DependenceAnalysis.h"
35 #include "llvm/Analysis/DominanceFrontier.h"
36 #include "llvm/Analysis/FunctionPropertiesAnalysis.h"
37 #include "llvm/Analysis/GlobalsModRef.h"
38 #include "llvm/Analysis/IRSimilarityIdentifier.h"
39 #include "llvm/Analysis/IVUsers.h"
40 #include "llvm/Analysis/InlineAdvisor.h"
41 #include "llvm/Analysis/InlineSizeEstimatorAnalysis.h"
42 #include "llvm/Analysis/InstCount.h"
43 #include "llvm/Analysis/LazyCallGraph.h"
44 #include "llvm/Analysis/LazyValueInfo.h"
45 #include "llvm/Analysis/Lint.h"
46 #include "llvm/Analysis/LoopAccessAnalysis.h"
47 #include "llvm/Analysis/LoopCacheAnalysis.h"
48 #include "llvm/Analysis/LoopInfo.h"
49 #include "llvm/Analysis/LoopNestAnalysis.h"
50 #include "llvm/Analysis/MemoryDependenceAnalysis.h"
51 #include "llvm/Analysis/MemorySSA.h"
52 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
53 #include "llvm/Analysis/ObjCARCAliasAnalysis.h"
54 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
55 #include "llvm/Analysis/PhiValues.h"
56 #include "llvm/Analysis/PostDominators.h"
57 #include "llvm/Analysis/ProfileSummaryInfo.h"
58 #include "llvm/Analysis/RegionInfo.h"
59 #include "llvm/Analysis/ScalarEvolution.h"
60 #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
61 #include "llvm/Analysis/ScopedNoAliasAA.h"
62 #include "llvm/Analysis/StackLifetime.h"
63 #include "llvm/Analysis/StackSafetyAnalysis.h"
64 #include "llvm/Analysis/TargetLibraryInfo.h"
65 #include "llvm/Analysis/TargetTransformInfo.h"
66 #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
67 #include "llvm/IR/Dominators.h"
68 #include "llvm/IR/IRPrintingPasses.h"
69 #include "llvm/IR/PassManager.h"
70 #include "llvm/IR/SafepointIRVerifier.h"
71 #include "llvm/IR/Verifier.h"
72 #include "llvm/Support/CommandLine.h"
73 #include "llvm/Support/Debug.h"
74 #include "llvm/Support/FormatVariadic.h"
75 #include "llvm/Support/Regex.h"
76 #include "llvm/Target/TargetMachine.h"
77 #include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
78 #include "llvm/Transforms/Coroutines/CoroCleanup.h"
79 #include "llvm/Transforms/Coroutines/CoroEarly.h"
80 #include "llvm/Transforms/Coroutines/CoroElide.h"
81 #include "llvm/Transforms/Coroutines/CoroSplit.h"
82 #include "llvm/Transforms/HelloNew/HelloWorld.h"
83 #include "llvm/Transforms/IPO/AlwaysInliner.h"
84 #include "llvm/Transforms/IPO/ArgumentPromotion.h"
85 #include "llvm/Transforms/IPO/Attributor.h"
86 #include "llvm/Transforms/IPO/CalledValuePropagation.h"
87 #include "llvm/Transforms/IPO/ConstantMerge.h"
88 #include "llvm/Transforms/IPO/CrossDSOCFI.h"
89 #include "llvm/Transforms/IPO/DeadArgumentElimination.h"
90 #include "llvm/Transforms/IPO/ElimAvailExtern.h"
91 #include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
92 #include "llvm/Transforms/IPO/FunctionAttrs.h"
93 #include "llvm/Transforms/IPO/FunctionImport.h"
94 #include "llvm/Transforms/IPO/GlobalDCE.h"
95 #include "llvm/Transforms/IPO/GlobalOpt.h"
96 #include "llvm/Transforms/IPO/GlobalSplit.h"
97 #include "llvm/Transforms/IPO/HotColdSplitting.h"
98 #include "llvm/Transforms/IPO/InferFunctionAttrs.h"
99 #include "llvm/Transforms/IPO/Inliner.h"
100 #include "llvm/Transforms/IPO/Internalize.h"
101 #include "llvm/Transforms/IPO/LowerTypeTests.h"
102 #include "llvm/Transforms/IPO/MergeFunctions.h"
103 #include "llvm/Transforms/IPO/OpenMPOpt.h"
104 #include "llvm/Transforms/IPO/PartialInlining.h"
105 #include "llvm/Transforms/IPO/SCCP.h"
106 #include "llvm/Transforms/IPO/SampleProfile.h"
107 #include "llvm/Transforms/IPO/StripDeadPrototypes.h"
108 #include "llvm/Transforms/IPO/StripSymbols.h"
109 #include "llvm/Transforms/IPO/SyntheticCountsPropagation.h"
110 #include "llvm/Transforms/IPO/WholeProgramDevirt.h"
111 #include "llvm/Transforms/InstCombine/InstCombine.h"
112 #include "llvm/Transforms/Instrumentation.h"
113 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
114 #include "llvm/Transforms/Instrumentation/BoundsChecking.h"
115 #include "llvm/Transforms/Instrumentation/CGProfile.h"
116 #include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
117 #include "llvm/Transforms/Instrumentation/DataFlowSanitizer.h"
118 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
119 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
120 #include "llvm/Transforms/Instrumentation/InstrOrderFile.h"
121 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
122 #include "llvm/Transforms/Instrumentation/MemProfiler.h"
123 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
124 #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
125 #include "llvm/Transforms/Instrumentation/PoisonChecking.h"
126 #include "llvm/Transforms/Instrumentation/SanitizerCoverage.h"
127 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
128 #include "llvm/Transforms/ObjCARC.h"
129 #include "llvm/Transforms/Scalar/ADCE.h"
130 #include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
131 #include "llvm/Transforms/Scalar/BDCE.h"
132 #include "llvm/Transforms/Scalar/CallSiteSplitting.h"
133 #include "llvm/Transforms/Scalar/ConstantHoisting.h"
134 #include "llvm/Transforms/Scalar/ConstraintElimination.h"
135 #include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h"
136 #include "llvm/Transforms/Scalar/DCE.h"
137 #include "llvm/Transforms/Scalar/DeadStoreElimination.h"
138 #include "llvm/Transforms/Scalar/DivRemPairs.h"
139 #include "llvm/Transforms/Scalar/EarlyCSE.h"
140 #include "llvm/Transforms/Scalar/Float2Int.h"
141 #include "llvm/Transforms/Scalar/GVN.h"
142 #include "llvm/Transforms/Scalar/GuardWidening.h"
143 #include "llvm/Transforms/Scalar/IVUsersPrinter.h"
144 #include "llvm/Transforms/Scalar/IndVarSimplify.h"
145 #include "llvm/Transforms/Scalar/InductiveRangeCheckElimination.h"
146 #include "llvm/Transforms/Scalar/InstSimplifyPass.h"
147 #include "llvm/Transforms/Scalar/JumpThreading.h"
148 #include "llvm/Transforms/Scalar/LICM.h"
149 #include "llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h"
150 #include "llvm/Transforms/Scalar/LoopDataPrefetch.h"
151 #include "llvm/Transforms/Scalar/LoopDeletion.h"
152 #include "llvm/Transforms/Scalar/LoopDistribute.h"
153 #include "llvm/Transforms/Scalar/LoopFlatten.h"
154 #include "llvm/Transforms/Scalar/LoopFuse.h"
155 #include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
156 #include "llvm/Transforms/Scalar/LoopInstSimplify.h"
157 #include "llvm/Transforms/Scalar/LoopLoadElimination.h"
158 #include "llvm/Transforms/Scalar/LoopPassManager.h"
159 #include "llvm/Transforms/Scalar/LoopPredication.h"
160 #include "llvm/Transforms/Scalar/LoopReroll.h"
161 #include "llvm/Transforms/Scalar/LoopRotation.h"
162 #include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"
163 #include "llvm/Transforms/Scalar/LoopSink.h"
164 #include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
165 #include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
166 #include "llvm/Transforms/Scalar/LoopUnrollPass.h"
167 #include "llvm/Transforms/Scalar/LowerAtomic.h"
168 #include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
169 #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
170 #include "llvm/Transforms/Scalar/LowerGuardIntrinsic.h"
171 #include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h"
172 #include "llvm/Transforms/Scalar/LowerWidenableCondition.h"
173 #include "llvm/Transforms/Scalar/MakeGuardsExplicit.h"
174 #include "llvm/Transforms/Scalar/MemCpyOptimizer.h"
175 #include "llvm/Transforms/Scalar/MergeICmps.h"
176 #include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h"
177 #include "llvm/Transforms/Scalar/NaryReassociate.h"
178 #include "llvm/Transforms/Scalar/NewGVN.h"
179 #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
180 #include "llvm/Transforms/Scalar/Reassociate.h"
181 #include "llvm/Transforms/Scalar/RewriteStatepointsForGC.h"
182 #include "llvm/Transforms/Scalar/SCCP.h"
183 #include "llvm/Transforms/Scalar/SROA.h"
184 #include "llvm/Transforms/Scalar/Scalarizer.h"
185 #include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
186 #include "llvm/Transforms/Scalar/SimplifyCFG.h"
187 #include "llvm/Transforms/Scalar/Sink.h"
188 #include "llvm/Transforms/Scalar/SpeculateAroundPHIs.h"
189 #include "llvm/Transforms/Scalar/SpeculativeExecution.h"
190 #include "llvm/Transforms/Scalar/TailRecursionElimination.h"
191 #include "llvm/Transforms/Scalar/WarnMissedTransforms.h"
192 #include "llvm/Transforms/Utils/AddDiscriminators.h"
193 #include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
194 #include "llvm/Transforms/Utils/BreakCriticalEdges.h"
195 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
196 #include "llvm/Transforms/Utils/CanonicalizeFreezeInLoops.h"
197 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
198 #include "llvm/Transforms/Utils/InjectTLIMappings.h"
199 #include "llvm/Transforms/Utils/LCSSA.h"
200 #include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
201 #include "llvm/Transforms/Utils/LoopSimplify.h"
202 #include "llvm/Transforms/Utils/LoopVersioning.h"
203 #include "llvm/Transforms/Utils/LowerInvoke.h"
204 #include "llvm/Transforms/Utils/LowerSwitch.h"
205 #include "llvm/Transforms/Utils/Mem2Reg.h"
206 #include "llvm/Transforms/Utils/MetaRenamer.h"
207 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
208 #include "llvm/Transforms/Utils/StripNonLineTableDebugInfo.h"
209 #include "llvm/Transforms/Utils/SymbolRewriter.h"
210 #include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
211 #include "llvm/Transforms/Vectorize/LoopVectorize.h"
212 #include "llvm/Transforms/Vectorize/SLPVectorizer.h"
213 #include "llvm/Transforms/Vectorize/VectorCombine.h"
214
215 using namespace llvm;
216
217 static cl::opt<unsigned> MaxDevirtIterations("pm-max-devirt-iterations",
218                                              cl::ReallyHidden, cl::init(4));
219 static cl::opt<bool>
220     RunPartialInlining("enable-npm-partial-inlining", cl::init(false),
221                        cl::Hidden, cl::ZeroOrMore,
222                        cl::desc("Run Partial inlinining pass"));
223
224 static cl::opt<int> PreInlineThreshold(
225     "npm-preinline-threshold", cl::Hidden, cl::init(75), cl::ZeroOrMore,
226     cl::desc("Control the amount of inlining in pre-instrumentation inliner "
227              "(default = 75)"));
228
229 static cl::opt<bool>
230     RunNewGVN("enable-npm-newgvn", cl::init(false),
231               cl::Hidden, cl::ZeroOrMore,
232               cl::desc("Run NewGVN instead of GVN"));
233
234 static cl::opt<bool> EnableGVNHoist(
235     "enable-npm-gvn-hoist", cl::init(false), cl::Hidden,
236     cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
237
238 static cl::opt<InliningAdvisorMode> UseInlineAdvisor(
239     "enable-ml-inliner", cl::init(InliningAdvisorMode::Default), cl::Hidden,
240     cl::desc("Enable ML policy for inliner. Currently trained for -Oz only"),
241     cl::values(clEnumValN(InliningAdvisorMode::Default, "default",
242                           "Heuristics-based inliner version."),
243                clEnumValN(InliningAdvisorMode::Development, "development",
244                           "Use development mode (runtime-loadable model)."),
245                clEnumValN(InliningAdvisorMode::Release, "release",
246                           "Use release mode (AOT-compiled model).")));
247
248 static cl::opt<bool> EnableGVNSink(
249     "enable-npm-gvn-sink", cl::init(false), cl::Hidden,
250     cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
251
252 static cl::opt<bool> EnableUnrollAndJam(
253     "enable-npm-unroll-and-jam", cl::init(false), cl::Hidden,
254     cl::desc("Enable the Unroll and Jam pass for the new PM (default = off)"));
255
256 static cl::opt<bool> EnableLoopFlatten(
257     "enable-npm-loop-flatten", cl::init(false), cl::Hidden,
258     cl::desc("Enable the Loop flattening pass for the new PM (default = off)"));
259
260 static cl::opt<bool> EnableSyntheticCounts(
261     "enable-npm-synthetic-counts", cl::init(false), cl::Hidden, cl::ZeroOrMore,
262     cl::desc("Run synthetic function entry count generation "
263              "pass"));
264
265 static const Regex DefaultAliasRegex(
266     "^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$");
267
268 // This option is used in simplifying testing SampleFDO optimizations for
269 // profile loading.
270 static cl::opt<bool>
271     EnableCHR("enable-chr-npm", cl::init(true), cl::Hidden,
272               cl::desc("Enable control height reduction optimization (CHR)"));
273
274 /// Flag to enable inline deferral during PGO.
275 static cl::opt<bool>
276     EnablePGOInlineDeferral("enable-npm-pgo-inline-deferral", cl::init(true),
277                             cl::Hidden,
278                             cl::desc("Enable inline deferral during PGO"));
279
280 static cl::opt<bool> EnableMemProfiler("enable-mem-prof", cl::init(false),
281                                        cl::Hidden, cl::ZeroOrMore,
282                                        cl::desc("Enable memory profiler"));
283
284 PipelineTuningOptions::PipelineTuningOptions() {
285   LoopInterleaving = true;
286   LoopVectorization = true;
287   SLPVectorization = false;
288   LoopUnrolling = true;
289   ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll;
290   Coroutines = false;
291   LicmMssaOptCap = SetLicmMssaOptCap;
292   LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap;
293   CallGraphProfile = true;
294 }
295
296 extern cl::opt<bool> EnableConstraintElimination;
297 extern cl::opt<bool> EnableHotColdSplit;
298 extern cl::opt<bool> EnableOrderFileInstrumentation;
299
300 extern cl::opt<bool> FlattenedProfileUsed;
301
302 extern cl::opt<AttributorRunOption> AttributorRun;
303 extern cl::opt<bool> EnableKnowledgeRetention;
304
305 extern cl::opt<bool> EnableMatrix;
306
307 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O0 = {
308     /*SpeedLevel*/ 0,
309     /*SizeLevel*/ 0};
310 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O1 = {
311     /*SpeedLevel*/ 1,
312     /*SizeLevel*/ 0};
313 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O2 = {
314     /*SpeedLevel*/ 2,
315     /*SizeLevel*/ 0};
316 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O3 = {
317     /*SpeedLevel*/ 3,
318     /*SizeLevel*/ 0};
319 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::Os = {
320     /*SpeedLevel*/ 2,
321     /*SizeLevel*/ 1};
322 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::Oz = {
323     /*SpeedLevel*/ 2,
324     /*SizeLevel*/ 2};
325
326 namespace {
327
328 // The following passes/analyses have custom names, otherwise their name will
329 // include `(anonymous namespace)`. These are special since they are only for
330 // testing purposes and don't live in a header file.
331
332 /// No-op module pass which does nothing.
333 struct NoOpModulePass : PassInfoMixin<NoOpModulePass> {
334   PreservedAnalyses run(Module &M, ModuleAnalysisManager &) {
335     return PreservedAnalyses::all();
336   }
337
338   static StringRef name() { return "NoOpModulePass"; }
339 };
340
341 /// No-op module analysis.
342 class NoOpModuleAnalysis : public AnalysisInfoMixin<NoOpModuleAnalysis> {
343   friend AnalysisInfoMixin<NoOpModuleAnalysis>;
344   static AnalysisKey Key;
345
346 public:
347   struct Result {};
348   Result run(Module &, ModuleAnalysisManager &) { return Result(); }
349   static StringRef name() { return "NoOpModuleAnalysis"; }
350 };
351
352 /// No-op CGSCC pass which does nothing.
353 struct NoOpCGSCCPass : PassInfoMixin<NoOpCGSCCPass> {
354   PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &,
355                         LazyCallGraph &, CGSCCUpdateResult &UR) {
356     return PreservedAnalyses::all();
357   }
358   static StringRef name() { return "NoOpCGSCCPass"; }
359 };
360
361 /// No-op CGSCC analysis.
362 class NoOpCGSCCAnalysis : public AnalysisInfoMixin<NoOpCGSCCAnalysis> {
363   friend AnalysisInfoMixin<NoOpCGSCCAnalysis>;
364   static AnalysisKey Key;
365
366 public:
367   struct Result {};
368   Result run(LazyCallGraph::SCC &, CGSCCAnalysisManager &, LazyCallGraph &G) {
369     return Result();
370   }
371   static StringRef name() { return "NoOpCGSCCAnalysis"; }
372 };
373
374 /// No-op function pass which does nothing.
375 struct NoOpFunctionPass : PassInfoMixin<NoOpFunctionPass> {
376   PreservedAnalyses run(Function &F, FunctionAnalysisManager &) {
377     return PreservedAnalyses::all();
378   }
379   static StringRef name() { return "NoOpFunctionPass"; }
380 };
381
382 /// No-op function analysis.
383 class NoOpFunctionAnalysis : public AnalysisInfoMixin<NoOpFunctionAnalysis> {
384   friend AnalysisInfoMixin<NoOpFunctionAnalysis>;
385   static AnalysisKey Key;
386
387 public:
388   struct Result {};
389   Result run(Function &, FunctionAnalysisManager &) { return Result(); }
390   static StringRef name() { return "NoOpFunctionAnalysis"; }
391 };
392
393 /// No-op loop pass which does nothing.
394 struct NoOpLoopPass : PassInfoMixin<NoOpLoopPass> {
395   PreservedAnalyses run(Loop &L, LoopAnalysisManager &,
396                         LoopStandardAnalysisResults &, LPMUpdater &) {
397     return PreservedAnalyses::all();
398   }
399   static StringRef name() { return "NoOpLoopPass"; }
400 };
401
402 /// No-op loop analysis.
403 class NoOpLoopAnalysis : public AnalysisInfoMixin<NoOpLoopAnalysis> {
404   friend AnalysisInfoMixin<NoOpLoopAnalysis>;
405   static AnalysisKey Key;
406
407 public:
408   struct Result {};
409   Result run(Loop &, LoopAnalysisManager &, LoopStandardAnalysisResults &) {
410     return Result();
411   }
412   static StringRef name() { return "NoOpLoopAnalysis"; }
413 };
414
415 AnalysisKey NoOpModuleAnalysis::Key;
416 AnalysisKey NoOpCGSCCAnalysis::Key;
417 AnalysisKey NoOpFunctionAnalysis::Key;
418 AnalysisKey NoOpLoopAnalysis::Key;
419
420 } // namespace
421
422 void PassBuilder::invokePeepholeEPCallbacks(
423     FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
424   for (auto &C : PeepholeEPCallbacks)
425     C(FPM, Level);
426 }
427
428 void PassBuilder::registerModuleAnalyses(ModuleAnalysisManager &MAM) {
429 #define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
430   MAM.registerPass([&] { return CREATE_PASS; });
431 #include "PassRegistry.def"
432
433   for (auto &C : ModuleAnalysisRegistrationCallbacks)
434     C(MAM);
435 }
436
437 void PassBuilder::registerCGSCCAnalyses(CGSCCAnalysisManager &CGAM) {
438 #define CGSCC_ANALYSIS(NAME, CREATE_PASS)                                      \
439   CGAM.registerPass([&] { return CREATE_PASS; });
440 #include "PassRegistry.def"
441
442   for (auto &C : CGSCCAnalysisRegistrationCallbacks)
443     C(CGAM);
444 }
445
446 void PassBuilder::registerFunctionAnalyses(FunctionAnalysisManager &FAM) {
447 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
448   FAM.registerPass([&] { return CREATE_PASS; });
449 #include "PassRegistry.def"
450
451   for (auto &C : FunctionAnalysisRegistrationCallbacks)
452     C(FAM);
453 }
454
455 void PassBuilder::registerLoopAnalyses(LoopAnalysisManager &LAM) {
456 #define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
457   LAM.registerPass([&] { return CREATE_PASS; });
458 #include "PassRegistry.def"
459
460   for (auto &C : LoopAnalysisRegistrationCallbacks)
461     C(LAM);
462 }
463
464 // TODO: Investigate the cost/benefit of tail call elimination on debugging.
465 FunctionPassManager PassBuilder::buildO1FunctionSimplificationPipeline(
466     OptimizationLevel Level, ThinLTOPhase Phase, bool DebugLogging) {
467
468   FunctionPassManager FPM(DebugLogging);
469
470   // Form SSA out of local memory accesses after breaking apart aggregates into
471   // scalars.
472   FPM.addPass(SROA());
473
474   // Catch trivial redundancies
475   FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
476
477   // Hoisting of scalars and load expressions.
478   FPM.addPass(SimplifyCFGPass());
479   FPM.addPass(InstCombinePass());
480
481   FPM.addPass(LibCallsShrinkWrapPass());
482
483   invokePeepholeEPCallbacks(FPM, Level);
484
485   FPM.addPass(SimplifyCFGPass());
486
487   // Form canonically associated expression trees, and simplify the trees using
488   // basic mathematical properties. For example, this will form (nearly)
489   // minimal multiplication trees.
490   FPM.addPass(ReassociatePass());
491
492   // Add the primary loop simplification pipeline.
493   // FIXME: Currently this is split into two loop pass pipelines because we run
494   // some function passes in between them. These can and should be removed
495   // and/or replaced by scheduling the loop pass equivalents in the correct
496   // positions. But those equivalent passes aren't powerful enough yet.
497   // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
498   // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
499   // fully replace `SimplifyCFGPass`, and the closest to the other we have is
500   // `LoopInstSimplify`.
501   LoopPassManager LPM1(DebugLogging), LPM2(DebugLogging);
502
503   // Simplify the loop body. We do this initially to clean up after other loop
504   // passes run, either when iterating on a loop or on inner loops with
505   // implications on the outer loop.
506   LPM1.addPass(LoopInstSimplifyPass());
507   LPM1.addPass(LoopSimplifyCFGPass());
508
509   LPM1.addPass(LoopRotatePass(/* Disable header duplication */ true));
510   // TODO: Investigate promotion cap for O1.
511   LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
512   LPM1.addPass(SimpleLoopUnswitchPass());
513   LPM2.addPass(IndVarSimplifyPass());
514   LPM2.addPass(LoopIdiomRecognizePass());
515
516   for (auto &C : LateLoopOptimizationsEPCallbacks)
517     C(LPM2, Level);
518
519   LPM2.addPass(LoopDeletionPass());
520   if (EnableLoopFlatten)
521     LPM2.addPass(LoopFlattenPass());
522   // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
523   // because it changes IR to makes profile annotation in back compile
524   // inaccurate. The normal unroller doesn't pay attention to forced full unroll
525   // attributes so we need to make sure and allow the full unroll pass to pay
526   // attention to it.
527   if (Phase != ThinLTOPhase::PreLink || !PGOOpt ||
528       PGOOpt->Action != PGOOptions::SampleUse)
529     LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
530                                     /* OnlyWhenForced= */ !PTO.LoopUnrolling,
531                                     PTO.ForgetAllSCEVInLoopUnroll));
532
533   for (auto &C : LoopOptimizerEndEPCallbacks)
534     C(LPM2, Level);
535
536   // We provide the opt remark emitter pass for LICM to use. We only need to do
537   // this once as it is immutable.
538   FPM.addPass(
539       RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
540   FPM.addPass(createFunctionToLoopPassAdaptor(
541       std::move(LPM1), EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true,
542       DebugLogging));
543   FPM.addPass(SimplifyCFGPass());
544   FPM.addPass(InstCombinePass());
545   // The loop passes in LPM2 (LoopFullUnrollPass) do not preserve MemorySSA.
546   // *All* loop passes must preserve it, in order to be able to use it.
547   FPM.addPass(createFunctionToLoopPassAdaptor(
548       std::move(LPM2), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false,
549       DebugLogging));
550
551   // Delete small array after loop unroll.
552   FPM.addPass(SROA());
553
554   // Specially optimize memory movement as it doesn't look like dataflow in SSA.
555   FPM.addPass(MemCpyOptPass());
556
557   // Sparse conditional constant propagation.
558   // FIXME: It isn't clear why we do this *after* loop passes rather than
559   // before...
560   FPM.addPass(SCCPPass());
561
562   // Delete dead bit computations (instcombine runs after to fold away the dead
563   // computations, and then ADCE will run later to exploit any new DCE
564   // opportunities that creates).
565   FPM.addPass(BDCEPass());
566
567   // Run instcombine after redundancy and dead bit elimination to exploit
568   // opportunities opened up by them.
569   FPM.addPass(InstCombinePass());
570   invokePeepholeEPCallbacks(FPM, Level);
571
572   if (PTO.Coroutines)
573     FPM.addPass(CoroElidePass());
574
575   for (auto &C : ScalarOptimizerLateEPCallbacks)
576     C(FPM, Level);
577
578   // Finally, do an expensive DCE pass to catch all the dead code exposed by
579   // the simplifications and basic cleanup after all the simplifications.
580   // TODO: Investigate if this is too expensive.
581   FPM.addPass(ADCEPass());
582   FPM.addPass(SimplifyCFGPass());
583   FPM.addPass(InstCombinePass());
584   invokePeepholeEPCallbacks(FPM, Level);
585
586   return FPM;
587 }
588
589 FunctionPassManager
590 PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
591                                                  ThinLTOPhase Phase,
592                                                  bool DebugLogging) {
593   assert(Level != OptimizationLevel::O0 && "Must request optimizations!");
594
595   // The O1 pipeline has a separate pipeline creation function to simplify
596   // construction readability.
597   if (Level.getSpeedupLevel() == 1)
598     return buildO1FunctionSimplificationPipeline(Level, Phase, DebugLogging);
599
600   FunctionPassManager FPM(DebugLogging);
601
602   // Form SSA out of local memory accesses after breaking apart aggregates into
603   // scalars.
604   FPM.addPass(SROA());
605
606   // Catch trivial redundancies
607   FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
608   if (EnableKnowledgeRetention)
609     FPM.addPass(AssumeSimplifyPass());
610
611   // Hoisting of scalars and load expressions.
612   if (EnableGVNHoist)
613     FPM.addPass(GVNHoistPass());
614
615   // Global value numbering based sinking.
616   if (EnableGVNSink) {
617     FPM.addPass(GVNSinkPass());
618     FPM.addPass(SimplifyCFGPass());
619   }
620
621   if (EnableConstraintElimination)
622     FPM.addPass(ConstraintEliminationPass());
623
624   // Speculative execution if the target has divergent branches; otherwise nop.
625   FPM.addPass(SpeculativeExecutionPass(/* OnlyIfDivergentTarget =*/true));
626
627   // Optimize based on known information about branches, and cleanup afterward.
628   FPM.addPass(JumpThreadingPass());
629   FPM.addPass(CorrelatedValuePropagationPass());
630
631   FPM.addPass(SimplifyCFGPass());
632   if (Level == OptimizationLevel::O3)
633     FPM.addPass(AggressiveInstCombinePass());
634   FPM.addPass(InstCombinePass());
635
636   if (!Level.isOptimizingForSize())
637     FPM.addPass(LibCallsShrinkWrapPass());
638
639   invokePeepholeEPCallbacks(FPM, Level);
640
641   // For PGO use pipeline, try to optimize memory intrinsics such as memcpy
642   // using the size value profile. Don't perform this when optimizing for size.
643   if (PGOOpt && PGOOpt->Action == PGOOptions::IRUse &&
644       !Level.isOptimizingForSize())
645     FPM.addPass(PGOMemOPSizeOpt());
646
647   FPM.addPass(TailCallElimPass());
648   FPM.addPass(SimplifyCFGPass());
649
650   // Form canonically associated expression trees, and simplify the trees using
651   // basic mathematical properties. For example, this will form (nearly)
652   // minimal multiplication trees.
653   FPM.addPass(ReassociatePass());
654
655   // Add the primary loop simplification pipeline.
656   // FIXME: Currently this is split into two loop pass pipelines because we run
657   // some function passes in between them. These can and should be removed
658   // and/or replaced by scheduling the loop pass equivalents in the correct
659   // positions. But those equivalent passes aren't powerful enough yet.
660   // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
661   // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
662   // fully replace `SimplifyCFGPass`, and the closest to the other we have is
663   // `LoopInstSimplify`.
664   LoopPassManager LPM1(DebugLogging), LPM2(DebugLogging);
665
666   // Simplify the loop body. We do this initially to clean up after other loop
667   // passes run, either when iterating on a loop or on inner loops with
668   // implications on the outer loop.
669   LPM1.addPass(LoopInstSimplifyPass());
670   LPM1.addPass(LoopSimplifyCFGPass());
671
672   // Rotate Loop - disable header duplication at -Oz
673   LPM1.addPass(LoopRotatePass(Level != OptimizationLevel::Oz));
674   // TODO: Investigate promotion cap for O1.
675   LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
676   LPM1.addPass(SimpleLoopUnswitchPass());
677   LPM2.addPass(IndVarSimplifyPass());
678   LPM2.addPass(LoopIdiomRecognizePass());
679
680   for (auto &C : LateLoopOptimizationsEPCallbacks)
681     C(LPM2, Level);
682
683   LPM2.addPass(LoopDeletionPass());
684   // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
685   // because it changes IR to makes profile annotation in back compile
686   // inaccurate. The normal unroller doesn't pay attention to forced full unroll
687   // attributes so we need to make sure and allow the full unroll pass to pay
688   // attention to it.
689   if (Phase != ThinLTOPhase::PreLink || !PGOOpt ||
690       PGOOpt->Action != PGOOptions::SampleUse)
691     LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
692                                     /* OnlyWhenForced= */ !PTO.LoopUnrolling,
693                                     PTO.ForgetAllSCEVInLoopUnroll));
694
695   for (auto &C : LoopOptimizerEndEPCallbacks)
696     C(LPM2, Level);
697
698   // We provide the opt remark emitter pass for LICM to use. We only need to do
699   // this once as it is immutable.
700   FPM.addPass(
701       RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
702   FPM.addPass(createFunctionToLoopPassAdaptor(
703       std::move(LPM1), EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true,
704       DebugLogging));
705   FPM.addPass(SimplifyCFGPass());
706   FPM.addPass(InstCombinePass());
707   // The loop passes in LPM2 (IndVarSimplifyPass, LoopIdiomRecognizePass,
708   // LoopDeletionPass and LoopFullUnrollPass) do not preserve MemorySSA.
709   // *All* loop passes must preserve it, in order to be able to use it.
710   FPM.addPass(createFunctionToLoopPassAdaptor(
711       std::move(LPM2), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false,
712       DebugLogging));
713
714   // Delete small array after loop unroll.
715   FPM.addPass(SROA());
716
717   // Eliminate redundancies.
718   FPM.addPass(MergedLoadStoreMotionPass());
719   if (RunNewGVN)
720     FPM.addPass(NewGVNPass());
721   else
722     FPM.addPass(GVN());
723
724   // Specially optimize memory movement as it doesn't look like dataflow in SSA.
725   FPM.addPass(MemCpyOptPass());
726
727   // Sparse conditional constant propagation.
728   // FIXME: It isn't clear why we do this *after* loop passes rather than
729   // before...
730   FPM.addPass(SCCPPass());
731
732   // Delete dead bit computations (instcombine runs after to fold away the dead
733   // computations, and then ADCE will run later to exploit any new DCE
734   // opportunities that creates).
735   FPM.addPass(BDCEPass());
736
737   // Run instcombine after redundancy and dead bit elimination to exploit
738   // opportunities opened up by them.
739   FPM.addPass(InstCombinePass());
740   invokePeepholeEPCallbacks(FPM, Level);
741
742   // Re-consider control flow based optimizations after redundancy elimination,
743   // redo DCE, etc.
744   FPM.addPass(JumpThreadingPass());
745   FPM.addPass(CorrelatedValuePropagationPass());
746   FPM.addPass(DSEPass());
747   FPM.addPass(createFunctionToLoopPassAdaptor(
748       LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
749       EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true, DebugLogging));
750
751   if (PTO.Coroutines)
752     FPM.addPass(CoroElidePass());
753
754   for (auto &C : ScalarOptimizerLateEPCallbacks)
755     C(FPM, Level);
756
757   // Finally, do an expensive DCE pass to catch all the dead code exposed by
758   // the simplifications and basic cleanup after all the simplifications.
759   // TODO: Investigate if this is too expensive.
760   FPM.addPass(ADCEPass());
761   FPM.addPass(SimplifyCFGPass());
762   FPM.addPass(InstCombinePass());
763   invokePeepholeEPCallbacks(FPM, Level);
764
765   if (EnableCHR && Level == OptimizationLevel::O3 && PGOOpt &&
766       (PGOOpt->Action == PGOOptions::IRUse ||
767        PGOOpt->Action == PGOOptions::SampleUse))
768     FPM.addPass(ControlHeightReductionPass());
769
770   return FPM;
771 }
772
773 void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging,
774                                     PassBuilder::OptimizationLevel Level,
775                                     bool RunProfileGen, bool IsCS,
776                                     std::string ProfileFile,
777                                     std::string ProfileRemappingFile) {
778   assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!");
779   // Generally running simplification passes and the inliner with an high
780   // threshold results in smaller executables, but there may be cases where
781   // the size grows, so let's be conservative here and skip this simplification
782   // at -Os/Oz. We will not do this  inline for context sensistive PGO (when
783   // IsCS is true).
784   if (!Level.isOptimizingForSize() && !IsCS) {
785     InlineParams IP;
786
787     IP.DefaultThreshold = PreInlineThreshold;
788
789     // FIXME: The hint threshold has the same value used by the regular inliner.
790     // This should probably be lowered after performance testing.
791     // FIXME: this comment is cargo culted from the old pass manager, revisit).
792     IP.HintThreshold = 325;
793     ModuleInlinerWrapperPass MIWP(IP, DebugLogging);
794     CGSCCPassManager &CGPipeline = MIWP.getPM();
795
796     FunctionPassManager FPM;
797     FPM.addPass(SROA());
798     FPM.addPass(EarlyCSEPass());    // Catch trivial redundancies.
799     FPM.addPass(SimplifyCFGPass()); // Merge & remove basic blocks.
800     FPM.addPass(InstCombinePass()); // Combine silly sequences.
801     invokePeepholeEPCallbacks(FPM, Level);
802
803     CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
804
805     MPM.addPass(std::move(MIWP));
806
807     // Delete anything that is now dead to make sure that we don't instrument
808     // dead code. Instrumentation can end up keeping dead code around and
809     // dramatically increase code size.
810     MPM.addPass(GlobalDCEPass());
811   }
812
813   if (!RunProfileGen) {
814     assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
815     MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
816     // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
817     // RequireAnalysisPass for PSI before subsequent non-module passes.
818     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
819     return;
820   }
821
822   // Perform PGO instrumentation.
823   MPM.addPass(PGOInstrumentationGen(IsCS));
824
825   FunctionPassManager FPM;
826   FPM.addPass(createFunctionToLoopPassAdaptor(
827       LoopRotatePass(), EnableMSSALoopDependency,
828       /*UseBlockFrequencyInfo=*/false, DebugLogging));
829   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
830
831   // Add the profile lowering pass.
832   InstrProfOptions Options;
833   if (!ProfileFile.empty())
834     Options.InstrProfileOutput = ProfileFile;
835   // Do counter promotion at Level greater than O0.
836   Options.DoCounterPromotion = true;
837   Options.UseBFIInPromotion = IsCS;
838   MPM.addPass(InstrProfiling(Options, IsCS));
839 }
840
841 void PassBuilder::addPGOInstrPassesForO0(ModulePassManager &MPM,
842                                          bool DebugLogging, bool RunProfileGen,
843                                          bool IsCS, std::string ProfileFile,
844                                          std::string ProfileRemappingFile) {
845   if (!RunProfileGen) {
846     assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
847     MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
848     // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
849     // RequireAnalysisPass for PSI before subsequent non-module passes.
850     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
851     return;
852   }
853
854   // Perform PGO instrumentation.
855   MPM.addPass(PGOInstrumentationGen(IsCS));
856   // Add the profile lowering pass.
857   InstrProfOptions Options;
858   if (!ProfileFile.empty())
859     Options.InstrProfileOutput = ProfileFile;
860   // Do not do counter promotion at O0.
861   Options.DoCounterPromotion = false;
862   Options.UseBFIInPromotion = IsCS;
863   MPM.addPass(InstrProfiling(Options, IsCS));
864 }
865
866 static InlineParams
867 getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level) {
868   return getInlineParams(Level.getSpeedupLevel(), Level.getSizeLevel());
869 }
870
871 ModuleInlinerWrapperPass
872 PassBuilder::buildInlinerPipeline(OptimizationLevel Level, ThinLTOPhase Phase,
873                                   bool DebugLogging) {
874   InlineParams IP = getInlineParamsFromOptLevel(Level);
875   if (Phase == PassBuilder::ThinLTOPhase::PreLink && PGOOpt &&
876       PGOOpt->Action == PGOOptions::SampleUse)
877     IP.HotCallSiteThreshold = 0;
878
879   if (PGOOpt)
880     IP.EnableDeferral = EnablePGOInlineDeferral;
881
882   ModuleInlinerWrapperPass MIWP(IP, DebugLogging, UseInlineAdvisor,
883                                 MaxDevirtIterations);
884
885   // Require the GlobalsAA analysis for the module so we can query it within
886   // the CGSCC pipeline.
887   MIWP.addRequiredModuleAnalysis<GlobalsAA>();
888
889   // Require the ProfileSummaryAnalysis for the module so we can query it within
890   // the inliner pass.
891   MIWP.addRequiredModuleAnalysis<ProfileSummaryAnalysis>();
892
893   // Now begin the main postorder CGSCC pipeline.
894   // FIXME: The current CGSCC pipeline has its origins in the legacy pass
895   // manager and trying to emulate its precise behavior. Much of this doesn't
896   // make a lot of sense and we should revisit the core CGSCC structure.
897   CGSCCPassManager &MainCGPipeline = MIWP.getPM();
898
899   // Note: historically, the PruneEH pass was run first to deduce nounwind and
900   // generally clean up exception handling overhead. It isn't clear this is
901   // valuable as the inliner doesn't currently care whether it is inlining an
902   // invoke or a call.
903
904   if (AttributorRun & AttributorRunOption::CGSCC)
905     MainCGPipeline.addPass(AttributorCGSCCPass());
906
907   if (PTO.Coroutines)
908     MainCGPipeline.addPass(CoroSplitPass(Level != OptimizationLevel::O0));
909
910   // Now deduce any function attributes based in the current code.
911   MainCGPipeline.addPass(PostOrderFunctionAttrsPass());
912
913   // When at O3 add argument promotion to the pass pipeline.
914   // FIXME: It isn't at all clear why this should be limited to O3.
915   if (Level == OptimizationLevel::O3)
916     MainCGPipeline.addPass(ArgumentPromotionPass());
917
918   // Try to perform OpenMP specific optimizations. This is a (quick!) no-op if
919   // there are no OpenMP runtime calls present in the module.
920   if (Level == OptimizationLevel::O2 || Level == OptimizationLevel::O3)
921     MainCGPipeline.addPass(OpenMPOptPass());
922
923   // Lastly, add the core function simplification pipeline nested inside the
924   // CGSCC walk.
925   MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor(
926       buildFunctionSimplificationPipeline(Level, Phase, DebugLogging)));
927
928   for (auto &C : CGSCCOptimizerLateEPCallbacks)
929     C(MainCGPipeline, Level);
930
931   return MIWP;
932 }
933
934 ModulePassManager PassBuilder::buildModuleSimplificationPipeline(
935     OptimizationLevel Level, ThinLTOPhase Phase, bool DebugLogging) {
936   ModulePassManager MPM(DebugLogging);
937
938   bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse);
939
940   // In ThinLTO mode, when flattened profile is used, all the available
941   // profile information will be annotated in PreLink phase so there is
942   // no need to load the profile again in PostLink.
943   bool LoadSampleProfile =
944       HasSampleProfile &&
945       !(FlattenedProfileUsed && Phase == ThinLTOPhase::PostLink);
946
947   // During the ThinLTO backend phase we perform early indirect call promotion
948   // here, before globalopt. Otherwise imported available_externally functions
949   // look unreferenced and are removed. If we are going to load the sample
950   // profile then defer until later.
951   // TODO: See if we can move later and consolidate with the location where
952   // we perform ICP when we are loading a sample profile.
953   // TODO: We pass HasSampleProfile (whether there was a sample profile file
954   // passed to the compile) to the SamplePGO flag of ICP. This is used to
955   // determine whether the new direct calls are annotated with prof metadata.
956   // Ideally this should be determined from whether the IR is annotated with
957   // sample profile, and not whether the a sample profile was provided on the
958   // command line. E.g. for flattened profiles where we will not be reloading
959   // the sample profile in the ThinLTO backend, we ideally shouldn't have to
960   // provide the sample profile file.
961   if (Phase == ThinLTOPhase::PostLink && !LoadSampleProfile)
962     MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile));
963
964   // Do basic inference of function attributes from known properties of system
965   // libraries and other oracles.
966   MPM.addPass(InferFunctionAttrsPass());
967
968   // Create an early function pass manager to cleanup the output of the
969   // frontend.
970   FunctionPassManager EarlyFPM(DebugLogging);
971   EarlyFPM.addPass(SimplifyCFGPass());
972   EarlyFPM.addPass(SROA());
973   EarlyFPM.addPass(EarlyCSEPass());
974   EarlyFPM.addPass(LowerExpectIntrinsicPass());
975   if (PTO.Coroutines)
976     EarlyFPM.addPass(CoroEarlyPass());
977   if (Level == OptimizationLevel::O3)
978     EarlyFPM.addPass(CallSiteSplittingPass());
979
980   // In SamplePGO ThinLTO backend, we need instcombine before profile annotation
981   // to convert bitcast to direct calls so that they can be inlined during the
982   // profile annotation prepration step.
983   // More details about SamplePGO design can be found in:
984   // https://research.google.com/pubs/pub45290.html
985   // FIXME: revisit how SampleProfileLoad/Inliner/ICP is structured.
986   if (LoadSampleProfile)
987     EarlyFPM.addPass(InstCombinePass());
988   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM)));
989
990   if (LoadSampleProfile) {
991     // Annotate sample profile right after early FPM to ensure freshness of
992     // the debug info.
993     MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
994                                         PGOOpt->ProfileRemappingFile,
995                                         Phase == ThinLTOPhase::PreLink));
996     // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
997     // RequireAnalysisPass for PSI before subsequent non-module passes.
998     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
999     // Do not invoke ICP in the ThinLTOPrelink phase as it makes it hard
1000     // for the profile annotation to be accurate in the ThinLTO backend.
1001     if (Phase != ThinLTOPhase::PreLink)
1002       // We perform early indirect call promotion here, before globalopt.
1003       // This is important for the ThinLTO backend phase because otherwise
1004       // imported available_externally functions look unreferenced and are
1005       // removed.
1006       MPM.addPass(PGOIndirectCallPromotion(Phase == ThinLTOPhase::PostLink,
1007                                            true /* SamplePGO */));
1008   }
1009
1010   if (AttributorRun & AttributorRunOption::MODULE)
1011     MPM.addPass(AttributorPass());
1012
1013   // Lower type metadata and the type.test intrinsic in the ThinLTO
1014   // post link pipeline after ICP. This is to enable usage of the type
1015   // tests in ICP sequences.
1016   if (Phase == ThinLTOPhase::PostLink)
1017     MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1018
1019   // Interprocedural constant propagation now that basic cleanup has occurred
1020   // and prior to optimizing globals.
1021   // FIXME: This position in the pipeline hasn't been carefully considered in
1022   // years, it should be re-analyzed.
1023   MPM.addPass(IPSCCPPass());
1024
1025   // Attach metadata to indirect call sites indicating the set of functions
1026   // they may target at run-time. This should follow IPSCCP.
1027   MPM.addPass(CalledValuePropagationPass());
1028
1029   // Optimize globals to try and fold them into constants.
1030   MPM.addPass(GlobalOptPass());
1031
1032   // Promote any localized globals to SSA registers.
1033   // FIXME: Should this instead by a run of SROA?
1034   // FIXME: We should probably run instcombine and simplify-cfg afterward to
1035   // delete control flows that are dead once globals have been folded to
1036   // constants.
1037   MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
1038
1039   // Remove any dead arguments exposed by cleanups and constant folding
1040   // globals.
1041   MPM.addPass(DeadArgumentEliminationPass());
1042
1043   // Create a small function pass pipeline to cleanup after all the global
1044   // optimizations.
1045   FunctionPassManager GlobalCleanupPM(DebugLogging);
1046   GlobalCleanupPM.addPass(InstCombinePass());
1047   invokePeepholeEPCallbacks(GlobalCleanupPM, Level);
1048
1049   GlobalCleanupPM.addPass(SimplifyCFGPass());
1050   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM)));
1051
1052   // Add all the requested passes for instrumentation PGO, if requested.
1053   if (PGOOpt && Phase != ThinLTOPhase::PostLink &&
1054       (PGOOpt->Action == PGOOptions::IRInstr ||
1055        PGOOpt->Action == PGOOptions::IRUse)) {
1056     addPGOInstrPasses(MPM, DebugLogging, Level,
1057                       /* RunProfileGen */ PGOOpt->Action == PGOOptions::IRInstr,
1058                       /* IsCS */ false, PGOOpt->ProfileFile,
1059                       PGOOpt->ProfileRemappingFile);
1060     MPM.addPass(PGOIndirectCallPromotion(false, false));
1061   }
1062   if (PGOOpt && Phase != ThinLTOPhase::PostLink &&
1063       PGOOpt->CSAction == PGOOptions::CSIRInstr)
1064     MPM.addPass(PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile));
1065
1066   // Synthesize function entry counts for non-PGO compilation.
1067   if (EnableSyntheticCounts && !PGOOpt)
1068     MPM.addPass(SyntheticCountsPropagation());
1069
1070   MPM.addPass(buildInlinerPipeline(Level, Phase, DebugLogging));
1071
1072   if (EnableMemProfiler && Phase != ThinLTOPhase::PreLink) {
1073     MPM.addPass(createModuleToFunctionPassAdaptor(MemProfilerPass()));
1074     MPM.addPass(ModuleMemProfilerPass());
1075   }
1076
1077   return MPM;
1078 }
1079
1080 ModulePassManager PassBuilder::buildModuleOptimizationPipeline(
1081     OptimizationLevel Level, bool DebugLogging, bool LTOPreLink) {
1082   ModulePassManager MPM(DebugLogging);
1083
1084   // Optimize globals now that the module is fully simplified.
1085   MPM.addPass(GlobalOptPass());
1086   MPM.addPass(GlobalDCEPass());
1087
1088   // Run partial inlining pass to partially inline functions that have
1089   // large bodies.
1090   if (RunPartialInlining)
1091     MPM.addPass(PartialInlinerPass());
1092
1093   // Remove avail extern fns and globals definitions since we aren't compiling
1094   // an object file for later LTO. For LTO we want to preserve these so they
1095   // are eligible for inlining at link-time. Note if they are unreferenced they
1096   // will be removed by GlobalDCE later, so this only impacts referenced
1097   // available externally globals. Eventually they will be suppressed during
1098   // codegen, but eliminating here enables more opportunity for GlobalDCE as it
1099   // may make globals referenced by available external functions dead and saves
1100   // running remaining passes on the eliminated functions. These should be
1101   // preserved during prelinking for link-time inlining decisions.
1102   if (!LTOPreLink)
1103     MPM.addPass(EliminateAvailableExternallyPass());
1104
1105   if (EnableOrderFileInstrumentation)
1106     MPM.addPass(InstrOrderFilePass());
1107
1108   // Do RPO function attribute inference across the module to forward-propagate
1109   // attributes where applicable.
1110   // FIXME: Is this really an optimization rather than a canonicalization?
1111   MPM.addPass(ReversePostOrderFunctionAttrsPass());
1112
1113   // Do a post inline PGO instrumentation and use pass. This is a context
1114   // sensitive PGO pass. We don't want to do this in LTOPreLink phrase as
1115   // cross-module inline has not been done yet. The context sensitive
1116   // instrumentation is after all the inlines are done.
1117   if (!LTOPreLink && PGOOpt) {
1118     if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1119       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ true,
1120                         /* IsCS */ true, PGOOpt->CSProfileGenFile,
1121                         PGOOpt->ProfileRemappingFile);
1122     else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1123       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ false,
1124                         /* IsCS */ true, PGOOpt->ProfileFile,
1125                         PGOOpt->ProfileRemappingFile);
1126   }
1127
1128   // Re-require GloblasAA here prior to function passes. This is particularly
1129   // useful as the above will have inlined, DCE'ed, and function-attr
1130   // propagated everything. We should at this point have a reasonably minimal
1131   // and richly annotated call graph. By computing aliasing and mod/ref
1132   // information for all local globals here, the late loop passes and notably
1133   // the vectorizer will be able to use them to help recognize vectorizable
1134   // memory operations.
1135   MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
1136
1137   FunctionPassManager OptimizePM(DebugLogging);
1138   OptimizePM.addPass(Float2IntPass());
1139   OptimizePM.addPass(LowerConstantIntrinsicsPass());
1140
1141   if (EnableMatrix) {
1142     OptimizePM.addPass(LowerMatrixIntrinsicsPass());
1143     OptimizePM.addPass(EarlyCSEPass());
1144   }
1145
1146   // FIXME: We need to run some loop optimizations to re-rotate loops after
1147   // simplify-cfg and others undo their rotation.
1148
1149   // Optimize the loop execution. These passes operate on entire loop nests
1150   // rather than on each loop in an inside-out manner, and so they are actually
1151   // function passes.
1152
1153   for (auto &C : VectorizerStartEPCallbacks)
1154     C(OptimizePM, Level);
1155
1156   // First rotate loops that may have been un-rotated by prior passes.
1157   OptimizePM.addPass(createFunctionToLoopPassAdaptor(
1158       LoopRotatePass(), EnableMSSALoopDependency,
1159       /*UseBlockFrequencyInfo=*/false, DebugLogging));
1160
1161   // Distribute loops to allow partial vectorization.  I.e. isolate dependences
1162   // into separate loop that would otherwise inhibit vectorization.  This is
1163   // currently only performed for loops marked with the metadata
1164   // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
1165   OptimizePM.addPass(LoopDistributePass());
1166
1167   // Populates the VFABI attribute with the scalar-to-vector mappings
1168   // from the TargetLibraryInfo.
1169   OptimizePM.addPass(InjectTLIMappings());
1170
1171   // Now run the core loop vectorizer.
1172   OptimizePM.addPass(LoopVectorizePass(
1173       LoopVectorizeOptions(!PTO.LoopInterleaving, !PTO.LoopVectorization)));
1174
1175   // Eliminate loads by forwarding stores from the previous iteration to loads
1176   // of the current iteration.
1177   OptimizePM.addPass(LoopLoadEliminationPass());
1178
1179   // Cleanup after the loop optimization passes.
1180   OptimizePM.addPass(InstCombinePass());
1181
1182   // Now that we've formed fast to execute loop structures, we do further
1183   // optimizations. These are run afterward as they might block doing complex
1184   // analyses and transforms such as what are needed for loop vectorization.
1185
1186   // Cleanup after loop vectorization, etc. Simplification passes like CVP and
1187   // GVN, loop transforms, and others have already run, so it's now better to
1188   // convert to more optimized IR using more aggressive simplify CFG options.
1189   // The extra sinking transform can create larger basic blocks, so do this
1190   // before SLP vectorization.
1191   // FIXME: study whether hoisting and/or sinking of common instructions should
1192   //        be delayed until after SLP vectorizer.
1193   OptimizePM.addPass(SimplifyCFGPass(SimplifyCFGOptions()
1194                                          .forwardSwitchCondToPhi(true)
1195                                          .convertSwitchToLookupTable(true)
1196                                          .needCanonicalLoops(false)
1197                                          .hoistCommonInsts(true)
1198                                          .sinkCommonInsts(true)));
1199
1200   // Optimize parallel scalar instruction chains into SIMD instructions.
1201   if (PTO.SLPVectorization)
1202     OptimizePM.addPass(SLPVectorizerPass());
1203
1204   // Enhance/cleanup vector code.
1205   OptimizePM.addPass(VectorCombinePass());
1206   OptimizePM.addPass(InstCombinePass());
1207
1208   // Unroll small loops to hide loop backedge latency and saturate any parallel
1209   // execution resources of an out-of-order processor. We also then need to
1210   // clean up redundancies and loop invariant code.
1211   // FIXME: It would be really good to use a loop-integrated instruction
1212   // combiner for cleanup here so that the unrolling and LICM can be pipelined
1213   // across the loop nests.
1214   // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
1215   if (EnableUnrollAndJam && PTO.LoopUnrolling) {
1216     OptimizePM.addPass(LoopUnrollAndJamPass(Level.getSpeedupLevel()));
1217   }
1218   OptimizePM.addPass(LoopUnrollPass(LoopUnrollOptions(
1219       Level.getSpeedupLevel(), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
1220       PTO.ForgetAllSCEVInLoopUnroll)));
1221   OptimizePM.addPass(WarnMissedTransformationsPass());
1222   OptimizePM.addPass(InstCombinePass());
1223   OptimizePM.addPass(RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
1224   OptimizePM.addPass(createFunctionToLoopPassAdaptor(
1225       LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
1226       EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true, DebugLogging));
1227
1228   // Now that we've vectorized and unrolled loops, we may have more refined
1229   // alignment information, try to re-derive it here.
1230   OptimizePM.addPass(AlignmentFromAssumptionsPass());
1231
1232   // Split out cold code. Splitting is done late to avoid hiding context from
1233   // other optimizations and inadvertently regressing performance. The tradeoff
1234   // is that this has a higher code size cost than splitting early.
1235   if (EnableHotColdSplit && !LTOPreLink)
1236     MPM.addPass(HotColdSplittingPass());
1237
1238   // LoopSink pass sinks instructions hoisted by LICM, which serves as a
1239   // canonicalization pass that enables other optimizations. As a result,
1240   // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
1241   // result too early.
1242   OptimizePM.addPass(LoopSinkPass());
1243
1244   // And finally clean up LCSSA form before generating code.
1245   OptimizePM.addPass(InstSimplifyPass());
1246
1247   // This hoists/decomposes div/rem ops. It should run after other sink/hoist
1248   // passes to avoid re-sinking, but before SimplifyCFG because it can allow
1249   // flattening of blocks.
1250   OptimizePM.addPass(DivRemPairsPass());
1251
1252   // LoopSink (and other loop passes since the last simplifyCFG) might have
1253   // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
1254   OptimizePM.addPass(SimplifyCFGPass());
1255
1256   // Optimize PHIs by speculating around them when profitable. Note that this
1257   // pass needs to be run after any PRE or similar pass as it is essentially
1258   // inserting redundancies into the program. This even includes SimplifyCFG.
1259   OptimizePM.addPass(SpeculateAroundPHIsPass());
1260
1261   if (PTO.Coroutines)
1262     OptimizePM.addPass(CoroCleanupPass());
1263
1264   // Add the core optimizing pipeline.
1265   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM)));
1266
1267   for (auto &C : OptimizerLastEPCallbacks)
1268     C(MPM, Level);
1269
1270   if (PTO.CallGraphProfile)
1271     MPM.addPass(CGProfilePass());
1272
1273   // Now we need to do some global optimization transforms.
1274   // FIXME: It would seem like these should come first in the optimization
1275   // pipeline and maybe be the bottom of the canonicalization pipeline? Weird
1276   // ordering here.
1277   MPM.addPass(GlobalDCEPass());
1278   MPM.addPass(ConstantMergePass());
1279
1280   return MPM;
1281 }
1282
1283 ModulePassManager
1284 PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level,
1285                                            bool DebugLogging, bool LTOPreLink) {
1286   assert(Level != OptimizationLevel::O0 &&
1287          "Must request optimizations for the default pipeline!");
1288
1289   ModulePassManager MPM(DebugLogging);
1290
1291   // Force any function attributes we want the rest of the pipeline to observe.
1292   MPM.addPass(ForceFunctionAttrsPass());
1293
1294   // Apply module pipeline start EP callback.
1295   for (auto &C : PipelineStartEPCallbacks)
1296     C(MPM);
1297
1298   if (PGOOpt && PGOOpt->SamplePGOSupport)
1299     MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
1300
1301   // Add the core simplification pipeline.
1302   MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::None,
1303                                                 DebugLogging));
1304
1305   // Now add the optimization pipeline.
1306   MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging, LTOPreLink));
1307
1308   return MPM;
1309 }
1310
1311 ModulePassManager
1312 PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level,
1313                                                 bool DebugLogging) {
1314   assert(Level != OptimizationLevel::O0 &&
1315          "Must request optimizations for the default pipeline!");
1316
1317   ModulePassManager MPM(DebugLogging);
1318
1319   // Force any function attributes we want the rest of the pipeline to observe.
1320   MPM.addPass(ForceFunctionAttrsPass());
1321
1322   if (PGOOpt && PGOOpt->SamplePGOSupport)
1323     MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
1324
1325   // Apply module pipeline start EP callback.
1326   for (auto &C : PipelineStartEPCallbacks)
1327     C(MPM);
1328
1329   // If we are planning to perform ThinLTO later, we don't bloat the code with
1330   // unrolling/vectorization/... now. Just simplify the module as much as we
1331   // can.
1332   MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PreLink,
1333                                                 DebugLogging));
1334
1335   // Run partial inlining pass to partially inline functions that have
1336   // large bodies.
1337   // FIXME: It isn't clear whether this is really the right place to run this
1338   // in ThinLTO. Because there is another canonicalization and simplification
1339   // phase that will run after the thin link, running this here ends up with
1340   // less information than will be available later and it may grow functions in
1341   // ways that aren't beneficial.
1342   if (RunPartialInlining)
1343     MPM.addPass(PartialInlinerPass());
1344
1345   // Reduce the size of the IR as much as possible.
1346   MPM.addPass(GlobalOptPass());
1347
1348   // Module simplification splits coroutines, but does not fully clean up
1349   // coroutine intrinsics. To ensure ThinLTO optimization passes don't trip up
1350   // on these, we schedule the cleanup here.
1351   if (PTO.Coroutines)
1352     MPM.addPass(createModuleToFunctionPassAdaptor(CoroCleanupPass()));
1353
1354   return MPM;
1355 }
1356
1357 ModulePassManager PassBuilder::buildThinLTODefaultPipeline(
1358     OptimizationLevel Level, bool DebugLogging,
1359     const ModuleSummaryIndex *ImportSummary) {
1360   ModulePassManager MPM(DebugLogging);
1361
1362   if (ImportSummary) {
1363     // These passes import type identifier resolutions for whole-program
1364     // devirtualization and CFI. They must run early because other passes may
1365     // disturb the specific instruction patterns that these passes look for,
1366     // creating dependencies on resolutions that may not appear in the summary.
1367     //
1368     // For example, GVN may transform the pattern assume(type.test) appearing in
1369     // two basic blocks into assume(phi(type.test, type.test)), which would
1370     // transform a dependency on a WPD resolution into a dependency on a type
1371     // identifier resolution for CFI.
1372     //
1373     // Also, WPD has access to more precise information than ICP and can
1374     // devirtualize more effectively, so it should operate on the IR first.
1375     //
1376     // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1377     // metadata and intrinsics.
1378     MPM.addPass(WholeProgramDevirtPass(nullptr, ImportSummary));
1379     MPM.addPass(LowerTypeTestsPass(nullptr, ImportSummary));
1380   }
1381
1382   if (Level == OptimizationLevel::O0)
1383     return MPM;
1384
1385   // Force any function attributes we want the rest of the pipeline to observe.
1386   MPM.addPass(ForceFunctionAttrsPass());
1387
1388   // Add the core simplification pipeline.
1389   MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PostLink,
1390                                                 DebugLogging));
1391
1392   // Now add the optimization pipeline.
1393   MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging));
1394
1395   return MPM;
1396 }
1397
1398 ModulePassManager
1399 PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level,
1400                                             bool DebugLogging) {
1401   assert(Level != OptimizationLevel::O0 &&
1402          "Must request optimizations for the default pipeline!");
1403   // FIXME: We should use a customized pre-link pipeline!
1404   return buildPerModuleDefaultPipeline(Level, DebugLogging,
1405                                        /* LTOPreLink */ true);
1406 }
1407
1408 ModulePassManager
1409 PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, bool DebugLogging,
1410                                      ModuleSummaryIndex *ExportSummary) {
1411   ModulePassManager MPM(DebugLogging);
1412
1413   if (Level == OptimizationLevel::O0) {
1414     // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1415     // metadata and intrinsics.
1416     MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1417     MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1418     // Run a second time to clean up any type tests left behind by WPD for use
1419     // in ICP.
1420     MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1421     return MPM;
1422   }
1423
1424   if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
1425     // Load sample profile before running the LTO optimization pipeline.
1426     MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
1427                                         PGOOpt->ProfileRemappingFile,
1428                                         false /* ThinLTOPhase::PreLink */));
1429     // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1430     // RequireAnalysisPass for PSI before subsequent non-module passes.
1431     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
1432   }
1433
1434   // Remove unused virtual tables to improve the quality of code generated by
1435   // whole-program devirtualization and bitset lowering.
1436   MPM.addPass(GlobalDCEPass());
1437
1438   // Force any function attributes we want the rest of the pipeline to observe.
1439   MPM.addPass(ForceFunctionAttrsPass());
1440
1441   // Do basic inference of function attributes from known properties of system
1442   // libraries and other oracles.
1443   MPM.addPass(InferFunctionAttrsPass());
1444
1445   if (Level.getSpeedupLevel() > 1) {
1446     FunctionPassManager EarlyFPM(DebugLogging);
1447     EarlyFPM.addPass(CallSiteSplittingPass());
1448     MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM)));
1449
1450     // Indirect call promotion. This should promote all the targets that are
1451     // left by the earlier promotion pass that promotes intra-module targets.
1452     // This two-step promotion is to save the compile time. For LTO, it should
1453     // produce the same result as if we only do promotion here.
1454     MPM.addPass(PGOIndirectCallPromotion(
1455         true /* InLTO */, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
1456     // Propagate constants at call sites into the functions they call.  This
1457     // opens opportunities for globalopt (and inlining) by substituting function
1458     // pointers passed as arguments to direct uses of functions.
1459     MPM.addPass(IPSCCPPass());
1460
1461     // Attach metadata to indirect call sites indicating the set of functions
1462     // they may target at run-time. This should follow IPSCCP.
1463     MPM.addPass(CalledValuePropagationPass());
1464   }
1465
1466   // Now deduce any function attributes based in the current code.
1467   MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1468               PostOrderFunctionAttrsPass()));
1469
1470   // Do RPO function attribute inference across the module to forward-propagate
1471   // attributes where applicable.
1472   // FIXME: Is this really an optimization rather than a canonicalization?
1473   MPM.addPass(ReversePostOrderFunctionAttrsPass());
1474
1475   // Use in-range annotations on GEP indices to split globals where beneficial.
1476   MPM.addPass(GlobalSplitPass());
1477
1478   // Run whole program optimization of virtual call when the list of callees
1479   // is fixed.
1480   MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1481
1482   // Stop here at -O1.
1483   if (Level == OptimizationLevel::O1) {
1484     // The LowerTypeTestsPass needs to run to lower type metadata and the
1485     // type.test intrinsics. The pass does nothing if CFI is disabled.
1486     MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1487     // Run a second time to clean up any type tests left behind by WPD for use
1488     // in ICP (which is performed earlier than this in the regular LTO
1489     // pipeline).
1490     MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1491     return MPM;
1492   }
1493
1494   // Optimize globals to try and fold them into constants.
1495   MPM.addPass(GlobalOptPass());
1496
1497   // Promote any localized globals to SSA registers.
1498   MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
1499
1500   // Linking modules together can lead to duplicate global constant, only
1501   // keep one copy of each constant.
1502   MPM.addPass(ConstantMergePass());
1503
1504   // Remove unused arguments from functions.
1505   MPM.addPass(DeadArgumentEliminationPass());
1506
1507   // Reduce the code after globalopt and ipsccp.  Both can open up significant
1508   // simplification opportunities, and both can propagate functions through
1509   // function pointers.  When this happens, we often have to resolve varargs
1510   // calls, etc, so let instcombine do this.
1511   FunctionPassManager PeepholeFPM(DebugLogging);
1512   if (Level == OptimizationLevel::O3)
1513     PeepholeFPM.addPass(AggressiveInstCombinePass());
1514   PeepholeFPM.addPass(InstCombinePass());
1515   invokePeepholeEPCallbacks(PeepholeFPM, Level);
1516
1517   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM)));
1518
1519   // Note: historically, the PruneEH pass was run first to deduce nounwind and
1520   // generally clean up exception handling overhead. It isn't clear this is
1521   // valuable as the inliner doesn't currently care whether it is inlining an
1522   // invoke or a call.
1523   // Run the inliner now.
1524   MPM.addPass(ModuleInlinerWrapperPass(getInlineParamsFromOptLevel(Level),
1525                                        DebugLogging));
1526
1527   // Optimize globals again after we ran the inliner.
1528   MPM.addPass(GlobalOptPass());
1529
1530   // Garbage collect dead functions.
1531   // FIXME: Add ArgumentPromotion pass after once it's ported.
1532   MPM.addPass(GlobalDCEPass());
1533
1534   FunctionPassManager FPM(DebugLogging);
1535   // The IPO Passes may leave cruft around. Clean up after them.
1536   FPM.addPass(InstCombinePass());
1537   invokePeepholeEPCallbacks(FPM, Level);
1538
1539   FPM.addPass(JumpThreadingPass(/*InsertFreezeWhenUnfoldingSelect*/ true));
1540
1541   // Do a post inline PGO instrumentation and use pass. This is a context
1542   // sensitive PGO pass.
1543   if (PGOOpt) {
1544     if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1545       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ true,
1546                         /* IsCS */ true, PGOOpt->CSProfileGenFile,
1547                         PGOOpt->ProfileRemappingFile);
1548     else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1549       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ false,
1550                         /* IsCS */ true, PGOOpt->ProfileFile,
1551                         PGOOpt->ProfileRemappingFile);
1552   }
1553
1554   // Break up allocas
1555   FPM.addPass(SROA());
1556
1557   // LTO provides additional opportunities for tailcall elimination due to
1558   // link-time inlining, and visibility of nocapture attribute.
1559   FPM.addPass(TailCallElimPass());
1560
1561   // Run a few AA driver optimizations here and now to cleanup the code.
1562   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
1563
1564   MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1565               PostOrderFunctionAttrsPass()));
1566   // FIXME: here we run IP alias analysis in the legacy PM.
1567
1568   FunctionPassManager MainFPM;
1569
1570   // FIXME: once we fix LoopPass Manager, add LICM here.
1571   // FIXME: once we provide support for enabling MLSM, add it here.
1572   if (RunNewGVN)
1573     MainFPM.addPass(NewGVNPass());
1574   else
1575     MainFPM.addPass(GVN());
1576
1577   // Remove dead memcpy()'s.
1578   MainFPM.addPass(MemCpyOptPass());
1579
1580   // Nuke dead stores.
1581   MainFPM.addPass(DSEPass());
1582
1583   // FIXME: at this point, we run a bunch of loop passes:
1584   // indVarSimplify, loopDeletion, loopInterchange, loopUnroll,
1585   // loopVectorize. Enable them once the remaining issue with LPM
1586   // are sorted out.
1587
1588   MainFPM.addPass(InstCombinePass());
1589   MainFPM.addPass(SimplifyCFGPass());
1590   MainFPM.addPass(SCCPPass());
1591   MainFPM.addPass(InstCombinePass());
1592   MainFPM.addPass(BDCEPass());
1593
1594   // FIXME: We may want to run SLPVectorizer here.
1595   // After vectorization, assume intrinsics may tell us more
1596   // about pointer alignments.
1597 #if 0
1598   MainFPM.add(AlignmentFromAssumptionsPass());
1599 #endif
1600
1601   // FIXME: Conditionally run LoadCombine here, after it's ported
1602   // (in case we still have this pass, given its questionable usefulness).
1603
1604   MainFPM.addPass(InstCombinePass());
1605   invokePeepholeEPCallbacks(MainFPM, Level);
1606   MainFPM.addPass(JumpThreadingPass(/*InsertFreezeWhenUnfoldingSelect*/ true));
1607   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM)));
1608
1609   // Create a function that performs CFI checks for cross-DSO calls with
1610   // targets in the current module.
1611   MPM.addPass(CrossDSOCFIPass());
1612
1613   // Lower type metadata and the type.test intrinsic. This pass supports
1614   // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs
1615   // to be run at link time if CFI is enabled. This pass does nothing if
1616   // CFI is disabled.
1617   MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1618   // Run a second time to clean up any type tests left behind by WPD for use
1619   // in ICP (which is performed earlier than this in the regular LTO pipeline).
1620   MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1621
1622   // Enable splitting late in the FullLTO post-link pipeline. This is done in
1623   // the same stage in the old pass manager (\ref addLateLTOOptimizationPasses).
1624   if (EnableHotColdSplit)
1625     MPM.addPass(HotColdSplittingPass());
1626
1627   // Add late LTO optimization passes.
1628   // Delete basic blocks, which optimization passes may have killed.
1629   MPM.addPass(createModuleToFunctionPassAdaptor(SimplifyCFGPass()));
1630
1631   // Drop bodies of available eternally objects to improve GlobalDCE.
1632   MPM.addPass(EliminateAvailableExternallyPass());
1633
1634   // Now that we have optimized the program, discard unreachable functions.
1635   MPM.addPass(GlobalDCEPass());
1636
1637   // FIXME: Maybe enable MergeFuncs conditionally after it's ported.
1638   return MPM;
1639 }
1640
1641 AAManager PassBuilder::buildDefaultAAPipeline() {
1642   AAManager AA;
1643
1644   // The order in which these are registered determines their priority when
1645   // being queried.
1646
1647   // First we register the basic alias analysis that provides the majority of
1648   // per-function local AA logic. This is a stateless, on-demand local set of
1649   // AA techniques.
1650   AA.registerFunctionAnalysis<BasicAA>();
1651
1652   // Next we query fast, specialized alias analyses that wrap IR-embedded
1653   // information about aliasing.
1654   AA.registerFunctionAnalysis<ScopedNoAliasAA>();
1655   AA.registerFunctionAnalysis<TypeBasedAA>();
1656
1657   // Add support for querying global aliasing information when available.
1658   // Because the `AAManager` is a function analysis and `GlobalsAA` is a module
1659   // analysis, all that the `AAManager` can do is query for any *cached*
1660   // results from `GlobalsAA` through a readonly proxy.
1661   AA.registerModuleAnalysis<GlobalsAA>();
1662
1663   return AA;
1664 }
1665
1666 static Optional<int> parseRepeatPassName(StringRef Name) {
1667   if (!Name.consume_front("repeat<") || !Name.consume_back(">"))
1668     return None;
1669   int Count;
1670   if (Name.getAsInteger(0, Count) || Count <= 0)
1671     return None;
1672   return Count;
1673 }
1674
1675 static Optional<int> parseDevirtPassName(StringRef Name) {
1676   if (!Name.consume_front("devirt<") || !Name.consume_back(">"))
1677     return None;
1678   int Count;
1679   if (Name.getAsInteger(0, Count) || Count <= 0)
1680     return None;
1681   return Count;
1682 }
1683
1684 static bool checkParametrizedPassName(StringRef Name, StringRef PassName) {
1685   if (!Name.consume_front(PassName))
1686     return false;
1687   // normal pass name w/o parameters == default parameters
1688   if (Name.empty())
1689     return true;
1690   return Name.startswith("<") && Name.endswith(">");
1691 }
1692
1693 namespace {
1694
1695 /// This performs customized parsing of pass name with parameters.
1696 ///
1697 /// We do not need parametrization of passes in textual pipeline very often,
1698 /// yet on a rare occasion ability to specify parameters right there can be
1699 /// useful.
1700 ///
1701 /// \p Name - parameterized specification of a pass from a textual pipeline
1702 /// is a string in a form of :
1703 ///      PassName '<' parameter-list '>'
1704 ///
1705 /// Parameter list is being parsed by the parser callable argument, \p Parser,
1706 /// It takes a string-ref of parameters and returns either StringError or a
1707 /// parameter list in a form of a custom parameters type, all wrapped into
1708 /// Expected<> template class.
1709 ///
1710 template <typename ParametersParseCallableT>
1711 auto parsePassParameters(ParametersParseCallableT &&Parser, StringRef Name,
1712                          StringRef PassName) -> decltype(Parser(StringRef{})) {
1713   using ParametersT = typename decltype(Parser(StringRef{}))::value_type;
1714
1715   StringRef Params = Name;
1716   if (!Params.consume_front(PassName)) {
1717     assert(false &&
1718            "unable to strip pass name from parametrized pass specification");
1719   }
1720   if (Params.empty())
1721     return ParametersT{};
1722   if (!Params.consume_front("<") || !Params.consume_back(">")) {
1723     assert(false && "invalid format for parametrized pass name");
1724   }
1725
1726   Expected<ParametersT> Result = Parser(Params);
1727   assert((Result || Result.template errorIsA<StringError>()) &&
1728          "Pass parameter parser can only return StringErrors.");
1729   return Result;
1730 }
1731
1732 /// Parser of parameters for LoopUnroll pass.
1733 Expected<LoopUnrollOptions> parseLoopUnrollOptions(StringRef Params) {
1734   LoopUnrollOptions UnrollOpts;
1735   while (!Params.empty()) {
1736     StringRef ParamName;
1737     std::tie(ParamName, Params) = Params.split(';');
1738     int OptLevel = StringSwitch<int>(ParamName)
1739                        .Case("O0", 0)
1740                        .Case("O1", 1)
1741                        .Case("O2", 2)
1742                        .Case("O3", 3)
1743                        .Default(-1);
1744     if (OptLevel >= 0) {
1745       UnrollOpts.setOptLevel(OptLevel);
1746       continue;
1747     }
1748     if (ParamName.consume_front("full-unroll-max=")) {
1749       int Count;
1750       if (ParamName.getAsInteger(0, Count))
1751         return make_error<StringError>(
1752             formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
1753             inconvertibleErrorCode());
1754       UnrollOpts.setFullUnrollMaxCount(Count);
1755       continue;
1756     }
1757
1758     bool Enable = !ParamName.consume_front("no-");
1759     if (ParamName == "partial") {
1760       UnrollOpts.setPartial(Enable);
1761     } else if (ParamName == "peeling") {
1762       UnrollOpts.setPeeling(Enable);
1763     } else if (ParamName == "profile-peeling") {
1764       UnrollOpts.setProfileBasedPeeling(Enable);
1765     } else if (ParamName == "runtime") {
1766       UnrollOpts.setRuntime(Enable);
1767     } else if (ParamName == "upperbound") {
1768       UnrollOpts.setUpperBound(Enable);
1769     } else {
1770       return make_error<StringError>(
1771           formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
1772           inconvertibleErrorCode());
1773     }
1774   }
1775   return UnrollOpts;
1776 }
1777
1778 Expected<MemorySanitizerOptions> parseMSanPassOptions(StringRef Params) {
1779   MemorySanitizerOptions Result;
1780   while (!Params.empty()) {
1781     StringRef ParamName;
1782     std::tie(ParamName, Params) = Params.split(';');
1783
1784     if (ParamName == "recover") {
1785       Result.Recover = true;
1786     } else if (ParamName == "kernel") {
1787       Result.Kernel = true;
1788     } else if (ParamName.consume_front("track-origins=")) {
1789       if (ParamName.getAsInteger(0, Result.TrackOrigins))
1790         return make_error<StringError>(
1791             formatv("invalid argument to MemorySanitizer pass track-origins "
1792                     "parameter: '{0}' ",
1793                     ParamName)
1794                 .str(),
1795             inconvertibleErrorCode());
1796     } else {
1797       return make_error<StringError>(
1798           formatv("invalid MemorySanitizer pass parameter '{0}' ", ParamName)
1799               .str(),
1800           inconvertibleErrorCode());
1801     }
1802   }
1803   return Result;
1804 }
1805
1806 /// Parser of parameters for SimplifyCFG pass.
1807 Expected<SimplifyCFGOptions> parseSimplifyCFGOptions(StringRef Params) {
1808   SimplifyCFGOptions Result;
1809   while (!Params.empty()) {
1810     StringRef ParamName;
1811     std::tie(ParamName, Params) = Params.split(';');
1812
1813     bool Enable = !ParamName.consume_front("no-");
1814     if (ParamName == "forward-switch-cond") {
1815       Result.forwardSwitchCondToPhi(Enable);
1816     } else if (ParamName == "switch-to-lookup") {
1817       Result.convertSwitchToLookupTable(Enable);
1818     } else if (ParamName == "keep-loops") {
1819       Result.needCanonicalLoops(Enable);
1820     } else if (ParamName == "hoist-common-insts") {
1821       Result.hoistCommonInsts(Enable);
1822     } else if (ParamName == "sink-common-insts") {
1823       Result.sinkCommonInsts(Enable);
1824     } else if (Enable && ParamName.consume_front("bonus-inst-threshold=")) {
1825       APInt BonusInstThreshold;
1826       if (ParamName.getAsInteger(0, BonusInstThreshold))
1827         return make_error<StringError>(
1828             formatv("invalid argument to SimplifyCFG pass bonus-threshold "
1829                     "parameter: '{0}' ",
1830                     ParamName).str(),
1831             inconvertibleErrorCode());
1832       Result.bonusInstThreshold(BonusInstThreshold.getSExtValue());
1833     } else {
1834       return make_error<StringError>(
1835           formatv("invalid SimplifyCFG pass parameter '{0}' ", ParamName).str(),
1836           inconvertibleErrorCode());
1837     }
1838   }
1839   return Result;
1840 }
1841
1842 /// Parser of parameters for LoopVectorize pass.
1843 Expected<LoopVectorizeOptions> parseLoopVectorizeOptions(StringRef Params) {
1844   LoopVectorizeOptions Opts;
1845   while (!Params.empty()) {
1846     StringRef ParamName;
1847     std::tie(ParamName, Params) = Params.split(';');
1848
1849     bool Enable = !ParamName.consume_front("no-");
1850     if (ParamName == "interleave-forced-only") {
1851       Opts.setInterleaveOnlyWhenForced(Enable);
1852     } else if (ParamName == "vectorize-forced-only") {
1853       Opts.setVectorizeOnlyWhenForced(Enable);
1854     } else {
1855       return make_error<StringError>(
1856           formatv("invalid LoopVectorize parameter '{0}' ", ParamName).str(),
1857           inconvertibleErrorCode());
1858     }
1859   }
1860   return Opts;
1861 }
1862
1863 Expected<bool> parseLoopUnswitchOptions(StringRef Params) {
1864   bool Result = false;
1865   while (!Params.empty()) {
1866     StringRef ParamName;
1867     std::tie(ParamName, Params) = Params.split(';');
1868
1869     bool Enable = !ParamName.consume_front("no-");
1870     if (ParamName == "nontrivial") {
1871       Result = Enable;
1872     } else {
1873       return make_error<StringError>(
1874           formatv("invalid LoopUnswitch pass parameter '{0}' ", ParamName)
1875               .str(),
1876           inconvertibleErrorCode());
1877     }
1878   }
1879   return Result;
1880 }
1881
1882 Expected<bool> parseMergedLoadStoreMotionOptions(StringRef Params) {
1883   bool Result = false;
1884   while (!Params.empty()) {
1885     StringRef ParamName;
1886     std::tie(ParamName, Params) = Params.split(';');
1887
1888     bool Enable = !ParamName.consume_front("no-");
1889     if (ParamName == "split-footer-bb") {
1890       Result = Enable;
1891     } else {
1892       return make_error<StringError>(
1893           formatv("invalid MergedLoadStoreMotion pass parameter '{0}' ",
1894                   ParamName)
1895               .str(),
1896           inconvertibleErrorCode());
1897     }
1898   }
1899   return Result;
1900 }
1901
1902 Expected<GVNOptions> parseGVNOptions(StringRef Params) {
1903   GVNOptions Result;
1904   while (!Params.empty()) {
1905     StringRef ParamName;
1906     std::tie(ParamName, Params) = Params.split(';');
1907
1908     bool Enable = !ParamName.consume_front("no-");
1909     if (ParamName == "pre") {
1910       Result.setPRE(Enable);
1911     } else if (ParamName == "load-pre") {
1912       Result.setLoadPRE(Enable);
1913     } else if (ParamName == "memdep") {
1914       Result.setMemDep(Enable);
1915     } else {
1916       return make_error<StringError>(
1917           formatv("invalid GVN pass parameter '{0}' ", ParamName).str(),
1918           inconvertibleErrorCode());
1919     }
1920   }
1921   return Result;
1922 }
1923
1924 Expected<StackLifetime::LivenessType>
1925 parseStackLifetimeOptions(StringRef Params) {
1926   StackLifetime::LivenessType Result = StackLifetime::LivenessType::May;
1927   while (!Params.empty()) {
1928     StringRef ParamName;
1929     std::tie(ParamName, Params) = Params.split(';');
1930
1931     if (ParamName == "may") {
1932       Result = StackLifetime::LivenessType::May;
1933     } else if (ParamName == "must") {
1934       Result = StackLifetime::LivenessType::Must;
1935     } else {
1936       return make_error<StringError>(
1937           formatv("invalid StackLifetime parameter '{0}' ", ParamName).str(),
1938           inconvertibleErrorCode());
1939     }
1940   }
1941   return Result;
1942 }
1943
1944 } // namespace
1945
1946 /// Tests whether a pass name starts with a valid prefix for a default pipeline
1947 /// alias.
1948 static bool startsWithDefaultPipelineAliasPrefix(StringRef Name) {
1949   return Name.startswith("default") || Name.startswith("thinlto") ||
1950          Name.startswith("lto");
1951 }
1952
1953 /// Tests whether registered callbacks will accept a given pass name.
1954 ///
1955 /// When parsing a pipeline text, the type of the outermost pipeline may be
1956 /// omitted, in which case the type is automatically determined from the first
1957 /// pass name in the text. This may be a name that is handled through one of the
1958 /// callbacks. We check this through the oridinary parsing callbacks by setting
1959 /// up a dummy PassManager in order to not force the client to also handle this
1960 /// type of query.
1961 template <typename PassManagerT, typename CallbacksT>
1962 static bool callbacksAcceptPassName(StringRef Name, CallbacksT &Callbacks) {
1963   if (!Callbacks.empty()) {
1964     PassManagerT DummyPM;
1965     for (auto &CB : Callbacks)
1966       if (CB(Name, DummyPM, {}))
1967         return true;
1968   }
1969   return false;
1970 }
1971
1972 template <typename CallbacksT>
1973 static bool isModulePassName(StringRef Name, CallbacksT &Callbacks) {
1974   // Manually handle aliases for pre-configured pipeline fragments.
1975   if (startsWithDefaultPipelineAliasPrefix(Name))
1976     return DefaultAliasRegex.match(Name);
1977
1978   // Explicitly handle pass manager names.
1979   if (Name == "module")
1980     return true;
1981   if (Name == "cgscc")
1982     return true;
1983   if (Name == "function")
1984     return true;
1985
1986   // Explicitly handle custom-parsed pass names.
1987   if (parseRepeatPassName(Name))
1988     return true;
1989
1990 #define MODULE_PASS(NAME, CREATE_PASS)                                         \
1991   if (Name == NAME)                                                            \
1992     return true;
1993 #define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
1994   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
1995     return true;
1996 #include "PassRegistry.def"
1997
1998   return callbacksAcceptPassName<ModulePassManager>(Name, Callbacks);
1999 }
2000
2001 template <typename CallbacksT>
2002 static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks) {
2003   // Explicitly handle pass manager names.
2004   if (Name == "cgscc")
2005     return true;
2006   if (Name == "function")
2007     return true;
2008
2009   // Explicitly handle custom-parsed pass names.
2010   if (parseRepeatPassName(Name))
2011     return true;
2012   if (parseDevirtPassName(Name))
2013     return true;
2014
2015 #define CGSCC_PASS(NAME, CREATE_PASS)                                          \
2016   if (Name == NAME)                                                            \
2017     return true;
2018 #define CGSCC_ANALYSIS(NAME, CREATE_PASS)                                      \
2019   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
2020     return true;
2021 #include "PassRegistry.def"
2022
2023   return callbacksAcceptPassName<CGSCCPassManager>(Name, Callbacks);
2024 }
2025
2026 template <typename CallbacksT>
2027 static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks) {
2028   // Explicitly handle pass manager names.
2029   if (Name == "function")
2030     return true;
2031   if (Name == "loop" || Name == "loop-mssa")
2032     return true;
2033
2034   // Explicitly handle custom-parsed pass names.
2035   if (parseRepeatPassName(Name))
2036     return true;
2037
2038 #define FUNCTION_PASS(NAME, CREATE_PASS)                                       \
2039   if (Name == NAME)                                                            \
2040     return true;
2041 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                   \
2042   if (checkParametrizedPassName(Name, NAME))                                   \
2043     return true;
2044 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
2045   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
2046     return true;
2047 #include "PassRegistry.def"
2048
2049   return callbacksAcceptPassName<FunctionPassManager>(Name, Callbacks);
2050 }
2051
2052 template <typename CallbacksT>
2053 static bool isLoopPassName(StringRef Name, CallbacksT &Callbacks) {
2054   // Explicitly handle pass manager names.
2055   if (Name == "loop" || Name == "loop-mssa")
2056     return true;
2057
2058   // Explicitly handle custom-parsed pass names.
2059   if (parseRepeatPassName(Name))
2060     return true;
2061
2062 #define LOOP_PASS(NAME, CREATE_PASS)                                           \
2063   if (Name == NAME)                                                            \
2064     return true;
2065 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                       \
2066   if (checkParametrizedPassName(Name, NAME))                                   \
2067     return true;
2068 #define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
2069   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
2070     return true;
2071 #include "PassRegistry.def"
2072
2073   return callbacksAcceptPassName<LoopPassManager>(Name, Callbacks);
2074 }
2075
2076 Optional<std::vector<PassBuilder::PipelineElement>>
2077 PassBuilder::parsePipelineText(StringRef Text) {
2078   std::vector<PipelineElement> ResultPipeline;
2079
2080   SmallVector<std::vector<PipelineElement> *, 4> PipelineStack = {
2081       &ResultPipeline};
2082   for (;;) {
2083     std::vector<PipelineElement> &Pipeline = *PipelineStack.back();
2084     size_t Pos = Text.find_first_of(",()");
2085     Pipeline.push_back({Text.substr(0, Pos), {}});
2086
2087     // If we have a single terminating name, we're done.
2088     if (Pos == Text.npos)
2089       break;
2090
2091     char Sep = Text[Pos];
2092     Text = Text.substr(Pos + 1);
2093     if (Sep == ',')
2094       // Just a name ending in a comma, continue.
2095       continue;
2096
2097     if (Sep == '(') {
2098       // Push the inner pipeline onto the stack to continue processing.
2099       PipelineStack.push_back(&Pipeline.back().InnerPipeline);
2100       continue;
2101     }
2102
2103     assert(Sep == ')' && "Bogus separator!");
2104     // When handling the close parenthesis, we greedily consume them to avoid
2105     // empty strings in the pipeline.
2106     do {
2107       // If we try to pop the outer pipeline we have unbalanced parentheses.
2108       if (PipelineStack.size() == 1)
2109         return None;
2110
2111       PipelineStack.pop_back();
2112     } while (Text.consume_front(")"));
2113
2114     // Check if we've finished parsing.
2115     if (Text.empty())
2116       break;
2117
2118     // Otherwise, the end of an inner pipeline always has to be followed by
2119     // a comma, and then we can continue.
2120     if (!Text.consume_front(","))
2121       return None;
2122   }
2123
2124   if (PipelineStack.size() > 1)
2125     // Unbalanced paretheses.
2126     return None;
2127
2128   assert(PipelineStack.back() == &ResultPipeline &&
2129          "Wrong pipeline at the bottom of the stack!");
2130   return {std::move(ResultPipeline)};
2131 }
2132
2133 Error PassBuilder::parseModulePass(ModulePassManager &MPM,
2134                                    const PipelineElement &E,
2135                                    bool VerifyEachPass, bool DebugLogging) {
2136   auto &Name = E.Name;
2137   auto &InnerPipeline = E.InnerPipeline;
2138
2139   // First handle complex passes like the pass managers which carry pipelines.
2140   if (!InnerPipeline.empty()) {
2141     if (Name == "module") {
2142       ModulePassManager NestedMPM(DebugLogging);
2143       if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline,
2144                                              VerifyEachPass, DebugLogging))
2145         return Err;
2146       MPM.addPass(std::move(NestedMPM));
2147       return Error::success();
2148     }
2149     if (Name == "cgscc") {
2150       CGSCCPassManager CGPM(DebugLogging);
2151       if (auto Err = parseCGSCCPassPipeline(CGPM, InnerPipeline, VerifyEachPass,
2152                                             DebugLogging))
2153         return Err;
2154       MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
2155       return Error::success();
2156     }
2157     if (Name == "function") {
2158       FunctionPassManager FPM(DebugLogging);
2159       if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline,
2160                                                VerifyEachPass, DebugLogging))
2161         return Err;
2162       MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
2163       return Error::success();
2164     }
2165     if (auto Count = parseRepeatPassName(Name)) {
2166       ModulePassManager NestedMPM(DebugLogging);
2167       if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline,
2168                                              VerifyEachPass, DebugLogging))
2169         return Err;
2170       MPM.addPass(createRepeatedPass(*Count, std::move(NestedMPM)));
2171       return Error::success();
2172     }
2173
2174     for (auto &C : ModulePipelineParsingCallbacks)
2175       if (C(Name, MPM, InnerPipeline))
2176         return Error::success();
2177
2178     // Normal passes can't have pipelines.
2179     return make_error<StringError>(
2180         formatv("invalid use of '{0}' pass as module pipeline", Name).str(),
2181         inconvertibleErrorCode());
2182     ;
2183   }
2184
2185   // Manually handle aliases for pre-configured pipeline fragments.
2186   if (startsWithDefaultPipelineAliasPrefix(Name)) {
2187     SmallVector<StringRef, 3> Matches;
2188     if (!DefaultAliasRegex.match(Name, &Matches))
2189       return make_error<StringError>(
2190           formatv("unknown default pipeline alias '{0}'", Name).str(),
2191           inconvertibleErrorCode());
2192
2193     assert(Matches.size() == 3 && "Must capture two matched strings!");
2194
2195     OptimizationLevel L = StringSwitch<OptimizationLevel>(Matches[2])
2196                               .Case("O0", OptimizationLevel::O0)
2197                               .Case("O1", OptimizationLevel::O1)
2198                               .Case("O2", OptimizationLevel::O2)
2199                               .Case("O3", OptimizationLevel::O3)
2200                               .Case("Os", OptimizationLevel::Os)
2201                               .Case("Oz", OptimizationLevel::Oz);
2202     if (L == OptimizationLevel::O0) {
2203       // Add instrumentation PGO passes -- at O0 we can still do PGO.
2204       if (PGOOpt && Matches[1] != "thinlto" &&
2205           (PGOOpt->Action == PGOOptions::IRInstr ||
2206            PGOOpt->Action == PGOOptions::IRUse))
2207         addPGOInstrPassesForO0(
2208             MPM, DebugLogging,
2209             /* RunProfileGen */ (PGOOpt->Action == PGOOptions::IRInstr),
2210             /* IsCS */ false, PGOOpt->ProfileFile,
2211             PGOOpt->ProfileRemappingFile);
2212
2213       // For IR that makes use of coroutines intrinsics, coroutine passes must
2214       // be run, even at -O0.
2215       if (PTO.Coroutines) {
2216         MPM.addPass(createModuleToFunctionPassAdaptor(CoroEarlyPass()));
2217
2218         CGSCCPassManager CGPM(DebugLogging);
2219         CGPM.addPass(CoroSplitPass());
2220         CGPM.addPass(createCGSCCToFunctionPassAdaptor(CoroElidePass()));
2221         MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
2222
2223         MPM.addPass(createModuleToFunctionPassAdaptor(CoroCleanupPass()));
2224       }
2225
2226       // Do nothing else at all!
2227       return Error::success();
2228     }
2229
2230     // This is consistent with old pass manager invoked via opt, but
2231     // inconsistent with clang. Clang doesn't enable loop vectorization
2232     // but does enable slp vectorization at Oz.
2233     PTO.LoopVectorization =
2234         L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz;
2235     PTO.SLPVectorization =
2236         L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz;
2237
2238     if (Matches[1] == "default") {
2239       MPM.addPass(buildPerModuleDefaultPipeline(L, DebugLogging));
2240     } else if (Matches[1] == "thinlto-pre-link") {
2241       MPM.addPass(buildThinLTOPreLinkDefaultPipeline(L, DebugLogging));
2242     } else if (Matches[1] == "thinlto") {
2243       MPM.addPass(buildThinLTODefaultPipeline(L, DebugLogging, nullptr));
2244     } else if (Matches[1] == "lto-pre-link") {
2245       MPM.addPass(buildLTOPreLinkDefaultPipeline(L, DebugLogging));
2246     } else {
2247       assert(Matches[1] == "lto" && "Not one of the matched options!");
2248       MPM.addPass(buildLTODefaultPipeline(L, DebugLogging, nullptr));
2249     }
2250     return Error::success();
2251   }
2252
2253   // Finally expand the basic registered passes from the .inc file.
2254 #define MODULE_PASS(NAME, CREATE_PASS)                                         \
2255   if (Name == NAME) {                                                          \
2256     MPM.addPass(CREATE_PASS);                                                  \
2257     return Error::success();                                                   \
2258   }
2259 #define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
2260   if (Name == "require<" NAME ">") {                                           \
2261     MPM.addPass(                                                               \
2262         RequireAnalysisPass<                                                   \
2263             std::remove_reference<decltype(CREATE_PASS)>::type, Module>());    \
2264     return Error::success();                                                   \
2265   }                                                                            \
2266   if (Name == "invalidate<" NAME ">") {                                        \
2267     MPM.addPass(InvalidateAnalysisPass<                                        \
2268                 std::remove_reference<decltype(CREATE_PASS)>::type>());        \
2269     return Error::success();                                                   \
2270   }
2271 #define CGSCC_PASS(NAME, CREATE_PASS)                                          \
2272   if (Name == NAME) {                                                          \
2273     MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(CREATE_PASS));         \
2274     return Error::success();                                                   \
2275   }
2276 #define FUNCTION_PASS(NAME, CREATE_PASS)                                       \
2277   if (Name == NAME) {                                                          \
2278     MPM.addPass(createModuleToFunctionPassAdaptor(CREATE_PASS));               \
2279     return Error::success();                                                   \
2280   }
2281 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                   \
2282   if (checkParametrizedPassName(Name, NAME)) {                                 \
2283     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
2284     if (!Params)                                                               \
2285       return Params.takeError();                                               \
2286     MPM.addPass(createModuleToFunctionPassAdaptor(CREATE_PASS(Params.get()))); \
2287     return Error::success();                                                   \
2288   }
2289 #define LOOP_PASS(NAME, CREATE_PASS)                                           \
2290   if (Name == NAME) {                                                          \
2291     MPM.addPass(                                                               \
2292         createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor(     \
2293             CREATE_PASS, false, false, DebugLogging)));                        \
2294     return Error::success();                                                   \
2295   }
2296 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                       \
2297   if (checkParametrizedPassName(Name, NAME)) {                                 \
2298     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
2299     if (!Params)                                                               \
2300       return Params.takeError();                                               \
2301     MPM.addPass(                                                               \
2302         createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor(     \
2303             CREATE_PASS(Params.get()), false, false, DebugLogging)));          \
2304     return Error::success();                                                   \
2305   }
2306 #include "PassRegistry.def"
2307
2308   for (auto &C : ModulePipelineParsingCallbacks)
2309     if (C(Name, MPM, InnerPipeline))
2310       return Error::success();
2311   return make_error<StringError>(
2312       formatv("unknown module pass '{0}'", Name).str(),
2313       inconvertibleErrorCode());
2314 }
2315
2316 Error PassBuilder::parseCGSCCPass(CGSCCPassManager &CGPM,
2317                                   const PipelineElement &E, bool VerifyEachPass,
2318                                   bool DebugLogging) {
2319   auto &Name = E.Name;
2320   auto &InnerPipeline = E.InnerPipeline;
2321
2322   // First handle complex passes like the pass managers which carry pipelines.
2323   if (!InnerPipeline.empty()) {
2324     if (Name == "cgscc") {
2325       CGSCCPassManager NestedCGPM(DebugLogging);
2326       if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline,
2327                                             VerifyEachPass, DebugLogging))
2328         return Err;
2329       // Add the nested pass manager with the appropriate adaptor.
2330       CGPM.addPass(std::move(NestedCGPM));
2331       return Error::success();
2332     }
2333     if (Name == "function") {
2334       FunctionPassManager FPM(DebugLogging);
2335       if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline,
2336                                                VerifyEachPass, DebugLogging))
2337         return Err;
2338       // Add the nested pass manager with the appropriate adaptor.
2339       CGPM.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
2340       return Error::success();
2341     }
2342     if (auto Count = parseRepeatPassName(Name)) {
2343       CGSCCPassManager NestedCGPM(DebugLogging);
2344       if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline,
2345                                             VerifyEachPass, DebugLogging))
2346         return Err;
2347       CGPM.addPass(createRepeatedPass(*Count, std::move(NestedCGPM)));
2348       return Error::success();
2349     }
2350     if (auto MaxRepetitions = parseDevirtPassName(Name)) {
2351       CGSCCPassManager NestedCGPM(DebugLogging);
2352       if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline,
2353                                             VerifyEachPass, DebugLogging))
2354         return Err;
2355       CGPM.addPass(
2356           createDevirtSCCRepeatedPass(std::move(NestedCGPM), *MaxRepetitions));
2357       return Error::success();
2358     }
2359
2360     for (auto &C : CGSCCPipelineParsingCallbacks)
2361       if (C(Name, CGPM, InnerPipeline))
2362         return Error::success();
2363
2364     // Normal passes can't have pipelines.
2365     return make_error<StringError>(
2366         formatv("invalid use of '{0}' pass as cgscc pipeline", Name).str(),
2367         inconvertibleErrorCode());
2368   }
2369
2370 // Now expand the basic registered passes from the .inc file.
2371 #define CGSCC_PASS(NAME, CREATE_PASS)                                          \
2372   if (Name == NAME) {                                                          \
2373     CGPM.addPass(CREATE_PASS);                                                 \
2374     return Error::success();                                                   \
2375   }
2376 #define CGSCC_ANALYSIS(NAME, CREATE_PASS)                                      \
2377   if (Name == "require<" NAME ">") {                                           \
2378     CGPM.addPass(RequireAnalysisPass<                                          \
2379                  std::remove_reference<decltype(CREATE_PASS)>::type,           \
2380                  LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &,    \
2381                  CGSCCUpdateResult &>());                                      \
2382     return Error::success();                                                   \
2383   }                                                                            \
2384   if (Name == "invalidate<" NAME ">") {                                        \
2385     CGPM.addPass(InvalidateAnalysisPass<                                       \
2386                  std::remove_reference<decltype(CREATE_PASS)>::type>());       \
2387     return Error::success();                                                   \
2388   }
2389 #define FUNCTION_PASS(NAME, CREATE_PASS)                                       \
2390   if (Name == NAME) {                                                          \
2391     CGPM.addPass(createCGSCCToFunctionPassAdaptor(CREATE_PASS));               \
2392     return Error::success();                                                   \
2393   }
2394 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                   \
2395   if (checkParametrizedPassName(Name, NAME)) {                                 \
2396     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
2397     if (!Params)                                                               \
2398       return Params.takeError();                                               \
2399     CGPM.addPass(createCGSCCToFunctionPassAdaptor(CREATE_PASS(Params.get()))); \
2400     return Error::success();                                                   \
2401   }
2402 #define LOOP_PASS(NAME, CREATE_PASS)                                           \
2403   if (Name == NAME) {                                                          \
2404     CGPM.addPass(                                                              \
2405         createCGSCCToFunctionPassAdaptor(createFunctionToLoopPassAdaptor(      \
2406             CREATE_PASS, false, false, DebugLogging)));                        \
2407     return Error::success();                                                   \
2408   }
2409 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                       \
2410   if (checkParametrizedPassName(Name, NAME)) {                                 \
2411     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
2412     if (!Params)                                                               \
2413       return Params.takeError();                                               \
2414     CGPM.addPass(                                                              \
2415         createCGSCCToFunctionPassAdaptor(createFunctionToLoopPassAdaptor(      \
2416             CREATE_PASS(Params.get()), false, false, DebugLogging)));          \
2417     return Error::success();                                                   \
2418   }
2419 #include "PassRegistry.def"
2420
2421   for (auto &C : CGSCCPipelineParsingCallbacks)
2422     if (C(Name, CGPM, InnerPipeline))
2423       return Error::success();
2424   return make_error<StringError>(
2425       formatv("unknown cgscc pass '{0}'", Name).str(),
2426       inconvertibleErrorCode());
2427 }
2428
2429 Error PassBuilder::parseFunctionPass(FunctionPassManager &FPM,
2430                                      const PipelineElement &E,
2431                                      bool VerifyEachPass, bool DebugLogging) {
2432   auto &Name = E.Name;
2433   auto &InnerPipeline = E.InnerPipeline;
2434
2435   // First handle complex passes like the pass managers which carry pipelines.
2436   if (!InnerPipeline.empty()) {
2437     if (Name == "function") {
2438       FunctionPassManager NestedFPM(DebugLogging);
2439       if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline,
2440                                                VerifyEachPass, DebugLogging))
2441         return Err;
2442       // Add the nested pass manager with the appropriate adaptor.
2443       FPM.addPass(std::move(NestedFPM));
2444       return Error::success();
2445     }
2446     if (Name == "loop" || Name == "loop-mssa") {
2447       LoopPassManager LPM(DebugLogging);
2448       if (auto Err = parseLoopPassPipeline(LPM, InnerPipeline, VerifyEachPass,
2449                                            DebugLogging))
2450         return Err;
2451       // Add the nested pass manager with the appropriate adaptor.
2452       bool UseMemorySSA = (Name == "loop-mssa");
2453       bool UseBFI =
2454           std::any_of(InnerPipeline.begin(), InnerPipeline.end(),
2455                       [](auto Pipeline) { return Pipeline.Name == "licm"; });
2456       FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), UseMemorySSA,
2457                                                   UseBFI, DebugLogging));
2458       return Error::success();
2459     }
2460     if (auto Count = parseRepeatPassName(Name)) {
2461       FunctionPassManager NestedFPM(DebugLogging);
2462       if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline,
2463                                                VerifyEachPass, DebugLogging))
2464         return Err;
2465       FPM.addPass(createRepeatedPass(*Count, std::move(NestedFPM)));
2466       return Error::success();
2467     }
2468
2469     for (auto &C : FunctionPipelineParsingCallbacks)
2470       if (C(Name, FPM, InnerPipeline))
2471         return Error::success();
2472
2473     // Normal passes can't have pipelines.
2474     return make_error<StringError>(
2475         formatv("invalid use of '{0}' pass as function pipeline", Name).str(),
2476         inconvertibleErrorCode());
2477   }
2478
2479 // Now expand the basic registered passes from the .inc file.
2480 #define FUNCTION_PASS(NAME, CREATE_PASS)                                       \
2481   if (Name == NAME) {                                                          \
2482     FPM.addPass(CREATE_PASS);                                                  \
2483     return Error::success();                                                   \
2484   }
2485 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                   \
2486   if (checkParametrizedPassName(Name, NAME)) {                                 \
2487     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
2488     if (!Params)                                                               \
2489       return Params.takeError();                                               \
2490     FPM.addPass(CREATE_PASS(Params.get()));                                    \
2491     return Error::success();                                                   \
2492   }
2493 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
2494   if (Name == "require<" NAME ">") {                                           \
2495     FPM.addPass(                                                               \
2496         RequireAnalysisPass<                                                   \
2497             std::remove_reference<decltype(CREATE_PASS)>::type, Function>());  \
2498     return Error::success();                                                   \
2499   }                                                                            \
2500   if (Name == "invalidate<" NAME ">") {                                        \
2501     FPM.addPass(InvalidateAnalysisPass<                                        \
2502                 std::remove_reference<decltype(CREATE_PASS)>::type>());        \
2503     return Error::success();                                                   \
2504   }
2505 // FIXME: UseMemorySSA is set to false. Maybe we could do things like:
2506 //        bool UseMemorySSA = !("canon-freeze" || "loop-predication" ||
2507 //                              "guard-widening");
2508 //        The risk is that it may become obsolete if we're not careful.
2509 #define LOOP_PASS(NAME, CREATE_PASS)                                           \
2510   if (Name == NAME) {                                                          \
2511     FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS, false, false,     \
2512                                                 DebugLogging));                \
2513     return Error::success();                                                   \
2514   }
2515 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                       \
2516   if (checkParametrizedPassName(Name, NAME)) {                                 \
2517     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
2518     if (!Params)                                                               \
2519       return Params.takeError();                                               \
2520     FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS(Params.get()),     \
2521                                                 false, false, DebugLogging));  \
2522     return Error::success();                                                   \
2523   }
2524 #include "PassRegistry.def"
2525
2526   for (auto &C : FunctionPipelineParsingCallbacks)
2527     if (C(Name, FPM, InnerPipeline))
2528       return Error::success();
2529   return make_error<StringError>(
2530       formatv("unknown function pass '{0}'", Name).str(),
2531       inconvertibleErrorCode());
2532 }
2533
2534 Error PassBuilder::parseLoopPass(LoopPassManager &LPM, const PipelineElement &E,
2535                                  bool VerifyEachPass, bool DebugLogging) {
2536   StringRef Name = E.Name;
2537   auto &InnerPipeline = E.InnerPipeline;
2538
2539   // First handle complex passes like the pass managers which carry pipelines.
2540   if (!InnerPipeline.empty()) {
2541     if (Name == "loop") {
2542       LoopPassManager NestedLPM(DebugLogging);
2543       if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline,
2544                                            VerifyEachPass, DebugLogging))
2545         return Err;
2546       // Add the nested pass manager with the appropriate adaptor.
2547       LPM.addPass(std::move(NestedLPM));
2548       return Error::success();
2549     }
2550     if (auto Count = parseRepeatPassName(Name)) {
2551       LoopPassManager NestedLPM(DebugLogging);
2552       if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline,
2553                                            VerifyEachPass, DebugLogging))
2554         return Err;
2555       LPM.addPass(createRepeatedPass(*Count, std::move(NestedLPM)));
2556       return Error::success();
2557     }
2558
2559     for (auto &C : LoopPipelineParsingCallbacks)
2560       if (C(Name, LPM, InnerPipeline))
2561         return Error::success();
2562
2563     // Normal passes can't have pipelines.
2564     return make_error<StringError>(
2565         formatv("invalid use of '{0}' pass as loop pipeline", Name).str(),
2566         inconvertibleErrorCode());
2567   }
2568
2569 // Now expand the basic registered passes from the .inc file.
2570 #define LOOP_PASS(NAME, CREATE_PASS)                                           \
2571   if (Name == NAME) {                                                          \
2572     LPM.addPass(CREATE_PASS);                                                  \
2573     return Error::success();                                                   \
2574   }
2575 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                       \
2576   if (checkParametrizedPassName(Name, NAME)) {                                 \
2577     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
2578     if (!Params)                                                               \
2579       return Params.takeError();                                               \
2580     LPM.addPass(CREATE_PASS(Params.get()));                                    \
2581     return Error::success();                                                   \
2582   }
2583 #define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
2584   if (Name == "require<" NAME ">") {                                           \
2585     LPM.addPass(RequireAnalysisPass<                                           \
2586                 std::remove_reference<decltype(CREATE_PASS)>::type, Loop,      \
2587                 LoopAnalysisManager, LoopStandardAnalysisResults &,            \
2588                 LPMUpdater &>());                                              \
2589     return Error::success();                                                   \
2590   }                                                                            \
2591   if (Name == "invalidate<" NAME ">") {                                        \
2592     LPM.addPass(InvalidateAnalysisPass<                                        \
2593                 std::remove_reference<decltype(CREATE_PASS)>::type>());        \
2594     return Error::success();                                                   \
2595   }
2596 #include "PassRegistry.def"
2597
2598   for (auto &C : LoopPipelineParsingCallbacks)
2599     if (C(Name, LPM, InnerPipeline))
2600       return Error::success();
2601   return make_error<StringError>(formatv("unknown loop pass '{0}'", Name).str(),
2602                                  inconvertibleErrorCode());
2603 }
2604
2605 bool PassBuilder::parseAAPassName(AAManager &AA, StringRef Name) {
2606 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS)                               \
2607   if (Name == NAME) {                                                          \
2608     AA.registerModuleAnalysis<                                                 \
2609         std::remove_reference<decltype(CREATE_PASS)>::type>();                 \
2610     return true;                                                               \
2611   }
2612 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS)                             \
2613   if (Name == NAME) {                                                          \
2614     AA.registerFunctionAnalysis<                                               \
2615         std::remove_reference<decltype(CREATE_PASS)>::type>();                 \
2616     return true;                                                               \
2617   }
2618 #include "PassRegistry.def"
2619
2620   for (auto &C : AAParsingCallbacks)
2621     if (C(Name, AA))
2622       return true;
2623   return false;
2624 }
2625
2626 Error PassBuilder::parseLoopPassPipeline(LoopPassManager &LPM,
2627                                          ArrayRef<PipelineElement> Pipeline,
2628                                          bool VerifyEachPass,
2629                                          bool DebugLogging) {
2630   for (const auto &Element : Pipeline) {
2631     if (auto Err = parseLoopPass(LPM, Element, VerifyEachPass, DebugLogging))
2632       return Err;
2633     // FIXME: No verifier support for Loop passes!
2634   }
2635   return Error::success();
2636 }
2637
2638 Error PassBuilder::parseFunctionPassPipeline(FunctionPassManager &FPM,
2639                                              ArrayRef<PipelineElement> Pipeline,
2640                                              bool VerifyEachPass,
2641                                              bool DebugLogging) {
2642   for (const auto &Element : Pipeline) {
2643     if (auto Err =
2644             parseFunctionPass(FPM, Element, VerifyEachPass, DebugLogging))
2645       return Err;
2646     if (VerifyEachPass)
2647       FPM.addPass(VerifierPass());
2648   }
2649   return Error::success();
2650 }
2651
2652 Error PassBuilder::parseCGSCCPassPipeline(CGSCCPassManager &CGPM,
2653                                           ArrayRef<PipelineElement> Pipeline,
2654                                           bool VerifyEachPass,
2655                                           bool DebugLogging) {
2656   for (const auto &Element : Pipeline) {
2657     if (auto Err = parseCGSCCPass(CGPM, Element, VerifyEachPass, DebugLogging))
2658       return Err;
2659     // FIXME: No verifier support for CGSCC passes!
2660   }
2661   return Error::success();
2662 }
2663
2664 void PassBuilder::crossRegisterProxies(LoopAnalysisManager &LAM,
2665                                        FunctionAnalysisManager &FAM,
2666                                        CGSCCAnalysisManager &CGAM,
2667                                        ModuleAnalysisManager &MAM) {
2668   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
2669   MAM.registerPass([&] { return CGSCCAnalysisManagerModuleProxy(CGAM); });
2670   CGAM.registerPass([&] { return ModuleAnalysisManagerCGSCCProxy(MAM); });
2671   FAM.registerPass([&] { return CGSCCAnalysisManagerFunctionProxy(CGAM); });
2672   FAM.registerPass([&] { return ModuleAnalysisManagerFunctionProxy(MAM); });
2673   FAM.registerPass([&] { return LoopAnalysisManagerFunctionProxy(LAM); });
2674   LAM.registerPass([&] { return FunctionAnalysisManagerLoopProxy(FAM); });
2675 }
2676
2677 Error PassBuilder::parseModulePassPipeline(ModulePassManager &MPM,
2678                                            ArrayRef<PipelineElement> Pipeline,
2679                                            bool VerifyEachPass,
2680                                            bool DebugLogging) {
2681   for (const auto &Element : Pipeline) {
2682     if (auto Err = parseModulePass(MPM, Element, VerifyEachPass, DebugLogging))
2683       return Err;
2684     if (VerifyEachPass)
2685       MPM.addPass(VerifierPass());
2686   }
2687   return Error::success();
2688 }
2689
2690 // Primary pass pipeline description parsing routine for a \c ModulePassManager
2691 // FIXME: Should this routine accept a TargetMachine or require the caller to
2692 // pre-populate the analysis managers with target-specific stuff?
2693 Error PassBuilder::parsePassPipeline(ModulePassManager &MPM,
2694                                      StringRef PipelineText,
2695                                      bool VerifyEachPass, bool DebugLogging) {
2696   auto Pipeline = parsePipelineText(PipelineText);
2697   if (!Pipeline || Pipeline->empty())
2698     return make_error<StringError>(
2699         formatv("invalid pipeline '{0}'", PipelineText).str(),
2700         inconvertibleErrorCode());
2701
2702   // If the first name isn't at the module layer, wrap the pipeline up
2703   // automatically.
2704   StringRef FirstName = Pipeline->front().Name;
2705
2706   if (!isModulePassName(FirstName, ModulePipelineParsingCallbacks)) {
2707     if (isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks)) {
2708       Pipeline = {{"cgscc", std::move(*Pipeline)}};
2709     } else if (isFunctionPassName(FirstName,
2710                                   FunctionPipelineParsingCallbacks)) {
2711       Pipeline = {{"function", std::move(*Pipeline)}};
2712     } else if (isLoopPassName(FirstName, LoopPipelineParsingCallbacks)) {
2713       Pipeline = {{"function", {{"loop", std::move(*Pipeline)}}}};
2714     } else {
2715       for (auto &C : TopLevelPipelineParsingCallbacks)
2716         if (C(MPM, *Pipeline, VerifyEachPass, DebugLogging))
2717           return Error::success();
2718
2719       // Unknown pass or pipeline name!
2720       auto &InnerPipeline = Pipeline->front().InnerPipeline;
2721       return make_error<StringError>(
2722           formatv("unknown {0} name '{1}'",
2723                   (InnerPipeline.empty() ? "pass" : "pipeline"), FirstName)
2724               .str(),
2725           inconvertibleErrorCode());
2726     }
2727   }
2728
2729   if (auto Err =
2730           parseModulePassPipeline(MPM, *Pipeline, VerifyEachPass, DebugLogging))
2731     return Err;
2732   return Error::success();
2733 }
2734
2735 // Primary pass pipeline description parsing routine for a \c CGSCCPassManager
2736 Error PassBuilder::parsePassPipeline(CGSCCPassManager &CGPM,
2737                                      StringRef PipelineText,
2738                                      bool VerifyEachPass, bool DebugLogging) {
2739   auto Pipeline = parsePipelineText(PipelineText);
2740   if (!Pipeline || Pipeline->empty())
2741     return make_error<StringError>(
2742         formatv("invalid pipeline '{0}'", PipelineText).str(),
2743         inconvertibleErrorCode());
2744
2745   StringRef FirstName = Pipeline->front().Name;
2746   if (!isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks))
2747     return make_error<StringError>(
2748         formatv("unknown cgscc pass '{0}' in pipeline '{1}'", FirstName,
2749                 PipelineText)
2750             .str(),
2751         inconvertibleErrorCode());
2752
2753   if (auto Err =
2754           parseCGSCCPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging))
2755     return Err;
2756   return Error::success();
2757 }
2758
2759 // Primary pass pipeline description parsing routine for a \c
2760 // FunctionPassManager
2761 Error PassBuilder::parsePassPipeline(FunctionPassManager &FPM,
2762                                      StringRef PipelineText,
2763                                      bool VerifyEachPass, bool DebugLogging) {
2764   auto Pipeline = parsePipelineText(PipelineText);
2765   if (!Pipeline || Pipeline->empty())
2766     return make_error<StringError>(
2767         formatv("invalid pipeline '{0}'", PipelineText).str(),
2768         inconvertibleErrorCode());
2769
2770   StringRef FirstName = Pipeline->front().Name;
2771   if (!isFunctionPassName(FirstName, FunctionPipelineParsingCallbacks))
2772     return make_error<StringError>(
2773         formatv("unknown function pass '{0}' in pipeline '{1}'", FirstName,
2774                 PipelineText)
2775             .str(),
2776         inconvertibleErrorCode());
2777
2778   if (auto Err = parseFunctionPassPipeline(FPM, *Pipeline, VerifyEachPass,
2779                                            DebugLogging))
2780     return Err;
2781   return Error::success();
2782 }
2783
2784 // Primary pass pipeline description parsing routine for a \c LoopPassManager
2785 Error PassBuilder::parsePassPipeline(LoopPassManager &CGPM,
2786                                      StringRef PipelineText,
2787                                      bool VerifyEachPass, bool DebugLogging) {
2788   auto Pipeline = parsePipelineText(PipelineText);
2789   if (!Pipeline || Pipeline->empty())
2790     return make_error<StringError>(
2791         formatv("invalid pipeline '{0}'", PipelineText).str(),
2792         inconvertibleErrorCode());
2793
2794   if (auto Err =
2795           parseLoopPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging))
2796     return Err;
2797
2798   return Error::success();
2799 }
2800
2801 Error PassBuilder::parseAAPipeline(AAManager &AA, StringRef PipelineText) {
2802   // If the pipeline just consists of the word 'default' just replace the AA
2803   // manager with our default one.
2804   if (PipelineText == "default") {
2805     AA = buildDefaultAAPipeline();
2806     return Error::success();
2807   }
2808
2809   while (!PipelineText.empty()) {
2810     StringRef Name;
2811     std::tie(Name, PipelineText) = PipelineText.split(',');
2812     if (!parseAAPassName(AA, Name))
2813       return make_error<StringError>(
2814           formatv("unknown alias analysis name '{0}'", Name).str(),
2815           inconvertibleErrorCode());
2816   }
2817
2818   return Error::success();
2819 }
2820
2821 bool PassBuilder::isAAPassName(StringRef PassName) {
2822 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS)                               \
2823   if (PassName == NAME)                                                        \
2824     return true;
2825 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS)                             \
2826   if (PassName == NAME)                                                        \
2827     return true;
2828 #include "PassRegistry.def"
2829   return false;
2830 }
2831
2832 bool PassBuilder::isAnalysisPassName(StringRef PassName) {
2833 #define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
2834   if (PassName == NAME)                                                        \
2835     return true;
2836 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
2837   if (PassName == NAME)                                                        \
2838     return true;
2839 #define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
2840   if (PassName == NAME)                                                        \
2841     return true;
2842 #define CGSSC_ANALYSIS(NAME, CREATE_PASS)                                      \
2843   if (PassName == NAME)                                                        \
2844     return true;
2845 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS)                               \
2846   if (PassName == NAME)                                                        \
2847     return true;
2848 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS)                             \
2849   if (PassName == NAME)                                                        \
2850     return true;
2851 #include "PassRegistry.def"
2852   return false;
2853 }
2854
2855 void PassBuilder::registerParseTopLevelPipelineCallback(
2856     const std::function<bool(ModulePassManager &, ArrayRef<PipelineElement>,
2857                              bool VerifyEachPass, bool DebugLogging)> &C) {
2858   TopLevelPipelineParsingCallbacks.push_back(C);
2859 }