Port StripGCRelocates pass to NPM
[lldb.git] / llvm / lib / Passes / PassBuilder.cpp
1 //===- Parsing, selection, and construction of pass pipelines -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 ///
10 /// This file provides the implementation of the PassBuilder based on our
11 /// static pass registry as well as related functionality. It also provides
12 /// helpers to aid in analyzing, debugging, and testing passes and pass
13 /// pipelines.
14 ///
15 //===----------------------------------------------------------------------===//
16
17 #include "llvm/Passes/PassBuilder.h"
18 #include "llvm/ADT/StringSwitch.h"
19 #include "llvm/Analysis/AliasAnalysis.h"
20 #include "llvm/Analysis/AliasAnalysisEvaluator.h"
21 #include "llvm/Analysis/AliasSetTracker.h"
22 #include "llvm/Analysis/AssumptionCache.h"
23 #include "llvm/Analysis/BasicAliasAnalysis.h"
24 #include "llvm/Analysis/BlockFrequencyInfo.h"
25 #include "llvm/Analysis/BranchProbabilityInfo.h"
26 #include "llvm/Analysis/CFGPrinter.h"
27 #include "llvm/Analysis/CFLAndersAliasAnalysis.h"
28 #include "llvm/Analysis/CFLSteensAliasAnalysis.h"
29 #include "llvm/Analysis/CGSCCPassManager.h"
30 #include "llvm/Analysis/CallGraph.h"
31 #include "llvm/Analysis/DDG.h"
32 #include "llvm/Analysis/Delinearization.h"
33 #include "llvm/Analysis/DemandedBits.h"
34 #include "llvm/Analysis/DependenceAnalysis.h"
35 #include "llvm/Analysis/DominanceFrontier.h"
36 #include "llvm/Analysis/FunctionPropertiesAnalysis.h"
37 #include "llvm/Analysis/GlobalsModRef.h"
38 #include "llvm/Analysis/IRSimilarityIdentifier.h"
39 #include "llvm/Analysis/IVUsers.h"
40 #include "llvm/Analysis/InlineAdvisor.h"
41 #include "llvm/Analysis/InlineSizeEstimatorAnalysis.h"
42 #include "llvm/Analysis/InstCount.h"
43 #include "llvm/Analysis/LazyCallGraph.h"
44 #include "llvm/Analysis/LazyValueInfo.h"
45 #include "llvm/Analysis/Lint.h"
46 #include "llvm/Analysis/LoopAccessAnalysis.h"
47 #include "llvm/Analysis/LoopCacheAnalysis.h"
48 #include "llvm/Analysis/LoopInfo.h"
49 #include "llvm/Analysis/LoopNestAnalysis.h"
50 #include "llvm/Analysis/MemoryDependenceAnalysis.h"
51 #include "llvm/Analysis/MemorySSA.h"
52 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
53 #include "llvm/Analysis/ObjCARCAliasAnalysis.h"
54 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
55 #include "llvm/Analysis/PhiValues.h"
56 #include "llvm/Analysis/PostDominators.h"
57 #include "llvm/Analysis/ProfileSummaryInfo.h"
58 #include "llvm/Analysis/RegionInfo.h"
59 #include "llvm/Analysis/ScalarEvolution.h"
60 #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
61 #include "llvm/Analysis/ScopedNoAliasAA.h"
62 #include "llvm/Analysis/StackLifetime.h"
63 #include "llvm/Analysis/StackSafetyAnalysis.h"
64 #include "llvm/Analysis/TargetLibraryInfo.h"
65 #include "llvm/Analysis/TargetTransformInfo.h"
66 #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
67 #include "llvm/IR/Dominators.h"
68 #include "llvm/IR/IRPrintingPasses.h"
69 #include "llvm/IR/PassManager.h"
70 #include "llvm/IR/SafepointIRVerifier.h"
71 #include "llvm/IR/Verifier.h"
72 #include "llvm/Support/CommandLine.h"
73 #include "llvm/Support/Debug.h"
74 #include "llvm/Support/FormatVariadic.h"
75 #include "llvm/Support/Regex.h"
76 #include "llvm/Target/TargetMachine.h"
77 #include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
78 #include "llvm/Transforms/Coroutines/CoroCleanup.h"
79 #include "llvm/Transforms/Coroutines/CoroEarly.h"
80 #include "llvm/Transforms/Coroutines/CoroElide.h"
81 #include "llvm/Transforms/Coroutines/CoroSplit.h"
82 #include "llvm/Transforms/HelloNew/HelloWorld.h"
83 #include "llvm/Transforms/IPO/AlwaysInliner.h"
84 #include "llvm/Transforms/IPO/ArgumentPromotion.h"
85 #include "llvm/Transforms/IPO/Attributor.h"
86 #include "llvm/Transforms/IPO/CalledValuePropagation.h"
87 #include "llvm/Transforms/IPO/ConstantMerge.h"
88 #include "llvm/Transforms/IPO/CrossDSOCFI.h"
89 #include "llvm/Transforms/IPO/DeadArgumentElimination.h"
90 #include "llvm/Transforms/IPO/ElimAvailExtern.h"
91 #include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
92 #include "llvm/Transforms/IPO/FunctionAttrs.h"
93 #include "llvm/Transforms/IPO/FunctionImport.h"
94 #include "llvm/Transforms/IPO/GlobalDCE.h"
95 #include "llvm/Transforms/IPO/GlobalOpt.h"
96 #include "llvm/Transforms/IPO/GlobalSplit.h"
97 #include "llvm/Transforms/IPO/HotColdSplitting.h"
98 #include "llvm/Transforms/IPO/InferFunctionAttrs.h"
99 #include "llvm/Transforms/IPO/Inliner.h"
100 #include "llvm/Transforms/IPO/Internalize.h"
101 #include "llvm/Transforms/IPO/LowerTypeTests.h"
102 #include "llvm/Transforms/IPO/MergeFunctions.h"
103 #include "llvm/Transforms/IPO/OpenMPOpt.h"
104 #include "llvm/Transforms/IPO/PartialInlining.h"
105 #include "llvm/Transforms/IPO/SCCP.h"
106 #include "llvm/Transforms/IPO/SampleProfile.h"
107 #include "llvm/Transforms/IPO/StripDeadPrototypes.h"
108 #include "llvm/Transforms/IPO/StripSymbols.h"
109 #include "llvm/Transforms/IPO/SyntheticCountsPropagation.h"
110 #include "llvm/Transforms/IPO/WholeProgramDevirt.h"
111 #include "llvm/Transforms/InstCombine/InstCombine.h"
112 #include "llvm/Transforms/Instrumentation.h"
113 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
114 #include "llvm/Transforms/Instrumentation/BoundsChecking.h"
115 #include "llvm/Transforms/Instrumentation/CGProfile.h"
116 #include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
117 #include "llvm/Transforms/Instrumentation/DataFlowSanitizer.h"
118 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
119 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
120 #include "llvm/Transforms/Instrumentation/InstrOrderFile.h"
121 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
122 #include "llvm/Transforms/Instrumentation/MemProfiler.h"
123 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
124 #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
125 #include "llvm/Transforms/Instrumentation/PoisonChecking.h"
126 #include "llvm/Transforms/Instrumentation/SanitizerCoverage.h"
127 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
128 #include "llvm/Transforms/ObjCARC.h"
129 #include "llvm/Transforms/Scalar/ADCE.h"
130 #include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
131 #include "llvm/Transforms/Scalar/BDCE.h"
132 #include "llvm/Transforms/Scalar/CallSiteSplitting.h"
133 #include "llvm/Transforms/Scalar/ConstantHoisting.h"
134 #include "llvm/Transforms/Scalar/ConstraintElimination.h"
135 #include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h"
136 #include "llvm/Transforms/Scalar/DCE.h"
137 #include "llvm/Transforms/Scalar/DeadStoreElimination.h"
138 #include "llvm/Transforms/Scalar/DivRemPairs.h"
139 #include "llvm/Transforms/Scalar/EarlyCSE.h"
140 #include "llvm/Transforms/Scalar/Float2Int.h"
141 #include "llvm/Transforms/Scalar/GVN.h"
142 #include "llvm/Transforms/Scalar/GuardWidening.h"
143 #include "llvm/Transforms/Scalar/IVUsersPrinter.h"
144 #include "llvm/Transforms/Scalar/IndVarSimplify.h"
145 #include "llvm/Transforms/Scalar/InductiveRangeCheckElimination.h"
146 #include "llvm/Transforms/Scalar/InstSimplifyPass.h"
147 #include "llvm/Transforms/Scalar/JumpThreading.h"
148 #include "llvm/Transforms/Scalar/LICM.h"
149 #include "llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h"
150 #include "llvm/Transforms/Scalar/LoopDataPrefetch.h"
151 #include "llvm/Transforms/Scalar/LoopDeletion.h"
152 #include "llvm/Transforms/Scalar/LoopDistribute.h"
153 #include "llvm/Transforms/Scalar/LoopFlatten.h"
154 #include "llvm/Transforms/Scalar/LoopFuse.h"
155 #include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
156 #include "llvm/Transforms/Scalar/LoopInstSimplify.h"
157 #include "llvm/Transforms/Scalar/LoopLoadElimination.h"
158 #include "llvm/Transforms/Scalar/LoopPassManager.h"
159 #include "llvm/Transforms/Scalar/LoopPredication.h"
160 #include "llvm/Transforms/Scalar/LoopReroll.h"
161 #include "llvm/Transforms/Scalar/LoopRotation.h"
162 #include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"
163 #include "llvm/Transforms/Scalar/LoopSink.h"
164 #include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
165 #include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
166 #include "llvm/Transforms/Scalar/LoopUnrollPass.h"
167 #include "llvm/Transforms/Scalar/LowerAtomic.h"
168 #include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
169 #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
170 #include "llvm/Transforms/Scalar/LowerGuardIntrinsic.h"
171 #include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h"
172 #include "llvm/Transforms/Scalar/LowerWidenableCondition.h"
173 #include "llvm/Transforms/Scalar/MakeGuardsExplicit.h"
174 #include "llvm/Transforms/Scalar/MemCpyOptimizer.h"
175 #include "llvm/Transforms/Scalar/MergeICmps.h"
176 #include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h"
177 #include "llvm/Transforms/Scalar/NaryReassociate.h"
178 #include "llvm/Transforms/Scalar/NewGVN.h"
179 #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
180 #include "llvm/Transforms/Scalar/Reassociate.h"
181 #include "llvm/Transforms/Scalar/RewriteStatepointsForGC.h"
182 #include "llvm/Transforms/Scalar/SCCP.h"
183 #include "llvm/Transforms/Scalar/SROA.h"
184 #include "llvm/Transforms/Scalar/Scalarizer.h"
185 #include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
186 #include "llvm/Transforms/Scalar/SimplifyCFG.h"
187 #include "llvm/Transforms/Scalar/Sink.h"
188 #include "llvm/Transforms/Scalar/SpeculateAroundPHIs.h"
189 #include "llvm/Transforms/Scalar/SpeculativeExecution.h"
190 #include "llvm/Transforms/Scalar/TailRecursionElimination.h"
191 #include "llvm/Transforms/Scalar/WarnMissedTransforms.h"
192 #include "llvm/Transforms/Utils/AddDiscriminators.h"
193 #include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
194 #include "llvm/Transforms/Utils/BreakCriticalEdges.h"
195 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
196 #include "llvm/Transforms/Utils/CanonicalizeFreezeInLoops.h"
197 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
198 #include "llvm/Transforms/Utils/InjectTLIMappings.h"
199 #include "llvm/Transforms/Utils/LCSSA.h"
200 #include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
201 #include "llvm/Transforms/Utils/LoopSimplify.h"
202 #include "llvm/Transforms/Utils/LoopVersioning.h"
203 #include "llvm/Transforms/Utils/LowerInvoke.h"
204 #include "llvm/Transforms/Utils/LowerSwitch.h"
205 #include "llvm/Transforms/Utils/Mem2Reg.h"
206 #include "llvm/Transforms/Utils/MetaRenamer.h"
207 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
208 #include "llvm/Transforms/Utils/StripGCRelocates.h"
209 #include "llvm/Transforms/Utils/StripNonLineTableDebugInfo.h"
210 #include "llvm/Transforms/Utils/SymbolRewriter.h"
211 #include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
212 #include "llvm/Transforms/Vectorize/LoopVectorize.h"
213 #include "llvm/Transforms/Vectorize/SLPVectorizer.h"
214 #include "llvm/Transforms/Vectorize/VectorCombine.h"
215
216 using namespace llvm;
217
218 static cl::opt<unsigned> MaxDevirtIterations("pm-max-devirt-iterations",
219                                              cl::ReallyHidden, cl::init(4));
220 static cl::opt<bool>
221     RunPartialInlining("enable-npm-partial-inlining", cl::init(false),
222                        cl::Hidden, cl::ZeroOrMore,
223                        cl::desc("Run Partial inlinining pass"));
224
225 static cl::opt<int> PreInlineThreshold(
226     "npm-preinline-threshold", cl::Hidden, cl::init(75), cl::ZeroOrMore,
227     cl::desc("Control the amount of inlining in pre-instrumentation inliner "
228              "(default = 75)"));
229
230 static cl::opt<bool>
231     RunNewGVN("enable-npm-newgvn", cl::init(false),
232               cl::Hidden, cl::ZeroOrMore,
233               cl::desc("Run NewGVN instead of GVN"));
234
235 static cl::opt<bool> EnableGVNHoist(
236     "enable-npm-gvn-hoist", cl::init(false), cl::Hidden,
237     cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
238
239 static cl::opt<InliningAdvisorMode> UseInlineAdvisor(
240     "enable-ml-inliner", cl::init(InliningAdvisorMode::Default), cl::Hidden,
241     cl::desc("Enable ML policy for inliner. Currently trained for -Oz only"),
242     cl::values(clEnumValN(InliningAdvisorMode::Default, "default",
243                           "Heuristics-based inliner version."),
244                clEnumValN(InliningAdvisorMode::Development, "development",
245                           "Use development mode (runtime-loadable model)."),
246                clEnumValN(InliningAdvisorMode::Release, "release",
247                           "Use release mode (AOT-compiled model).")));
248
249 static cl::opt<bool> EnableGVNSink(
250     "enable-npm-gvn-sink", cl::init(false), cl::Hidden,
251     cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
252
253 static cl::opt<bool> EnableUnrollAndJam(
254     "enable-npm-unroll-and-jam", cl::init(false), cl::Hidden,
255     cl::desc("Enable the Unroll and Jam pass for the new PM (default = off)"));
256
257 static cl::opt<bool> EnableLoopFlatten(
258     "enable-npm-loop-flatten", cl::init(false), cl::Hidden,
259     cl::desc("Enable the Loop flattening pass for the new PM (default = off)"));
260
261 static cl::opt<bool> EnableSyntheticCounts(
262     "enable-npm-synthetic-counts", cl::init(false), cl::Hidden, cl::ZeroOrMore,
263     cl::desc("Run synthetic function entry count generation "
264              "pass"));
265
266 static const Regex DefaultAliasRegex(
267     "^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$");
268
269 // This option is used in simplifying testing SampleFDO optimizations for
270 // profile loading.
271 static cl::opt<bool>
272     EnableCHR("enable-chr-npm", cl::init(true), cl::Hidden,
273               cl::desc("Enable control height reduction optimization (CHR)"));
274
275 /// Flag to enable inline deferral during PGO.
276 static cl::opt<bool>
277     EnablePGOInlineDeferral("enable-npm-pgo-inline-deferral", cl::init(true),
278                             cl::Hidden,
279                             cl::desc("Enable inline deferral during PGO"));
280
281 static cl::opt<bool> EnableMemProfiler("enable-mem-prof", cl::init(false),
282                                        cl::Hidden, cl::ZeroOrMore,
283                                        cl::desc("Enable memory profiler"));
284
285 PipelineTuningOptions::PipelineTuningOptions() {
286   LoopInterleaving = true;
287   LoopVectorization = true;
288   SLPVectorization = false;
289   LoopUnrolling = true;
290   ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll;
291   Coroutines = false;
292   LicmMssaOptCap = SetLicmMssaOptCap;
293   LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap;
294   CallGraphProfile = true;
295 }
296
297 extern cl::opt<bool> EnableConstraintElimination;
298 extern cl::opt<bool> EnableHotColdSplit;
299 extern cl::opt<bool> EnableOrderFileInstrumentation;
300
301 extern cl::opt<bool> FlattenedProfileUsed;
302
303 extern cl::opt<AttributorRunOption> AttributorRun;
304 extern cl::opt<bool> EnableKnowledgeRetention;
305
306 extern cl::opt<bool> EnableMatrix;
307
308 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O0 = {
309     /*SpeedLevel*/ 0,
310     /*SizeLevel*/ 0};
311 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O1 = {
312     /*SpeedLevel*/ 1,
313     /*SizeLevel*/ 0};
314 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O2 = {
315     /*SpeedLevel*/ 2,
316     /*SizeLevel*/ 0};
317 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O3 = {
318     /*SpeedLevel*/ 3,
319     /*SizeLevel*/ 0};
320 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::Os = {
321     /*SpeedLevel*/ 2,
322     /*SizeLevel*/ 1};
323 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::Oz = {
324     /*SpeedLevel*/ 2,
325     /*SizeLevel*/ 2};
326
327 namespace {
328
329 // The following passes/analyses have custom names, otherwise their name will
330 // include `(anonymous namespace)`. These are special since they are only for
331 // testing purposes and don't live in a header file.
332
333 /// No-op module pass which does nothing.
334 struct NoOpModulePass : PassInfoMixin<NoOpModulePass> {
335   PreservedAnalyses run(Module &M, ModuleAnalysisManager &) {
336     return PreservedAnalyses::all();
337   }
338
339   static StringRef name() { return "NoOpModulePass"; }
340 };
341
342 /// No-op module analysis.
343 class NoOpModuleAnalysis : public AnalysisInfoMixin<NoOpModuleAnalysis> {
344   friend AnalysisInfoMixin<NoOpModuleAnalysis>;
345   static AnalysisKey Key;
346
347 public:
348   struct Result {};
349   Result run(Module &, ModuleAnalysisManager &) { return Result(); }
350   static StringRef name() { return "NoOpModuleAnalysis"; }
351 };
352
353 /// No-op CGSCC pass which does nothing.
354 struct NoOpCGSCCPass : PassInfoMixin<NoOpCGSCCPass> {
355   PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &,
356                         LazyCallGraph &, CGSCCUpdateResult &UR) {
357     return PreservedAnalyses::all();
358   }
359   static StringRef name() { return "NoOpCGSCCPass"; }
360 };
361
362 /// No-op CGSCC analysis.
363 class NoOpCGSCCAnalysis : public AnalysisInfoMixin<NoOpCGSCCAnalysis> {
364   friend AnalysisInfoMixin<NoOpCGSCCAnalysis>;
365   static AnalysisKey Key;
366
367 public:
368   struct Result {};
369   Result run(LazyCallGraph::SCC &, CGSCCAnalysisManager &, LazyCallGraph &G) {
370     return Result();
371   }
372   static StringRef name() { return "NoOpCGSCCAnalysis"; }
373 };
374
375 /// No-op function pass which does nothing.
376 struct NoOpFunctionPass : PassInfoMixin<NoOpFunctionPass> {
377   PreservedAnalyses run(Function &F, FunctionAnalysisManager &) {
378     return PreservedAnalyses::all();
379   }
380   static StringRef name() { return "NoOpFunctionPass"; }
381 };
382
383 /// No-op function analysis.
384 class NoOpFunctionAnalysis : public AnalysisInfoMixin<NoOpFunctionAnalysis> {
385   friend AnalysisInfoMixin<NoOpFunctionAnalysis>;
386   static AnalysisKey Key;
387
388 public:
389   struct Result {};
390   Result run(Function &, FunctionAnalysisManager &) { return Result(); }
391   static StringRef name() { return "NoOpFunctionAnalysis"; }
392 };
393
394 /// No-op loop pass which does nothing.
395 struct NoOpLoopPass : PassInfoMixin<NoOpLoopPass> {
396   PreservedAnalyses run(Loop &L, LoopAnalysisManager &,
397                         LoopStandardAnalysisResults &, LPMUpdater &) {
398     return PreservedAnalyses::all();
399   }
400   static StringRef name() { return "NoOpLoopPass"; }
401 };
402
403 /// No-op loop analysis.
404 class NoOpLoopAnalysis : public AnalysisInfoMixin<NoOpLoopAnalysis> {
405   friend AnalysisInfoMixin<NoOpLoopAnalysis>;
406   static AnalysisKey Key;
407
408 public:
409   struct Result {};
410   Result run(Loop &, LoopAnalysisManager &, LoopStandardAnalysisResults &) {
411     return Result();
412   }
413   static StringRef name() { return "NoOpLoopAnalysis"; }
414 };
415
416 AnalysisKey NoOpModuleAnalysis::Key;
417 AnalysisKey NoOpCGSCCAnalysis::Key;
418 AnalysisKey NoOpFunctionAnalysis::Key;
419 AnalysisKey NoOpLoopAnalysis::Key;
420
421 } // namespace
422
423 void PassBuilder::invokePeepholeEPCallbacks(
424     FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
425   for (auto &C : PeepholeEPCallbacks)
426     C(FPM, Level);
427 }
428
429 void PassBuilder::registerModuleAnalyses(ModuleAnalysisManager &MAM) {
430 #define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
431   MAM.registerPass([&] { return CREATE_PASS; });
432 #include "PassRegistry.def"
433
434   for (auto &C : ModuleAnalysisRegistrationCallbacks)
435     C(MAM);
436 }
437
438 void PassBuilder::registerCGSCCAnalyses(CGSCCAnalysisManager &CGAM) {
439 #define CGSCC_ANALYSIS(NAME, CREATE_PASS)                                      \
440   CGAM.registerPass([&] { return CREATE_PASS; });
441 #include "PassRegistry.def"
442
443   for (auto &C : CGSCCAnalysisRegistrationCallbacks)
444     C(CGAM);
445 }
446
447 void PassBuilder::registerFunctionAnalyses(FunctionAnalysisManager &FAM) {
448 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
449   FAM.registerPass([&] { return CREATE_PASS; });
450 #include "PassRegistry.def"
451
452   for (auto &C : FunctionAnalysisRegistrationCallbacks)
453     C(FAM);
454 }
455
456 void PassBuilder::registerLoopAnalyses(LoopAnalysisManager &LAM) {
457 #define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
458   LAM.registerPass([&] { return CREATE_PASS; });
459 #include "PassRegistry.def"
460
461   for (auto &C : LoopAnalysisRegistrationCallbacks)
462     C(LAM);
463 }
464
465 // TODO: Investigate the cost/benefit of tail call elimination on debugging.
466 FunctionPassManager PassBuilder::buildO1FunctionSimplificationPipeline(
467     OptimizationLevel Level, ThinLTOPhase Phase, bool DebugLogging) {
468
469   FunctionPassManager FPM(DebugLogging);
470
471   // Form SSA out of local memory accesses after breaking apart aggregates into
472   // scalars.
473   FPM.addPass(SROA());
474
475   // Catch trivial redundancies
476   FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
477
478   // Hoisting of scalars and load expressions.
479   FPM.addPass(SimplifyCFGPass());
480   FPM.addPass(InstCombinePass());
481
482   FPM.addPass(LibCallsShrinkWrapPass());
483
484   invokePeepholeEPCallbacks(FPM, Level);
485
486   FPM.addPass(SimplifyCFGPass());
487
488   // Form canonically associated expression trees, and simplify the trees using
489   // basic mathematical properties. For example, this will form (nearly)
490   // minimal multiplication trees.
491   FPM.addPass(ReassociatePass());
492
493   // Add the primary loop simplification pipeline.
494   // FIXME: Currently this is split into two loop pass pipelines because we run
495   // some function passes in between them. These can and should be removed
496   // and/or replaced by scheduling the loop pass equivalents in the correct
497   // positions. But those equivalent passes aren't powerful enough yet.
498   // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
499   // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
500   // fully replace `SimplifyCFGPass`, and the closest to the other we have is
501   // `LoopInstSimplify`.
502   LoopPassManager LPM1(DebugLogging), LPM2(DebugLogging);
503
504   // Simplify the loop body. We do this initially to clean up after other loop
505   // passes run, either when iterating on a loop or on inner loops with
506   // implications on the outer loop.
507   LPM1.addPass(LoopInstSimplifyPass());
508   LPM1.addPass(LoopSimplifyCFGPass());
509
510   LPM1.addPass(LoopRotatePass(/* Disable header duplication */ true));
511   // TODO: Investigate promotion cap for O1.
512   LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
513   LPM1.addPass(SimpleLoopUnswitchPass());
514   LPM2.addPass(IndVarSimplifyPass());
515   LPM2.addPass(LoopIdiomRecognizePass());
516
517   for (auto &C : LateLoopOptimizationsEPCallbacks)
518     C(LPM2, Level);
519
520   LPM2.addPass(LoopDeletionPass());
521   if (EnableLoopFlatten)
522     LPM2.addPass(LoopFlattenPass());
523   // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
524   // because it changes IR to makes profile annotation in back compile
525   // inaccurate. The normal unroller doesn't pay attention to forced full unroll
526   // attributes so we need to make sure and allow the full unroll pass to pay
527   // attention to it.
528   if (Phase != ThinLTOPhase::PreLink || !PGOOpt ||
529       PGOOpt->Action != PGOOptions::SampleUse)
530     LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
531                                     /* OnlyWhenForced= */ !PTO.LoopUnrolling,
532                                     PTO.ForgetAllSCEVInLoopUnroll));
533
534   for (auto &C : LoopOptimizerEndEPCallbacks)
535     C(LPM2, Level);
536
537   // We provide the opt remark emitter pass for LICM to use. We only need to do
538   // this once as it is immutable.
539   FPM.addPass(
540       RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
541   FPM.addPass(createFunctionToLoopPassAdaptor(
542       std::move(LPM1), EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true,
543       DebugLogging));
544   FPM.addPass(SimplifyCFGPass());
545   FPM.addPass(InstCombinePass());
546   // The loop passes in LPM2 (LoopFullUnrollPass) do not preserve MemorySSA.
547   // *All* loop passes must preserve it, in order to be able to use it.
548   FPM.addPass(createFunctionToLoopPassAdaptor(
549       std::move(LPM2), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false,
550       DebugLogging));
551
552   // Delete small array after loop unroll.
553   FPM.addPass(SROA());
554
555   // Specially optimize memory movement as it doesn't look like dataflow in SSA.
556   FPM.addPass(MemCpyOptPass());
557
558   // Sparse conditional constant propagation.
559   // FIXME: It isn't clear why we do this *after* loop passes rather than
560   // before...
561   FPM.addPass(SCCPPass());
562
563   // Delete dead bit computations (instcombine runs after to fold away the dead
564   // computations, and then ADCE will run later to exploit any new DCE
565   // opportunities that creates).
566   FPM.addPass(BDCEPass());
567
568   // Run instcombine after redundancy and dead bit elimination to exploit
569   // opportunities opened up by them.
570   FPM.addPass(InstCombinePass());
571   invokePeepholeEPCallbacks(FPM, Level);
572
573   if (PTO.Coroutines)
574     FPM.addPass(CoroElidePass());
575
576   for (auto &C : ScalarOptimizerLateEPCallbacks)
577     C(FPM, Level);
578
579   // Finally, do an expensive DCE pass to catch all the dead code exposed by
580   // the simplifications and basic cleanup after all the simplifications.
581   // TODO: Investigate if this is too expensive.
582   FPM.addPass(ADCEPass());
583   FPM.addPass(SimplifyCFGPass());
584   FPM.addPass(InstCombinePass());
585   invokePeepholeEPCallbacks(FPM, Level);
586
587   return FPM;
588 }
589
590 FunctionPassManager
591 PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
592                                                  ThinLTOPhase Phase,
593                                                  bool DebugLogging) {
594   assert(Level != OptimizationLevel::O0 && "Must request optimizations!");
595
596   // The O1 pipeline has a separate pipeline creation function to simplify
597   // construction readability.
598   if (Level.getSpeedupLevel() == 1)
599     return buildO1FunctionSimplificationPipeline(Level, Phase, DebugLogging);
600
601   FunctionPassManager FPM(DebugLogging);
602
603   // Form SSA out of local memory accesses after breaking apart aggregates into
604   // scalars.
605   FPM.addPass(SROA());
606
607   // Catch trivial redundancies
608   FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
609   if (EnableKnowledgeRetention)
610     FPM.addPass(AssumeSimplifyPass());
611
612   // Hoisting of scalars and load expressions.
613   if (EnableGVNHoist)
614     FPM.addPass(GVNHoistPass());
615
616   // Global value numbering based sinking.
617   if (EnableGVNSink) {
618     FPM.addPass(GVNSinkPass());
619     FPM.addPass(SimplifyCFGPass());
620   }
621
622   if (EnableConstraintElimination)
623     FPM.addPass(ConstraintEliminationPass());
624
625   // Speculative execution if the target has divergent branches; otherwise nop.
626   FPM.addPass(SpeculativeExecutionPass(/* OnlyIfDivergentTarget =*/true));
627
628   // Optimize based on known information about branches, and cleanup afterward.
629   FPM.addPass(JumpThreadingPass());
630   FPM.addPass(CorrelatedValuePropagationPass());
631
632   FPM.addPass(SimplifyCFGPass());
633   if (Level == OptimizationLevel::O3)
634     FPM.addPass(AggressiveInstCombinePass());
635   FPM.addPass(InstCombinePass());
636
637   if (!Level.isOptimizingForSize())
638     FPM.addPass(LibCallsShrinkWrapPass());
639
640   invokePeepholeEPCallbacks(FPM, Level);
641
642   // For PGO use pipeline, try to optimize memory intrinsics such as memcpy
643   // using the size value profile. Don't perform this when optimizing for size.
644   if (PGOOpt && PGOOpt->Action == PGOOptions::IRUse &&
645       !Level.isOptimizingForSize())
646     FPM.addPass(PGOMemOPSizeOpt());
647
648   FPM.addPass(TailCallElimPass());
649   FPM.addPass(SimplifyCFGPass());
650
651   // Form canonically associated expression trees, and simplify the trees using
652   // basic mathematical properties. For example, this will form (nearly)
653   // minimal multiplication trees.
654   FPM.addPass(ReassociatePass());
655
656   // Add the primary loop simplification pipeline.
657   // FIXME: Currently this is split into two loop pass pipelines because we run
658   // some function passes in between them. These can and should be removed
659   // and/or replaced by scheduling the loop pass equivalents in the correct
660   // positions. But those equivalent passes aren't powerful enough yet.
661   // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
662   // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
663   // fully replace `SimplifyCFGPass`, and the closest to the other we have is
664   // `LoopInstSimplify`.
665   LoopPassManager LPM1(DebugLogging), LPM2(DebugLogging);
666
667   // Simplify the loop body. We do this initially to clean up after other loop
668   // passes run, either when iterating on a loop or on inner loops with
669   // implications on the outer loop.
670   LPM1.addPass(LoopInstSimplifyPass());
671   LPM1.addPass(LoopSimplifyCFGPass());
672
673   // Rotate Loop - disable header duplication at -Oz
674   LPM1.addPass(LoopRotatePass(Level != OptimizationLevel::Oz));
675   // TODO: Investigate promotion cap for O1.
676   LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
677   LPM1.addPass(SimpleLoopUnswitchPass());
678   LPM2.addPass(IndVarSimplifyPass());
679   LPM2.addPass(LoopIdiomRecognizePass());
680
681   for (auto &C : LateLoopOptimizationsEPCallbacks)
682     C(LPM2, Level);
683
684   LPM2.addPass(LoopDeletionPass());
685   // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
686   // because it changes IR to makes profile annotation in back compile
687   // inaccurate. The normal unroller doesn't pay attention to forced full unroll
688   // attributes so we need to make sure and allow the full unroll pass to pay
689   // attention to it.
690   if (Phase != ThinLTOPhase::PreLink || !PGOOpt ||
691       PGOOpt->Action != PGOOptions::SampleUse)
692     LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
693                                     /* OnlyWhenForced= */ !PTO.LoopUnrolling,
694                                     PTO.ForgetAllSCEVInLoopUnroll));
695
696   for (auto &C : LoopOptimizerEndEPCallbacks)
697     C(LPM2, Level);
698
699   // We provide the opt remark emitter pass for LICM to use. We only need to do
700   // this once as it is immutable.
701   FPM.addPass(
702       RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
703   FPM.addPass(createFunctionToLoopPassAdaptor(
704       std::move(LPM1), EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true,
705       DebugLogging));
706   FPM.addPass(SimplifyCFGPass());
707   FPM.addPass(InstCombinePass());
708   // The loop passes in LPM2 (IndVarSimplifyPass, LoopIdiomRecognizePass,
709   // LoopDeletionPass and LoopFullUnrollPass) do not preserve MemorySSA.
710   // *All* loop passes must preserve it, in order to be able to use it.
711   FPM.addPass(createFunctionToLoopPassAdaptor(
712       std::move(LPM2), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false,
713       DebugLogging));
714
715   // Delete small array after loop unroll.
716   FPM.addPass(SROA());
717
718   // Eliminate redundancies.
719   FPM.addPass(MergedLoadStoreMotionPass());
720   if (RunNewGVN)
721     FPM.addPass(NewGVNPass());
722   else
723     FPM.addPass(GVN());
724
725   // Specially optimize memory movement as it doesn't look like dataflow in SSA.
726   FPM.addPass(MemCpyOptPass());
727
728   // Sparse conditional constant propagation.
729   // FIXME: It isn't clear why we do this *after* loop passes rather than
730   // before...
731   FPM.addPass(SCCPPass());
732
733   // Delete dead bit computations (instcombine runs after to fold away the dead
734   // computations, and then ADCE will run later to exploit any new DCE
735   // opportunities that creates).
736   FPM.addPass(BDCEPass());
737
738   // Run instcombine after redundancy and dead bit elimination to exploit
739   // opportunities opened up by them.
740   FPM.addPass(InstCombinePass());
741   invokePeepholeEPCallbacks(FPM, Level);
742
743   // Re-consider control flow based optimizations after redundancy elimination,
744   // redo DCE, etc.
745   FPM.addPass(JumpThreadingPass());
746   FPM.addPass(CorrelatedValuePropagationPass());
747   FPM.addPass(DSEPass());
748   FPM.addPass(createFunctionToLoopPassAdaptor(
749       LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
750       EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true, DebugLogging));
751
752   if (PTO.Coroutines)
753     FPM.addPass(CoroElidePass());
754
755   for (auto &C : ScalarOptimizerLateEPCallbacks)
756     C(FPM, Level);
757
758   // Finally, do an expensive DCE pass to catch all the dead code exposed by
759   // the simplifications and basic cleanup after all the simplifications.
760   // TODO: Investigate if this is too expensive.
761   FPM.addPass(ADCEPass());
762   FPM.addPass(SimplifyCFGPass());
763   FPM.addPass(InstCombinePass());
764   invokePeepholeEPCallbacks(FPM, Level);
765
766   if (EnableCHR && Level == OptimizationLevel::O3 && PGOOpt &&
767       (PGOOpt->Action == PGOOptions::IRUse ||
768        PGOOpt->Action == PGOOptions::SampleUse))
769     FPM.addPass(ControlHeightReductionPass());
770
771   return FPM;
772 }
773
774 void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging,
775                                     PassBuilder::OptimizationLevel Level,
776                                     bool RunProfileGen, bool IsCS,
777                                     std::string ProfileFile,
778                                     std::string ProfileRemappingFile) {
779   assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!");
780   // Generally running simplification passes and the inliner with an high
781   // threshold results in smaller executables, but there may be cases where
782   // the size grows, so let's be conservative here and skip this simplification
783   // at -Os/Oz. We will not do this  inline for context sensistive PGO (when
784   // IsCS is true).
785   if (!Level.isOptimizingForSize() && !IsCS) {
786     InlineParams IP;
787
788     IP.DefaultThreshold = PreInlineThreshold;
789
790     // FIXME: The hint threshold has the same value used by the regular inliner.
791     // This should probably be lowered after performance testing.
792     // FIXME: this comment is cargo culted from the old pass manager, revisit).
793     IP.HintThreshold = 325;
794     ModuleInlinerWrapperPass MIWP(IP, DebugLogging);
795     CGSCCPassManager &CGPipeline = MIWP.getPM();
796
797     FunctionPassManager FPM;
798     FPM.addPass(SROA());
799     FPM.addPass(EarlyCSEPass());    // Catch trivial redundancies.
800     FPM.addPass(SimplifyCFGPass()); // Merge & remove basic blocks.
801     FPM.addPass(InstCombinePass()); // Combine silly sequences.
802     invokePeepholeEPCallbacks(FPM, Level);
803
804     CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
805
806     MPM.addPass(std::move(MIWP));
807
808     // Delete anything that is now dead to make sure that we don't instrument
809     // dead code. Instrumentation can end up keeping dead code around and
810     // dramatically increase code size.
811     MPM.addPass(GlobalDCEPass());
812   }
813
814   if (!RunProfileGen) {
815     assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
816     MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
817     // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
818     // RequireAnalysisPass for PSI before subsequent non-module passes.
819     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
820     return;
821   }
822
823   // Perform PGO instrumentation.
824   MPM.addPass(PGOInstrumentationGen(IsCS));
825
826   FunctionPassManager FPM;
827   FPM.addPass(createFunctionToLoopPassAdaptor(
828       LoopRotatePass(), EnableMSSALoopDependency,
829       /*UseBlockFrequencyInfo=*/false, DebugLogging));
830   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
831
832   // Add the profile lowering pass.
833   InstrProfOptions Options;
834   if (!ProfileFile.empty())
835     Options.InstrProfileOutput = ProfileFile;
836   // Do counter promotion at Level greater than O0.
837   Options.DoCounterPromotion = true;
838   Options.UseBFIInPromotion = IsCS;
839   MPM.addPass(InstrProfiling(Options, IsCS));
840 }
841
842 void PassBuilder::addPGOInstrPassesForO0(ModulePassManager &MPM,
843                                          bool DebugLogging, bool RunProfileGen,
844                                          bool IsCS, std::string ProfileFile,
845                                          std::string ProfileRemappingFile) {
846   if (!RunProfileGen) {
847     assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
848     MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
849     // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
850     // RequireAnalysisPass for PSI before subsequent non-module passes.
851     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
852     return;
853   }
854
855   // Perform PGO instrumentation.
856   MPM.addPass(PGOInstrumentationGen(IsCS));
857   // Add the profile lowering pass.
858   InstrProfOptions Options;
859   if (!ProfileFile.empty())
860     Options.InstrProfileOutput = ProfileFile;
861   // Do not do counter promotion at O0.
862   Options.DoCounterPromotion = false;
863   Options.UseBFIInPromotion = IsCS;
864   MPM.addPass(InstrProfiling(Options, IsCS));
865 }
866
867 static InlineParams
868 getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level) {
869   return getInlineParams(Level.getSpeedupLevel(), Level.getSizeLevel());
870 }
871
872 ModuleInlinerWrapperPass
873 PassBuilder::buildInlinerPipeline(OptimizationLevel Level, ThinLTOPhase Phase,
874                                   bool DebugLogging) {
875   InlineParams IP = getInlineParamsFromOptLevel(Level);
876   if (Phase == PassBuilder::ThinLTOPhase::PreLink && PGOOpt &&
877       PGOOpt->Action == PGOOptions::SampleUse)
878     IP.HotCallSiteThreshold = 0;
879
880   if (PGOOpt)
881     IP.EnableDeferral = EnablePGOInlineDeferral;
882
883   ModuleInlinerWrapperPass MIWP(IP, DebugLogging, UseInlineAdvisor,
884                                 MaxDevirtIterations);
885
886   // Require the GlobalsAA analysis for the module so we can query it within
887   // the CGSCC pipeline.
888   MIWP.addRequiredModuleAnalysis<GlobalsAA>();
889
890   // Require the ProfileSummaryAnalysis for the module so we can query it within
891   // the inliner pass.
892   MIWP.addRequiredModuleAnalysis<ProfileSummaryAnalysis>();
893
894   // Now begin the main postorder CGSCC pipeline.
895   // FIXME: The current CGSCC pipeline has its origins in the legacy pass
896   // manager and trying to emulate its precise behavior. Much of this doesn't
897   // make a lot of sense and we should revisit the core CGSCC structure.
898   CGSCCPassManager &MainCGPipeline = MIWP.getPM();
899
900   // Note: historically, the PruneEH pass was run first to deduce nounwind and
901   // generally clean up exception handling overhead. It isn't clear this is
902   // valuable as the inliner doesn't currently care whether it is inlining an
903   // invoke or a call.
904
905   if (AttributorRun & AttributorRunOption::CGSCC)
906     MainCGPipeline.addPass(AttributorCGSCCPass());
907
908   if (PTO.Coroutines)
909     MainCGPipeline.addPass(CoroSplitPass(Level != OptimizationLevel::O0));
910
911   // Now deduce any function attributes based in the current code.
912   MainCGPipeline.addPass(PostOrderFunctionAttrsPass());
913
914   // When at O3 add argument promotion to the pass pipeline.
915   // FIXME: It isn't at all clear why this should be limited to O3.
916   if (Level == OptimizationLevel::O3)
917     MainCGPipeline.addPass(ArgumentPromotionPass());
918
919   // Try to perform OpenMP specific optimizations. This is a (quick!) no-op if
920   // there are no OpenMP runtime calls present in the module.
921   if (Level == OptimizationLevel::O2 || Level == OptimizationLevel::O3)
922     MainCGPipeline.addPass(OpenMPOptPass());
923
924   // Lastly, add the core function simplification pipeline nested inside the
925   // CGSCC walk.
926   MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor(
927       buildFunctionSimplificationPipeline(Level, Phase, DebugLogging)));
928
929   for (auto &C : CGSCCOptimizerLateEPCallbacks)
930     C(MainCGPipeline, Level);
931
932   return MIWP;
933 }
934
935 ModulePassManager PassBuilder::buildModuleSimplificationPipeline(
936     OptimizationLevel Level, ThinLTOPhase Phase, bool DebugLogging) {
937   ModulePassManager MPM(DebugLogging);
938
939   bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse);
940
941   // In ThinLTO mode, when flattened profile is used, all the available
942   // profile information will be annotated in PreLink phase so there is
943   // no need to load the profile again in PostLink.
944   bool LoadSampleProfile =
945       HasSampleProfile &&
946       !(FlattenedProfileUsed && Phase == ThinLTOPhase::PostLink);
947
948   // During the ThinLTO backend phase we perform early indirect call promotion
949   // here, before globalopt. Otherwise imported available_externally functions
950   // look unreferenced and are removed. If we are going to load the sample
951   // profile then defer until later.
952   // TODO: See if we can move later and consolidate with the location where
953   // we perform ICP when we are loading a sample profile.
954   // TODO: We pass HasSampleProfile (whether there was a sample profile file
955   // passed to the compile) to the SamplePGO flag of ICP. This is used to
956   // determine whether the new direct calls are annotated with prof metadata.
957   // Ideally this should be determined from whether the IR is annotated with
958   // sample profile, and not whether the a sample profile was provided on the
959   // command line. E.g. for flattened profiles where we will not be reloading
960   // the sample profile in the ThinLTO backend, we ideally shouldn't have to
961   // provide the sample profile file.
962   if (Phase == ThinLTOPhase::PostLink && !LoadSampleProfile)
963     MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile));
964
965   // Do basic inference of function attributes from known properties of system
966   // libraries and other oracles.
967   MPM.addPass(InferFunctionAttrsPass());
968
969   // Create an early function pass manager to cleanup the output of the
970   // frontend.
971   FunctionPassManager EarlyFPM(DebugLogging);
972   EarlyFPM.addPass(SimplifyCFGPass());
973   EarlyFPM.addPass(SROA());
974   EarlyFPM.addPass(EarlyCSEPass());
975   EarlyFPM.addPass(LowerExpectIntrinsicPass());
976   if (PTO.Coroutines)
977     EarlyFPM.addPass(CoroEarlyPass());
978   if (Level == OptimizationLevel::O3)
979     EarlyFPM.addPass(CallSiteSplittingPass());
980
981   // In SamplePGO ThinLTO backend, we need instcombine before profile annotation
982   // to convert bitcast to direct calls so that they can be inlined during the
983   // profile annotation prepration step.
984   // More details about SamplePGO design can be found in:
985   // https://research.google.com/pubs/pub45290.html
986   // FIXME: revisit how SampleProfileLoad/Inliner/ICP is structured.
987   if (LoadSampleProfile)
988     EarlyFPM.addPass(InstCombinePass());
989   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM)));
990
991   if (LoadSampleProfile) {
992     // Annotate sample profile right after early FPM to ensure freshness of
993     // the debug info.
994     MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
995                                         PGOOpt->ProfileRemappingFile,
996                                         Phase == ThinLTOPhase::PreLink));
997     // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
998     // RequireAnalysisPass for PSI before subsequent non-module passes.
999     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
1000     // Do not invoke ICP in the ThinLTOPrelink phase as it makes it hard
1001     // for the profile annotation to be accurate in the ThinLTO backend.
1002     if (Phase != ThinLTOPhase::PreLink)
1003       // We perform early indirect call promotion here, before globalopt.
1004       // This is important for the ThinLTO backend phase because otherwise
1005       // imported available_externally functions look unreferenced and are
1006       // removed.
1007       MPM.addPass(PGOIndirectCallPromotion(Phase == ThinLTOPhase::PostLink,
1008                                            true /* SamplePGO */));
1009   }
1010
1011   if (AttributorRun & AttributorRunOption::MODULE)
1012     MPM.addPass(AttributorPass());
1013
1014   // Lower type metadata and the type.test intrinsic in the ThinLTO
1015   // post link pipeline after ICP. This is to enable usage of the type
1016   // tests in ICP sequences.
1017   if (Phase == ThinLTOPhase::PostLink)
1018     MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1019
1020   // Interprocedural constant propagation now that basic cleanup has occurred
1021   // and prior to optimizing globals.
1022   // FIXME: This position in the pipeline hasn't been carefully considered in
1023   // years, it should be re-analyzed.
1024   MPM.addPass(IPSCCPPass());
1025
1026   // Attach metadata to indirect call sites indicating the set of functions
1027   // they may target at run-time. This should follow IPSCCP.
1028   MPM.addPass(CalledValuePropagationPass());
1029
1030   // Optimize globals to try and fold them into constants.
1031   MPM.addPass(GlobalOptPass());
1032
1033   // Promote any localized globals to SSA registers.
1034   // FIXME: Should this instead by a run of SROA?
1035   // FIXME: We should probably run instcombine and simplify-cfg afterward to
1036   // delete control flows that are dead once globals have been folded to
1037   // constants.
1038   MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
1039
1040   // Remove any dead arguments exposed by cleanups and constant folding
1041   // globals.
1042   MPM.addPass(DeadArgumentEliminationPass());
1043
1044   // Create a small function pass pipeline to cleanup after all the global
1045   // optimizations.
1046   FunctionPassManager GlobalCleanupPM(DebugLogging);
1047   GlobalCleanupPM.addPass(InstCombinePass());
1048   invokePeepholeEPCallbacks(GlobalCleanupPM, Level);
1049
1050   GlobalCleanupPM.addPass(SimplifyCFGPass());
1051   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM)));
1052
1053   // Add all the requested passes for instrumentation PGO, if requested.
1054   if (PGOOpt && Phase != ThinLTOPhase::PostLink &&
1055       (PGOOpt->Action == PGOOptions::IRInstr ||
1056        PGOOpt->Action == PGOOptions::IRUse)) {
1057     addPGOInstrPasses(MPM, DebugLogging, Level,
1058                       /* RunProfileGen */ PGOOpt->Action == PGOOptions::IRInstr,
1059                       /* IsCS */ false, PGOOpt->ProfileFile,
1060                       PGOOpt->ProfileRemappingFile);
1061     MPM.addPass(PGOIndirectCallPromotion(false, false));
1062   }
1063   if (PGOOpt && Phase != ThinLTOPhase::PostLink &&
1064       PGOOpt->CSAction == PGOOptions::CSIRInstr)
1065     MPM.addPass(PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile));
1066
1067   // Synthesize function entry counts for non-PGO compilation.
1068   if (EnableSyntheticCounts && !PGOOpt)
1069     MPM.addPass(SyntheticCountsPropagation());
1070
1071   MPM.addPass(buildInlinerPipeline(Level, Phase, DebugLogging));
1072
1073   if (EnableMemProfiler && Phase != ThinLTOPhase::PreLink) {
1074     MPM.addPass(createModuleToFunctionPassAdaptor(MemProfilerPass()));
1075     MPM.addPass(ModuleMemProfilerPass());
1076   }
1077
1078   return MPM;
1079 }
1080
1081 ModulePassManager PassBuilder::buildModuleOptimizationPipeline(
1082     OptimizationLevel Level, bool DebugLogging, bool LTOPreLink) {
1083   ModulePassManager MPM(DebugLogging);
1084
1085   // Optimize globals now that the module is fully simplified.
1086   MPM.addPass(GlobalOptPass());
1087   MPM.addPass(GlobalDCEPass());
1088
1089   // Run partial inlining pass to partially inline functions that have
1090   // large bodies.
1091   if (RunPartialInlining)
1092     MPM.addPass(PartialInlinerPass());
1093
1094   // Remove avail extern fns and globals definitions since we aren't compiling
1095   // an object file for later LTO. For LTO we want to preserve these so they
1096   // are eligible for inlining at link-time. Note if they are unreferenced they
1097   // will be removed by GlobalDCE later, so this only impacts referenced
1098   // available externally globals. Eventually they will be suppressed during
1099   // codegen, but eliminating here enables more opportunity for GlobalDCE as it
1100   // may make globals referenced by available external functions dead and saves
1101   // running remaining passes on the eliminated functions. These should be
1102   // preserved during prelinking for link-time inlining decisions.
1103   if (!LTOPreLink)
1104     MPM.addPass(EliminateAvailableExternallyPass());
1105
1106   if (EnableOrderFileInstrumentation)
1107     MPM.addPass(InstrOrderFilePass());
1108
1109   // Do RPO function attribute inference across the module to forward-propagate
1110   // attributes where applicable.
1111   // FIXME: Is this really an optimization rather than a canonicalization?
1112   MPM.addPass(ReversePostOrderFunctionAttrsPass());
1113
1114   // Do a post inline PGO instrumentation and use pass. This is a context
1115   // sensitive PGO pass. We don't want to do this in LTOPreLink phrase as
1116   // cross-module inline has not been done yet. The context sensitive
1117   // instrumentation is after all the inlines are done.
1118   if (!LTOPreLink && PGOOpt) {
1119     if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1120       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ true,
1121                         /* IsCS */ true, PGOOpt->CSProfileGenFile,
1122                         PGOOpt->ProfileRemappingFile);
1123     else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1124       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ false,
1125                         /* IsCS */ true, PGOOpt->ProfileFile,
1126                         PGOOpt->ProfileRemappingFile);
1127   }
1128
1129   // Re-require GloblasAA here prior to function passes. This is particularly
1130   // useful as the above will have inlined, DCE'ed, and function-attr
1131   // propagated everything. We should at this point have a reasonably minimal
1132   // and richly annotated call graph. By computing aliasing and mod/ref
1133   // information for all local globals here, the late loop passes and notably
1134   // the vectorizer will be able to use them to help recognize vectorizable
1135   // memory operations.
1136   MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
1137
1138   FunctionPassManager OptimizePM(DebugLogging);
1139   OptimizePM.addPass(Float2IntPass());
1140   OptimizePM.addPass(LowerConstantIntrinsicsPass());
1141
1142   if (EnableMatrix) {
1143     OptimizePM.addPass(LowerMatrixIntrinsicsPass());
1144     OptimizePM.addPass(EarlyCSEPass());
1145   }
1146
1147   // FIXME: We need to run some loop optimizations to re-rotate loops after
1148   // simplify-cfg and others undo their rotation.
1149
1150   // Optimize the loop execution. These passes operate on entire loop nests
1151   // rather than on each loop in an inside-out manner, and so they are actually
1152   // function passes.
1153
1154   for (auto &C : VectorizerStartEPCallbacks)
1155     C(OptimizePM, Level);
1156
1157   // First rotate loops that may have been un-rotated by prior passes.
1158   OptimizePM.addPass(createFunctionToLoopPassAdaptor(
1159       LoopRotatePass(), EnableMSSALoopDependency,
1160       /*UseBlockFrequencyInfo=*/false, DebugLogging));
1161
1162   // Distribute loops to allow partial vectorization.  I.e. isolate dependences
1163   // into separate loop that would otherwise inhibit vectorization.  This is
1164   // currently only performed for loops marked with the metadata
1165   // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
1166   OptimizePM.addPass(LoopDistributePass());
1167
1168   // Populates the VFABI attribute with the scalar-to-vector mappings
1169   // from the TargetLibraryInfo.
1170   OptimizePM.addPass(InjectTLIMappings());
1171
1172   // Now run the core loop vectorizer.
1173   OptimizePM.addPass(LoopVectorizePass(
1174       LoopVectorizeOptions(!PTO.LoopInterleaving, !PTO.LoopVectorization)));
1175
1176   // Eliminate loads by forwarding stores from the previous iteration to loads
1177   // of the current iteration.
1178   OptimizePM.addPass(LoopLoadEliminationPass());
1179
1180   // Cleanup after the loop optimization passes.
1181   OptimizePM.addPass(InstCombinePass());
1182
1183   // Now that we've formed fast to execute loop structures, we do further
1184   // optimizations. These are run afterward as they might block doing complex
1185   // analyses and transforms such as what are needed for loop vectorization.
1186
1187   // Cleanup after loop vectorization, etc. Simplification passes like CVP and
1188   // GVN, loop transforms, and others have already run, so it's now better to
1189   // convert to more optimized IR using more aggressive simplify CFG options.
1190   // The extra sinking transform can create larger basic blocks, so do this
1191   // before SLP vectorization.
1192   // FIXME: study whether hoisting and/or sinking of common instructions should
1193   //        be delayed until after SLP vectorizer.
1194   OptimizePM.addPass(SimplifyCFGPass(SimplifyCFGOptions()
1195                                          .forwardSwitchCondToPhi(true)
1196                                          .convertSwitchToLookupTable(true)
1197                                          .needCanonicalLoops(false)
1198                                          .hoistCommonInsts(true)
1199                                          .sinkCommonInsts(true)));
1200
1201   // Optimize parallel scalar instruction chains into SIMD instructions.
1202   if (PTO.SLPVectorization)
1203     OptimizePM.addPass(SLPVectorizerPass());
1204
1205   // Enhance/cleanup vector code.
1206   OptimizePM.addPass(VectorCombinePass());
1207   OptimizePM.addPass(InstCombinePass());
1208
1209   // Unroll small loops to hide loop backedge latency and saturate any parallel
1210   // execution resources of an out-of-order processor. We also then need to
1211   // clean up redundancies and loop invariant code.
1212   // FIXME: It would be really good to use a loop-integrated instruction
1213   // combiner for cleanup here so that the unrolling and LICM can be pipelined
1214   // across the loop nests.
1215   // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
1216   if (EnableUnrollAndJam && PTO.LoopUnrolling) {
1217     OptimizePM.addPass(LoopUnrollAndJamPass(Level.getSpeedupLevel()));
1218   }
1219   OptimizePM.addPass(LoopUnrollPass(LoopUnrollOptions(
1220       Level.getSpeedupLevel(), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
1221       PTO.ForgetAllSCEVInLoopUnroll)));
1222   OptimizePM.addPass(WarnMissedTransformationsPass());
1223   OptimizePM.addPass(InstCombinePass());
1224   OptimizePM.addPass(RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
1225   OptimizePM.addPass(createFunctionToLoopPassAdaptor(
1226       LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
1227       EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true, DebugLogging));
1228
1229   // Now that we've vectorized and unrolled loops, we may have more refined
1230   // alignment information, try to re-derive it here.
1231   OptimizePM.addPass(AlignmentFromAssumptionsPass());
1232
1233   // Split out cold code. Splitting is done late to avoid hiding context from
1234   // other optimizations and inadvertently regressing performance. The tradeoff
1235   // is that this has a higher code size cost than splitting early.
1236   if (EnableHotColdSplit && !LTOPreLink)
1237     MPM.addPass(HotColdSplittingPass());
1238
1239   // LoopSink pass sinks instructions hoisted by LICM, which serves as a
1240   // canonicalization pass that enables other optimizations. As a result,
1241   // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
1242   // result too early.
1243   OptimizePM.addPass(LoopSinkPass());
1244
1245   // And finally clean up LCSSA form before generating code.
1246   OptimizePM.addPass(InstSimplifyPass());
1247
1248   // This hoists/decomposes div/rem ops. It should run after other sink/hoist
1249   // passes to avoid re-sinking, but before SimplifyCFG because it can allow
1250   // flattening of blocks.
1251   OptimizePM.addPass(DivRemPairsPass());
1252
1253   // LoopSink (and other loop passes since the last simplifyCFG) might have
1254   // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
1255   OptimizePM.addPass(SimplifyCFGPass());
1256
1257   // Optimize PHIs by speculating around them when profitable. Note that this
1258   // pass needs to be run after any PRE or similar pass as it is essentially
1259   // inserting redundancies into the program. This even includes SimplifyCFG.
1260   OptimizePM.addPass(SpeculateAroundPHIsPass());
1261
1262   if (PTO.Coroutines)
1263     OptimizePM.addPass(CoroCleanupPass());
1264
1265   // Add the core optimizing pipeline.
1266   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM)));
1267
1268   for (auto &C : OptimizerLastEPCallbacks)
1269     C(MPM, Level);
1270
1271   if (PTO.CallGraphProfile)
1272     MPM.addPass(CGProfilePass());
1273
1274   // Now we need to do some global optimization transforms.
1275   // FIXME: It would seem like these should come first in the optimization
1276   // pipeline and maybe be the bottom of the canonicalization pipeline? Weird
1277   // ordering here.
1278   MPM.addPass(GlobalDCEPass());
1279   MPM.addPass(ConstantMergePass());
1280
1281   return MPM;
1282 }
1283
1284 ModulePassManager
1285 PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level,
1286                                            bool DebugLogging, bool LTOPreLink) {
1287   assert(Level != OptimizationLevel::O0 &&
1288          "Must request optimizations for the default pipeline!");
1289
1290   ModulePassManager MPM(DebugLogging);
1291
1292   // Force any function attributes we want the rest of the pipeline to observe.
1293   MPM.addPass(ForceFunctionAttrsPass());
1294
1295   // Apply module pipeline start EP callback.
1296   for (auto &C : PipelineStartEPCallbacks)
1297     C(MPM);
1298
1299   if (PGOOpt && PGOOpt->SamplePGOSupport)
1300     MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
1301
1302   // Add the core simplification pipeline.
1303   MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::None,
1304                                                 DebugLogging));
1305
1306   // Now add the optimization pipeline.
1307   MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging, LTOPreLink));
1308
1309   return MPM;
1310 }
1311
1312 ModulePassManager
1313 PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level,
1314                                                 bool DebugLogging) {
1315   assert(Level != OptimizationLevel::O0 &&
1316          "Must request optimizations for the default pipeline!");
1317
1318   ModulePassManager MPM(DebugLogging);
1319
1320   // Force any function attributes we want the rest of the pipeline to observe.
1321   MPM.addPass(ForceFunctionAttrsPass());
1322
1323   if (PGOOpt && PGOOpt->SamplePGOSupport)
1324     MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
1325
1326   // Apply module pipeline start EP callback.
1327   for (auto &C : PipelineStartEPCallbacks)
1328     C(MPM);
1329
1330   // If we are planning to perform ThinLTO later, we don't bloat the code with
1331   // unrolling/vectorization/... now. Just simplify the module as much as we
1332   // can.
1333   MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PreLink,
1334                                                 DebugLogging));
1335
1336   // Run partial inlining pass to partially inline functions that have
1337   // large bodies.
1338   // FIXME: It isn't clear whether this is really the right place to run this
1339   // in ThinLTO. Because there is another canonicalization and simplification
1340   // phase that will run after the thin link, running this here ends up with
1341   // less information than will be available later and it may grow functions in
1342   // ways that aren't beneficial.
1343   if (RunPartialInlining)
1344     MPM.addPass(PartialInlinerPass());
1345
1346   // Reduce the size of the IR as much as possible.
1347   MPM.addPass(GlobalOptPass());
1348
1349   // Module simplification splits coroutines, but does not fully clean up
1350   // coroutine intrinsics. To ensure ThinLTO optimization passes don't trip up
1351   // on these, we schedule the cleanup here.
1352   if (PTO.Coroutines)
1353     MPM.addPass(createModuleToFunctionPassAdaptor(CoroCleanupPass()));
1354
1355   return MPM;
1356 }
1357
1358 ModulePassManager PassBuilder::buildThinLTODefaultPipeline(
1359     OptimizationLevel Level, bool DebugLogging,
1360     const ModuleSummaryIndex *ImportSummary) {
1361   ModulePassManager MPM(DebugLogging);
1362
1363   if (ImportSummary) {
1364     // These passes import type identifier resolutions for whole-program
1365     // devirtualization and CFI. They must run early because other passes may
1366     // disturb the specific instruction patterns that these passes look for,
1367     // creating dependencies on resolutions that may not appear in the summary.
1368     //
1369     // For example, GVN may transform the pattern assume(type.test) appearing in
1370     // two basic blocks into assume(phi(type.test, type.test)), which would
1371     // transform a dependency on a WPD resolution into a dependency on a type
1372     // identifier resolution for CFI.
1373     //
1374     // Also, WPD has access to more precise information than ICP and can
1375     // devirtualize more effectively, so it should operate on the IR first.
1376     //
1377     // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1378     // metadata and intrinsics.
1379     MPM.addPass(WholeProgramDevirtPass(nullptr, ImportSummary));
1380     MPM.addPass(LowerTypeTestsPass(nullptr, ImportSummary));
1381   }
1382
1383   if (Level == OptimizationLevel::O0)
1384     return MPM;
1385
1386   // Force any function attributes we want the rest of the pipeline to observe.
1387   MPM.addPass(ForceFunctionAttrsPass());
1388
1389   // Add the core simplification pipeline.
1390   MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PostLink,
1391                                                 DebugLogging));
1392
1393   // Now add the optimization pipeline.
1394   MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging));
1395
1396   return MPM;
1397 }
1398
1399 ModulePassManager
1400 PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level,
1401                                             bool DebugLogging) {
1402   assert(Level != OptimizationLevel::O0 &&
1403          "Must request optimizations for the default pipeline!");
1404   // FIXME: We should use a customized pre-link pipeline!
1405   return buildPerModuleDefaultPipeline(Level, DebugLogging,
1406                                        /* LTOPreLink */ true);
1407 }
1408
1409 ModulePassManager
1410 PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, bool DebugLogging,
1411                                      ModuleSummaryIndex *ExportSummary) {
1412   ModulePassManager MPM(DebugLogging);
1413
1414   if (Level == OptimizationLevel::O0) {
1415     // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1416     // metadata and intrinsics.
1417     MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1418     MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1419     // Run a second time to clean up any type tests left behind by WPD for use
1420     // in ICP.
1421     MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1422     return MPM;
1423   }
1424
1425   if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
1426     // Load sample profile before running the LTO optimization pipeline.
1427     MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
1428                                         PGOOpt->ProfileRemappingFile,
1429                                         false /* ThinLTOPhase::PreLink */));
1430     // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1431     // RequireAnalysisPass for PSI before subsequent non-module passes.
1432     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
1433   }
1434
1435   // Remove unused virtual tables to improve the quality of code generated by
1436   // whole-program devirtualization and bitset lowering.
1437   MPM.addPass(GlobalDCEPass());
1438
1439   // Force any function attributes we want the rest of the pipeline to observe.
1440   MPM.addPass(ForceFunctionAttrsPass());
1441
1442   // Do basic inference of function attributes from known properties of system
1443   // libraries and other oracles.
1444   MPM.addPass(InferFunctionAttrsPass());
1445
1446   if (Level.getSpeedupLevel() > 1) {
1447     FunctionPassManager EarlyFPM(DebugLogging);
1448     EarlyFPM.addPass(CallSiteSplittingPass());
1449     MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM)));
1450
1451     // Indirect call promotion. This should promote all the targets that are
1452     // left by the earlier promotion pass that promotes intra-module targets.
1453     // This two-step promotion is to save the compile time. For LTO, it should
1454     // produce the same result as if we only do promotion here.
1455     MPM.addPass(PGOIndirectCallPromotion(
1456         true /* InLTO */, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
1457     // Propagate constants at call sites into the functions they call.  This
1458     // opens opportunities for globalopt (and inlining) by substituting function
1459     // pointers passed as arguments to direct uses of functions.
1460     MPM.addPass(IPSCCPPass());
1461
1462     // Attach metadata to indirect call sites indicating the set of functions
1463     // they may target at run-time. This should follow IPSCCP.
1464     MPM.addPass(CalledValuePropagationPass());
1465   }
1466
1467   // Now deduce any function attributes based in the current code.
1468   MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1469               PostOrderFunctionAttrsPass()));
1470
1471   // Do RPO function attribute inference across the module to forward-propagate
1472   // attributes where applicable.
1473   // FIXME: Is this really an optimization rather than a canonicalization?
1474   MPM.addPass(ReversePostOrderFunctionAttrsPass());
1475
1476   // Use in-range annotations on GEP indices to split globals where beneficial.
1477   MPM.addPass(GlobalSplitPass());
1478
1479   // Run whole program optimization of virtual call when the list of callees
1480   // is fixed.
1481   MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1482
1483   // Stop here at -O1.
1484   if (Level == OptimizationLevel::O1) {
1485     // The LowerTypeTestsPass needs to run to lower type metadata and the
1486     // type.test intrinsics. The pass does nothing if CFI is disabled.
1487     MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1488     // Run a second time to clean up any type tests left behind by WPD for use
1489     // in ICP (which is performed earlier than this in the regular LTO
1490     // pipeline).
1491     MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1492     return MPM;
1493   }
1494
1495   // Optimize globals to try and fold them into constants.
1496   MPM.addPass(GlobalOptPass());
1497
1498   // Promote any localized globals to SSA registers.
1499   MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
1500
1501   // Linking modules together can lead to duplicate global constant, only
1502   // keep one copy of each constant.
1503   MPM.addPass(ConstantMergePass());
1504
1505   // Remove unused arguments from functions.
1506   MPM.addPass(DeadArgumentEliminationPass());
1507
1508   // Reduce the code after globalopt and ipsccp.  Both can open up significant
1509   // simplification opportunities, and both can propagate functions through
1510   // function pointers.  When this happens, we often have to resolve varargs
1511   // calls, etc, so let instcombine do this.
1512   FunctionPassManager PeepholeFPM(DebugLogging);
1513   if (Level == OptimizationLevel::O3)
1514     PeepholeFPM.addPass(AggressiveInstCombinePass());
1515   PeepholeFPM.addPass(InstCombinePass());
1516   invokePeepholeEPCallbacks(PeepholeFPM, Level);
1517
1518   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM)));
1519
1520   // Note: historically, the PruneEH pass was run first to deduce nounwind and
1521   // generally clean up exception handling overhead. It isn't clear this is
1522   // valuable as the inliner doesn't currently care whether it is inlining an
1523   // invoke or a call.
1524   // Run the inliner now.
1525   MPM.addPass(ModuleInlinerWrapperPass(getInlineParamsFromOptLevel(Level),
1526                                        DebugLogging));
1527
1528   // Optimize globals again after we ran the inliner.
1529   MPM.addPass(GlobalOptPass());
1530
1531   // Garbage collect dead functions.
1532   // FIXME: Add ArgumentPromotion pass after once it's ported.
1533   MPM.addPass(GlobalDCEPass());
1534
1535   FunctionPassManager FPM(DebugLogging);
1536   // The IPO Passes may leave cruft around. Clean up after them.
1537   FPM.addPass(InstCombinePass());
1538   invokePeepholeEPCallbacks(FPM, Level);
1539
1540   FPM.addPass(JumpThreadingPass(/*InsertFreezeWhenUnfoldingSelect*/ true));
1541
1542   // Do a post inline PGO instrumentation and use pass. This is a context
1543   // sensitive PGO pass.
1544   if (PGOOpt) {
1545     if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1546       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ true,
1547                         /* IsCS */ true, PGOOpt->CSProfileGenFile,
1548                         PGOOpt->ProfileRemappingFile);
1549     else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1550       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ false,
1551                         /* IsCS */ true, PGOOpt->ProfileFile,
1552                         PGOOpt->ProfileRemappingFile);
1553   }
1554
1555   // Break up allocas
1556   FPM.addPass(SROA());
1557
1558   // LTO provides additional opportunities for tailcall elimination due to
1559   // link-time inlining, and visibility of nocapture attribute.
1560   FPM.addPass(TailCallElimPass());
1561
1562   // Run a few AA driver optimizations here and now to cleanup the code.
1563   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
1564
1565   MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1566               PostOrderFunctionAttrsPass()));
1567   // FIXME: here we run IP alias analysis in the legacy PM.
1568
1569   FunctionPassManager MainFPM;
1570
1571   // FIXME: once we fix LoopPass Manager, add LICM here.
1572   // FIXME: once we provide support for enabling MLSM, add it here.
1573   if (RunNewGVN)
1574     MainFPM.addPass(NewGVNPass());
1575   else
1576     MainFPM.addPass(GVN());
1577
1578   // Remove dead memcpy()'s.
1579   MainFPM.addPass(MemCpyOptPass());
1580
1581   // Nuke dead stores.
1582   MainFPM.addPass(DSEPass());
1583
1584   // FIXME: at this point, we run a bunch of loop passes:
1585   // indVarSimplify, loopDeletion, loopInterchange, loopUnroll,
1586   // loopVectorize. Enable them once the remaining issue with LPM
1587   // are sorted out.
1588
1589   MainFPM.addPass(InstCombinePass());
1590   MainFPM.addPass(SimplifyCFGPass());
1591   MainFPM.addPass(SCCPPass());
1592   MainFPM.addPass(InstCombinePass());
1593   MainFPM.addPass(BDCEPass());
1594
1595   // FIXME: We may want to run SLPVectorizer here.
1596   // After vectorization, assume intrinsics may tell us more
1597   // about pointer alignments.
1598 #if 0
1599   MainFPM.add(AlignmentFromAssumptionsPass());
1600 #endif
1601
1602   // FIXME: Conditionally run LoadCombine here, after it's ported
1603   // (in case we still have this pass, given its questionable usefulness).
1604
1605   MainFPM.addPass(InstCombinePass());
1606   invokePeepholeEPCallbacks(MainFPM, Level);
1607   MainFPM.addPass(JumpThreadingPass(/*InsertFreezeWhenUnfoldingSelect*/ true));
1608   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM)));
1609
1610   // Create a function that performs CFI checks for cross-DSO calls with
1611   // targets in the current module.
1612   MPM.addPass(CrossDSOCFIPass());
1613
1614   // Lower type metadata and the type.test intrinsic. This pass supports
1615   // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs
1616   // to be run at link time if CFI is enabled. This pass does nothing if
1617   // CFI is disabled.
1618   MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1619   // Run a second time to clean up any type tests left behind by WPD for use
1620   // in ICP (which is performed earlier than this in the regular LTO pipeline).
1621   MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1622
1623   // Enable splitting late in the FullLTO post-link pipeline. This is done in
1624   // the same stage in the old pass manager (\ref addLateLTOOptimizationPasses).
1625   if (EnableHotColdSplit)
1626     MPM.addPass(HotColdSplittingPass());
1627
1628   // Add late LTO optimization passes.
1629   // Delete basic blocks, which optimization passes may have killed.
1630   MPM.addPass(createModuleToFunctionPassAdaptor(SimplifyCFGPass()));
1631
1632   // Drop bodies of available eternally objects to improve GlobalDCE.
1633   MPM.addPass(EliminateAvailableExternallyPass());
1634
1635   // Now that we have optimized the program, discard unreachable functions.
1636   MPM.addPass(GlobalDCEPass());
1637
1638   // FIXME: Maybe enable MergeFuncs conditionally after it's ported.
1639   return MPM;
1640 }
1641
1642 AAManager PassBuilder::buildDefaultAAPipeline() {
1643   AAManager AA;
1644
1645   // The order in which these are registered determines their priority when
1646   // being queried.
1647
1648   // First we register the basic alias analysis that provides the majority of
1649   // per-function local AA logic. This is a stateless, on-demand local set of
1650   // AA techniques.
1651   AA.registerFunctionAnalysis<BasicAA>();
1652
1653   // Next we query fast, specialized alias analyses that wrap IR-embedded
1654   // information about aliasing.
1655   AA.registerFunctionAnalysis<ScopedNoAliasAA>();
1656   AA.registerFunctionAnalysis<TypeBasedAA>();
1657
1658   // Add support for querying global aliasing information when available.
1659   // Because the `AAManager` is a function analysis and `GlobalsAA` is a module
1660   // analysis, all that the `AAManager` can do is query for any *cached*
1661   // results from `GlobalsAA` through a readonly proxy.
1662   AA.registerModuleAnalysis<GlobalsAA>();
1663
1664   return AA;
1665 }
1666
1667 static Optional<int> parseRepeatPassName(StringRef Name) {
1668   if (!Name.consume_front("repeat<") || !Name.consume_back(">"))
1669     return None;
1670   int Count;
1671   if (Name.getAsInteger(0, Count) || Count <= 0)
1672     return None;
1673   return Count;
1674 }
1675
1676 static Optional<int> parseDevirtPassName(StringRef Name) {
1677   if (!Name.consume_front("devirt<") || !Name.consume_back(">"))
1678     return None;
1679   int Count;
1680   if (Name.getAsInteger(0, Count) || Count <= 0)
1681     return None;
1682   return Count;
1683 }
1684
1685 static bool checkParametrizedPassName(StringRef Name, StringRef PassName) {
1686   if (!Name.consume_front(PassName))
1687     return false;
1688   // normal pass name w/o parameters == default parameters
1689   if (Name.empty())
1690     return true;
1691   return Name.startswith("<") && Name.endswith(">");
1692 }
1693
1694 namespace {
1695
1696 /// This performs customized parsing of pass name with parameters.
1697 ///
1698 /// We do not need parametrization of passes in textual pipeline very often,
1699 /// yet on a rare occasion ability to specify parameters right there can be
1700 /// useful.
1701 ///
1702 /// \p Name - parameterized specification of a pass from a textual pipeline
1703 /// is a string in a form of :
1704 ///      PassName '<' parameter-list '>'
1705 ///
1706 /// Parameter list is being parsed by the parser callable argument, \p Parser,
1707 /// It takes a string-ref of parameters and returns either StringError or a
1708 /// parameter list in a form of a custom parameters type, all wrapped into
1709 /// Expected<> template class.
1710 ///
1711 template <typename ParametersParseCallableT>
1712 auto parsePassParameters(ParametersParseCallableT &&Parser, StringRef Name,
1713                          StringRef PassName) -> decltype(Parser(StringRef{})) {
1714   using ParametersT = typename decltype(Parser(StringRef{}))::value_type;
1715
1716   StringRef Params = Name;
1717   if (!Params.consume_front(PassName)) {
1718     assert(false &&
1719            "unable to strip pass name from parametrized pass specification");
1720   }
1721   if (Params.empty())
1722     return ParametersT{};
1723   if (!Params.consume_front("<") || !Params.consume_back(">")) {
1724     assert(false && "invalid format for parametrized pass name");
1725   }
1726
1727   Expected<ParametersT> Result = Parser(Params);
1728   assert((Result || Result.template errorIsA<StringError>()) &&
1729          "Pass parameter parser can only return StringErrors.");
1730   return Result;
1731 }
1732
1733 /// Parser of parameters for LoopUnroll pass.
1734 Expected<LoopUnrollOptions> parseLoopUnrollOptions(StringRef Params) {
1735   LoopUnrollOptions UnrollOpts;
1736   while (!Params.empty()) {
1737     StringRef ParamName;
1738     std::tie(ParamName, Params) = Params.split(';');
1739     int OptLevel = StringSwitch<int>(ParamName)
1740                        .Case("O0", 0)
1741                        .Case("O1", 1)
1742                        .Case("O2", 2)
1743                        .Case("O3", 3)
1744                        .Default(-1);
1745     if (OptLevel >= 0) {
1746       UnrollOpts.setOptLevel(OptLevel);
1747       continue;
1748     }
1749     if (ParamName.consume_front("full-unroll-max=")) {
1750       int Count;
1751       if (ParamName.getAsInteger(0, Count))
1752         return make_error<StringError>(
1753             formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
1754             inconvertibleErrorCode());
1755       UnrollOpts.setFullUnrollMaxCount(Count);
1756       continue;
1757     }
1758
1759     bool Enable = !ParamName.consume_front("no-");
1760     if (ParamName == "partial") {
1761       UnrollOpts.setPartial(Enable);
1762     } else if (ParamName == "peeling") {
1763       UnrollOpts.setPeeling(Enable);
1764     } else if (ParamName == "profile-peeling") {
1765       UnrollOpts.setProfileBasedPeeling(Enable);
1766     } else if (ParamName == "runtime") {
1767       UnrollOpts.setRuntime(Enable);
1768     } else if (ParamName == "upperbound") {
1769       UnrollOpts.setUpperBound(Enable);
1770     } else {
1771       return make_error<StringError>(
1772           formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
1773           inconvertibleErrorCode());
1774     }
1775   }
1776   return UnrollOpts;
1777 }
1778
1779 Expected<MemorySanitizerOptions> parseMSanPassOptions(StringRef Params) {
1780   MemorySanitizerOptions Result;
1781   while (!Params.empty()) {
1782     StringRef ParamName;
1783     std::tie(ParamName, Params) = Params.split(';');
1784
1785     if (ParamName == "recover") {
1786       Result.Recover = true;
1787     } else if (ParamName == "kernel") {
1788       Result.Kernel = true;
1789     } else if (ParamName.consume_front("track-origins=")) {
1790       if (ParamName.getAsInteger(0, Result.TrackOrigins))
1791         return make_error<StringError>(
1792             formatv("invalid argument to MemorySanitizer pass track-origins "
1793                     "parameter: '{0}' ",
1794                     ParamName)
1795                 .str(),
1796             inconvertibleErrorCode());
1797     } else {
1798       return make_error<StringError>(
1799           formatv("invalid MemorySanitizer pass parameter '{0}' ", ParamName)
1800               .str(),
1801           inconvertibleErrorCode());
1802     }
1803   }
1804   return Result;
1805 }
1806
1807 /// Parser of parameters for SimplifyCFG pass.
1808 Expected<SimplifyCFGOptions> parseSimplifyCFGOptions(StringRef Params) {
1809   SimplifyCFGOptions Result;
1810   while (!Params.empty()) {
1811     StringRef ParamName;
1812     std::tie(ParamName, Params) = Params.split(';');
1813
1814     bool Enable = !ParamName.consume_front("no-");
1815     if (ParamName == "forward-switch-cond") {
1816       Result.forwardSwitchCondToPhi(Enable);
1817     } else if (ParamName == "switch-to-lookup") {
1818       Result.convertSwitchToLookupTable(Enable);
1819     } else if (ParamName == "keep-loops") {
1820       Result.needCanonicalLoops(Enable);
1821     } else if (ParamName == "hoist-common-insts") {
1822       Result.hoistCommonInsts(Enable);
1823     } else if (ParamName == "sink-common-insts") {
1824       Result.sinkCommonInsts(Enable);
1825     } else if (Enable && ParamName.consume_front("bonus-inst-threshold=")) {
1826       APInt BonusInstThreshold;
1827       if (ParamName.getAsInteger(0, BonusInstThreshold))
1828         return make_error<StringError>(
1829             formatv("invalid argument to SimplifyCFG pass bonus-threshold "
1830                     "parameter: '{0}' ",
1831                     ParamName).str(),
1832             inconvertibleErrorCode());
1833       Result.bonusInstThreshold(BonusInstThreshold.getSExtValue());
1834     } else {
1835       return make_error<StringError>(
1836           formatv("invalid SimplifyCFG pass parameter '{0}' ", ParamName).str(),
1837           inconvertibleErrorCode());
1838     }
1839   }
1840   return Result;
1841 }
1842
1843 /// Parser of parameters for LoopVectorize pass.
1844 Expected<LoopVectorizeOptions> parseLoopVectorizeOptions(StringRef Params) {
1845   LoopVectorizeOptions Opts;
1846   while (!Params.empty()) {
1847     StringRef ParamName;
1848     std::tie(ParamName, Params) = Params.split(';');
1849
1850     bool Enable = !ParamName.consume_front("no-");
1851     if (ParamName == "interleave-forced-only") {
1852       Opts.setInterleaveOnlyWhenForced(Enable);
1853     } else if (ParamName == "vectorize-forced-only") {
1854       Opts.setVectorizeOnlyWhenForced(Enable);
1855     } else {
1856       return make_error<StringError>(
1857           formatv("invalid LoopVectorize parameter '{0}' ", ParamName).str(),
1858           inconvertibleErrorCode());
1859     }
1860   }
1861   return Opts;
1862 }
1863
1864 Expected<bool> parseLoopUnswitchOptions(StringRef Params) {
1865   bool Result = false;
1866   while (!Params.empty()) {
1867     StringRef ParamName;
1868     std::tie(ParamName, Params) = Params.split(';');
1869
1870     bool Enable = !ParamName.consume_front("no-");
1871     if (ParamName == "nontrivial") {
1872       Result = Enable;
1873     } else {
1874       return make_error<StringError>(
1875           formatv("invalid LoopUnswitch pass parameter '{0}' ", ParamName)
1876               .str(),
1877           inconvertibleErrorCode());
1878     }
1879   }
1880   return Result;
1881 }
1882
1883 Expected<bool> parseMergedLoadStoreMotionOptions(StringRef Params) {
1884   bool Result = false;
1885   while (!Params.empty()) {
1886     StringRef ParamName;
1887     std::tie(ParamName, Params) = Params.split(';');
1888
1889     bool Enable = !ParamName.consume_front("no-");
1890     if (ParamName == "split-footer-bb") {
1891       Result = Enable;
1892     } else {
1893       return make_error<StringError>(
1894           formatv("invalid MergedLoadStoreMotion pass parameter '{0}' ",
1895                   ParamName)
1896               .str(),
1897           inconvertibleErrorCode());
1898     }
1899   }
1900   return Result;
1901 }
1902
1903 Expected<GVNOptions> parseGVNOptions(StringRef Params) {
1904   GVNOptions Result;
1905   while (!Params.empty()) {
1906     StringRef ParamName;
1907     std::tie(ParamName, Params) = Params.split(';');
1908
1909     bool Enable = !ParamName.consume_front("no-");
1910     if (ParamName == "pre") {
1911       Result.setPRE(Enable);
1912     } else if (ParamName == "load-pre") {
1913       Result.setLoadPRE(Enable);
1914     } else if (ParamName == "memdep") {
1915       Result.setMemDep(Enable);
1916     } else {
1917       return make_error<StringError>(
1918           formatv("invalid GVN pass parameter '{0}' ", ParamName).str(),
1919           inconvertibleErrorCode());
1920     }
1921   }
1922   return Result;
1923 }
1924
1925 Expected<StackLifetime::LivenessType>
1926 parseStackLifetimeOptions(StringRef Params) {
1927   StackLifetime::LivenessType Result = StackLifetime::LivenessType::May;
1928   while (!Params.empty()) {
1929     StringRef ParamName;
1930     std::tie(ParamName, Params) = Params.split(';');
1931
1932     if (ParamName == "may") {
1933       Result = StackLifetime::LivenessType::May;
1934     } else if (ParamName == "must") {
1935       Result = StackLifetime::LivenessType::Must;
1936     } else {
1937       return make_error<StringError>(
1938           formatv("invalid StackLifetime parameter '{0}' ", ParamName).str(),
1939           inconvertibleErrorCode());
1940     }
1941   }
1942   return Result;
1943 }
1944
1945 } // namespace
1946
1947 /// Tests whether a pass name starts with a valid prefix for a default pipeline
1948 /// alias.
1949 static bool startsWithDefaultPipelineAliasPrefix(StringRef Name) {
1950   return Name.startswith("default") || Name.startswith("thinlto") ||
1951          Name.startswith("lto");
1952 }
1953
1954 /// Tests whether registered callbacks will accept a given pass name.
1955 ///
1956 /// When parsing a pipeline text, the type of the outermost pipeline may be
1957 /// omitted, in which case the type is automatically determined from the first
1958 /// pass name in the text. This may be a name that is handled through one of the
1959 /// callbacks. We check this through the oridinary parsing callbacks by setting
1960 /// up a dummy PassManager in order to not force the client to also handle this
1961 /// type of query.
1962 template <typename PassManagerT, typename CallbacksT>
1963 static bool callbacksAcceptPassName(StringRef Name, CallbacksT &Callbacks) {
1964   if (!Callbacks.empty()) {
1965     PassManagerT DummyPM;
1966     for (auto &CB : Callbacks)
1967       if (CB(Name, DummyPM, {}))
1968         return true;
1969   }
1970   return false;
1971 }
1972
1973 template <typename CallbacksT>
1974 static bool isModulePassName(StringRef Name, CallbacksT &Callbacks) {
1975   // Manually handle aliases for pre-configured pipeline fragments.
1976   if (startsWithDefaultPipelineAliasPrefix(Name))
1977     return DefaultAliasRegex.match(Name);
1978
1979   // Explicitly handle pass manager names.
1980   if (Name == "module")
1981     return true;
1982   if (Name == "cgscc")
1983     return true;
1984   if (Name == "function")
1985     return true;
1986
1987   // Explicitly handle custom-parsed pass names.
1988   if (parseRepeatPassName(Name))
1989     return true;
1990
1991 #define MODULE_PASS(NAME, CREATE_PASS)                                         \
1992   if (Name == NAME)                                                            \
1993     return true;
1994 #define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
1995   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
1996     return true;
1997 #include "PassRegistry.def"
1998
1999   return callbacksAcceptPassName<ModulePassManager>(Name, Callbacks);
2000 }
2001
2002 template <typename CallbacksT>
2003 static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks) {
2004   // Explicitly handle pass manager names.
2005   if (Name == "cgscc")
2006     return true;
2007   if (Name == "function")
2008     return true;
2009
2010   // Explicitly handle custom-parsed pass names.
2011   if (parseRepeatPassName(Name))
2012     return true;
2013   if (parseDevirtPassName(Name))
2014     return true;
2015
2016 #define CGSCC_PASS(NAME, CREATE_PASS)                                          \
2017   if (Name == NAME)                                                            \
2018     return true;
2019 #define CGSCC_ANALYSIS(NAME, CREATE_PASS)                                      \
2020   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
2021     return true;
2022 #include "PassRegistry.def"
2023
2024   return callbacksAcceptPassName<CGSCCPassManager>(Name, Callbacks);
2025 }
2026
2027 template <typename CallbacksT>
2028 static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks) {
2029   // Explicitly handle pass manager names.
2030   if (Name == "function")
2031     return true;
2032   if (Name == "loop" || Name == "loop-mssa")
2033     return true;
2034
2035   // Explicitly handle custom-parsed pass names.
2036   if (parseRepeatPassName(Name))
2037     return true;
2038
2039 #define FUNCTION_PASS(NAME, CREATE_PASS)                                       \
2040   if (Name == NAME)                                                            \
2041     return true;
2042 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                   \
2043   if (checkParametrizedPassName(Name, NAME))                                   \
2044     return true;
2045 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
2046   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
2047     return true;
2048 #include "PassRegistry.def"
2049
2050   return callbacksAcceptPassName<FunctionPassManager>(Name, Callbacks);
2051 }
2052
2053 template <typename CallbacksT>
2054 static bool isLoopPassName(StringRef Name, CallbacksT &Callbacks) {
2055   // Explicitly handle pass manager names.
2056   if (Name == "loop" || Name == "loop-mssa")
2057     return true;
2058
2059   // Explicitly handle custom-parsed pass names.
2060   if (parseRepeatPassName(Name))
2061     return true;
2062
2063 #define LOOP_PASS(NAME, CREATE_PASS)                                           \
2064   if (Name == NAME)                                                            \
2065     return true;
2066 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                       \
2067   if (checkParametrizedPassName(Name, NAME))                                   \
2068     return true;
2069 #define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
2070   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
2071     return true;
2072 #include "PassRegistry.def"
2073
2074   return callbacksAcceptPassName<LoopPassManager>(Name, Callbacks);
2075 }
2076
2077 Optional<std::vector<PassBuilder::PipelineElement>>
2078 PassBuilder::parsePipelineText(StringRef Text) {
2079   std::vector<PipelineElement> ResultPipeline;
2080
2081   SmallVector<std::vector<PipelineElement> *, 4> PipelineStack = {
2082       &ResultPipeline};
2083   for (;;) {
2084     std::vector<PipelineElement> &Pipeline = *PipelineStack.back();
2085     size_t Pos = Text.find_first_of(",()");
2086     Pipeline.push_back({Text.substr(0, Pos), {}});
2087
2088     // If we have a single terminating name, we're done.
2089     if (Pos == Text.npos)
2090       break;
2091
2092     char Sep = Text[Pos];
2093     Text = Text.substr(Pos + 1);
2094     if (Sep == ',')
2095       // Just a name ending in a comma, continue.
2096       continue;
2097
2098     if (Sep == '(') {
2099       // Push the inner pipeline onto the stack to continue processing.
2100       PipelineStack.push_back(&Pipeline.back().InnerPipeline);
2101       continue;
2102     }
2103
2104     assert(Sep == ')' && "Bogus separator!");
2105     // When handling the close parenthesis, we greedily consume them to avoid
2106     // empty strings in the pipeline.
2107     do {
2108       // If we try to pop the outer pipeline we have unbalanced parentheses.
2109       if (PipelineStack.size() == 1)
2110         return None;
2111
2112       PipelineStack.pop_back();
2113     } while (Text.consume_front(")"));
2114
2115     // Check if we've finished parsing.
2116     if (Text.empty())
2117       break;
2118
2119     // Otherwise, the end of an inner pipeline always has to be followed by
2120     // a comma, and then we can continue.
2121     if (!Text.consume_front(","))
2122       return None;
2123   }
2124
2125   if (PipelineStack.size() > 1)
2126     // Unbalanced paretheses.
2127     return None;
2128
2129   assert(PipelineStack.back() == &ResultPipeline &&
2130          "Wrong pipeline at the bottom of the stack!");
2131   return {std::move(ResultPipeline)};
2132 }
2133
2134 Error PassBuilder::parseModulePass(ModulePassManager &MPM,
2135                                    const PipelineElement &E,
2136                                    bool VerifyEachPass, bool DebugLogging) {
2137   auto &Name = E.Name;
2138   auto &InnerPipeline = E.InnerPipeline;
2139
2140   // First handle complex passes like the pass managers which carry pipelines.
2141   if (!InnerPipeline.empty()) {
2142     if (Name == "module") {
2143       ModulePassManager NestedMPM(DebugLogging);
2144       if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline,
2145                                              VerifyEachPass, DebugLogging))
2146         return Err;
2147       MPM.addPass(std::move(NestedMPM));
2148       return Error::success();
2149     }
2150     if (Name == "cgscc") {
2151       CGSCCPassManager CGPM(DebugLogging);
2152       if (auto Err = parseCGSCCPassPipeline(CGPM, InnerPipeline, VerifyEachPass,
2153                                             DebugLogging))
2154         return Err;
2155       MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
2156       return Error::success();
2157     }
2158     if (Name == "function") {
2159       FunctionPassManager FPM(DebugLogging);
2160       if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline,
2161                                                VerifyEachPass, DebugLogging))
2162         return Err;
2163       MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
2164       return Error::success();
2165     }
2166     if (auto Count = parseRepeatPassName(Name)) {
2167       ModulePassManager NestedMPM(DebugLogging);
2168       if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline,
2169                                              VerifyEachPass, DebugLogging))
2170         return Err;
2171       MPM.addPass(createRepeatedPass(*Count, std::move(NestedMPM)));
2172       return Error::success();
2173     }
2174
2175     for (auto &C : ModulePipelineParsingCallbacks)
2176       if (C(Name, MPM, InnerPipeline))
2177         return Error::success();
2178
2179     // Normal passes can't have pipelines.
2180     return make_error<StringError>(
2181         formatv("invalid use of '{0}' pass as module pipeline", Name).str(),
2182         inconvertibleErrorCode());
2183     ;
2184   }
2185
2186   // Manually handle aliases for pre-configured pipeline fragments.
2187   if (startsWithDefaultPipelineAliasPrefix(Name)) {
2188     SmallVector<StringRef, 3> Matches;
2189     if (!DefaultAliasRegex.match(Name, &Matches))
2190       return make_error<StringError>(
2191           formatv("unknown default pipeline alias '{0}'", Name).str(),
2192           inconvertibleErrorCode());
2193
2194     assert(Matches.size() == 3 && "Must capture two matched strings!");
2195
2196     OptimizationLevel L = StringSwitch<OptimizationLevel>(Matches[2])
2197                               .Case("O0", OptimizationLevel::O0)
2198                               .Case("O1", OptimizationLevel::O1)
2199                               .Case("O2", OptimizationLevel::O2)
2200                               .Case("O3", OptimizationLevel::O3)
2201                               .Case("Os", OptimizationLevel::Os)
2202                               .Case("Oz", OptimizationLevel::Oz);
2203     if (L == OptimizationLevel::O0) {
2204       // Add instrumentation PGO passes -- at O0 we can still do PGO.
2205       if (PGOOpt && Matches[1] != "thinlto" &&
2206           (PGOOpt->Action == PGOOptions::IRInstr ||
2207            PGOOpt->Action == PGOOptions::IRUse))
2208         addPGOInstrPassesForO0(
2209             MPM, DebugLogging,
2210             /* RunProfileGen */ (PGOOpt->Action == PGOOptions::IRInstr),
2211             /* IsCS */ false, PGOOpt->ProfileFile,
2212             PGOOpt->ProfileRemappingFile);
2213
2214       // For IR that makes use of coroutines intrinsics, coroutine passes must
2215       // be run, even at -O0.
2216       if (PTO.Coroutines) {
2217         MPM.addPass(createModuleToFunctionPassAdaptor(CoroEarlyPass()));
2218
2219         CGSCCPassManager CGPM(DebugLogging);
2220         CGPM.addPass(CoroSplitPass());
2221         CGPM.addPass(createCGSCCToFunctionPassAdaptor(CoroElidePass()));
2222         MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
2223
2224         MPM.addPass(createModuleToFunctionPassAdaptor(CoroCleanupPass()));
2225       }
2226
2227       // Do nothing else at all!
2228       return Error::success();
2229     }
2230
2231     // This is consistent with old pass manager invoked via opt, but
2232     // inconsistent with clang. Clang doesn't enable loop vectorization
2233     // but does enable slp vectorization at Oz.
2234     PTO.LoopVectorization =
2235         L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz;
2236     PTO.SLPVectorization =
2237         L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz;
2238
2239     if (Matches[1] == "default") {
2240       MPM.addPass(buildPerModuleDefaultPipeline(L, DebugLogging));
2241     } else if (Matches[1] == "thinlto-pre-link") {
2242       MPM.addPass(buildThinLTOPreLinkDefaultPipeline(L, DebugLogging));
2243     } else if (Matches[1] == "thinlto") {
2244       MPM.addPass(buildThinLTODefaultPipeline(L, DebugLogging, nullptr));
2245     } else if (Matches[1] == "lto-pre-link") {
2246       MPM.addPass(buildLTOPreLinkDefaultPipeline(L, DebugLogging));
2247     } else {
2248       assert(Matches[1] == "lto" && "Not one of the matched options!");
2249       MPM.addPass(buildLTODefaultPipeline(L, DebugLogging, nullptr));
2250     }
2251     return Error::success();
2252   }
2253
2254   // Finally expand the basic registered passes from the .inc file.
2255 #define MODULE_PASS(NAME, CREATE_PASS)                                         \
2256   if (Name == NAME) {                                                          \
2257     MPM.addPass(CREATE_PASS);                                                  \
2258     return Error::success();                                                   \
2259   }
2260 #define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
2261   if (Name == "require<" NAME ">") {                                           \
2262     MPM.addPass(                                                               \
2263         RequireAnalysisPass<                                                   \
2264             std::remove_reference<decltype(CREATE_PASS)>::type, Module>());    \
2265     return Error::success();                                                   \
2266   }                                                                            \
2267   if (Name == "invalidate<" NAME ">") {                                        \
2268     MPM.addPass(InvalidateAnalysisPass<                                        \
2269                 std::remove_reference<decltype(CREATE_PASS)>::type>());        \
2270     return Error::success();                                                   \
2271   }
2272 #define CGSCC_PASS(NAME, CREATE_PASS)                                          \
2273   if (Name == NAME) {                                                          \
2274     MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(CREATE_PASS));         \
2275     return Error::success();                                                   \
2276   }
2277 #define FUNCTION_PASS(NAME, CREATE_PASS)                                       \
2278   if (Name == NAME) {                                                          \
2279     MPM.addPass(createModuleToFunctionPassAdaptor(CREATE_PASS));               \
2280     return Error::success();                                                   \
2281   }
2282 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                   \
2283   if (checkParametrizedPassName(Name, NAME)) {                                 \
2284     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
2285     if (!Params)                                                               \
2286       return Params.takeError();                                               \
2287     MPM.addPass(createModuleToFunctionPassAdaptor(CREATE_PASS(Params.get()))); \
2288     return Error::success();                                                   \
2289   }
2290 #define LOOP_PASS(NAME, CREATE_PASS)                                           \
2291   if (Name == NAME) {                                                          \
2292     MPM.addPass(                                                               \
2293         createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor(     \
2294             CREATE_PASS, false, false, DebugLogging)));                        \
2295     return Error::success();                                                   \
2296   }
2297 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                       \
2298   if (checkParametrizedPassName(Name, NAME)) {                                 \
2299     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
2300     if (!Params)                                                               \
2301       return Params.takeError();                                               \
2302     MPM.addPass(                                                               \
2303         createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor(     \
2304             CREATE_PASS(Params.get()), false, false, DebugLogging)));          \
2305     return Error::success();                                                   \
2306   }
2307 #include "PassRegistry.def"
2308
2309   for (auto &C : ModulePipelineParsingCallbacks)
2310     if (C(Name, MPM, InnerPipeline))
2311       return Error::success();
2312   return make_error<StringError>(
2313       formatv("unknown module pass '{0}'", Name).str(),
2314       inconvertibleErrorCode());
2315 }
2316
2317 Error PassBuilder::parseCGSCCPass(CGSCCPassManager &CGPM,
2318                                   const PipelineElement &E, bool VerifyEachPass,
2319                                   bool DebugLogging) {
2320   auto &Name = E.Name;
2321   auto &InnerPipeline = E.InnerPipeline;
2322
2323   // First handle complex passes like the pass managers which carry pipelines.
2324   if (!InnerPipeline.empty()) {
2325     if (Name == "cgscc") {
2326       CGSCCPassManager NestedCGPM(DebugLogging);
2327       if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline,
2328                                             VerifyEachPass, DebugLogging))
2329         return Err;
2330       // Add the nested pass manager with the appropriate adaptor.
2331       CGPM.addPass(std::move(NestedCGPM));
2332       return Error::success();
2333     }
2334     if (Name == "function") {
2335       FunctionPassManager FPM(DebugLogging);
2336       if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline,
2337                                                VerifyEachPass, DebugLogging))
2338         return Err;
2339       // Add the nested pass manager with the appropriate adaptor.
2340       CGPM.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
2341       return Error::success();
2342     }
2343     if (auto Count = parseRepeatPassName(Name)) {
2344       CGSCCPassManager NestedCGPM(DebugLogging);
2345       if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline,
2346                                             VerifyEachPass, DebugLogging))
2347         return Err;
2348       CGPM.addPass(createRepeatedPass(*Count, std::move(NestedCGPM)));
2349       return Error::success();
2350     }
2351     if (auto MaxRepetitions = parseDevirtPassName(Name)) {
2352       CGSCCPassManager NestedCGPM(DebugLogging);
2353       if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline,
2354                                             VerifyEachPass, DebugLogging))
2355         return Err;
2356       CGPM.addPass(
2357           createDevirtSCCRepeatedPass(std::move(NestedCGPM), *MaxRepetitions));
2358       return Error::success();
2359     }
2360
2361     for (auto &C : CGSCCPipelineParsingCallbacks)
2362       if (C(Name, CGPM, InnerPipeline))
2363         return Error::success();
2364
2365     // Normal passes can't have pipelines.
2366     return make_error<StringError>(
2367         formatv("invalid use of '{0}' pass as cgscc pipeline", Name).str(),
2368         inconvertibleErrorCode());
2369   }
2370
2371 // Now expand the basic registered passes from the .inc file.
2372 #define CGSCC_PASS(NAME, CREATE_PASS)                                          \
2373   if (Name == NAME) {                                                          \
2374     CGPM.addPass(CREATE_PASS);                                                 \
2375     return Error::success();                                                   \
2376   }
2377 #define CGSCC_ANALYSIS(NAME, CREATE_PASS)                                      \
2378   if (Name == "require<" NAME ">") {                                           \
2379     CGPM.addPass(RequireAnalysisPass<                                          \
2380                  std::remove_reference<decltype(CREATE_PASS)>::type,           \
2381                  LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &,    \
2382                  CGSCCUpdateResult &>());                                      \
2383     return Error::success();                                                   \
2384   }                                                                            \
2385   if (Name == "invalidate<" NAME ">") {                                        \
2386     CGPM.addPass(InvalidateAnalysisPass<                                       \
2387                  std::remove_reference<decltype(CREATE_PASS)>::type>());       \
2388     return Error::success();                                                   \
2389   }
2390 #define FUNCTION_PASS(NAME, CREATE_PASS)                                       \
2391   if (Name == NAME) {                                                          \
2392     CGPM.addPass(createCGSCCToFunctionPassAdaptor(CREATE_PASS));               \
2393     return Error::success();                                                   \
2394   }
2395 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                   \
2396   if (checkParametrizedPassName(Name, NAME)) {                                 \
2397     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
2398     if (!Params)                                                               \
2399       return Params.takeError();                                               \
2400     CGPM.addPass(createCGSCCToFunctionPassAdaptor(CREATE_PASS(Params.get()))); \
2401     return Error::success();                                                   \
2402   }
2403 #define LOOP_PASS(NAME, CREATE_PASS)                                           \
2404   if (Name == NAME) {                                                          \
2405     CGPM.addPass(                                                              \
2406         createCGSCCToFunctionPassAdaptor(createFunctionToLoopPassAdaptor(      \
2407             CREATE_PASS, false, false, DebugLogging)));                        \
2408     return Error::success();                                                   \
2409   }
2410 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                       \
2411   if (checkParametrizedPassName(Name, NAME)) {                                 \
2412     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
2413     if (!Params)                                                               \
2414       return Params.takeError();                                               \
2415     CGPM.addPass(                                                              \
2416         createCGSCCToFunctionPassAdaptor(createFunctionToLoopPassAdaptor(      \
2417             CREATE_PASS(Params.get()), false, false, DebugLogging)));          \
2418     return Error::success();                                                   \
2419   }
2420 #include "PassRegistry.def"
2421
2422   for (auto &C : CGSCCPipelineParsingCallbacks)
2423     if (C(Name, CGPM, InnerPipeline))
2424       return Error::success();
2425   return make_error<StringError>(
2426       formatv("unknown cgscc pass '{0}'", Name).str(),
2427       inconvertibleErrorCode());
2428 }
2429
2430 Error PassBuilder::parseFunctionPass(FunctionPassManager &FPM,
2431                                      const PipelineElement &E,
2432                                      bool VerifyEachPass, bool DebugLogging) {
2433   auto &Name = E.Name;
2434   auto &InnerPipeline = E.InnerPipeline;
2435
2436   // First handle complex passes like the pass managers which carry pipelines.
2437   if (!InnerPipeline.empty()) {
2438     if (Name == "function") {
2439       FunctionPassManager NestedFPM(DebugLogging);
2440       if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline,
2441                                                VerifyEachPass, DebugLogging))
2442         return Err;
2443       // Add the nested pass manager with the appropriate adaptor.
2444       FPM.addPass(std::move(NestedFPM));
2445       return Error::success();
2446     }
2447     if (Name == "loop" || Name == "loop-mssa") {
2448       LoopPassManager LPM(DebugLogging);
2449       if (auto Err = parseLoopPassPipeline(LPM, InnerPipeline, VerifyEachPass,
2450                                            DebugLogging))
2451         return Err;
2452       // Add the nested pass manager with the appropriate adaptor.
2453       bool UseMemorySSA = (Name == "loop-mssa");
2454       bool UseBFI =
2455           std::any_of(InnerPipeline.begin(), InnerPipeline.end(),
2456                       [](auto Pipeline) { return Pipeline.Name == "licm"; });
2457       FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), UseMemorySSA,
2458                                                   UseBFI, DebugLogging));
2459       return Error::success();
2460     }
2461     if (auto Count = parseRepeatPassName(Name)) {
2462       FunctionPassManager NestedFPM(DebugLogging);
2463       if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline,
2464                                                VerifyEachPass, DebugLogging))
2465         return Err;
2466       FPM.addPass(createRepeatedPass(*Count, std::move(NestedFPM)));
2467       return Error::success();
2468     }
2469
2470     for (auto &C : FunctionPipelineParsingCallbacks)
2471       if (C(Name, FPM, InnerPipeline))
2472         return Error::success();
2473
2474     // Normal passes can't have pipelines.
2475     return make_error<StringError>(
2476         formatv("invalid use of '{0}' pass as function pipeline", Name).str(),
2477         inconvertibleErrorCode());
2478   }
2479
2480 // Now expand the basic registered passes from the .inc file.
2481 #define FUNCTION_PASS(NAME, CREATE_PASS)                                       \
2482   if (Name == NAME) {                                                          \
2483     FPM.addPass(CREATE_PASS);                                                  \
2484     return Error::success();                                                   \
2485   }
2486 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                   \
2487   if (checkParametrizedPassName(Name, NAME)) {                                 \
2488     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
2489     if (!Params)                                                               \
2490       return Params.takeError();                                               \
2491     FPM.addPass(CREATE_PASS(Params.get()));                                    \
2492     return Error::success();                                                   \
2493   }
2494 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
2495   if (Name == "require<" NAME ">") {                                           \
2496     FPM.addPass(                                                               \
2497         RequireAnalysisPass<                                                   \
2498             std::remove_reference<decltype(CREATE_PASS)>::type, Function>());  \
2499     return Error::success();                                                   \
2500   }                                                                            \
2501   if (Name == "invalidate<" NAME ">") {                                        \
2502     FPM.addPass(InvalidateAnalysisPass<                                        \
2503                 std::remove_reference<decltype(CREATE_PASS)>::type>());        \
2504     return Error::success();                                                   \
2505   }
2506 // FIXME: UseMemorySSA is set to false. Maybe we could do things like:
2507 //        bool UseMemorySSA = !("canon-freeze" || "loop-predication" ||
2508 //                              "guard-widening");
2509 //        The risk is that it may become obsolete if we're not careful.
2510 #define LOOP_PASS(NAME, CREATE_PASS)                                           \
2511   if (Name == NAME) {                                                          \
2512     FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS, false, false,     \
2513                                                 DebugLogging));                \
2514     return Error::success();                                                   \
2515   }
2516 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                       \
2517   if (checkParametrizedPassName(Name, NAME)) {                                 \
2518     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
2519     if (!Params)                                                               \
2520       return Params.takeError();                                               \
2521     FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS(Params.get()),     \
2522                                                 false, false, DebugLogging));  \
2523     return Error::success();                                                   \
2524   }
2525 #include "PassRegistry.def"
2526
2527   for (auto &C : FunctionPipelineParsingCallbacks)
2528     if (C(Name, FPM, InnerPipeline))
2529       return Error::success();
2530   return make_error<StringError>(
2531       formatv("unknown function pass '{0}'", Name).str(),
2532       inconvertibleErrorCode());
2533 }
2534
2535 Error PassBuilder::parseLoopPass(LoopPassManager &LPM, const PipelineElement &E,
2536                                  bool VerifyEachPass, bool DebugLogging) {
2537   StringRef Name = E.Name;
2538   auto &InnerPipeline = E.InnerPipeline;
2539
2540   // First handle complex passes like the pass managers which carry pipelines.
2541   if (!InnerPipeline.empty()) {
2542     if (Name == "loop") {
2543       LoopPassManager NestedLPM(DebugLogging);
2544       if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline,
2545                                            VerifyEachPass, DebugLogging))
2546         return Err;
2547       // Add the nested pass manager with the appropriate adaptor.
2548       LPM.addPass(std::move(NestedLPM));
2549       return Error::success();
2550     }
2551     if (auto Count = parseRepeatPassName(Name)) {
2552       LoopPassManager NestedLPM(DebugLogging);
2553       if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline,
2554                                            VerifyEachPass, DebugLogging))
2555         return Err;
2556       LPM.addPass(createRepeatedPass(*Count, std::move(NestedLPM)));
2557       return Error::success();
2558     }
2559
2560     for (auto &C : LoopPipelineParsingCallbacks)
2561       if (C(Name, LPM, InnerPipeline))
2562         return Error::success();
2563
2564     // Normal passes can't have pipelines.
2565     return make_error<StringError>(
2566         formatv("invalid use of '{0}' pass as loop pipeline", Name).str(),
2567         inconvertibleErrorCode());
2568   }
2569
2570 // Now expand the basic registered passes from the .inc file.
2571 #define LOOP_PASS(NAME, CREATE_PASS)                                           \
2572   if (Name == NAME) {                                                          \
2573     LPM.addPass(CREATE_PASS);                                                  \
2574     return Error::success();                                                   \
2575   }
2576 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                       \
2577   if (checkParametrizedPassName(Name, NAME)) {                                 \
2578     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
2579     if (!Params)                                                               \
2580       return Params.takeError();                                               \
2581     LPM.addPass(CREATE_PASS(Params.get()));                                    \
2582     return Error::success();                                                   \
2583   }
2584 #define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
2585   if (Name == "require<" NAME ">") {                                           \
2586     LPM.addPass(RequireAnalysisPass<                                           \
2587                 std::remove_reference<decltype(CREATE_PASS)>::type, Loop,      \
2588                 LoopAnalysisManager, LoopStandardAnalysisResults &,            \
2589                 LPMUpdater &>());                                              \
2590     return Error::success();                                                   \
2591   }                                                                            \
2592   if (Name == "invalidate<" NAME ">") {                                        \
2593     LPM.addPass(InvalidateAnalysisPass<                                        \
2594                 std::remove_reference<decltype(CREATE_PASS)>::type>());        \
2595     return Error::success();                                                   \
2596   }
2597 #include "PassRegistry.def"
2598
2599   for (auto &C : LoopPipelineParsingCallbacks)
2600     if (C(Name, LPM, InnerPipeline))
2601       return Error::success();
2602   return make_error<StringError>(formatv("unknown loop pass '{0}'", Name).str(),
2603                                  inconvertibleErrorCode());
2604 }
2605
2606 bool PassBuilder::parseAAPassName(AAManager &AA, StringRef Name) {
2607 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS)                               \
2608   if (Name == NAME) {                                                          \
2609     AA.registerModuleAnalysis<                                                 \
2610         std::remove_reference<decltype(CREATE_PASS)>::type>();                 \
2611     return true;                                                               \
2612   }
2613 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS)                             \
2614   if (Name == NAME) {                                                          \
2615     AA.registerFunctionAnalysis<                                               \
2616         std::remove_reference<decltype(CREATE_PASS)>::type>();                 \
2617     return true;                                                               \
2618   }
2619 #include "PassRegistry.def"
2620
2621   for (auto &C : AAParsingCallbacks)
2622     if (C(Name, AA))
2623       return true;
2624   return false;
2625 }
2626
2627 Error PassBuilder::parseLoopPassPipeline(LoopPassManager &LPM,
2628                                          ArrayRef<PipelineElement> Pipeline,
2629                                          bool VerifyEachPass,
2630                                          bool DebugLogging) {
2631   for (const auto &Element : Pipeline) {
2632     if (auto Err = parseLoopPass(LPM, Element, VerifyEachPass, DebugLogging))
2633       return Err;
2634     // FIXME: No verifier support for Loop passes!
2635   }
2636   return Error::success();
2637 }
2638
2639 Error PassBuilder::parseFunctionPassPipeline(FunctionPassManager &FPM,
2640                                              ArrayRef<PipelineElement> Pipeline,
2641                                              bool VerifyEachPass,
2642                                              bool DebugLogging) {
2643   for (const auto &Element : Pipeline) {
2644     if (auto Err =
2645             parseFunctionPass(FPM, Element, VerifyEachPass, DebugLogging))
2646       return Err;
2647     if (VerifyEachPass)
2648       FPM.addPass(VerifierPass());
2649   }
2650   return Error::success();
2651 }
2652
2653 Error PassBuilder::parseCGSCCPassPipeline(CGSCCPassManager &CGPM,
2654                                           ArrayRef<PipelineElement> Pipeline,
2655                                           bool VerifyEachPass,
2656                                           bool DebugLogging) {
2657   for (const auto &Element : Pipeline) {
2658     if (auto Err = parseCGSCCPass(CGPM, Element, VerifyEachPass, DebugLogging))
2659       return Err;
2660     // FIXME: No verifier support for CGSCC passes!
2661   }
2662   return Error::success();
2663 }
2664
2665 void PassBuilder::crossRegisterProxies(LoopAnalysisManager &LAM,
2666                                        FunctionAnalysisManager &FAM,
2667                                        CGSCCAnalysisManager &CGAM,
2668                                        ModuleAnalysisManager &MAM) {
2669   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
2670   MAM.registerPass([&] { return CGSCCAnalysisManagerModuleProxy(CGAM); });
2671   CGAM.registerPass([&] { return ModuleAnalysisManagerCGSCCProxy(MAM); });
2672   FAM.registerPass([&] { return CGSCCAnalysisManagerFunctionProxy(CGAM); });
2673   FAM.registerPass([&] { return ModuleAnalysisManagerFunctionProxy(MAM); });
2674   FAM.registerPass([&] { return LoopAnalysisManagerFunctionProxy(LAM); });
2675   LAM.registerPass([&] { return FunctionAnalysisManagerLoopProxy(FAM); });
2676 }
2677
2678 Error PassBuilder::parseModulePassPipeline(ModulePassManager &MPM,
2679                                            ArrayRef<PipelineElement> Pipeline,
2680                                            bool VerifyEachPass,
2681                                            bool DebugLogging) {
2682   for (const auto &Element : Pipeline) {
2683     if (auto Err = parseModulePass(MPM, Element, VerifyEachPass, DebugLogging))
2684       return Err;
2685     if (VerifyEachPass)
2686       MPM.addPass(VerifierPass());
2687   }
2688   return Error::success();
2689 }
2690
2691 // Primary pass pipeline description parsing routine for a \c ModulePassManager
2692 // FIXME: Should this routine accept a TargetMachine or require the caller to
2693 // pre-populate the analysis managers with target-specific stuff?
2694 Error PassBuilder::parsePassPipeline(ModulePassManager &MPM,
2695                                      StringRef PipelineText,
2696                                      bool VerifyEachPass, bool DebugLogging) {
2697   auto Pipeline = parsePipelineText(PipelineText);
2698   if (!Pipeline || Pipeline->empty())
2699     return make_error<StringError>(
2700         formatv("invalid pipeline '{0}'", PipelineText).str(),
2701         inconvertibleErrorCode());
2702
2703   // If the first name isn't at the module layer, wrap the pipeline up
2704   // automatically.
2705   StringRef FirstName = Pipeline->front().Name;
2706
2707   if (!isModulePassName(FirstName, ModulePipelineParsingCallbacks)) {
2708     if (isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks)) {
2709       Pipeline = {{"cgscc", std::move(*Pipeline)}};
2710     } else if (isFunctionPassName(FirstName,
2711                                   FunctionPipelineParsingCallbacks)) {
2712       Pipeline = {{"function", std::move(*Pipeline)}};
2713     } else if (isLoopPassName(FirstName, LoopPipelineParsingCallbacks)) {
2714       Pipeline = {{"function", {{"loop", std::move(*Pipeline)}}}};
2715     } else {
2716       for (auto &C : TopLevelPipelineParsingCallbacks)
2717         if (C(MPM, *Pipeline, VerifyEachPass, DebugLogging))
2718           return Error::success();
2719
2720       // Unknown pass or pipeline name!
2721       auto &InnerPipeline = Pipeline->front().InnerPipeline;
2722       return make_error<StringError>(
2723           formatv("unknown {0} name '{1}'",
2724                   (InnerPipeline.empty() ? "pass" : "pipeline"), FirstName)
2725               .str(),
2726           inconvertibleErrorCode());
2727     }
2728   }
2729
2730   if (auto Err =
2731           parseModulePassPipeline(MPM, *Pipeline, VerifyEachPass, DebugLogging))
2732     return Err;
2733   return Error::success();
2734 }
2735
2736 // Primary pass pipeline description parsing routine for a \c CGSCCPassManager
2737 Error PassBuilder::parsePassPipeline(CGSCCPassManager &CGPM,
2738                                      StringRef PipelineText,
2739                                      bool VerifyEachPass, bool DebugLogging) {
2740   auto Pipeline = parsePipelineText(PipelineText);
2741   if (!Pipeline || Pipeline->empty())
2742     return make_error<StringError>(
2743         formatv("invalid pipeline '{0}'", PipelineText).str(),
2744         inconvertibleErrorCode());
2745
2746   StringRef FirstName = Pipeline->front().Name;
2747   if (!isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks))
2748     return make_error<StringError>(
2749         formatv("unknown cgscc pass '{0}' in pipeline '{1}'", FirstName,
2750                 PipelineText)
2751             .str(),
2752         inconvertibleErrorCode());
2753
2754   if (auto Err =
2755           parseCGSCCPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging))
2756     return Err;
2757   return Error::success();
2758 }
2759
2760 // Primary pass pipeline description parsing routine for a \c
2761 // FunctionPassManager
2762 Error PassBuilder::parsePassPipeline(FunctionPassManager &FPM,
2763                                      StringRef PipelineText,
2764                                      bool VerifyEachPass, bool DebugLogging) {
2765   auto Pipeline = parsePipelineText(PipelineText);
2766   if (!Pipeline || Pipeline->empty())
2767     return make_error<StringError>(
2768         formatv("invalid pipeline '{0}'", PipelineText).str(),
2769         inconvertibleErrorCode());
2770
2771   StringRef FirstName = Pipeline->front().Name;
2772   if (!isFunctionPassName(FirstName, FunctionPipelineParsingCallbacks))
2773     return make_error<StringError>(
2774         formatv("unknown function pass '{0}' in pipeline '{1}'", FirstName,
2775                 PipelineText)
2776             .str(),
2777         inconvertibleErrorCode());
2778
2779   if (auto Err = parseFunctionPassPipeline(FPM, *Pipeline, VerifyEachPass,
2780                                            DebugLogging))
2781     return Err;
2782   return Error::success();
2783 }
2784
2785 // Primary pass pipeline description parsing routine for a \c LoopPassManager
2786 Error PassBuilder::parsePassPipeline(LoopPassManager &CGPM,
2787                                      StringRef PipelineText,
2788                                      bool VerifyEachPass, bool DebugLogging) {
2789   auto Pipeline = parsePipelineText(PipelineText);
2790   if (!Pipeline || Pipeline->empty())
2791     return make_error<StringError>(
2792         formatv("invalid pipeline '{0}'", PipelineText).str(),
2793         inconvertibleErrorCode());
2794
2795   if (auto Err =
2796           parseLoopPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging))
2797     return Err;
2798
2799   return Error::success();
2800 }
2801
2802 Error PassBuilder::parseAAPipeline(AAManager &AA, StringRef PipelineText) {
2803   // If the pipeline just consists of the word 'default' just replace the AA
2804   // manager with our default one.
2805   if (PipelineText == "default") {
2806     AA = buildDefaultAAPipeline();
2807     return Error::success();
2808   }
2809
2810   while (!PipelineText.empty()) {
2811     StringRef Name;
2812     std::tie(Name, PipelineText) = PipelineText.split(',');
2813     if (!parseAAPassName(AA, Name))
2814       return make_error<StringError>(
2815           formatv("unknown alias analysis name '{0}'", Name).str(),
2816           inconvertibleErrorCode());
2817   }
2818
2819   return Error::success();
2820 }
2821
2822 bool PassBuilder::isAAPassName(StringRef PassName) {
2823 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS)                               \
2824   if (PassName == NAME)                                                        \
2825     return true;
2826 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS)                             \
2827   if (PassName == NAME)                                                        \
2828     return true;
2829 #include "PassRegistry.def"
2830   return false;
2831 }
2832
2833 bool PassBuilder::isAnalysisPassName(StringRef PassName) {
2834 #define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
2835   if (PassName == NAME)                                                        \
2836     return true;
2837 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
2838   if (PassName == NAME)                                                        \
2839     return true;
2840 #define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
2841   if (PassName == NAME)                                                        \
2842     return true;
2843 #define CGSSC_ANALYSIS(NAME, CREATE_PASS)                                      \
2844   if (PassName == NAME)                                                        \
2845     return true;
2846 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS)                               \
2847   if (PassName == NAME)                                                        \
2848     return true;
2849 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS)                             \
2850   if (PassName == NAME)                                                        \
2851     return true;
2852 #include "PassRegistry.def"
2853   return false;
2854 }
2855
2856 void PassBuilder::registerParseTopLevelPipelineCallback(
2857     const std::function<bool(ModulePassManager &, ArrayRef<PipelineElement>,
2858                              bool VerifyEachPass, bool DebugLogging)> &C) {
2859   TopLevelPipelineParsingCallbacks.push_back(C);
2860 }