1 | //===- TargetPassConfig.cpp - Target independent code generation passes ---===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file defines interfaces to access the target independent code |
10 | // generation passes provided by the LLVM backend. |
11 | // |
12 | //===---------------------------------------------------------------------===// |
13 | |
14 | #include "llvm/CodeGen/TargetPassConfig.h" |
15 | #include "llvm/ADT/DenseMap.h" |
16 | #include "llvm/ADT/SmallVector.h" |
17 | #include "llvm/ADT/StringRef.h" |
18 | #include "llvm/Analysis/BasicAliasAnalysis.h" |
19 | #include "llvm/Analysis/CallGraphSCCPass.h" |
20 | #include "llvm/Analysis/ScopedNoAliasAA.h" |
21 | #include "llvm/Analysis/TargetTransformInfo.h" |
22 | #include "llvm/Analysis/TypeBasedAliasAnalysis.h" |
23 | #include "llvm/CodeGen/BasicBlockSectionsProfileReader.h" |
24 | #include "llvm/CodeGen/CSEConfigBase.h" |
25 | #include "llvm/CodeGen/MachineFunctionPass.h" |
26 | #include "llvm/CodeGen/MachinePassRegistry.h" |
27 | #include "llvm/CodeGen/Passes.h" |
28 | #include "llvm/CodeGen/RegAllocRegistry.h" |
29 | #include "llvm/IR/IRPrintingPasses.h" |
30 | #include "llvm/IR/LegacyPassManager.h" |
31 | #include "llvm/IR/PassInstrumentation.h" |
32 | #include "llvm/IR/Verifier.h" |
33 | #include "llvm/InitializePasses.h" |
34 | #include "llvm/MC/MCAsmInfo.h" |
35 | #include "llvm/MC/MCTargetOptions.h" |
36 | #include "llvm/Pass.h" |
37 | #include "llvm/Support/CodeGen.h" |
38 | #include "llvm/Support/CommandLine.h" |
39 | #include "llvm/Support/Compiler.h" |
40 | #include "llvm/Support/Debug.h" |
41 | #include "llvm/Support/Discriminator.h" |
42 | #include "llvm/Support/ErrorHandling.h" |
43 | #include "llvm/Support/SaveAndRestore.h" |
44 | #include "llvm/Support/Threading.h" |
45 | #include "llvm/Support/VirtualFileSystem.h" |
46 | #include "llvm/Support/WithColor.h" |
47 | #include "llvm/Target/CGPassBuilderOption.h" |
48 | #include "llvm/Target/TargetMachine.h" |
49 | #include "llvm/Transforms/Scalar.h" |
50 | #include "llvm/Transforms/Utils.h" |
51 | #include <cassert> |
52 | #include <optional> |
53 | #include <string> |
54 | |
55 | using namespace llvm; |
56 | |
57 | static cl::opt<bool> |
58 | EnableIPRA("enable-ipra" , cl::init(Val: false), cl::Hidden, |
59 | cl::desc("Enable interprocedural register allocation " |
60 | "to reduce load/store at procedure calls." )); |
61 | static cl::opt<bool> DisablePostRASched("disable-post-ra" , cl::Hidden, |
62 | cl::desc("Disable Post Regalloc Scheduler" )); |
63 | static cl::opt<bool> DisableBranchFold("disable-branch-fold" , cl::Hidden, |
64 | cl::desc("Disable branch folding" )); |
65 | static cl::opt<bool> DisableTailDuplicate("disable-tail-duplicate" , cl::Hidden, |
66 | cl::desc("Disable tail duplication" )); |
67 | static cl::opt<bool> DisableEarlyTailDup("disable-early-taildup" , cl::Hidden, |
68 | cl::desc("Disable pre-register allocation tail duplication" )); |
69 | static cl::opt<bool> DisableBlockPlacement("disable-block-placement" , |
70 | cl::Hidden, cl::desc("Disable probability-driven block placement" )); |
71 | static cl::opt<bool> EnableBlockPlacementStats("enable-block-placement-stats" , |
72 | cl::Hidden, cl::desc("Collect probability-driven block placement stats" )); |
73 | static cl::opt<bool> DisableSSC("disable-ssc" , cl::Hidden, |
74 | cl::desc("Disable Stack Slot Coloring" )); |
75 | static cl::opt<bool> DisableMachineDCE("disable-machine-dce" , cl::Hidden, |
76 | cl::desc("Disable Machine Dead Code Elimination" )); |
77 | static cl::opt<bool> DisableEarlyIfConversion("disable-early-ifcvt" , cl::Hidden, |
78 | cl::desc("Disable Early If-conversion" )); |
79 | static cl::opt<bool> DisableMachineLICM("disable-machine-licm" , cl::Hidden, |
80 | cl::desc("Disable Machine LICM" )); |
81 | static cl::opt<bool> DisableMachineCSE("disable-machine-cse" , cl::Hidden, |
82 | cl::desc("Disable Machine Common Subexpression Elimination" )); |
83 | static cl::opt<cl::boolOrDefault> OptimizeRegAlloc( |
84 | "optimize-regalloc" , cl::Hidden, |
85 | cl::desc("Enable optimized register allocation compilation path." )); |
86 | static cl::opt<bool> DisablePostRAMachineLICM("disable-postra-machine-licm" , |
87 | cl::Hidden, |
88 | cl::desc("Disable Machine LICM" )); |
89 | static cl::opt<bool> DisableMachineSink("disable-machine-sink" , cl::Hidden, |
90 | cl::desc("Disable Machine Sinking" )); |
91 | static cl::opt<bool> DisablePostRAMachineSink("disable-postra-machine-sink" , |
92 | cl::Hidden, |
93 | cl::desc("Disable PostRA Machine Sinking" )); |
94 | static cl::opt<bool> DisableLSR("disable-lsr" , cl::Hidden, |
95 | cl::desc("Disable Loop Strength Reduction Pass" )); |
96 | static cl::opt<bool> DisableConstantHoisting("disable-constant-hoisting" , |
97 | cl::Hidden, cl::desc("Disable ConstantHoisting" )); |
98 | static cl::opt<bool> DisableCGP("disable-cgp" , cl::Hidden, |
99 | cl::desc("Disable Codegen Prepare" )); |
100 | static cl::opt<bool> DisableCopyProp("disable-copyprop" , cl::Hidden, |
101 | cl::desc("Disable Copy Propagation pass" )); |
102 | static cl::opt<bool> DisablePartialLibcallInlining("disable-partial-libcall-inlining" , |
103 | cl::Hidden, cl::desc("Disable Partial Libcall Inlining" )); |
104 | static cl::opt<bool> DisableAtExitBasedGlobalDtorLowering( |
105 | "disable-atexit-based-global-dtor-lowering" , cl::Hidden, |
106 | cl::desc("For MachO, disable atexit()-based global destructor lowering" )); |
107 | static cl::opt<bool> EnableImplicitNullChecks( |
108 | "enable-implicit-null-checks" , |
109 | cl::desc("Fold null checks into faulting memory operations" ), |
110 | cl::init(Val: false), cl::Hidden); |
111 | static cl::opt<bool> DisableMergeICmps("disable-mergeicmps" , |
112 | cl::desc("Disable MergeICmps Pass" ), |
113 | cl::init(Val: false), cl::Hidden); |
114 | static cl::opt<bool> PrintLSR("print-lsr-output" , cl::Hidden, |
115 | cl::desc("Print LLVM IR produced by the loop-reduce pass" )); |
116 | static cl::opt<bool> |
117 | PrintISelInput("print-isel-input" , cl::Hidden, |
118 | cl::desc("Print LLVM IR input to isel pass" )); |
119 | static cl::opt<cl::boolOrDefault> |
120 | VerifyMachineCode("verify-machineinstrs" , cl::Hidden, |
121 | cl::desc("Verify generated machine code" )); |
122 | static cl::opt<cl::boolOrDefault> |
123 | DebugifyAndStripAll("debugify-and-strip-all-safe" , cl::Hidden, |
124 | cl::desc("Debugify MIR before and Strip debug after " |
125 | "each pass except those known to be unsafe " |
126 | "when debug info is present" )); |
127 | static cl::opt<cl::boolOrDefault> DebugifyCheckAndStripAll( |
128 | "debugify-check-and-strip-all-safe" , cl::Hidden, |
129 | cl::desc( |
130 | "Debugify MIR before, by checking and stripping the debug info after, " |
131 | "each pass except those known to be unsafe when debug info is " |
132 | "present" )); |
133 | // Enable or disable the MachineOutliner. |
134 | static cl::opt<RunOutliner> EnableMachineOutliner( |
135 | "enable-machine-outliner" , cl::desc("Enable the machine outliner" ), |
136 | cl::Hidden, cl::ValueOptional, cl::init(Val: RunOutliner::TargetDefault), |
137 | cl::values(clEnumValN(RunOutliner::AlwaysOutline, "always" , |
138 | "Run on all functions guaranteed to be beneficial" ), |
139 | clEnumValN(RunOutliner::NeverOutline, "never" , |
140 | "Disable all outlining" ), |
141 | // Sentinel value for unspecified option. |
142 | clEnumValN(RunOutliner::AlwaysOutline, "" , "" ))); |
143 | // Disable the pass to fix unwind information. Whether the pass is included in |
144 | // the pipeline is controlled via the target options, this option serves as |
145 | // manual override. |
146 | static cl::opt<bool> DisableCFIFixup("disable-cfi-fixup" , cl::Hidden, |
147 | cl::desc("Disable the CFI fixup pass" )); |
148 | // Enable or disable FastISel. Both options are needed, because |
149 | // FastISel is enabled by default with -fast, and we wish to be |
150 | // able to enable or disable fast-isel independently from -O0. |
151 | static cl::opt<cl::boolOrDefault> |
152 | EnableFastISelOption("fast-isel" , cl::Hidden, |
153 | cl::desc("Enable the \"fast\" instruction selector" )); |
154 | |
155 | static cl::opt<cl::boolOrDefault> EnableGlobalISelOption( |
156 | "global-isel" , cl::Hidden, |
157 | cl::desc("Enable the \"global\" instruction selector" )); |
158 | |
159 | // FIXME: remove this after switching to NPM or GlobalISel, whichever gets there |
160 | // first... |
161 | static cl::opt<bool> |
162 | PrintAfterISel("print-after-isel" , cl::init(Val: false), cl::Hidden, |
163 | cl::desc("Print machine instrs after ISel" )); |
164 | |
165 | static cl::opt<GlobalISelAbortMode> EnableGlobalISelAbort( |
166 | "global-isel-abort" , cl::Hidden, |
167 | cl::desc("Enable abort calls when \"global\" instruction selection " |
168 | "fails to lower/select an instruction" ), |
169 | cl::values( |
170 | clEnumValN(GlobalISelAbortMode::Disable, "0" , "Disable the abort" ), |
171 | clEnumValN(GlobalISelAbortMode::Enable, "1" , "Enable the abort" ), |
172 | clEnumValN(GlobalISelAbortMode::DisableWithDiag, "2" , |
173 | "Disable the abort but emit a diagnostic on failure" ))); |
174 | |
175 | // Disable MIRProfileLoader before RegAlloc. This is for for debugging and |
176 | // tuning purpose. |
177 | static cl::opt<bool> DisableRAFSProfileLoader( |
178 | "disable-ra-fsprofile-loader" , cl::init(Val: false), cl::Hidden, |
179 | cl::desc("Disable MIRProfileLoader before RegAlloc" )); |
180 | // Disable MIRProfileLoader before BloackPlacement. This is for for debugging |
181 | // and tuning purpose. |
182 | static cl::opt<bool> DisableLayoutFSProfileLoader( |
183 | "disable-layout-fsprofile-loader" , cl::init(Val: false), cl::Hidden, |
184 | cl::desc("Disable MIRProfileLoader before BlockPlacement" )); |
185 | // Specify FSProfile file name. |
186 | static cl::opt<std::string> |
187 | FSProfileFile("fs-profile-file" , cl::init(Val: "" ), cl::value_desc("filename" ), |
188 | cl::desc("Flow Sensitive profile file name." ), cl::Hidden); |
189 | // Specify Remapping file for FSProfile. |
190 | static cl::opt<std::string> FSRemappingFile( |
191 | "fs-remapping-file" , cl::init(Val: "" ), cl::value_desc("filename" ), |
192 | cl::desc("Flow Sensitive profile remapping file name." ), cl::Hidden); |
193 | |
194 | // Temporary option to allow experimenting with MachineScheduler as a post-RA |
195 | // scheduler. Targets can "properly" enable this with |
196 | // substitutePass(&PostRASchedulerID, &PostMachineSchedulerID). |
197 | // Targets can return true in targetSchedulesPostRAScheduling() and |
198 | // insert a PostRA scheduling pass wherever it wants. |
199 | static cl::opt<bool> MISchedPostRA( |
200 | "misched-postra" , cl::Hidden, |
201 | cl::desc( |
202 | "Run MachineScheduler post regalloc (independent of preRA sched)" )); |
203 | |
204 | // Experimental option to run live interval analysis early. |
205 | static cl::opt<bool> EarlyLiveIntervals("early-live-intervals" , cl::Hidden, |
206 | cl::desc("Run live interval analysis earlier in the pipeline" )); |
207 | |
208 | /// Option names for limiting the codegen pipeline. |
209 | /// Those are used in error reporting and we didn't want |
210 | /// to duplicate their names all over the place. |
211 | static const char StartAfterOptName[] = "start-after" ; |
212 | static const char StartBeforeOptName[] = "start-before" ; |
213 | static const char StopAfterOptName[] = "stop-after" ; |
214 | static const char StopBeforeOptName[] = "stop-before" ; |
215 | |
216 | static cl::opt<std::string> |
217 | StartAfterOpt(StringRef(StartAfterOptName), |
218 | cl::desc("Resume compilation after a specific pass" ), |
219 | cl::value_desc("pass-name" ), cl::init(Val: "" ), cl::Hidden); |
220 | |
221 | static cl::opt<std::string> |
222 | StartBeforeOpt(StringRef(StartBeforeOptName), |
223 | cl::desc("Resume compilation before a specific pass" ), |
224 | cl::value_desc("pass-name" ), cl::init(Val: "" ), cl::Hidden); |
225 | |
226 | static cl::opt<std::string> |
227 | StopAfterOpt(StringRef(StopAfterOptName), |
228 | cl::desc("Stop compilation after a specific pass" ), |
229 | cl::value_desc("pass-name" ), cl::init(Val: "" ), cl::Hidden); |
230 | |
231 | static cl::opt<std::string> |
232 | StopBeforeOpt(StringRef(StopBeforeOptName), |
233 | cl::desc("Stop compilation before a specific pass" ), |
234 | cl::value_desc("pass-name" ), cl::init(Val: "" ), cl::Hidden); |
235 | |
236 | /// Enable the machine function splitter pass. |
237 | static cl::opt<bool> EnableMachineFunctionSplitter( |
238 | "enable-split-machine-functions" , cl::Hidden, |
239 | cl::desc("Split out cold blocks from machine functions based on profile " |
240 | "information." )); |
241 | |
242 | /// Disable the expand reductions pass for testing. |
243 | static cl::opt<bool> DisableExpandReductions( |
244 | "disable-expand-reductions" , cl::init(Val: false), cl::Hidden, |
245 | cl::desc("Disable the expand reduction intrinsics pass from running" )); |
246 | |
247 | /// Disable the select optimization pass. |
248 | static cl::opt<bool> DisableSelectOptimize( |
249 | "disable-select-optimize" , cl::init(Val: true), cl::Hidden, |
250 | cl::desc("Disable the select-optimization pass from running" )); |
251 | |
252 | /// Enable garbage-collecting empty basic blocks. |
253 | static cl::opt<bool> |
254 | GCEmptyBlocks("gc-empty-basic-blocks" , cl::init(Val: false), cl::Hidden, |
255 | cl::desc("Enable garbage-collecting empty basic blocks" )); |
256 | |
257 | /// Allow standard passes to be disabled by command line options. This supports |
258 | /// simple binary flags that either suppress the pass or do nothing. |
259 | /// i.e. -disable-mypass=false has no effect. |
260 | /// These should be converted to boolOrDefault in order to use applyOverride. |
261 | static IdentifyingPassPtr applyDisable(IdentifyingPassPtr PassID, |
262 | bool Override) { |
263 | if (Override) |
264 | return IdentifyingPassPtr(); |
265 | return PassID; |
266 | } |
267 | |
268 | /// Allow standard passes to be disabled by the command line, regardless of who |
269 | /// is adding the pass. |
270 | /// |
271 | /// StandardID is the pass identified in the standard pass pipeline and provided |
272 | /// to addPass(). It may be a target-specific ID in the case that the target |
273 | /// directly adds its own pass, but in that case we harmlessly fall through. |
274 | /// |
275 | /// TargetID is the pass that the target has configured to override StandardID. |
276 | /// |
277 | /// StandardID may be a pseudo ID. In that case TargetID is the name of the real |
278 | /// pass to run. This allows multiple options to control a single pass depending |
279 | /// on where in the pipeline that pass is added. |
280 | static IdentifyingPassPtr overridePass(AnalysisID StandardID, |
281 | IdentifyingPassPtr TargetID) { |
282 | if (StandardID == &PostRASchedulerID) |
283 | return applyDisable(PassID: TargetID, Override: DisablePostRASched); |
284 | |
285 | if (StandardID == &BranchFolderPassID) |
286 | return applyDisable(PassID: TargetID, Override: DisableBranchFold); |
287 | |
288 | if (StandardID == &TailDuplicateID) |
289 | return applyDisable(PassID: TargetID, Override: DisableTailDuplicate); |
290 | |
291 | if (StandardID == &EarlyTailDuplicateID) |
292 | return applyDisable(PassID: TargetID, Override: DisableEarlyTailDup); |
293 | |
294 | if (StandardID == &MachineBlockPlacementID) |
295 | return applyDisable(PassID: TargetID, Override: DisableBlockPlacement); |
296 | |
297 | if (StandardID == &StackSlotColoringID) |
298 | return applyDisable(PassID: TargetID, Override: DisableSSC); |
299 | |
300 | if (StandardID == &DeadMachineInstructionElimID) |
301 | return applyDisable(PassID: TargetID, Override: DisableMachineDCE); |
302 | |
303 | if (StandardID == &EarlyIfConverterID) |
304 | return applyDisable(PassID: TargetID, Override: DisableEarlyIfConversion); |
305 | |
306 | if (StandardID == &EarlyMachineLICMID) |
307 | return applyDisable(PassID: TargetID, Override: DisableMachineLICM); |
308 | |
309 | if (StandardID == &MachineCSEID) |
310 | return applyDisable(PassID: TargetID, Override: DisableMachineCSE); |
311 | |
312 | if (StandardID == &MachineLICMID) |
313 | return applyDisable(PassID: TargetID, Override: DisablePostRAMachineLICM); |
314 | |
315 | if (StandardID == &MachineSinkingID) |
316 | return applyDisable(PassID: TargetID, Override: DisableMachineSink); |
317 | |
318 | if (StandardID == &PostRAMachineSinkingID) |
319 | return applyDisable(PassID: TargetID, Override: DisablePostRAMachineSink); |
320 | |
321 | if (StandardID == &MachineCopyPropagationID) |
322 | return applyDisable(PassID: TargetID, Override: DisableCopyProp); |
323 | |
324 | return TargetID; |
325 | } |
326 | |
327 | // Find the FSProfile file name. The internal option takes the precedence |
328 | // before getting from TargetMachine. |
329 | static std::string getFSProfileFile(const TargetMachine *TM) { |
330 | if (!FSProfileFile.empty()) |
331 | return FSProfileFile.getValue(); |
332 | const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption(); |
333 | if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse) |
334 | return std::string(); |
335 | return PGOOpt->ProfileFile; |
336 | } |
337 | |
338 | // Find the Profile remapping file name. The internal option takes the |
339 | // precedence before getting from TargetMachine. |
340 | static std::string getFSRemappingFile(const TargetMachine *TM) { |
341 | if (!FSRemappingFile.empty()) |
342 | return FSRemappingFile.getValue(); |
343 | const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption(); |
344 | if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse) |
345 | return std::string(); |
346 | return PGOOpt->ProfileRemappingFile; |
347 | } |
348 | |
349 | //===---------------------------------------------------------------------===// |
350 | /// TargetPassConfig |
351 | //===---------------------------------------------------------------------===// |
352 | |
353 | INITIALIZE_PASS(TargetPassConfig, "targetpassconfig" , |
354 | "Target Pass Configuration" , false, false) |
355 | char TargetPassConfig::ID = 0; |
356 | |
357 | namespace { |
358 | |
359 | struct InsertedPass { |
360 | AnalysisID TargetPassID; |
361 | IdentifyingPassPtr InsertedPassID; |
362 | |
363 | InsertedPass(AnalysisID TargetPassID, IdentifyingPassPtr InsertedPassID) |
364 | : TargetPassID(TargetPassID), InsertedPassID(InsertedPassID) {} |
365 | |
366 | Pass *getInsertedPass() const { |
367 | assert(InsertedPassID.isValid() && "Illegal Pass ID!" ); |
368 | if (InsertedPassID.isInstance()) |
369 | return InsertedPassID.getInstance(); |
370 | Pass *NP = Pass::createPass(ID: InsertedPassID.getID()); |
371 | assert(NP && "Pass ID not registered" ); |
372 | return NP; |
373 | } |
374 | }; |
375 | |
376 | } // end anonymous namespace |
377 | |
378 | namespace llvm { |
379 | |
380 | extern cl::opt<bool> EnableFSDiscriminator; |
381 | |
382 | class PassConfigImpl { |
383 | public: |
384 | // List of passes explicitly substituted by this target. Normally this is |
385 | // empty, but it is a convenient way to suppress or replace specific passes |
386 | // that are part of a standard pass pipeline without overridding the entire |
387 | // pipeline. This mechanism allows target options to inherit a standard pass's |
388 | // user interface. For example, a target may disable a standard pass by |
389 | // default by substituting a pass ID of zero, and the user may still enable |
390 | // that standard pass with an explicit command line option. |
391 | DenseMap<AnalysisID,IdentifyingPassPtr> TargetPasses; |
392 | |
393 | /// Store the pairs of <AnalysisID, AnalysisID> of which the second pass |
394 | /// is inserted after each instance of the first one. |
395 | SmallVector<InsertedPass, 4> InsertedPasses; |
396 | }; |
397 | |
398 | } // end namespace llvm |
399 | |
400 | // Out of line virtual method. |
401 | TargetPassConfig::~TargetPassConfig() { |
402 | delete Impl; |
403 | } |
404 | |
405 | static const PassInfo *getPassInfo(StringRef PassName) { |
406 | if (PassName.empty()) |
407 | return nullptr; |
408 | |
409 | const PassRegistry &PR = *PassRegistry::getPassRegistry(); |
410 | const PassInfo *PI = PR.getPassInfo(Arg: PassName); |
411 | if (!PI) |
412 | report_fatal_error(reason: Twine('\"') + Twine(PassName) + |
413 | Twine("\" pass is not registered." )); |
414 | return PI; |
415 | } |
416 | |
417 | static AnalysisID getPassIDFromName(StringRef PassName) { |
418 | const PassInfo *PI = getPassInfo(PassName); |
419 | return PI ? PI->getTypeInfo() : nullptr; |
420 | } |
421 | |
422 | static std::pair<StringRef, unsigned> |
423 | getPassNameAndInstanceNum(StringRef PassName) { |
424 | StringRef Name, InstanceNumStr; |
425 | std::tie(args&: Name, args&: InstanceNumStr) = PassName.split(Separator: ','); |
426 | |
427 | unsigned InstanceNum = 0; |
428 | if (!InstanceNumStr.empty() && InstanceNumStr.getAsInteger(Radix: 10, Result&: InstanceNum)) |
429 | report_fatal_error(reason: "invalid pass instance specifier " + PassName); |
430 | |
431 | return std::make_pair(x&: Name, y&: InstanceNum); |
432 | } |
433 | |
434 | void TargetPassConfig::setStartStopPasses() { |
435 | StringRef StartBeforeName; |
436 | std::tie(args&: StartBeforeName, args&: StartBeforeInstanceNum) = |
437 | getPassNameAndInstanceNum(PassName: StartBeforeOpt); |
438 | |
439 | StringRef StartAfterName; |
440 | std::tie(args&: StartAfterName, args&: StartAfterInstanceNum) = |
441 | getPassNameAndInstanceNum(PassName: StartAfterOpt); |
442 | |
443 | StringRef StopBeforeName; |
444 | std::tie(args&: StopBeforeName, args&: StopBeforeInstanceNum) |
445 | = getPassNameAndInstanceNum(PassName: StopBeforeOpt); |
446 | |
447 | StringRef StopAfterName; |
448 | std::tie(args&: StopAfterName, args&: StopAfterInstanceNum) |
449 | = getPassNameAndInstanceNum(PassName: StopAfterOpt); |
450 | |
451 | StartBefore = getPassIDFromName(PassName: StartBeforeName); |
452 | StartAfter = getPassIDFromName(PassName: StartAfterName); |
453 | StopBefore = getPassIDFromName(PassName: StopBeforeName); |
454 | StopAfter = getPassIDFromName(PassName: StopAfterName); |
455 | if (StartBefore && StartAfter) |
456 | report_fatal_error(reason: Twine(StartBeforeOptName) + Twine(" and " ) + |
457 | Twine(StartAfterOptName) + Twine(" specified!" )); |
458 | if (StopBefore && StopAfter) |
459 | report_fatal_error(reason: Twine(StopBeforeOptName) + Twine(" and " ) + |
460 | Twine(StopAfterOptName) + Twine(" specified!" )); |
461 | Started = (StartAfter == nullptr) && (StartBefore == nullptr); |
462 | } |
463 | |
464 | CGPassBuilderOption llvm::getCGPassBuilderOption() { |
465 | CGPassBuilderOption Opt; |
466 | |
467 | #define SET_OPTION(Option) \ |
468 | if (Option.getNumOccurrences()) \ |
469 | Opt.Option = Option; |
470 | |
471 | SET_OPTION(EnableFastISelOption) |
472 | SET_OPTION(EnableGlobalISelAbort) |
473 | SET_OPTION(EnableGlobalISelOption) |
474 | SET_OPTION(EnableIPRA) |
475 | SET_OPTION(OptimizeRegAlloc) |
476 | SET_OPTION(VerifyMachineCode) |
477 | SET_OPTION(DisableAtExitBasedGlobalDtorLowering) |
478 | SET_OPTION(DisableExpandReductions) |
479 | SET_OPTION(PrintAfterISel) |
480 | SET_OPTION(FSProfileFile) |
481 | SET_OPTION(GCEmptyBlocks) |
482 | |
483 | #define SET_BOOLEAN_OPTION(Option) Opt.Option = Option; |
484 | |
485 | SET_BOOLEAN_OPTION(EarlyLiveIntervals) |
486 | SET_BOOLEAN_OPTION(EnableBlockPlacementStats) |
487 | SET_BOOLEAN_OPTION(EnableImplicitNullChecks) |
488 | SET_BOOLEAN_OPTION(EnableMachineOutliner) |
489 | SET_BOOLEAN_OPTION(MISchedPostRA) |
490 | SET_BOOLEAN_OPTION(DisableMergeICmps) |
491 | SET_BOOLEAN_OPTION(DisableLSR) |
492 | SET_BOOLEAN_OPTION(DisableConstantHoisting) |
493 | SET_BOOLEAN_OPTION(DisableCGP) |
494 | SET_BOOLEAN_OPTION(DisablePartialLibcallInlining) |
495 | SET_BOOLEAN_OPTION(DisableSelectOptimize) |
496 | SET_BOOLEAN_OPTION(PrintLSR) |
497 | SET_BOOLEAN_OPTION(PrintISelInput) |
498 | SET_BOOLEAN_OPTION(DebugifyAndStripAll) |
499 | SET_BOOLEAN_OPTION(DebugifyCheckAndStripAll) |
500 | SET_BOOLEAN_OPTION(DisableRAFSProfileLoader) |
501 | SET_BOOLEAN_OPTION(DisableCFIFixup) |
502 | SET_BOOLEAN_OPTION(EnableMachineFunctionSplitter) |
503 | |
504 | return Opt; |
505 | } |
506 | |
507 | void llvm::registerCodeGenCallback(PassInstrumentationCallbacks &PIC, |
508 | LLVMTargetMachine &LLVMTM) { |
509 | |
510 | // Register a callback for disabling passes. |
511 | PIC.registerShouldRunOptionalPassCallback(C: [](StringRef P, Any) { |
512 | |
513 | #define DISABLE_PASS(Option, Name) \ |
514 | if (Option && P.contains(#Name)) \ |
515 | return false; |
516 | DISABLE_PASS(DisableBlockPlacement, MachineBlockPlacementPass) |
517 | DISABLE_PASS(DisableBranchFold, BranchFolderPass) |
518 | DISABLE_PASS(DisableCopyProp, MachineCopyPropagationPass) |
519 | DISABLE_PASS(DisableEarlyIfConversion, EarlyIfConverterPass) |
520 | DISABLE_PASS(DisableEarlyTailDup, EarlyTailDuplicatePass) |
521 | DISABLE_PASS(DisableMachineCSE, MachineCSEPass) |
522 | DISABLE_PASS(DisableMachineDCE, DeadMachineInstructionElimPass) |
523 | DISABLE_PASS(DisableMachineLICM, EarlyMachineLICMPass) |
524 | DISABLE_PASS(DisableMachineSink, MachineSinkingPass) |
525 | DISABLE_PASS(DisablePostRAMachineLICM, MachineLICMPass) |
526 | DISABLE_PASS(DisablePostRAMachineSink, PostRAMachineSinkingPass) |
527 | DISABLE_PASS(DisablePostRASched, PostRASchedulerPass) |
528 | DISABLE_PASS(DisableSSC, StackSlotColoringPass) |
529 | DISABLE_PASS(DisableTailDuplicate, TailDuplicatePass) |
530 | |
531 | return true; |
532 | }); |
533 | } |
534 | |
535 | Expected<TargetPassConfig::StartStopInfo> |
536 | TargetPassConfig::getStartStopInfo(PassInstrumentationCallbacks &PIC) { |
537 | auto [StartBefore, StartBeforeInstanceNum] = |
538 | getPassNameAndInstanceNum(PassName: StartBeforeOpt); |
539 | auto [StartAfter, StartAfterInstanceNum] = |
540 | getPassNameAndInstanceNum(PassName: StartAfterOpt); |
541 | auto [StopBefore, StopBeforeInstanceNum] = |
542 | getPassNameAndInstanceNum(PassName: StopBeforeOpt); |
543 | auto [StopAfter, StopAfterInstanceNum] = |
544 | getPassNameAndInstanceNum(PassName: StopAfterOpt); |
545 | |
546 | if (!StartBefore.empty() && !StartAfter.empty()) |
547 | return make_error<StringError>( |
548 | Args: Twine(StartBeforeOptName) + " and " + StartAfterOptName + " specified!" , |
549 | Args: std::make_error_code(e: std::errc::invalid_argument)); |
550 | if (!StopBefore.empty() && !StopAfter.empty()) |
551 | return make_error<StringError>( |
552 | Args: Twine(StopBeforeOptName) + " and " + StopAfterOptName + " specified!" , |
553 | Args: std::make_error_code(e: std::errc::invalid_argument)); |
554 | |
555 | StartStopInfo Result; |
556 | Result.StartPass = StartBefore.empty() ? StartAfter : StartBefore; |
557 | Result.StopPass = StopBefore.empty() ? StopAfter : StopBefore; |
558 | Result.StartInstanceNum = |
559 | StartBefore.empty() ? StartAfterInstanceNum : StartBeforeInstanceNum; |
560 | Result.StopInstanceNum = |
561 | StopBefore.empty() ? StopAfterInstanceNum : StopBeforeInstanceNum; |
562 | Result.StartAfter = !StartAfter.empty(); |
563 | Result.StopAfter = !StopAfter.empty(); |
564 | Result.StartInstanceNum += Result.StartInstanceNum == 0; |
565 | Result.StopInstanceNum += Result.StopInstanceNum == 0; |
566 | return Result; |
567 | } |
568 | |
569 | // Out of line constructor provides default values for pass options and |
570 | // registers all common codegen passes. |
571 | TargetPassConfig::TargetPassConfig(LLVMTargetMachine &TM, PassManagerBase &pm) |
572 | : ImmutablePass(ID), PM(&pm), TM(&TM) { |
573 | Impl = new PassConfigImpl(); |
574 | |
575 | // Register all target independent codegen passes to activate their PassIDs, |
576 | // including this pass itself. |
577 | initializeCodeGen(*PassRegistry::getPassRegistry()); |
578 | |
579 | // Also register alias analysis passes required by codegen passes. |
580 | initializeBasicAAWrapperPassPass(*PassRegistry::getPassRegistry()); |
581 | initializeAAResultsWrapperPassPass(*PassRegistry::getPassRegistry()); |
582 | |
583 | if (EnableIPRA.getNumOccurrences()) |
584 | TM.Options.EnableIPRA = EnableIPRA; |
585 | else { |
586 | // If not explicitly specified, use target default. |
587 | TM.Options.EnableIPRA |= TM.useIPRA(); |
588 | } |
589 | |
590 | if (TM.Options.EnableIPRA) |
591 | setRequiresCodeGenSCCOrder(); |
592 | |
593 | if (EnableGlobalISelAbort.getNumOccurrences()) |
594 | TM.Options.GlobalISelAbort = EnableGlobalISelAbort; |
595 | |
596 | setStartStopPasses(); |
597 | } |
598 | |
599 | CodeGenOptLevel TargetPassConfig::getOptLevel() const { |
600 | return TM->getOptLevel(); |
601 | } |
602 | |
603 | /// Insert InsertedPassID pass after TargetPassID. |
604 | void TargetPassConfig::insertPass(AnalysisID TargetPassID, |
605 | IdentifyingPassPtr InsertedPassID) { |
606 | assert(((!InsertedPassID.isInstance() && |
607 | TargetPassID != InsertedPassID.getID()) || |
608 | (InsertedPassID.isInstance() && |
609 | TargetPassID != InsertedPassID.getInstance()->getPassID())) && |
610 | "Insert a pass after itself!" ); |
611 | Impl->InsertedPasses.emplace_back(Args&: TargetPassID, Args&: InsertedPassID); |
612 | } |
613 | |
614 | /// createPassConfig - Create a pass configuration object to be used by |
615 | /// addPassToEmitX methods for generating a pipeline of CodeGen passes. |
616 | /// |
617 | /// Targets may override this to extend TargetPassConfig. |
618 | TargetPassConfig *LLVMTargetMachine::createPassConfig(PassManagerBase &PM) { |
619 | return new TargetPassConfig(*this, PM); |
620 | } |
621 | |
622 | TargetPassConfig::TargetPassConfig() |
623 | : ImmutablePass(ID) { |
624 | report_fatal_error(reason: "Trying to construct TargetPassConfig without a target " |
625 | "machine. Scheduling a CodeGen pass without a target " |
626 | "triple set?" ); |
627 | } |
628 | |
629 | bool TargetPassConfig::willCompleteCodeGenPipeline() { |
630 | return StopBeforeOpt.empty() && StopAfterOpt.empty(); |
631 | } |
632 | |
633 | bool TargetPassConfig::hasLimitedCodeGenPipeline() { |
634 | return !StartBeforeOpt.empty() || !StartAfterOpt.empty() || |
635 | !willCompleteCodeGenPipeline(); |
636 | } |
637 | |
638 | std::string TargetPassConfig::getLimitedCodeGenPipelineReason() { |
639 | if (!hasLimitedCodeGenPipeline()) |
640 | return std::string(); |
641 | std::string Res; |
642 | static cl::opt<std::string> *PassNames[] = {&StartAfterOpt, &StartBeforeOpt, |
643 | &StopAfterOpt, &StopBeforeOpt}; |
644 | static const char *OptNames[] = {StartAfterOptName, StartBeforeOptName, |
645 | StopAfterOptName, StopBeforeOptName}; |
646 | bool IsFirst = true; |
647 | for (int Idx = 0; Idx < 4; ++Idx) |
648 | if (!PassNames[Idx]->empty()) { |
649 | if (!IsFirst) |
650 | Res += " and " ; |
651 | IsFirst = false; |
652 | Res += OptNames[Idx]; |
653 | } |
654 | return Res; |
655 | } |
656 | |
657 | // Helper to verify the analysis is really immutable. |
658 | void TargetPassConfig::setOpt(bool &Opt, bool Val) { |
659 | assert(!Initialized && "PassConfig is immutable" ); |
660 | Opt = Val; |
661 | } |
662 | |
663 | void TargetPassConfig::substitutePass(AnalysisID StandardID, |
664 | IdentifyingPassPtr TargetID) { |
665 | Impl->TargetPasses[StandardID] = TargetID; |
666 | } |
667 | |
668 | IdentifyingPassPtr TargetPassConfig::getPassSubstitution(AnalysisID ID) const { |
669 | DenseMap<AnalysisID, IdentifyingPassPtr>::const_iterator |
670 | I = Impl->TargetPasses.find(Val: ID); |
671 | if (I == Impl->TargetPasses.end()) |
672 | return ID; |
673 | return I->second; |
674 | } |
675 | |
676 | bool TargetPassConfig::isPassSubstitutedOrOverridden(AnalysisID ID) const { |
677 | IdentifyingPassPtr TargetID = getPassSubstitution(ID); |
678 | IdentifyingPassPtr FinalPtr = overridePass(StandardID: ID, TargetID); |
679 | return !FinalPtr.isValid() || FinalPtr.isInstance() || |
680 | FinalPtr.getID() != ID; |
681 | } |
682 | |
683 | /// Add a pass to the PassManager if that pass is supposed to be run. If the |
684 | /// Started/Stopped flags indicate either that the compilation should start at |
685 | /// a later pass or that it should stop after an earlier pass, then do not add |
686 | /// the pass. Finally, compare the current pass against the StartAfter |
687 | /// and StopAfter options and change the Started/Stopped flags accordingly. |
688 | void TargetPassConfig::addPass(Pass *P) { |
689 | assert(!Initialized && "PassConfig is immutable" ); |
690 | |
691 | // Cache the Pass ID here in case the pass manager finds this pass is |
692 | // redundant with ones already scheduled / available, and deletes it. |
693 | // Fundamentally, once we add the pass to the manager, we no longer own it |
694 | // and shouldn't reference it. |
695 | AnalysisID PassID = P->getPassID(); |
696 | |
697 | if (StartBefore == PassID && StartBeforeCount++ == StartBeforeInstanceNum) |
698 | Started = true; |
699 | if (StopBefore == PassID && StopBeforeCount++ == StopBeforeInstanceNum) |
700 | Stopped = true; |
701 | if (Started && !Stopped) { |
702 | if (AddingMachinePasses) { |
703 | // Construct banner message before PM->add() as that may delete the pass. |
704 | std::string Banner = |
705 | std::string("After " ) + std::string(P->getPassName()); |
706 | addMachinePrePasses(); |
707 | PM->add(P); |
708 | addMachinePostPasses(Banner); |
709 | } else { |
710 | PM->add(P); |
711 | } |
712 | |
713 | // Add the passes after the pass P if there is any. |
714 | for (const auto &IP : Impl->InsertedPasses) |
715 | if (IP.TargetPassID == PassID) |
716 | addPass(P: IP.getInsertedPass()); |
717 | } else { |
718 | delete P; |
719 | } |
720 | |
721 | if (StopAfter == PassID && StopAfterCount++ == StopAfterInstanceNum) |
722 | Stopped = true; |
723 | |
724 | if (StartAfter == PassID && StartAfterCount++ == StartAfterInstanceNum) |
725 | Started = true; |
726 | if (Stopped && !Started) |
727 | report_fatal_error(reason: "Cannot stop compilation after pass that is not run" ); |
728 | } |
729 | |
730 | /// Add a CodeGen pass at this point in the pipeline after checking for target |
731 | /// and command line overrides. |
732 | /// |
733 | /// addPass cannot return a pointer to the pass instance because is internal the |
734 | /// PassManager and the instance we create here may already be freed. |
735 | AnalysisID TargetPassConfig::addPass(AnalysisID PassID) { |
736 | IdentifyingPassPtr TargetID = getPassSubstitution(ID: PassID); |
737 | IdentifyingPassPtr FinalPtr = overridePass(StandardID: PassID, TargetID); |
738 | if (!FinalPtr.isValid()) |
739 | return nullptr; |
740 | |
741 | Pass *P; |
742 | if (FinalPtr.isInstance()) |
743 | P = FinalPtr.getInstance(); |
744 | else { |
745 | P = Pass::createPass(ID: FinalPtr.getID()); |
746 | if (!P) |
747 | llvm_unreachable("Pass ID not registered" ); |
748 | } |
749 | AnalysisID FinalID = P->getPassID(); |
750 | addPass(P); // Ends the lifetime of P. |
751 | |
752 | return FinalID; |
753 | } |
754 | |
755 | void TargetPassConfig::printAndVerify(const std::string &Banner) { |
756 | addPrintPass(Banner); |
757 | addVerifyPass(Banner); |
758 | } |
759 | |
760 | void TargetPassConfig::addPrintPass(const std::string &Banner) { |
761 | if (PrintAfterISel) |
762 | PM->add(P: createMachineFunctionPrinterPass(OS&: dbgs(), Banner)); |
763 | } |
764 | |
765 | void TargetPassConfig::addVerifyPass(const std::string &Banner) { |
766 | bool Verify = VerifyMachineCode == cl::BOU_TRUE; |
767 | #ifdef EXPENSIVE_CHECKS |
768 | if (VerifyMachineCode == cl::BOU_UNSET) |
769 | Verify = TM->isMachineVerifierClean(); |
770 | #endif |
771 | if (Verify) |
772 | PM->add(P: createMachineVerifierPass(Banner)); |
773 | } |
774 | |
775 | void TargetPassConfig::addDebugifyPass() { |
776 | PM->add(P: createDebugifyMachineModulePass()); |
777 | } |
778 | |
779 | void TargetPassConfig::addStripDebugPass() { |
780 | PM->add(P: createStripDebugMachineModulePass(/*OnlyDebugified=*/true)); |
781 | } |
782 | |
783 | void TargetPassConfig::addCheckDebugPass() { |
784 | PM->add(P: createCheckDebugMachineModulePass()); |
785 | } |
786 | |
787 | void TargetPassConfig::addMachinePrePasses(bool AllowDebugify) { |
788 | if (AllowDebugify && DebugifyIsSafe && |
789 | (DebugifyAndStripAll == cl::BOU_TRUE || |
790 | DebugifyCheckAndStripAll == cl::BOU_TRUE)) |
791 | addDebugifyPass(); |
792 | } |
793 | |
794 | void TargetPassConfig::addMachinePostPasses(const std::string &Banner) { |
795 | if (DebugifyIsSafe) { |
796 | if (DebugifyCheckAndStripAll == cl::BOU_TRUE) { |
797 | addCheckDebugPass(); |
798 | addStripDebugPass(); |
799 | } else if (DebugifyAndStripAll == cl::BOU_TRUE) |
800 | addStripDebugPass(); |
801 | } |
802 | addVerifyPass(Banner); |
803 | } |
804 | |
805 | /// Add common target configurable passes that perform LLVM IR to IR transforms |
806 | /// following machine independent optimization. |
807 | void TargetPassConfig::addIRPasses() { |
808 | // Before running any passes, run the verifier to determine if the input |
809 | // coming from the front-end and/or optimizer is valid. |
810 | if (!DisableVerify) |
811 | addPass(P: createVerifierPass()); |
812 | |
813 | if (getOptLevel() != CodeGenOptLevel::None) { |
814 | // Basic AliasAnalysis support. |
815 | // Add TypeBasedAliasAnalysis before BasicAliasAnalysis so that |
816 | // BasicAliasAnalysis wins if they disagree. This is intended to help |
817 | // support "obvious" type-punning idioms. |
818 | addPass(P: createTypeBasedAAWrapperPass()); |
819 | addPass(P: createScopedNoAliasAAWrapperPass()); |
820 | addPass(P: createBasicAAWrapperPass()); |
821 | |
822 | // Run loop strength reduction before anything else. |
823 | if (!DisableLSR) { |
824 | addPass(P: createCanonicalizeFreezeInLoopsPass()); |
825 | addPass(P: createLoopStrengthReducePass()); |
826 | if (PrintLSR) |
827 | addPass(P: createPrintFunctionPass(OS&: dbgs(), |
828 | Banner: "\n\n*** Code after LSR ***\n" )); |
829 | } |
830 | |
831 | // The MergeICmpsPass tries to create memcmp calls by grouping sequences of |
832 | // loads and compares. ExpandMemCmpPass then tries to expand those calls |
833 | // into optimally-sized loads and compares. The transforms are enabled by a |
834 | // target lowering hook. |
835 | if (!DisableMergeICmps) |
836 | addPass(P: createMergeICmpsLegacyPass()); |
837 | addPass(P: createExpandMemCmpLegacyPass()); |
838 | } |
839 | |
840 | // Run GC lowering passes for builtin collectors |
841 | // TODO: add a pass insertion point here |
842 | addPass(PassID: &GCLoweringID); |
843 | addPass(PassID: &ShadowStackGCLoweringID); |
844 | addPass(P: createLowerConstantIntrinsicsPass()); |
845 | |
846 | // For MachO, lower @llvm.global_dtors into @llvm.global_ctors with |
847 | // __cxa_atexit() calls to avoid emitting the deprecated __mod_term_func. |
848 | if (TM->getTargetTriple().isOSBinFormatMachO() && |
849 | !DisableAtExitBasedGlobalDtorLowering) |
850 | addPass(P: createLowerGlobalDtorsLegacyPass()); |
851 | |
852 | // Make sure that no unreachable blocks are instruction selected. |
853 | addPass(P: createUnreachableBlockEliminationPass()); |
854 | |
855 | // Prepare expensive constants for SelectionDAG. |
856 | if (getOptLevel() != CodeGenOptLevel::None && !DisableConstantHoisting) |
857 | addPass(P: createConstantHoistingPass()); |
858 | |
859 | if (getOptLevel() != CodeGenOptLevel::None) |
860 | addPass(P: createReplaceWithVeclibLegacyPass()); |
861 | |
862 | if (getOptLevel() != CodeGenOptLevel::None && !DisablePartialLibcallInlining) |
863 | addPass(P: createPartiallyInlineLibCallsPass()); |
864 | |
865 | // Expand vector predication intrinsics into standard IR instructions. |
866 | // This pass has to run before ScalarizeMaskedMemIntrin and ExpandReduction |
867 | // passes since it emits those kinds of intrinsics. |
868 | addPass(P: createExpandVectorPredicationPass()); |
869 | |
870 | // Add scalarization of target's unsupported masked memory intrinsics pass. |
871 | // the unsupported intrinsic will be replaced with a chain of basic blocks, |
872 | // that stores/loads element one-by-one if the appropriate mask bit is set. |
873 | addPass(P: createScalarizeMaskedMemIntrinLegacyPass()); |
874 | |
875 | // Expand reduction intrinsics into shuffle sequences if the target wants to. |
876 | // Allow disabling it for testing purposes. |
877 | if (!DisableExpandReductions) |
878 | addPass(P: createExpandReductionsPass()); |
879 | |
880 | if (getOptLevel() != CodeGenOptLevel::None) |
881 | addPass(P: createTLSVariableHoistPass()); |
882 | |
883 | // Convert conditional moves to conditional jumps when profitable. |
884 | if (getOptLevel() != CodeGenOptLevel::None && !DisableSelectOptimize) |
885 | addPass(P: createSelectOptimizePass()); |
886 | } |
887 | |
888 | /// Turn exception handling constructs into something the code generators can |
889 | /// handle. |
890 | void TargetPassConfig::addPassesToHandleExceptions() { |
891 | const MCAsmInfo *MCAI = TM->getMCAsmInfo(); |
892 | assert(MCAI && "No MCAsmInfo" ); |
893 | switch (MCAI->getExceptionHandlingType()) { |
894 | case ExceptionHandling::SjLj: |
895 | // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both |
896 | // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise, |
897 | // catch info can get misplaced when a selector ends up more than one block |
898 | // removed from the parent invoke(s). This could happen when a landing |
899 | // pad is shared by multiple invokes and is also a target of a normal |
900 | // edge from elsewhere. |
901 | addPass(P: createSjLjEHPreparePass(TM)); |
902 | [[fallthrough]]; |
903 | case ExceptionHandling::DwarfCFI: |
904 | case ExceptionHandling::ARM: |
905 | case ExceptionHandling::AIX: |
906 | case ExceptionHandling::ZOS: |
907 | addPass(P: createDwarfEHPass(OptLevel: getOptLevel())); |
908 | break; |
909 | case ExceptionHandling::WinEH: |
910 | // We support using both GCC-style and MSVC-style exceptions on Windows, so |
911 | // add both preparation passes. Each pass will only actually run if it |
912 | // recognizes the personality function. |
913 | addPass(P: createWinEHPass()); |
914 | addPass(P: createDwarfEHPass(OptLevel: getOptLevel())); |
915 | break; |
916 | case ExceptionHandling::Wasm: |
917 | // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs |
918 | // on catchpads and cleanuppads because it does not outline them into |
919 | // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we |
920 | // should remove PHIs there. |
921 | addPass(P: createWinEHPass(/*DemoteCatchSwitchPHIOnly=*/true)); |
922 | addPass(P: createWasmEHPass()); |
923 | break; |
924 | case ExceptionHandling::None: |
925 | addPass(P: createLowerInvokePass()); |
926 | |
927 | // The lower invoke pass may create unreachable code. Remove it. |
928 | addPass(P: createUnreachableBlockEliminationPass()); |
929 | break; |
930 | } |
931 | } |
932 | |
933 | /// Add pass to prepare the LLVM IR for code generation. This should be done |
934 | /// before exception handling preparation passes. |
935 | void TargetPassConfig::addCodeGenPrepare() { |
936 | if (getOptLevel() != CodeGenOptLevel::None && !DisableCGP) |
937 | addPass(P: createCodeGenPrepareLegacyPass()); |
938 | } |
939 | |
940 | /// Add common passes that perform LLVM IR to IR transforms in preparation for |
941 | /// instruction selection. |
942 | void TargetPassConfig::addISelPrepare() { |
943 | addPreISel(); |
944 | |
945 | // Force codegen to run according to the callgraph. |
946 | if (requiresCodeGenSCCOrder()) |
947 | addPass(P: new DummyCGSCCPass); |
948 | |
949 | addPass(P: createCallBrPass()); |
950 | |
951 | // Add both the safe stack and the stack protection passes: each of them will |
952 | // only protect functions that have corresponding attributes. |
953 | addPass(P: createSafeStackPass()); |
954 | addPass(P: createStackProtectorPass()); |
955 | |
956 | if (PrintISelInput) |
957 | addPass(P: createPrintFunctionPass( |
958 | OS&: dbgs(), Banner: "\n\n*** Final LLVM Code input to ISel ***\n" )); |
959 | |
960 | // All passes which modify the LLVM IR are now complete; run the verifier |
961 | // to ensure that the IR is valid. |
962 | if (!DisableVerify) |
963 | addPass(P: createVerifierPass()); |
964 | } |
965 | |
966 | bool TargetPassConfig::addCoreISelPasses() { |
967 | // Enable FastISel with -fast-isel, but allow that to be overridden. |
968 | TM->setO0WantsFastISel(EnableFastISelOption != cl::BOU_FALSE); |
969 | |
970 | // Determine an instruction selector. |
971 | enum class SelectorType { SelectionDAG, FastISel, GlobalISel }; |
972 | SelectorType Selector; |
973 | |
974 | if (EnableFastISelOption == cl::BOU_TRUE) |
975 | Selector = SelectorType::FastISel; |
976 | else if (EnableGlobalISelOption == cl::BOU_TRUE || |
977 | (TM->Options.EnableGlobalISel && |
978 | EnableGlobalISelOption != cl::BOU_FALSE)) |
979 | Selector = SelectorType::GlobalISel; |
980 | else if (TM->getOptLevel() == CodeGenOptLevel::None && |
981 | TM->getO0WantsFastISel()) |
982 | Selector = SelectorType::FastISel; |
983 | else |
984 | Selector = SelectorType::SelectionDAG; |
985 | |
986 | // Set consistently TM->Options.EnableFastISel and EnableGlobalISel. |
987 | if (Selector == SelectorType::FastISel) { |
988 | TM->setFastISel(true); |
989 | TM->setGlobalISel(false); |
990 | } else if (Selector == SelectorType::GlobalISel) { |
991 | TM->setFastISel(false); |
992 | TM->setGlobalISel(true); |
993 | } |
994 | |
995 | // FIXME: Injecting into the DAGISel pipeline seems to cause issues with |
996 | // analyses needing to be re-run. This can result in being unable to |
997 | // schedule passes (particularly with 'Function Alias Analysis |
998 | // Results'). It's not entirely clear why but AFAICT this seems to be |
999 | // due to one FunctionPassManager not being able to use analyses from a |
1000 | // previous one. As we're injecting a ModulePass we break the usual |
1001 | // pass manager into two. GlobalISel with the fallback path disabled |
1002 | // and -run-pass seem to be unaffected. The majority of GlobalISel |
1003 | // testing uses -run-pass so this probably isn't too bad. |
1004 | SaveAndRestore SavedDebugifyIsSafe(DebugifyIsSafe); |
1005 | if (Selector != SelectorType::GlobalISel || !isGlobalISelAbortEnabled()) |
1006 | DebugifyIsSafe = false; |
1007 | |
1008 | // Add instruction selector passes. |
1009 | if (Selector == SelectorType::GlobalISel) { |
1010 | SaveAndRestore SavedAddingMachinePasses(AddingMachinePasses, true); |
1011 | if (addIRTranslator()) |
1012 | return true; |
1013 | |
1014 | addPreLegalizeMachineIR(); |
1015 | |
1016 | if (addLegalizeMachineIR()) |
1017 | return true; |
1018 | |
1019 | // Before running the register bank selector, ask the target if it |
1020 | // wants to run some passes. |
1021 | addPreRegBankSelect(); |
1022 | |
1023 | if (addRegBankSelect()) |
1024 | return true; |
1025 | |
1026 | addPreGlobalInstructionSelect(); |
1027 | |
1028 | if (addGlobalInstructionSelect()) |
1029 | return true; |
1030 | |
1031 | // Pass to reset the MachineFunction if the ISel failed. |
1032 | addPass(P: createResetMachineFunctionPass( |
1033 | EmitFallbackDiag: reportDiagnosticWhenGlobalISelFallback(), AbortOnFailedISel: isGlobalISelAbortEnabled())); |
1034 | |
1035 | // Provide a fallback path when we do not want to abort on |
1036 | // not-yet-supported input. |
1037 | if (!isGlobalISelAbortEnabled() && addInstSelector()) |
1038 | return true; |
1039 | |
1040 | } else if (addInstSelector()) |
1041 | return true; |
1042 | |
1043 | // Expand pseudo-instructions emitted by ISel. Don't run the verifier before |
1044 | // FinalizeISel. |
1045 | addPass(PassID: &FinalizeISelID); |
1046 | |
1047 | // Print the instruction selected machine code... |
1048 | printAndVerify(Banner: "After Instruction Selection" ); |
1049 | |
1050 | return false; |
1051 | } |
1052 | |
1053 | bool TargetPassConfig::addISelPasses() { |
1054 | if (TM->useEmulatedTLS()) |
1055 | addPass(P: createLowerEmuTLSPass()); |
1056 | |
1057 | PM->add(P: createTargetTransformInfoWrapperPass(TIRA: TM->getTargetIRAnalysis())); |
1058 | addPass(P: createPreISelIntrinsicLoweringPass()); |
1059 | addPass(P: createExpandLargeDivRemPass()); |
1060 | addPass(P: createExpandLargeFpConvertPass()); |
1061 | addIRPasses(); |
1062 | addCodeGenPrepare(); |
1063 | addPassesToHandleExceptions(); |
1064 | addISelPrepare(); |
1065 | |
1066 | return addCoreISelPasses(); |
1067 | } |
1068 | |
1069 | /// -regalloc=... command line option. |
1070 | static FunctionPass *useDefaultRegisterAllocator() { return nullptr; } |
1071 | static cl::opt<RegisterRegAlloc::FunctionPassCtor, false, |
1072 | RegisterPassParser<RegisterRegAlloc>> |
1073 | RegAlloc("regalloc" , cl::Hidden, cl::init(Val: &useDefaultRegisterAllocator), |
1074 | cl::desc("Register allocator to use" )); |
1075 | |
1076 | /// Add the complete set of target-independent postISel code generator passes. |
1077 | /// |
1078 | /// This can be read as the standard order of major LLVM CodeGen stages. Stages |
1079 | /// with nontrivial configuration or multiple passes are broken out below in |
1080 | /// add%Stage routines. |
1081 | /// |
1082 | /// Any TargetPassConfig::addXX routine may be overriden by the Target. The |
1083 | /// addPre/Post methods with empty header implementations allow injecting |
1084 | /// target-specific fixups just before or after major stages. Additionally, |
1085 | /// targets have the flexibility to change pass order within a stage by |
1086 | /// overriding default implementation of add%Stage routines below. Each |
1087 | /// technique has maintainability tradeoffs because alternate pass orders are |
1088 | /// not well supported. addPre/Post works better if the target pass is easily |
1089 | /// tied to a common pass. But if it has subtle dependencies on multiple passes, |
1090 | /// the target should override the stage instead. |
1091 | /// |
1092 | /// TODO: We could use a single addPre/Post(ID) hook to allow pass injection |
1093 | /// before/after any target-independent pass. But it's currently overkill. |
1094 | void TargetPassConfig::addMachinePasses() { |
1095 | AddingMachinePasses = true; |
1096 | |
1097 | // Add passes that optimize machine instructions in SSA form. |
1098 | if (getOptLevel() != CodeGenOptLevel::None) { |
1099 | addMachineSSAOptimization(); |
1100 | } else { |
1101 | // If the target requests it, assign local variables to stack slots relative |
1102 | // to one another and simplify frame index references where possible. |
1103 | addPass(PassID: &LocalStackSlotAllocationID); |
1104 | } |
1105 | |
1106 | if (TM->Options.EnableIPRA) |
1107 | addPass(P: createRegUsageInfoPropPass()); |
1108 | |
1109 | // Run pre-ra passes. |
1110 | addPreRegAlloc(); |
1111 | |
1112 | // Debugifying the register allocator passes seems to provoke some |
1113 | // non-determinism that affects CodeGen and there doesn't seem to be a point |
1114 | // where it becomes safe again so stop debugifying here. |
1115 | DebugifyIsSafe = false; |
1116 | |
1117 | // Add a FSDiscriminator pass right before RA, so that we could get |
1118 | // more precise SampleFDO profile for RA. |
1119 | if (EnableFSDiscriminator) { |
1120 | addPass(P: createMIRAddFSDiscriminatorsPass( |
1121 | P: sampleprof::FSDiscriminatorPass::Pass1)); |
1122 | const std::string ProfileFile = getFSProfileFile(TM); |
1123 | if (!ProfileFile.empty() && !DisableRAFSProfileLoader) |
1124 | addPass(P: createMIRProfileLoaderPass(File: ProfileFile, RemappingFile: getFSRemappingFile(TM), |
1125 | P: sampleprof::FSDiscriminatorPass::Pass1, |
1126 | FS: nullptr)); |
1127 | } |
1128 | |
1129 | // Run register allocation and passes that are tightly coupled with it, |
1130 | // including phi elimination and scheduling. |
1131 | if (getOptimizeRegAlloc()) |
1132 | addOptimizedRegAlloc(); |
1133 | else |
1134 | addFastRegAlloc(); |
1135 | |
1136 | // Run post-ra passes. |
1137 | addPostRegAlloc(); |
1138 | |
1139 | addPass(PassID: &RemoveRedundantDebugValuesID); |
1140 | |
1141 | addPass(PassID: &FixupStatepointCallerSavedID); |
1142 | |
1143 | // Insert prolog/epilog code. Eliminate abstract frame index references... |
1144 | if (getOptLevel() != CodeGenOptLevel::None) { |
1145 | addPass(PassID: &PostRAMachineSinkingID); |
1146 | addPass(PassID: &ShrinkWrapID); |
1147 | } |
1148 | |
1149 | // Prolog/Epilog inserter needs a TargetMachine to instantiate. But only |
1150 | // do so if it hasn't been disabled, substituted, or overridden. |
1151 | if (!isPassSubstitutedOrOverridden(ID: &PrologEpilogCodeInserterID)) |
1152 | addPass(P: createPrologEpilogInserterPass()); |
1153 | |
1154 | /// Add passes that optimize machine instructions after register allocation. |
1155 | if (getOptLevel() != CodeGenOptLevel::None) |
1156 | addMachineLateOptimization(); |
1157 | |
1158 | // Expand pseudo instructions before second scheduling pass. |
1159 | addPass(PassID: &ExpandPostRAPseudosID); |
1160 | |
1161 | // Run pre-sched2 passes. |
1162 | addPreSched2(); |
1163 | |
1164 | if (EnableImplicitNullChecks) |
1165 | addPass(PassID: &ImplicitNullChecksID); |
1166 | |
1167 | // Second pass scheduler. |
1168 | // Let Target optionally insert this pass by itself at some other |
1169 | // point. |
1170 | if (getOptLevel() != CodeGenOptLevel::None && |
1171 | !TM->targetSchedulesPostRAScheduling()) { |
1172 | if (MISchedPostRA) |
1173 | addPass(PassID: &PostMachineSchedulerID); |
1174 | else |
1175 | addPass(PassID: &PostRASchedulerID); |
1176 | } |
1177 | |
1178 | // GC |
1179 | addGCPasses(); |
1180 | |
1181 | // Basic block placement. |
1182 | if (getOptLevel() != CodeGenOptLevel::None) |
1183 | addBlockPlacement(); |
1184 | |
1185 | // Insert before XRay Instrumentation. |
1186 | addPass(PassID: &FEntryInserterID); |
1187 | |
1188 | addPass(PassID: &XRayInstrumentationID); |
1189 | addPass(PassID: &PatchableFunctionID); |
1190 | |
1191 | addPreEmitPass(); |
1192 | |
1193 | if (TM->Options.EnableIPRA) |
1194 | // Collect register usage information and produce a register mask of |
1195 | // clobbered registers, to be used to optimize call sites. |
1196 | addPass(P: createRegUsageInfoCollector()); |
1197 | |
1198 | // FIXME: Some backends are incompatible with running the verifier after |
1199 | // addPreEmitPass. Maybe only pass "false" here for those targets? |
1200 | addPass(PassID: &FuncletLayoutID); |
1201 | |
1202 | addPass(PassID: &StackMapLivenessID); |
1203 | addPass(PassID: &LiveDebugValuesID); |
1204 | addPass(PassID: &MachineSanitizerBinaryMetadataID); |
1205 | |
1206 | if (TM->Options.EnableMachineOutliner && |
1207 | getOptLevel() != CodeGenOptLevel::None && |
1208 | EnableMachineOutliner != RunOutliner::NeverOutline) { |
1209 | bool RunOnAllFunctions = |
1210 | (EnableMachineOutliner == RunOutliner::AlwaysOutline); |
1211 | bool AddOutliner = |
1212 | RunOnAllFunctions || TM->Options.SupportsDefaultOutlining; |
1213 | if (AddOutliner) |
1214 | addPass(P: createMachineOutlinerPass(RunOnAllFunctions)); |
1215 | } |
1216 | |
1217 | if (GCEmptyBlocks) |
1218 | addPass(P: llvm::createGCEmptyBasicBlocksPass()); |
1219 | |
1220 | if (EnableFSDiscriminator) |
1221 | addPass(P: createMIRAddFSDiscriminatorsPass( |
1222 | P: sampleprof::FSDiscriminatorPass::PassLast)); |
1223 | |
1224 | bool NeedsBBSections = |
1225 | TM->getBBSectionsType() != llvm::BasicBlockSection::None; |
1226 | // Machine function splitter uses the basic block sections feature. Both |
1227 | // cannot be enabled at the same time. We do not apply machine function |
1228 | // splitter if -basic-block-sections is requested. |
1229 | if (!NeedsBBSections && (TM->Options.EnableMachineFunctionSplitter || |
1230 | EnableMachineFunctionSplitter)) { |
1231 | const std::string ProfileFile = getFSProfileFile(TM); |
1232 | if (!ProfileFile.empty()) { |
1233 | if (EnableFSDiscriminator) { |
1234 | addPass(P: createMIRProfileLoaderPass( |
1235 | File: ProfileFile, RemappingFile: getFSRemappingFile(TM), |
1236 | P: sampleprof::FSDiscriminatorPass::PassLast, FS: nullptr)); |
1237 | } else { |
1238 | // Sample profile is given, but FSDiscriminator is not |
1239 | // enabled, this may result in performance regression. |
1240 | WithColor::warning() |
1241 | << "Using AutoFDO without FSDiscriminator for MFS may regress " |
1242 | "performance.\n" ; |
1243 | } |
1244 | } |
1245 | addPass(P: createMachineFunctionSplitterPass()); |
1246 | } |
1247 | // We run the BasicBlockSections pass if either we need BB sections or BB |
1248 | // address map (or both). |
1249 | if (NeedsBBSections || TM->Options.BBAddrMap) { |
1250 | if (TM->getBBSectionsType() == llvm::BasicBlockSection::List) { |
1251 | addPass(P: llvm::createBasicBlockSectionsProfileReaderWrapperPass( |
1252 | Buf: TM->getBBSectionsFuncListBuf())); |
1253 | addPass(P: llvm::createBasicBlockPathCloningPass()); |
1254 | } |
1255 | addPass(P: llvm::createBasicBlockSectionsPass()); |
1256 | } |
1257 | |
1258 | addPostBBSections(); |
1259 | |
1260 | if (!DisableCFIFixup && TM->Options.EnableCFIFixup) |
1261 | addPass(P: createCFIFixup()); |
1262 | |
1263 | PM->add(P: createStackFrameLayoutAnalysisPass()); |
1264 | |
1265 | // Add passes that directly emit MI after all other MI passes. |
1266 | addPreEmitPass2(); |
1267 | |
1268 | AddingMachinePasses = false; |
1269 | } |
1270 | |
1271 | /// Add passes that optimize machine instructions in SSA form. |
1272 | void TargetPassConfig::addMachineSSAOptimization() { |
1273 | // Pre-ra tail duplication. |
1274 | addPass(PassID: &EarlyTailDuplicateID); |
1275 | |
1276 | // Optimize PHIs before DCE: removing dead PHI cycles may make more |
1277 | // instructions dead. |
1278 | addPass(PassID: &OptimizePHIsID); |
1279 | |
1280 | // This pass merges large allocas. StackSlotColoring is a different pass |
1281 | // which merges spill slots. |
1282 | addPass(PassID: &StackColoringID); |
1283 | |
1284 | // If the target requests it, assign local variables to stack slots relative |
1285 | // to one another and simplify frame index references where possible. |
1286 | addPass(PassID: &LocalStackSlotAllocationID); |
1287 | |
1288 | // With optimization, dead code should already be eliminated. However |
1289 | // there is one known exception: lowered code for arguments that are only |
1290 | // used by tail calls, where the tail calls reuse the incoming stack |
1291 | // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll). |
1292 | addPass(PassID: &DeadMachineInstructionElimID); |
1293 | |
1294 | // Allow targets to insert passes that improve instruction level parallelism, |
1295 | // like if-conversion. Such passes will typically need dominator trees and |
1296 | // loop info, just like LICM and CSE below. |
1297 | addILPOpts(); |
1298 | |
1299 | addPass(PassID: &EarlyMachineLICMID); |
1300 | addPass(PassID: &MachineCSEID); |
1301 | |
1302 | addPass(PassID: &MachineSinkingID); |
1303 | |
1304 | addPass(PassID: &PeepholeOptimizerID); |
1305 | // Clean-up the dead code that may have been generated by peephole |
1306 | // rewriting. |
1307 | addPass(PassID: &DeadMachineInstructionElimID); |
1308 | } |
1309 | |
1310 | //===---------------------------------------------------------------------===// |
1311 | /// Register Allocation Pass Configuration |
1312 | //===---------------------------------------------------------------------===// |
1313 | |
1314 | bool TargetPassConfig::getOptimizeRegAlloc() const { |
1315 | switch (OptimizeRegAlloc) { |
1316 | case cl::BOU_UNSET: |
1317 | return getOptLevel() != CodeGenOptLevel::None; |
1318 | case cl::BOU_TRUE: return true; |
1319 | case cl::BOU_FALSE: return false; |
1320 | } |
1321 | llvm_unreachable("Invalid optimize-regalloc state" ); |
1322 | } |
1323 | |
1324 | /// A dummy default pass factory indicates whether the register allocator is |
1325 | /// overridden on the command line. |
1326 | static llvm::once_flag InitializeDefaultRegisterAllocatorFlag; |
1327 | |
1328 | static RegisterRegAlloc |
1329 | defaultRegAlloc("default" , |
1330 | "pick register allocator based on -O option" , |
1331 | useDefaultRegisterAllocator); |
1332 | |
1333 | static void initializeDefaultRegisterAllocatorOnce() { |
1334 | if (!RegisterRegAlloc::getDefault()) |
1335 | RegisterRegAlloc::setDefault(RegAlloc); |
1336 | } |
1337 | |
1338 | /// Instantiate the default register allocator pass for this target for either |
1339 | /// the optimized or unoptimized allocation path. This will be added to the pass |
1340 | /// manager by addFastRegAlloc in the unoptimized case or addOptimizedRegAlloc |
1341 | /// in the optimized case. |
1342 | /// |
1343 | /// A target that uses the standard regalloc pass order for fast or optimized |
1344 | /// allocation may still override this for per-target regalloc |
1345 | /// selection. But -regalloc=... always takes precedence. |
1346 | FunctionPass *TargetPassConfig::createTargetRegisterAllocator(bool Optimized) { |
1347 | if (Optimized) |
1348 | return createGreedyRegisterAllocator(); |
1349 | else |
1350 | return createFastRegisterAllocator(); |
1351 | } |
1352 | |
1353 | /// Find and instantiate the register allocation pass requested by this target |
1354 | /// at the current optimization level. Different register allocators are |
1355 | /// defined as separate passes because they may require different analysis. |
1356 | /// |
1357 | /// This helper ensures that the regalloc= option is always available, |
1358 | /// even for targets that override the default allocator. |
1359 | /// |
1360 | /// FIXME: When MachinePassRegistry register pass IDs instead of function ptrs, |
1361 | /// this can be folded into addPass. |
1362 | FunctionPass *TargetPassConfig::createRegAllocPass(bool Optimized) { |
1363 | // Initialize the global default. |
1364 | llvm::call_once(flag&: InitializeDefaultRegisterAllocatorFlag, |
1365 | F&: initializeDefaultRegisterAllocatorOnce); |
1366 | |
1367 | RegisterRegAlloc::FunctionPassCtor Ctor = RegisterRegAlloc::getDefault(); |
1368 | if (Ctor != useDefaultRegisterAllocator) |
1369 | return Ctor(); |
1370 | |
1371 | // With no -regalloc= override, ask the target for a regalloc pass. |
1372 | return createTargetRegisterAllocator(Optimized); |
1373 | } |
1374 | |
1375 | bool TargetPassConfig::isCustomizedRegAlloc() { |
1376 | return RegAlloc != |
1377 | (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator; |
1378 | } |
1379 | |
1380 | bool TargetPassConfig::addRegAssignAndRewriteFast() { |
1381 | if (RegAlloc != (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator && |
1382 | RegAlloc != (RegisterRegAlloc::FunctionPassCtor)&createFastRegisterAllocator) |
1383 | report_fatal_error(reason: "Must use fast (default) register allocator for unoptimized regalloc." ); |
1384 | |
1385 | addPass(P: createRegAllocPass(Optimized: false)); |
1386 | |
1387 | // Allow targets to change the register assignments after |
1388 | // fast register allocation. |
1389 | addPostFastRegAllocRewrite(); |
1390 | return true; |
1391 | } |
1392 | |
1393 | bool TargetPassConfig::addRegAssignAndRewriteOptimized() { |
1394 | // Add the selected register allocation pass. |
1395 | addPass(P: createRegAllocPass(Optimized: true)); |
1396 | |
1397 | // Allow targets to change the register assignments before rewriting. |
1398 | addPreRewrite(); |
1399 | |
1400 | // Finally rewrite virtual registers. |
1401 | addPass(PassID: &VirtRegRewriterID); |
1402 | |
1403 | // Regalloc scoring for ML-driven eviction - noop except when learning a new |
1404 | // eviction policy. |
1405 | addPass(P: createRegAllocScoringPass()); |
1406 | return true; |
1407 | } |
1408 | |
1409 | /// Return true if the default global register allocator is in use and |
1410 | /// has not be overriden on the command line with '-regalloc=...' |
1411 | bool TargetPassConfig::usingDefaultRegAlloc() const { |
1412 | return RegAlloc.getNumOccurrences() == 0; |
1413 | } |
1414 | |
1415 | /// Add the minimum set of target-independent passes that are required for |
1416 | /// register allocation. No coalescing or scheduling. |
1417 | void TargetPassConfig::addFastRegAlloc() { |
1418 | addPass(PassID: &PHIEliminationID); |
1419 | addPass(PassID: &TwoAddressInstructionPassID); |
1420 | |
1421 | addRegAssignAndRewriteFast(); |
1422 | } |
1423 | |
1424 | /// Add standard target-independent passes that are tightly coupled with |
1425 | /// optimized register allocation, including coalescing, machine instruction |
1426 | /// scheduling, and register allocation itself. |
1427 | void TargetPassConfig::addOptimizedRegAlloc() { |
1428 | addPass(PassID: &DetectDeadLanesID); |
1429 | |
1430 | addPass(PassID: &ProcessImplicitDefsID); |
1431 | |
1432 | // LiveVariables currently requires pure SSA form. |
1433 | // |
1434 | // FIXME: Once TwoAddressInstruction pass no longer uses kill flags, |
1435 | // LiveVariables can be removed completely, and LiveIntervals can be directly |
1436 | // computed. (We still either need to regenerate kill flags after regalloc, or |
1437 | // preferably fix the scavenger to not depend on them). |
1438 | // FIXME: UnreachableMachineBlockElim is a dependant pass of LiveVariables. |
1439 | // When LiveVariables is removed this has to be removed/moved either. |
1440 | // Explicit addition of UnreachableMachineBlockElim allows stopping before or |
1441 | // after it with -stop-before/-stop-after. |
1442 | addPass(PassID: &UnreachableMachineBlockElimID); |
1443 | addPass(PassID: &LiveVariablesID); |
1444 | |
1445 | // Edge splitting is smarter with machine loop info. |
1446 | addPass(PassID: &MachineLoopInfoID); |
1447 | addPass(PassID: &PHIEliminationID); |
1448 | |
1449 | // Eventually, we want to run LiveIntervals before PHI elimination. |
1450 | if (EarlyLiveIntervals) |
1451 | addPass(PassID: &LiveIntervalsID); |
1452 | |
1453 | addPass(PassID: &TwoAddressInstructionPassID); |
1454 | addPass(PassID: &RegisterCoalescerID); |
1455 | |
1456 | // The machine scheduler may accidentally create disconnected components |
1457 | // when moving subregister definitions around, avoid this by splitting them to |
1458 | // separate vregs before. Splitting can also improve reg. allocation quality. |
1459 | addPass(PassID: &RenameIndependentSubregsID); |
1460 | |
1461 | // PreRA instruction scheduling. |
1462 | addPass(PassID: &MachineSchedulerID); |
1463 | |
1464 | if (addRegAssignAndRewriteOptimized()) { |
1465 | // Perform stack slot coloring and post-ra machine LICM. |
1466 | addPass(PassID: &StackSlotColoringID); |
1467 | |
1468 | // Allow targets to expand pseudo instructions depending on the choice of |
1469 | // registers before MachineCopyPropagation. |
1470 | addPostRewrite(); |
1471 | |
1472 | // Copy propagate to forward register uses and try to eliminate COPYs that |
1473 | // were not coalesced. |
1474 | addPass(PassID: &MachineCopyPropagationID); |
1475 | |
1476 | // Run post-ra machine LICM to hoist reloads / remats. |
1477 | // |
1478 | // FIXME: can this move into MachineLateOptimization? |
1479 | addPass(PassID: &MachineLICMID); |
1480 | } |
1481 | } |
1482 | |
1483 | //===---------------------------------------------------------------------===// |
1484 | /// Post RegAlloc Pass Configuration |
1485 | //===---------------------------------------------------------------------===// |
1486 | |
1487 | /// Add passes that optimize machine instructions after register allocation. |
1488 | void TargetPassConfig::addMachineLateOptimization() { |
1489 | // Cleanup of redundant immediate/address loads. |
1490 | addPass(PassID: &MachineLateInstrsCleanupID); |
1491 | |
1492 | // Branch folding must be run after regalloc and prolog/epilog insertion. |
1493 | addPass(PassID: &BranchFolderPassID); |
1494 | |
1495 | // Tail duplication. |
1496 | // Note that duplicating tail just increases code size and degrades |
1497 | // performance for targets that require Structured Control Flow. |
1498 | // In addition it can also make CFG irreducible. Thus we disable it. |
1499 | if (!TM->requiresStructuredCFG()) |
1500 | addPass(PassID: &TailDuplicateID); |
1501 | |
1502 | // Copy propagation. |
1503 | addPass(PassID: &MachineCopyPropagationID); |
1504 | } |
1505 | |
1506 | /// Add standard GC passes. |
1507 | bool TargetPassConfig::addGCPasses() { |
1508 | addPass(PassID: &GCMachineCodeAnalysisID); |
1509 | return true; |
1510 | } |
1511 | |
1512 | /// Add standard basic block placement passes. |
1513 | void TargetPassConfig::addBlockPlacement() { |
1514 | if (EnableFSDiscriminator) { |
1515 | addPass(P: createMIRAddFSDiscriminatorsPass( |
1516 | P: sampleprof::FSDiscriminatorPass::Pass2)); |
1517 | const std::string ProfileFile = getFSProfileFile(TM); |
1518 | if (!ProfileFile.empty() && !DisableLayoutFSProfileLoader) |
1519 | addPass(P: createMIRProfileLoaderPass(File: ProfileFile, RemappingFile: getFSRemappingFile(TM), |
1520 | P: sampleprof::FSDiscriminatorPass::Pass2, |
1521 | FS: nullptr)); |
1522 | } |
1523 | if (addPass(PassID: &MachineBlockPlacementID)) { |
1524 | // Run a separate pass to collect block placement statistics. |
1525 | if (EnableBlockPlacementStats) |
1526 | addPass(PassID: &MachineBlockPlacementStatsID); |
1527 | } |
1528 | } |
1529 | |
1530 | //===---------------------------------------------------------------------===// |
1531 | /// GlobalISel Configuration |
1532 | //===---------------------------------------------------------------------===// |
1533 | bool TargetPassConfig::isGlobalISelAbortEnabled() const { |
1534 | return TM->Options.GlobalISelAbort == GlobalISelAbortMode::Enable; |
1535 | } |
1536 | |
1537 | bool TargetPassConfig::reportDiagnosticWhenGlobalISelFallback() const { |
1538 | return TM->Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag; |
1539 | } |
1540 | |
1541 | bool TargetPassConfig::isGISelCSEEnabled() const { |
1542 | return true; |
1543 | } |
1544 | |
1545 | std::unique_ptr<CSEConfigBase> TargetPassConfig::getCSEConfig() const { |
1546 | return std::make_unique<CSEConfigBase>(); |
1547 | } |
1548 | |