1 | //===-- X86TargetMachine.cpp - Define TargetMachine for the X86 -----------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file defines the X86 specific subclass of TargetMachine. |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | |
13 | #include "X86TargetMachine.h" |
14 | #include "MCTargetDesc/X86MCTargetDesc.h" |
15 | #include "TargetInfo/X86TargetInfo.h" |
16 | #include "X86.h" |
17 | #include "X86MachineFunctionInfo.h" |
18 | #include "X86MacroFusion.h" |
19 | #include "X86Subtarget.h" |
20 | #include "X86TargetObjectFile.h" |
21 | #include "X86TargetTransformInfo.h" |
22 | #include "llvm/ADT/STLExtras.h" |
23 | #include "llvm/ADT/SmallString.h" |
24 | #include "llvm/ADT/StringRef.h" |
25 | #include "llvm/Analysis/TargetTransformInfo.h" |
26 | #include "llvm/CodeGen/ExecutionDomainFix.h" |
27 | #include "llvm/CodeGen/GlobalISel/CSEInfo.h" |
28 | #include "llvm/CodeGen/GlobalISel/CallLowering.h" |
29 | #include "llvm/CodeGen/GlobalISel/IRTranslator.h" |
30 | #include "llvm/CodeGen/GlobalISel/InstructionSelect.h" |
31 | #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" |
32 | #include "llvm/CodeGen/GlobalISel/Legalizer.h" |
33 | #include "llvm/CodeGen/GlobalISel/RegBankSelect.h" |
34 | #include "llvm/CodeGen/MachineScheduler.h" |
35 | #include "llvm/CodeGen/Passes.h" |
36 | #include "llvm/CodeGen/RegAllocRegistry.h" |
37 | #include "llvm/CodeGen/TargetPassConfig.h" |
38 | #include "llvm/IR/Attributes.h" |
39 | #include "llvm/IR/DataLayout.h" |
40 | #include "llvm/IR/Function.h" |
41 | #include "llvm/MC/MCAsmInfo.h" |
42 | #include "llvm/MC/TargetRegistry.h" |
43 | #include "llvm/Pass.h" |
44 | #include "llvm/Support/CodeGen.h" |
45 | #include "llvm/Support/CommandLine.h" |
46 | #include "llvm/Support/ErrorHandling.h" |
47 | #include "llvm/Target/TargetLoweringObjectFile.h" |
48 | #include "llvm/Target/TargetOptions.h" |
49 | #include "llvm/TargetParser/Triple.h" |
50 | #include "llvm/Transforms/CFGuard.h" |
51 | #include <memory> |
52 | #include <optional> |
53 | #include <string> |
54 | |
55 | using namespace llvm; |
56 | |
57 | static cl::opt<bool> EnableMachineCombinerPass("x86-machine-combiner" , |
58 | cl::desc("Enable the machine combiner pass" ), |
59 | cl::init(Val: true), cl::Hidden); |
60 | |
61 | static cl::opt<bool> |
62 | EnableTileRAPass("x86-tile-ra" , |
63 | cl::desc("Enable the tile register allocation pass" ), |
64 | cl::init(Val: true), cl::Hidden); |
65 | |
66 | extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeX86Target() { |
67 | // Register the target. |
68 | RegisterTargetMachine<X86TargetMachine> X(getTheX86_32Target()); |
69 | RegisterTargetMachine<X86TargetMachine> Y(getTheX86_64Target()); |
70 | |
71 | PassRegistry &PR = *PassRegistry::getPassRegistry(); |
72 | initializeX86LowerAMXIntrinsicsLegacyPassPass(PR); |
73 | initializeX86LowerAMXTypeLegacyPassPass(PR); |
74 | initializeX86PreTileConfigPass(PR); |
75 | initializeGlobalISel(PR); |
76 | initializeWinEHStatePassPass(PR); |
77 | initializeFixupBWInstPassPass(PR); |
78 | initializeCompressEVEXPassPass(PR); |
79 | initializeFixupLEAPassPass(PR); |
80 | initializeFPSPass(PR); |
81 | initializeX86FixupSetCCPassPass(PR); |
82 | initializeX86CallFrameOptimizationPass(PR); |
83 | initializeX86CmovConverterPassPass(PR); |
84 | initializeX86TileConfigPass(PR); |
85 | initializeX86FastPreTileConfigPass(PR); |
86 | initializeX86FastTileConfigPass(PR); |
87 | initializeKCFIPass(PR); |
88 | initializeX86LowerTileCopyPass(PR); |
89 | initializeX86ExpandPseudoPass(PR); |
90 | initializeX86ExecutionDomainFixPass(PR); |
91 | initializeX86DomainReassignmentPass(PR); |
92 | initializeX86AvoidSFBPassPass(PR); |
93 | initializeX86AvoidTrailingCallPassPass(PR); |
94 | initializeX86SpeculativeLoadHardeningPassPass(PR); |
95 | initializeX86SpeculativeExecutionSideEffectSuppressionPass(PR); |
96 | initializeX86FlagsCopyLoweringPassPass(PR); |
97 | initializeX86LoadValueInjectionLoadHardeningPassPass(PR); |
98 | initializeX86LoadValueInjectionRetHardeningPassPass(PR); |
99 | initializeX86OptimizeLEAPassPass(PR); |
100 | initializeX86PartialReductionPass(PR); |
101 | initializePseudoProbeInserterPass(PR); |
102 | initializeX86ReturnThunksPass(PR); |
103 | initializeX86DAGToDAGISelPass(PR); |
104 | initializeX86ArgumentStackSlotPassPass(PR); |
105 | initializeX86FixupInstTuningPassPass(PR); |
106 | initializeX86FixupVectorConstantsPassPass(PR); |
107 | } |
108 | |
109 | static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) { |
110 | if (TT.isOSBinFormatMachO()) { |
111 | if (TT.getArch() == Triple::x86_64) |
112 | return std::make_unique<X86_64MachoTargetObjectFile>(); |
113 | return std::make_unique<TargetLoweringObjectFileMachO>(); |
114 | } |
115 | |
116 | if (TT.isOSBinFormatCOFF()) |
117 | return std::make_unique<TargetLoweringObjectFileCOFF>(); |
118 | |
119 | if (TT.getArch() == Triple::x86_64) |
120 | return std::make_unique<X86_64ELFTargetObjectFile>(); |
121 | return std::make_unique<X86ELFTargetObjectFile>(); |
122 | } |
123 | |
124 | static std::string computeDataLayout(const Triple &TT) { |
125 | // X86 is little endian |
126 | std::string Ret = "e" ; |
127 | |
128 | Ret += DataLayout::getManglingComponent(T: TT); |
129 | // X86 and x32 have 32 bit pointers. |
130 | if (!TT.isArch64Bit() || TT.isX32() || TT.isOSNaCl()) |
131 | Ret += "-p:32:32" ; |
132 | |
133 | // Address spaces for 32 bit signed, 32 bit unsigned, and 64 bit pointers. |
134 | Ret += "-p270:32:32-p271:32:32-p272:64:64" ; |
135 | |
136 | // Some ABIs align 64 bit integers and doubles to 64 bits, others to 32. |
137 | // 128 bit integers are not specified in the 32-bit ABIs but are used |
138 | // internally for lowering f128, so we match the alignment to that. |
139 | if (TT.isArch64Bit() || TT.isOSWindows() || TT.isOSNaCl()) |
140 | Ret += "-i64:64-i128:128" ; |
141 | else if (TT.isOSIAMCU()) |
142 | Ret += "-i64:32-f64:32" ; |
143 | else |
144 | Ret += "-i128:128-f64:32:64" ; |
145 | |
146 | // Some ABIs align long double to 128 bits, others to 32. |
147 | if (TT.isOSNaCl() || TT.isOSIAMCU()) |
148 | ; // No f80 |
149 | else if (TT.isArch64Bit() || TT.isOSDarwin() || TT.isWindowsMSVCEnvironment()) |
150 | Ret += "-f80:128" ; |
151 | else |
152 | Ret += "-f80:32" ; |
153 | |
154 | if (TT.isOSIAMCU()) |
155 | Ret += "-f128:32" ; |
156 | |
157 | // The registers can hold 8, 16, 32 or, in x86-64, 64 bits. |
158 | if (TT.isArch64Bit()) |
159 | Ret += "-n8:16:32:64" ; |
160 | else |
161 | Ret += "-n8:16:32" ; |
162 | |
163 | // The stack is aligned to 32 bits on some ABIs and 128 bits on others. |
164 | if ((!TT.isArch64Bit() && TT.isOSWindows()) || TT.isOSIAMCU()) |
165 | Ret += "-a:0:32-S32" ; |
166 | else |
167 | Ret += "-S128" ; |
168 | |
169 | return Ret; |
170 | } |
171 | |
172 | static Reloc::Model getEffectiveRelocModel(const Triple &TT, bool JIT, |
173 | std::optional<Reloc::Model> RM) { |
174 | bool is64Bit = TT.getArch() == Triple::x86_64; |
175 | if (!RM) { |
176 | // JIT codegen should use static relocations by default, since it's |
177 | // typically executed in process and not relocatable. |
178 | if (JIT) |
179 | return Reloc::Static; |
180 | |
181 | // Darwin defaults to PIC in 64 bit mode and dynamic-no-pic in 32 bit mode. |
182 | // Win64 requires rip-rel addressing, thus we force it to PIC. Otherwise we |
183 | // use static relocation model by default. |
184 | if (TT.isOSDarwin()) { |
185 | if (is64Bit) |
186 | return Reloc::PIC_; |
187 | return Reloc::DynamicNoPIC; |
188 | } |
189 | if (TT.isOSWindows() && is64Bit) |
190 | return Reloc::PIC_; |
191 | return Reloc::Static; |
192 | } |
193 | |
194 | // ELF and X86-64 don't have a distinct DynamicNoPIC model. DynamicNoPIC |
195 | // is defined as a model for code which may be used in static or dynamic |
196 | // executables but not necessarily a shared library. On X86-32 we just |
197 | // compile in -static mode, in x86-64 we use PIC. |
198 | if (*RM == Reloc::DynamicNoPIC) { |
199 | if (is64Bit) |
200 | return Reloc::PIC_; |
201 | if (!TT.isOSDarwin()) |
202 | return Reloc::Static; |
203 | } |
204 | |
205 | // If we are on Darwin, disallow static relocation model in X86-64 mode, since |
206 | // the Mach-O file format doesn't support it. |
207 | if (*RM == Reloc::Static && TT.isOSDarwin() && is64Bit) |
208 | return Reloc::PIC_; |
209 | |
210 | return *RM; |
211 | } |
212 | |
213 | static CodeModel::Model |
214 | getEffectiveX86CodeModel(const Triple &TT, std::optional<CodeModel::Model> CM, |
215 | bool JIT) { |
216 | bool Is64Bit = TT.getArch() == Triple::x86_64; |
217 | if (CM) { |
218 | if (*CM == CodeModel::Tiny) |
219 | report_fatal_error(reason: "Target does not support the tiny CodeModel" , gen_crash_diag: false); |
220 | return *CM; |
221 | } |
222 | if (JIT) |
223 | return Is64Bit ? CodeModel::Large : CodeModel::Small; |
224 | return CodeModel::Small; |
225 | } |
226 | |
227 | /// Create an X86 target. |
228 | /// |
229 | X86TargetMachine::X86TargetMachine(const Target &T, const Triple &TT, |
230 | StringRef CPU, StringRef FS, |
231 | const TargetOptions &Options, |
232 | std::optional<Reloc::Model> RM, |
233 | std::optional<CodeModel::Model> CM, |
234 | CodeGenOptLevel OL, bool JIT) |
235 | : LLVMTargetMachine( |
236 | T, computeDataLayout(TT), TT, CPU, FS, Options, |
237 | getEffectiveRelocModel(TT, JIT, RM), |
238 | getEffectiveX86CodeModel(TT, CM, JIT), |
239 | OL), |
240 | TLOF(createTLOF(TT: getTargetTriple())), IsJIT(JIT) { |
241 | // On PS4/PS5, the "return address" of a 'noreturn' call must still be within |
242 | // the calling function, and TrapUnreachable is an easy way to get that. |
243 | if (TT.isPS() || TT.isOSBinFormatMachO()) { |
244 | this->Options.TrapUnreachable = true; |
245 | this->Options.NoTrapAfterNoreturn = TT.isOSBinFormatMachO(); |
246 | } |
247 | |
248 | setMachineOutliner(true); |
249 | |
250 | // x86 supports the debug entry values. |
251 | setSupportsDebugEntryValues(true); |
252 | |
253 | initAsmInfo(); |
254 | } |
255 | |
256 | X86TargetMachine::~X86TargetMachine() = default; |
257 | |
258 | const X86Subtarget * |
259 | X86TargetMachine::getSubtargetImpl(const Function &F) const { |
260 | Attribute CPUAttr = F.getFnAttribute(Kind: "target-cpu" ); |
261 | Attribute TuneAttr = F.getFnAttribute(Kind: "tune-cpu" ); |
262 | Attribute FSAttr = F.getFnAttribute(Kind: "target-features" ); |
263 | |
264 | StringRef CPU = |
265 | CPUAttr.isValid() ? CPUAttr.getValueAsString() : (StringRef)TargetCPU; |
266 | // "x86-64" is a default target setting for many front ends. In these cases, |
267 | // they actually request for "generic" tuning unless the "tune-cpu" was |
268 | // specified. |
269 | StringRef TuneCPU = TuneAttr.isValid() ? TuneAttr.getValueAsString() |
270 | : CPU == "x86-64" ? "generic" |
271 | : (StringRef)CPU; |
272 | StringRef FS = |
273 | FSAttr.isValid() ? FSAttr.getValueAsString() : (StringRef)TargetFS; |
274 | |
275 | SmallString<512> Key; |
276 | // The additions here are ordered so that the definitely short strings are |
277 | // added first so we won't exceed the small size. We append the |
278 | // much longer FS string at the end so that we only heap allocate at most |
279 | // one time. |
280 | |
281 | // Extract prefer-vector-width attribute. |
282 | unsigned PreferVectorWidthOverride = 0; |
283 | Attribute PreferVecWidthAttr = F.getFnAttribute(Kind: "prefer-vector-width" ); |
284 | if (PreferVecWidthAttr.isValid()) { |
285 | StringRef Val = PreferVecWidthAttr.getValueAsString(); |
286 | unsigned Width; |
287 | if (!Val.getAsInteger(Radix: 0, Result&: Width)) { |
288 | Key += 'p'; |
289 | Key += Val; |
290 | PreferVectorWidthOverride = Width; |
291 | } |
292 | } |
293 | |
294 | // Extract min-legal-vector-width attribute. |
295 | unsigned RequiredVectorWidth = UINT32_MAX; |
296 | Attribute MinLegalVecWidthAttr = F.getFnAttribute(Kind: "min-legal-vector-width" ); |
297 | if (MinLegalVecWidthAttr.isValid()) { |
298 | StringRef Val = MinLegalVecWidthAttr.getValueAsString(); |
299 | unsigned Width; |
300 | if (!Val.getAsInteger(Radix: 0, Result&: Width)) { |
301 | Key += 'm'; |
302 | Key += Val; |
303 | RequiredVectorWidth = Width; |
304 | } |
305 | } |
306 | |
307 | // Add CPU to the Key. |
308 | Key += CPU; |
309 | |
310 | // Add tune CPU to the Key. |
311 | Key += TuneCPU; |
312 | |
313 | // Keep track of the start of the feature portion of the string. |
314 | unsigned FSStart = Key.size(); |
315 | |
316 | // FIXME: This is related to the code below to reset the target options, |
317 | // we need to know whether or not the soft float flag is set on the |
318 | // function before we can generate a subtarget. We also need to use |
319 | // it as a key for the subtarget since that can be the only difference |
320 | // between two functions. |
321 | bool SoftFloat = F.getFnAttribute(Kind: "use-soft-float" ).getValueAsBool(); |
322 | // If the soft float attribute is set on the function turn on the soft float |
323 | // subtarget feature. |
324 | if (SoftFloat) |
325 | Key += FS.empty() ? "+soft-float" : "+soft-float," ; |
326 | |
327 | Key += FS; |
328 | |
329 | // We may have added +soft-float to the features so move the StringRef to |
330 | // point to the full string in the Key. |
331 | FS = Key.substr(Start: FSStart); |
332 | |
333 | auto &I = SubtargetMap[Key]; |
334 | if (!I) { |
335 | // This needs to be done before we create a new subtarget since any |
336 | // creation will depend on the TM and the code generation flags on the |
337 | // function that reside in TargetOptions. |
338 | resetTargetOptions(F); |
339 | I = std::make_unique<X86Subtarget>( |
340 | args: TargetTriple, args&: CPU, args&: TuneCPU, args&: FS, args: *this, |
341 | args: MaybeAlign(F.getParent()->getOverrideStackAlignment()), |
342 | args&: PreferVectorWidthOverride, args&: RequiredVectorWidth); |
343 | } |
344 | return I.get(); |
345 | } |
346 | |
347 | bool X86TargetMachine::isNoopAddrSpaceCast(unsigned SrcAS, |
348 | unsigned DestAS) const { |
349 | assert(SrcAS != DestAS && "Expected different address spaces!" ); |
350 | if (getPointerSize(AS: SrcAS) != getPointerSize(AS: DestAS)) |
351 | return false; |
352 | return SrcAS < 256 && DestAS < 256; |
353 | } |
354 | |
355 | //===----------------------------------------------------------------------===// |
356 | // X86 TTI query. |
357 | //===----------------------------------------------------------------------===// |
358 | |
359 | TargetTransformInfo |
360 | X86TargetMachine::getTargetTransformInfo(const Function &F) const { |
361 | return TargetTransformInfo(X86TTIImpl(this, F)); |
362 | } |
363 | |
364 | //===----------------------------------------------------------------------===// |
365 | // Pass Pipeline Configuration |
366 | //===----------------------------------------------------------------------===// |
367 | |
368 | namespace { |
369 | |
370 | /// X86 Code Generator Pass Configuration Options. |
371 | class X86PassConfig : public TargetPassConfig { |
372 | public: |
373 | X86PassConfig(X86TargetMachine &TM, PassManagerBase &PM) |
374 | : TargetPassConfig(TM, PM) {} |
375 | |
376 | X86TargetMachine &getX86TargetMachine() const { |
377 | return getTM<X86TargetMachine>(); |
378 | } |
379 | |
380 | ScheduleDAGInstrs * |
381 | createMachineScheduler(MachineSchedContext *C) const override { |
382 | ScheduleDAGMILive *DAG = createGenericSchedLive(C); |
383 | DAG->addMutation(Mutation: createX86MacroFusionDAGMutation()); |
384 | return DAG; |
385 | } |
386 | |
387 | ScheduleDAGInstrs * |
388 | createPostMachineScheduler(MachineSchedContext *C) const override { |
389 | ScheduleDAGMI *DAG = createGenericSchedPostRA(C); |
390 | DAG->addMutation(Mutation: createX86MacroFusionDAGMutation()); |
391 | return DAG; |
392 | } |
393 | |
394 | void addIRPasses() override; |
395 | bool addInstSelector() override; |
396 | bool addIRTranslator() override; |
397 | bool addLegalizeMachineIR() override; |
398 | bool addRegBankSelect() override; |
399 | bool addGlobalInstructionSelect() override; |
400 | bool addILPOpts() override; |
401 | bool addPreISel() override; |
402 | void addMachineSSAOptimization() override; |
403 | void addPreRegAlloc() override; |
404 | bool addPostFastRegAllocRewrite() override; |
405 | void addPostRegAlloc() override; |
406 | void addPreEmitPass() override; |
407 | void addPreEmitPass2() override; |
408 | void addPreSched2() override; |
409 | bool addRegAssignAndRewriteOptimized() override; |
410 | |
411 | std::unique_ptr<CSEConfigBase> getCSEConfig() const override; |
412 | }; |
413 | |
414 | class X86ExecutionDomainFix : public ExecutionDomainFix { |
415 | public: |
416 | static char ID; |
417 | X86ExecutionDomainFix() : ExecutionDomainFix(ID, X86::VR128XRegClass) {} |
418 | StringRef getPassName() const override { |
419 | return "X86 Execution Dependency Fix" ; |
420 | } |
421 | }; |
422 | char X86ExecutionDomainFix::ID; |
423 | |
424 | } // end anonymous namespace |
425 | |
426 | INITIALIZE_PASS_BEGIN(X86ExecutionDomainFix, "x86-execution-domain-fix" , |
427 | "X86 Execution Domain Fix" , false, false) |
428 | INITIALIZE_PASS_DEPENDENCY(ReachingDefAnalysis) |
429 | INITIALIZE_PASS_END(X86ExecutionDomainFix, "x86-execution-domain-fix" , |
430 | "X86 Execution Domain Fix" , false, false) |
431 | |
432 | TargetPassConfig *X86TargetMachine::createPassConfig(PassManagerBase &PM) { |
433 | return new X86PassConfig(*this, PM); |
434 | } |
435 | |
436 | MachineFunctionInfo *X86TargetMachine::createMachineFunctionInfo( |
437 | BumpPtrAllocator &Allocator, const Function &F, |
438 | const TargetSubtargetInfo *STI) const { |
439 | return X86MachineFunctionInfo::create<X86MachineFunctionInfo>(Allocator, F, |
440 | STI); |
441 | } |
442 | |
443 | void X86PassConfig::addIRPasses() { |
444 | addPass(P: createAtomicExpandLegacyPass()); |
445 | |
446 | // We add both pass anyway and when these two passes run, we skip the pass |
447 | // based on the option level and option attribute. |
448 | addPass(P: createX86LowerAMXIntrinsicsPass()); |
449 | addPass(P: createX86LowerAMXTypePass()); |
450 | |
451 | TargetPassConfig::addIRPasses(); |
452 | |
453 | if (TM->getOptLevel() != CodeGenOptLevel::None) { |
454 | addPass(P: createInterleavedAccessPass()); |
455 | addPass(P: createX86PartialReductionPass()); |
456 | } |
457 | |
458 | // Add passes that handle indirect branch removal and insertion of a retpoline |
459 | // thunk. These will be a no-op unless a function subtarget has the retpoline |
460 | // feature enabled. |
461 | addPass(P: createIndirectBrExpandPass()); |
462 | |
463 | // Add Control Flow Guard checks. |
464 | const Triple &TT = TM->getTargetTriple(); |
465 | if (TT.isOSWindows()) { |
466 | if (TT.getArch() == Triple::x86_64) { |
467 | addPass(P: createCFGuardDispatchPass()); |
468 | } else { |
469 | addPass(P: createCFGuardCheckPass()); |
470 | } |
471 | } |
472 | |
473 | if (TM->Options.JMCInstrument) |
474 | addPass(P: createJMCInstrumenterPass()); |
475 | } |
476 | |
477 | bool X86PassConfig::addInstSelector() { |
478 | // Install an instruction selector. |
479 | addPass(P: createX86ISelDag(TM&: getX86TargetMachine(), OptLevel: getOptLevel())); |
480 | |
481 | // For ELF, cleanup any local-dynamic TLS accesses. |
482 | if (TM->getTargetTriple().isOSBinFormatELF() && |
483 | getOptLevel() != CodeGenOptLevel::None) |
484 | addPass(P: createCleanupLocalDynamicTLSPass()); |
485 | |
486 | addPass(P: createX86GlobalBaseRegPass()); |
487 | addPass(P: createX86ArgumentStackSlotPass()); |
488 | return false; |
489 | } |
490 | |
491 | bool X86PassConfig::addIRTranslator() { |
492 | addPass(P: new IRTranslator(getOptLevel())); |
493 | return false; |
494 | } |
495 | |
496 | bool X86PassConfig::addLegalizeMachineIR() { |
497 | addPass(P: new Legalizer()); |
498 | return false; |
499 | } |
500 | |
501 | bool X86PassConfig::addRegBankSelect() { |
502 | addPass(P: new RegBankSelect()); |
503 | return false; |
504 | } |
505 | |
506 | bool X86PassConfig::addGlobalInstructionSelect() { |
507 | addPass(P: new InstructionSelect(getOptLevel())); |
508 | return false; |
509 | } |
510 | |
511 | bool X86PassConfig::addILPOpts() { |
512 | addPass(PassID: &EarlyIfConverterID); |
513 | if (EnableMachineCombinerPass) |
514 | addPass(PassID: &MachineCombinerID); |
515 | addPass(P: createX86CmovConverterPass()); |
516 | return true; |
517 | } |
518 | |
519 | bool X86PassConfig::addPreISel() { |
520 | // Only add this pass for 32-bit x86 Windows. |
521 | const Triple &TT = TM->getTargetTriple(); |
522 | if (TT.isOSWindows() && TT.getArch() == Triple::x86) |
523 | addPass(P: createX86WinEHStatePass()); |
524 | return true; |
525 | } |
526 | |
527 | void X86PassConfig::addPreRegAlloc() { |
528 | if (getOptLevel() != CodeGenOptLevel::None) { |
529 | addPass(PassID: &LiveRangeShrinkID); |
530 | addPass(P: createX86FixupSetCC()); |
531 | addPass(P: createX86OptimizeLEAs()); |
532 | addPass(P: createX86CallFrameOptimization()); |
533 | addPass(P: createX86AvoidStoreForwardingBlocks()); |
534 | } |
535 | |
536 | addPass(P: createX86SpeculativeLoadHardeningPass()); |
537 | addPass(P: createX86FlagsCopyLoweringPass()); |
538 | addPass(P: createX86DynAllocaExpander()); |
539 | |
540 | if (getOptLevel() != CodeGenOptLevel::None) |
541 | addPass(P: createX86PreTileConfigPass()); |
542 | else |
543 | addPass(P: createX86FastPreTileConfigPass()); |
544 | } |
545 | |
546 | void X86PassConfig::addMachineSSAOptimization() { |
547 | addPass(P: createX86DomainReassignmentPass()); |
548 | TargetPassConfig::addMachineSSAOptimization(); |
549 | } |
550 | |
551 | void X86PassConfig::addPostRegAlloc() { |
552 | addPass(P: createX86LowerTileCopyPass()); |
553 | addPass(P: createX86FloatingPointStackifierPass()); |
554 | // When -O0 is enabled, the Load Value Injection Hardening pass will fall back |
555 | // to using the Speculative Execution Side Effect Suppression pass for |
556 | // mitigation. This is to prevent slow downs due to |
557 | // analyses needed by the LVIHardening pass when compiling at -O0. |
558 | if (getOptLevel() != CodeGenOptLevel::None) |
559 | addPass(P: createX86LoadValueInjectionLoadHardeningPass()); |
560 | } |
561 | |
562 | void X86PassConfig::addPreSched2() { |
563 | addPass(P: createX86ExpandPseudoPass()); |
564 | addPass(P: createKCFIPass()); |
565 | } |
566 | |
567 | void X86PassConfig::addPreEmitPass() { |
568 | if (getOptLevel() != CodeGenOptLevel::None) { |
569 | addPass(P: new X86ExecutionDomainFix()); |
570 | addPass(P: createBreakFalseDeps()); |
571 | } |
572 | |
573 | addPass(P: createX86IndirectBranchTrackingPass()); |
574 | |
575 | addPass(P: createX86IssueVZeroUpperPass()); |
576 | |
577 | if (getOptLevel() != CodeGenOptLevel::None) { |
578 | addPass(P: createX86FixupBWInsts()); |
579 | addPass(P: createX86PadShortFunctions()); |
580 | addPass(P: createX86FixupLEAs()); |
581 | addPass(P: createX86FixupInstTuning()); |
582 | addPass(P: createX86FixupVectorConstants()); |
583 | } |
584 | addPass(P: createX86CompressEVEXPass()); |
585 | addPass(P: createX86DiscriminateMemOpsPass()); |
586 | addPass(P: createX86InsertPrefetchPass()); |
587 | addPass(P: createX86InsertX87waitPass()); |
588 | } |
589 | |
590 | void X86PassConfig::addPreEmitPass2() { |
591 | const Triple &TT = TM->getTargetTriple(); |
592 | const MCAsmInfo *MAI = TM->getMCAsmInfo(); |
593 | |
594 | // The X86 Speculative Execution Pass must run after all control |
595 | // flow graph modifying passes. As a result it was listed to run right before |
596 | // the X86 Retpoline Thunks pass. The reason it must run after control flow |
597 | // graph modifications is that the model of LFENCE in LLVM has to be updated |
598 | // (FIXME: https://bugs.llvm.org/show_bug.cgi?id=45167). Currently the |
599 | // placement of this pass was hand checked to ensure that the subsequent |
600 | // passes don't move the code around the LFENCEs in a way that will hurt the |
601 | // correctness of this pass. This placement has been shown to work based on |
602 | // hand inspection of the codegen output. |
603 | addPass(P: createX86SpeculativeExecutionSideEffectSuppression()); |
604 | addPass(P: createX86IndirectThunksPass()); |
605 | addPass(P: createX86ReturnThunksPass()); |
606 | |
607 | // Insert extra int3 instructions after trailing call instructions to avoid |
608 | // issues in the unwinder. |
609 | if (TT.isOSWindows() && TT.getArch() == Triple::x86_64) |
610 | addPass(P: createX86AvoidTrailingCallPass()); |
611 | |
612 | // Verify basic block incoming and outgoing cfa offset and register values and |
613 | // correct CFA calculation rule where needed by inserting appropriate CFI |
614 | // instructions. |
615 | if (!TT.isOSDarwin() && |
616 | (!TT.isOSWindows() || |
617 | MAI->getExceptionHandlingType() == ExceptionHandling::DwarfCFI)) |
618 | addPass(P: createCFIInstrInserter()); |
619 | |
620 | if (TT.isOSWindows()) { |
621 | // Identify valid longjmp targets for Windows Control Flow Guard. |
622 | addPass(P: createCFGuardLongjmpPass()); |
623 | // Identify valid eh continuation targets for Windows EHCont Guard. |
624 | addPass(P: createEHContGuardCatchretPass()); |
625 | } |
626 | addPass(P: createX86LoadValueInjectionRetHardeningPass()); |
627 | |
628 | // Insert pseudo probe annotation for callsite profiling |
629 | addPass(P: createPseudoProbeInserter()); |
630 | |
631 | // KCFI indirect call checks are lowered to a bundle, and on Darwin platforms, |
632 | // also CALL_RVMARKER. |
633 | addPass(P: createUnpackMachineBundles(Ftor: [&TT](const MachineFunction &MF) { |
634 | // Only run bundle expansion if the module uses kcfi, or there are relevant |
635 | // ObjC runtime functions present in the module. |
636 | const Function &F = MF.getFunction(); |
637 | const Module *M = F.getParent(); |
638 | return M->getModuleFlag(Key: "kcfi" ) || |
639 | (TT.isOSDarwin() && |
640 | (M->getFunction(Name: "objc_retainAutoreleasedReturnValue" ) || |
641 | M->getFunction(Name: "objc_unsafeClaimAutoreleasedReturnValue" ))); |
642 | })); |
643 | } |
644 | |
645 | bool X86PassConfig::addPostFastRegAllocRewrite() { |
646 | addPass(P: createX86FastTileConfigPass()); |
647 | return true; |
648 | } |
649 | |
650 | std::unique_ptr<CSEConfigBase> X86PassConfig::getCSEConfig() const { |
651 | return getStandardCSEConfigForOpt(Level: TM->getOptLevel()); |
652 | } |
653 | |
654 | static bool onlyAllocateTileRegisters(const TargetRegisterInfo &TRI, |
655 | const TargetRegisterClass &RC) { |
656 | return static_cast<const X86RegisterInfo &>(TRI).isTileRegisterClass(&RC); |
657 | } |
658 | |
659 | bool X86PassConfig::addRegAssignAndRewriteOptimized() { |
660 | // Don't support tile RA when RA is specified by command line "-regalloc". |
661 | if (!isCustomizedRegAlloc() && EnableTileRAPass) { |
662 | // Allocate tile register first. |
663 | addPass(P: createGreedyRegisterAllocator(F: onlyAllocateTileRegisters)); |
664 | addPass(P: createX86TileConfigPass()); |
665 | } |
666 | return TargetPassConfig::addRegAssignAndRewriteOptimized(); |
667 | } |
668 | |