1 | //===- PrologEpilogInserter.cpp - Insert Prolog/Epilog code in function ---===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This pass is responsible for finalizing the functions frame layout, saving |
10 | // callee saved registers, and for emitting prolog & epilog code for the |
11 | // function. |
12 | // |
13 | // This pass must be run after register allocation. After this pass is |
14 | // executed, it is illegal to construct MO_FrameIndex operands. |
15 | // |
16 | //===----------------------------------------------------------------------===// |
17 | |
18 | #include "llvm/ADT/ArrayRef.h" |
19 | #include "llvm/ADT/BitVector.h" |
20 | #include "llvm/ADT/STLExtras.h" |
21 | #include "llvm/ADT/SetVector.h" |
22 | #include "llvm/ADT/SmallPtrSet.h" |
23 | #include "llvm/ADT/SmallSet.h" |
24 | #include "llvm/ADT/SmallVector.h" |
25 | #include "llvm/ADT/Statistic.h" |
26 | #include "llvm/Analysis/OptimizationRemarkEmitter.h" |
27 | #include "llvm/CodeGen/MachineBasicBlock.h" |
28 | #include "llvm/CodeGen/MachineDominators.h" |
29 | #include "llvm/CodeGen/MachineFrameInfo.h" |
30 | #include "llvm/CodeGen/MachineFunction.h" |
31 | #include "llvm/CodeGen/MachineFunctionPass.h" |
32 | #include "llvm/CodeGen/MachineInstr.h" |
33 | #include "llvm/CodeGen/MachineInstrBuilder.h" |
34 | #include "llvm/CodeGen/MachineLoopInfo.h" |
35 | #include "llvm/CodeGen/MachineModuleInfo.h" |
36 | #include "llvm/CodeGen/MachineOperand.h" |
37 | #include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h" |
38 | #include "llvm/CodeGen/MachineRegisterInfo.h" |
39 | #include "llvm/CodeGen/RegisterScavenging.h" |
40 | #include "llvm/CodeGen/TargetFrameLowering.h" |
41 | #include "llvm/CodeGen/TargetInstrInfo.h" |
42 | #include "llvm/CodeGen/TargetOpcodes.h" |
43 | #include "llvm/CodeGen/TargetRegisterInfo.h" |
44 | #include "llvm/CodeGen/TargetSubtargetInfo.h" |
45 | #include "llvm/CodeGen/WinEHFuncInfo.h" |
46 | #include "llvm/IR/Attributes.h" |
47 | #include "llvm/IR/CallingConv.h" |
48 | #include "llvm/IR/DebugInfoMetadata.h" |
49 | #include "llvm/IR/DiagnosticInfo.h" |
50 | #include "llvm/IR/Function.h" |
51 | #include "llvm/IR/InlineAsm.h" |
52 | #include "llvm/IR/LLVMContext.h" |
53 | #include "llvm/InitializePasses.h" |
54 | #include "llvm/MC/MCRegisterInfo.h" |
55 | #include "llvm/Pass.h" |
56 | #include "llvm/Support/CodeGen.h" |
57 | #include "llvm/Support/Debug.h" |
58 | #include "llvm/Support/ErrorHandling.h" |
59 | #include "llvm/Support/FormatVariadic.h" |
60 | #include "llvm/Support/raw_ostream.h" |
61 | #include "llvm/Target/TargetMachine.h" |
62 | #include "llvm/Target/TargetOptions.h" |
63 | #include <algorithm> |
64 | #include <cassert> |
65 | #include <cstdint> |
66 | #include <functional> |
67 | #include <limits> |
68 | #include <utility> |
69 | #include <vector> |
70 | |
71 | using namespace llvm; |
72 | |
73 | #define DEBUG_TYPE "prologepilog" |
74 | |
75 | using MBBVector = SmallVector<MachineBasicBlock *, 4>; |
76 | |
77 | STATISTIC(NumLeafFuncWithSpills, "Number of leaf functions with CSRs" ); |
78 | STATISTIC(NumFuncSeen, "Number of functions seen in PEI" ); |
79 | |
80 | |
81 | namespace { |
82 | |
83 | class PEI : public MachineFunctionPass { |
84 | public: |
85 | static char ID; |
86 | |
87 | PEI() : MachineFunctionPass(ID) { |
88 | initializePEIPass(*PassRegistry::getPassRegistry()); |
89 | } |
90 | |
91 | void getAnalysisUsage(AnalysisUsage &AU) const override; |
92 | |
93 | /// runOnMachineFunction - Insert prolog/epilog code and replace abstract |
94 | /// frame indexes with appropriate references. |
95 | bool runOnMachineFunction(MachineFunction &MF) override; |
96 | |
97 | private: |
98 | RegScavenger *RS = nullptr; |
99 | |
100 | // MinCSFrameIndex, MaxCSFrameIndex - Keeps the range of callee saved |
101 | // stack frame indexes. |
102 | unsigned MinCSFrameIndex = std::numeric_limits<unsigned>::max(); |
103 | unsigned MaxCSFrameIndex = 0; |
104 | |
105 | // Save and Restore blocks of the current function. Typically there is a |
106 | // single save block, unless Windows EH funclets are involved. |
107 | MBBVector SaveBlocks; |
108 | MBBVector RestoreBlocks; |
109 | |
110 | // Flag to control whether to use the register scavenger to resolve |
111 | // frame index materialization registers. Set according to |
112 | // TRI->requiresFrameIndexScavenging() for the current function. |
113 | bool FrameIndexVirtualScavenging = false; |
114 | |
115 | // Flag to control whether the scavenger should be passed even though |
116 | // FrameIndexVirtualScavenging is used. |
117 | bool FrameIndexEliminationScavenging = false; |
118 | |
119 | // Emit remarks. |
120 | MachineOptimizationRemarkEmitter *ORE = nullptr; |
121 | |
122 | void calculateCallFrameInfo(MachineFunction &MF); |
123 | void calculateSaveRestoreBlocks(MachineFunction &MF); |
124 | void spillCalleeSavedRegs(MachineFunction &MF); |
125 | |
126 | void calculateFrameObjectOffsets(MachineFunction &MF); |
127 | void replaceFrameIndices(MachineFunction &MF); |
128 | void replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &MF, |
129 | int &SPAdj); |
130 | // Frame indices in debug values are encoded in a target independent |
131 | // way with simply the frame index and offset rather than any |
132 | // target-specific addressing mode. |
133 | bool replaceFrameIndexDebugInstr(MachineFunction &MF, MachineInstr &MI, |
134 | unsigned OpIdx, int SPAdj = 0); |
135 | // Does same as replaceFrameIndices but using the backward MIR walk and |
136 | // backward register scavenger walk. |
137 | void replaceFrameIndicesBackward(MachineFunction &MF); |
138 | void replaceFrameIndicesBackward(MachineBasicBlock *BB, MachineFunction &MF, |
139 | int &SPAdj); |
140 | |
141 | void insertPrologEpilogCode(MachineFunction &MF); |
142 | void insertZeroCallUsedRegs(MachineFunction &MF); |
143 | }; |
144 | |
145 | } // end anonymous namespace |
146 | |
147 | char PEI::ID = 0; |
148 | |
149 | char &llvm::PrologEpilogCodeInserterID = PEI::ID; |
150 | |
151 | INITIALIZE_PASS_BEGIN(PEI, DEBUG_TYPE, "Prologue/Epilogue Insertion" , false, |
152 | false) |
153 | INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) |
154 | INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree) |
155 | INITIALIZE_PASS_DEPENDENCY(MachineOptimizationRemarkEmitterPass) |
156 | INITIALIZE_PASS_END(PEI, DEBUG_TYPE, |
157 | "Prologue/Epilogue Insertion & Frame Finalization" , false, |
158 | false) |
159 | |
160 | MachineFunctionPass *llvm::createPrologEpilogInserterPass() { |
161 | return new PEI(); |
162 | } |
163 | |
164 | STATISTIC(NumBytesStackSpace, |
165 | "Number of bytes used for stack in all functions" ); |
166 | |
167 | void PEI::getAnalysisUsage(AnalysisUsage &AU) const { |
168 | AU.setPreservesCFG(); |
169 | AU.addPreserved<MachineLoopInfo>(); |
170 | AU.addPreserved<MachineDominatorTree>(); |
171 | AU.addRequired<MachineOptimizationRemarkEmitterPass>(); |
172 | MachineFunctionPass::getAnalysisUsage(AU); |
173 | } |
174 | |
175 | /// StackObjSet - A set of stack object indexes |
176 | using StackObjSet = SmallSetVector<int, 8>; |
177 | |
178 | using SavedDbgValuesMap = |
179 | SmallDenseMap<MachineBasicBlock *, SmallVector<MachineInstr *, 4>, 4>; |
180 | |
181 | /// Stash DBG_VALUEs that describe parameters and which are placed at the start |
182 | /// of the block. Later on, after the prologue code has been emitted, the |
183 | /// stashed DBG_VALUEs will be reinserted at the start of the block. |
184 | static void stashEntryDbgValues(MachineBasicBlock &MBB, |
185 | SavedDbgValuesMap &EntryDbgValues) { |
186 | SmallVector<const MachineInstr *, 4> FrameIndexValues; |
187 | |
188 | for (auto &MI : MBB) { |
189 | if (!MI.isDebugInstr()) |
190 | break; |
191 | if (!MI.isDebugValue() || !MI.getDebugVariable()->isParameter()) |
192 | continue; |
193 | if (any_of(Range: MI.debug_operands(), |
194 | P: [](const MachineOperand &MO) { return MO.isFI(); })) { |
195 | // We can only emit valid locations for frame indices after the frame |
196 | // setup, so do not stash away them. |
197 | FrameIndexValues.push_back(Elt: &MI); |
198 | continue; |
199 | } |
200 | const DILocalVariable *Var = MI.getDebugVariable(); |
201 | const DIExpression *Expr = MI.getDebugExpression(); |
202 | auto Overlaps = [Var, Expr](const MachineInstr *DV) { |
203 | return Var == DV->getDebugVariable() && |
204 | Expr->fragmentsOverlap(Other: DV->getDebugExpression()); |
205 | }; |
206 | // See if the debug value overlaps with any preceding debug value that will |
207 | // not be stashed. If that is the case, then we can't stash this value, as |
208 | // we would then reorder the values at reinsertion. |
209 | if (llvm::none_of(Range&: FrameIndexValues, P: Overlaps)) |
210 | EntryDbgValues[&MBB].push_back(Elt: &MI); |
211 | } |
212 | |
213 | // Remove stashed debug values from the block. |
214 | if (EntryDbgValues.count(Val: &MBB)) |
215 | for (auto *MI : EntryDbgValues[&MBB]) |
216 | MI->removeFromParent(); |
217 | } |
218 | |
219 | /// runOnMachineFunction - Insert prolog/epilog code and replace abstract |
220 | /// frame indexes with appropriate references. |
221 | bool PEI::runOnMachineFunction(MachineFunction &MF) { |
222 | NumFuncSeen++; |
223 | const Function &F = MF.getFunction(); |
224 | const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); |
225 | const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering(); |
226 | |
227 | RS = TRI->requiresRegisterScavenging(MF) ? new RegScavenger() : nullptr; |
228 | FrameIndexVirtualScavenging = TRI->requiresFrameIndexScavenging(MF); |
229 | ORE = &getAnalysis<MachineOptimizationRemarkEmitterPass>().getORE(); |
230 | |
231 | // Calculate the MaxCallFrameSize and AdjustsStack variables for the |
232 | // function's frame information. Also eliminates call frame pseudo |
233 | // instructions. |
234 | calculateCallFrameInfo(MF); |
235 | |
236 | // Determine placement of CSR spill/restore code and prolog/epilog code: |
237 | // place all spills in the entry block, all restores in return blocks. |
238 | calculateSaveRestoreBlocks(MF); |
239 | |
240 | // Stash away DBG_VALUEs that should not be moved by insertion of prolog code. |
241 | SavedDbgValuesMap EntryDbgValues; |
242 | for (MachineBasicBlock *SaveBlock : SaveBlocks) |
243 | stashEntryDbgValues(MBB&: *SaveBlock, EntryDbgValues); |
244 | |
245 | // Handle CSR spilling and restoring, for targets that need it. |
246 | if (MF.getTarget().usesPhysRegsForValues()) |
247 | spillCalleeSavedRegs(MF); |
248 | |
249 | // Allow the target machine to make final modifications to the function |
250 | // before the frame layout is finalized. |
251 | TFI->processFunctionBeforeFrameFinalized(MF, RS); |
252 | |
253 | // Calculate actual frame offsets for all abstract stack objects... |
254 | calculateFrameObjectOffsets(MF); |
255 | |
256 | // Add prolog and epilog code to the function. This function is required |
257 | // to align the stack frame as necessary for any stack variables or |
258 | // called functions. Because of this, calculateCalleeSavedRegisters() |
259 | // must be called before this function in order to set the AdjustsStack |
260 | // and MaxCallFrameSize variables. |
261 | if (!F.hasFnAttribute(Attribute::Naked)) |
262 | insertPrologEpilogCode(MF); |
263 | |
264 | // Reinsert stashed debug values at the start of the entry blocks. |
265 | for (auto &I : EntryDbgValues) |
266 | I.first->insert(I: I.first->begin(), S: I.second.begin(), E: I.second.end()); |
267 | |
268 | // Allow the target machine to make final modifications to the function |
269 | // before the frame layout is finalized. |
270 | TFI->processFunctionBeforeFrameIndicesReplaced(MF, RS); |
271 | |
272 | // Replace all MO_FrameIndex operands with physical register references |
273 | // and actual offsets. |
274 | if (TFI->needsFrameIndexResolution(MF)) { |
275 | // Allow the target to determine this after knowing the frame size. |
276 | FrameIndexEliminationScavenging = |
277 | (RS && !FrameIndexVirtualScavenging) || |
278 | TRI->requiresFrameIndexReplacementScavenging(MF); |
279 | |
280 | if (TRI->eliminateFrameIndicesBackwards()) |
281 | replaceFrameIndicesBackward(MF); |
282 | else |
283 | replaceFrameIndices(MF); |
284 | } |
285 | |
286 | // If register scavenging is needed, as we've enabled doing it as a |
287 | // post-pass, scavenge the virtual registers that frame index elimination |
288 | // inserted. |
289 | if (TRI->requiresRegisterScavenging(MF) && FrameIndexVirtualScavenging) |
290 | scavengeFrameVirtualRegs(MF, RS&: *RS); |
291 | |
292 | // Warn on stack size when we exceeds the given limit. |
293 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
294 | uint64_t StackSize = MFI.getStackSize(); |
295 | |
296 | uint64_t Threshold = TFI->getStackThreshold(); |
297 | if (MF.getFunction().hasFnAttribute(Kind: "warn-stack-size" )) { |
298 | bool Failed = MF.getFunction() |
299 | .getFnAttribute(Kind: "warn-stack-size" ) |
300 | .getValueAsString() |
301 | .getAsInteger(Radix: 10, Result&: Threshold); |
302 | // Verifier should have caught this. |
303 | assert(!Failed && "Invalid warn-stack-size fn attr value" ); |
304 | (void)Failed; |
305 | } |
306 | uint64_t UnsafeStackSize = MFI.getUnsafeStackSize(); |
307 | if (MF.getFunction().hasFnAttribute(Attribute::SafeStack)) |
308 | StackSize += UnsafeStackSize; |
309 | |
310 | if (StackSize > Threshold) { |
311 | DiagnosticInfoStackSize DiagStackSize(F, StackSize, Threshold, DS_Warning); |
312 | F.getContext().diagnose(DI: DiagStackSize); |
313 | int64_t SpillSize = 0; |
314 | for (int Idx = MFI.getObjectIndexBegin(), End = MFI.getObjectIndexEnd(); |
315 | Idx != End; ++Idx) { |
316 | if (MFI.isSpillSlotObjectIndex(ObjectIdx: Idx)) |
317 | SpillSize += MFI.getObjectSize(ObjectIdx: Idx); |
318 | } |
319 | |
320 | [[maybe_unused]] float SpillPct = |
321 | static_cast<float>(SpillSize) / static_cast<float>(StackSize); |
322 | LLVM_DEBUG( |
323 | dbgs() << formatv("{0}/{1} ({3:P}) spills, {2}/{1} ({4:P}) variables" , |
324 | SpillSize, StackSize, StackSize - SpillSize, SpillPct, |
325 | 1.0f - SpillPct)); |
326 | if (UnsafeStackSize != 0) { |
327 | LLVM_DEBUG(dbgs() << formatv(", {0}/{2} ({1:P}) unsafe stack" , |
328 | UnsafeStackSize, |
329 | static_cast<float>(UnsafeStackSize) / |
330 | static_cast<float>(StackSize), |
331 | StackSize)); |
332 | } |
333 | LLVM_DEBUG(dbgs() << "\n" ); |
334 | } |
335 | |
336 | ORE->emit(RemarkBuilder: [&]() { |
337 | return MachineOptimizationRemarkAnalysis(DEBUG_TYPE, "StackSize" , |
338 | MF.getFunction().getSubprogram(), |
339 | &MF.front()) |
340 | << ore::NV("NumStackBytes" , StackSize) |
341 | << " stack bytes in function '" |
342 | << ore::NV("Function" , MF.getFunction().getName()) << "'" ; |
343 | }); |
344 | |
345 | delete RS; |
346 | SaveBlocks.clear(); |
347 | RestoreBlocks.clear(); |
348 | MFI.setSavePoint(nullptr); |
349 | MFI.setRestorePoint(nullptr); |
350 | return true; |
351 | } |
352 | |
353 | /// Calculate the MaxCallFrameSize and AdjustsStack |
354 | /// variables for the function's frame information and eliminate call frame |
355 | /// pseudo instructions. |
356 | void PEI::calculateCallFrameInfo(MachineFunction &MF) { |
357 | const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo(); |
358 | const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering(); |
359 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
360 | |
361 | unsigned MaxCallFrameSize = 0; |
362 | bool AdjustsStack = MFI.adjustsStack(); |
363 | |
364 | // Get the function call frame set-up and tear-down instruction opcode |
365 | unsigned FrameSetupOpcode = TII.getCallFrameSetupOpcode(); |
366 | unsigned FrameDestroyOpcode = TII.getCallFrameDestroyOpcode(); |
367 | |
368 | // Early exit for targets which have no call frame setup/destroy pseudo |
369 | // instructions. |
370 | if (FrameSetupOpcode == ~0u && FrameDestroyOpcode == ~0u) |
371 | return; |
372 | |
373 | std::vector<MachineBasicBlock::iterator> FrameSDOps; |
374 | for (MachineBasicBlock &BB : MF) |
375 | for (MachineBasicBlock::iterator I = BB.begin(); I != BB.end(); ++I) |
376 | if (TII.isFrameInstr(I: *I)) { |
377 | unsigned Size = TII.getFrameSize(I: *I); |
378 | if (Size > MaxCallFrameSize) MaxCallFrameSize = Size; |
379 | AdjustsStack = true; |
380 | FrameSDOps.push_back(x: I); |
381 | } else if (I->isInlineAsm()) { |
382 | // Some inline asm's need a stack frame, as indicated by operand 1. |
383 | unsigned = I->getOperand(i: InlineAsm::MIOp_ExtraInfo).getImm(); |
384 | if (ExtraInfo & InlineAsm::Extra_IsAlignStack) |
385 | AdjustsStack = true; |
386 | } |
387 | |
388 | assert(!MFI.isMaxCallFrameSizeComputed() || |
389 | (MFI.getMaxCallFrameSize() >= MaxCallFrameSize && |
390 | !(AdjustsStack && !MFI.adjustsStack()))); |
391 | MFI.setAdjustsStack(AdjustsStack); |
392 | MFI.setMaxCallFrameSize(MaxCallFrameSize); |
393 | |
394 | if (TFI->canSimplifyCallFramePseudos(MF)) { |
395 | // If call frames are not being included as part of the stack frame, and |
396 | // the target doesn't indicate otherwise, remove the call frame pseudos |
397 | // here. The sub/add sp instruction pairs are still inserted, but we don't |
398 | // need to track the SP adjustment for frame index elimination. |
399 | for (MachineBasicBlock::iterator I : FrameSDOps) |
400 | TFI->eliminateCallFramePseudoInstr(MF, MBB&: *I->getParent(), MI: I); |
401 | |
402 | // We can't track the call frame size after call frame pseudos have been |
403 | // eliminated. Set it to zero everywhere to keep MachineVerifier happy. |
404 | for (MachineBasicBlock &MBB : MF) |
405 | MBB.setCallFrameSize(0); |
406 | } |
407 | } |
408 | |
409 | /// Compute the sets of entry and return blocks for saving and restoring |
410 | /// callee-saved registers, and placing prolog and epilog code. |
411 | void PEI::calculateSaveRestoreBlocks(MachineFunction &MF) { |
412 | const MachineFrameInfo &MFI = MF.getFrameInfo(); |
413 | |
414 | // Even when we do not change any CSR, we still want to insert the |
415 | // prologue and epilogue of the function. |
416 | // So set the save points for those. |
417 | |
418 | // Use the points found by shrink-wrapping, if any. |
419 | if (MFI.getSavePoint()) { |
420 | SaveBlocks.push_back(Elt: MFI.getSavePoint()); |
421 | assert(MFI.getRestorePoint() && "Both restore and save must be set" ); |
422 | MachineBasicBlock *RestoreBlock = MFI.getRestorePoint(); |
423 | // If RestoreBlock does not have any successor and is not a return block |
424 | // then the end point is unreachable and we do not need to insert any |
425 | // epilogue. |
426 | if (!RestoreBlock->succ_empty() || RestoreBlock->isReturnBlock()) |
427 | RestoreBlocks.push_back(Elt: RestoreBlock); |
428 | return; |
429 | } |
430 | |
431 | // Save refs to entry and return blocks. |
432 | SaveBlocks.push_back(Elt: &MF.front()); |
433 | for (MachineBasicBlock &MBB : MF) { |
434 | if (MBB.isEHFuncletEntry()) |
435 | SaveBlocks.push_back(Elt: &MBB); |
436 | if (MBB.isReturnBlock()) |
437 | RestoreBlocks.push_back(Elt: &MBB); |
438 | } |
439 | } |
440 | |
441 | static void assignCalleeSavedSpillSlots(MachineFunction &F, |
442 | const BitVector &SavedRegs, |
443 | unsigned &MinCSFrameIndex, |
444 | unsigned &MaxCSFrameIndex) { |
445 | if (SavedRegs.empty()) |
446 | return; |
447 | |
448 | const TargetRegisterInfo *RegInfo = F.getSubtarget().getRegisterInfo(); |
449 | const MCPhysReg *CSRegs = F.getRegInfo().getCalleeSavedRegs(); |
450 | BitVector CSMask(SavedRegs.size()); |
451 | |
452 | for (unsigned i = 0; CSRegs[i]; ++i) |
453 | CSMask.set(CSRegs[i]); |
454 | |
455 | std::vector<CalleeSavedInfo> CSI; |
456 | for (unsigned i = 0; CSRegs[i]; ++i) { |
457 | unsigned Reg = CSRegs[i]; |
458 | if (SavedRegs.test(Idx: Reg)) { |
459 | bool SavedSuper = false; |
460 | for (const MCPhysReg &SuperReg : RegInfo->superregs(Reg)) { |
461 | // Some backends set all aliases for some registers as saved, such as |
462 | // Mips's $fp, so they appear in SavedRegs but not CSRegs. |
463 | if (SavedRegs.test(Idx: SuperReg) && CSMask.test(Idx: SuperReg)) { |
464 | SavedSuper = true; |
465 | break; |
466 | } |
467 | } |
468 | |
469 | if (!SavedSuper) |
470 | CSI.push_back(x: CalleeSavedInfo(Reg)); |
471 | } |
472 | } |
473 | |
474 | const TargetFrameLowering *TFI = F.getSubtarget().getFrameLowering(); |
475 | MachineFrameInfo &MFI = F.getFrameInfo(); |
476 | if (!TFI->assignCalleeSavedSpillSlots(MF&: F, TRI: RegInfo, CSI, MinCSFrameIndex, |
477 | MaxCSFrameIndex)) { |
478 | // If target doesn't implement this, use generic code. |
479 | |
480 | if (CSI.empty()) |
481 | return; // Early exit if no callee saved registers are modified! |
482 | |
483 | unsigned NumFixedSpillSlots; |
484 | const TargetFrameLowering::SpillSlot *FixedSpillSlots = |
485 | TFI->getCalleeSavedSpillSlots(NumEntries&: NumFixedSpillSlots); |
486 | |
487 | // Now that we know which registers need to be saved and restored, allocate |
488 | // stack slots for them. |
489 | for (auto &CS : CSI) { |
490 | // If the target has spilled this register to another register, we don't |
491 | // need to allocate a stack slot. |
492 | if (CS.isSpilledToReg()) |
493 | continue; |
494 | |
495 | unsigned Reg = CS.getReg(); |
496 | const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg); |
497 | |
498 | int FrameIdx; |
499 | if (RegInfo->hasReservedSpillSlot(MF: F, Reg, FrameIdx)) { |
500 | CS.setFrameIdx(FrameIdx); |
501 | continue; |
502 | } |
503 | |
504 | // Check to see if this physreg must be spilled to a particular stack slot |
505 | // on this target. |
506 | const TargetFrameLowering::SpillSlot *FixedSlot = FixedSpillSlots; |
507 | while (FixedSlot != FixedSpillSlots + NumFixedSpillSlots && |
508 | FixedSlot->Reg != Reg) |
509 | ++FixedSlot; |
510 | |
511 | unsigned Size = RegInfo->getSpillSize(RC: *RC); |
512 | if (FixedSlot == FixedSpillSlots + NumFixedSpillSlots) { |
513 | // Nope, just spill it anywhere convenient. |
514 | Align Alignment = RegInfo->getSpillAlign(RC: *RC); |
515 | // We may not be able to satisfy the desired alignment specification of |
516 | // the TargetRegisterClass if the stack alignment is smaller. Use the |
517 | // min. |
518 | Alignment = std::min(a: Alignment, b: TFI->getStackAlign()); |
519 | FrameIdx = MFI.CreateStackObject(Size, Alignment, isSpillSlot: true); |
520 | if ((unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx; |
521 | if ((unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx; |
522 | } else { |
523 | // Spill it to the stack where we must. |
524 | FrameIdx = MFI.CreateFixedSpillStackObject(Size, SPOffset: FixedSlot->Offset); |
525 | } |
526 | |
527 | CS.setFrameIdx(FrameIdx); |
528 | } |
529 | } |
530 | |
531 | MFI.setCalleeSavedInfo(CSI); |
532 | } |
533 | |
534 | /// Helper function to update the liveness information for the callee-saved |
535 | /// registers. |
536 | static void updateLiveness(MachineFunction &MF) { |
537 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
538 | // Visited will contain all the basic blocks that are in the region |
539 | // where the callee saved registers are alive: |
540 | // - Anything that is not Save or Restore -> LiveThrough. |
541 | // - Save -> LiveIn. |
542 | // - Restore -> LiveOut. |
543 | // The live-out is not attached to the block, so no need to keep |
544 | // Restore in this set. |
545 | SmallPtrSet<MachineBasicBlock *, 8> Visited; |
546 | SmallVector<MachineBasicBlock *, 8> WorkList; |
547 | MachineBasicBlock *Entry = &MF.front(); |
548 | MachineBasicBlock *Save = MFI.getSavePoint(); |
549 | |
550 | if (!Save) |
551 | Save = Entry; |
552 | |
553 | if (Entry != Save) { |
554 | WorkList.push_back(Elt: Entry); |
555 | Visited.insert(Ptr: Entry); |
556 | } |
557 | Visited.insert(Ptr: Save); |
558 | |
559 | MachineBasicBlock *Restore = MFI.getRestorePoint(); |
560 | if (Restore) |
561 | // By construction Restore cannot be visited, otherwise it |
562 | // means there exists a path to Restore that does not go |
563 | // through Save. |
564 | WorkList.push_back(Elt: Restore); |
565 | |
566 | while (!WorkList.empty()) { |
567 | const MachineBasicBlock *CurBB = WorkList.pop_back_val(); |
568 | // By construction, the region that is after the save point is |
569 | // dominated by the Save and post-dominated by the Restore. |
570 | if (CurBB == Save && Save != Restore) |
571 | continue; |
572 | // Enqueue all the successors not already visited. |
573 | // Those are by construction either before Save or after Restore. |
574 | for (MachineBasicBlock *SuccBB : CurBB->successors()) |
575 | if (Visited.insert(Ptr: SuccBB).second) |
576 | WorkList.push_back(Elt: SuccBB); |
577 | } |
578 | |
579 | const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo(); |
580 | |
581 | MachineRegisterInfo &MRI = MF.getRegInfo(); |
582 | for (const CalleeSavedInfo &I : CSI) { |
583 | for (MachineBasicBlock *MBB : Visited) { |
584 | MCPhysReg Reg = I.getReg(); |
585 | // Add the callee-saved register as live-in. |
586 | // It's killed at the spill. |
587 | if (!MRI.isReserved(PhysReg: Reg) && !MBB->isLiveIn(Reg)) |
588 | MBB->addLiveIn(PhysReg: Reg); |
589 | } |
590 | // If callee-saved register is spilled to another register rather than |
591 | // spilling to stack, the destination register has to be marked as live for |
592 | // each MBB between the prologue and epilogue so that it is not clobbered |
593 | // before it is reloaded in the epilogue. The Visited set contains all |
594 | // blocks outside of the region delimited by prologue/epilogue. |
595 | if (I.isSpilledToReg()) { |
596 | for (MachineBasicBlock &MBB : MF) { |
597 | if (Visited.count(Ptr: &MBB)) |
598 | continue; |
599 | MCPhysReg DstReg = I.getDstReg(); |
600 | if (!MBB.isLiveIn(Reg: DstReg)) |
601 | MBB.addLiveIn(PhysReg: DstReg); |
602 | } |
603 | } |
604 | } |
605 | } |
606 | |
607 | /// Insert spill code for the callee-saved registers used in the function. |
608 | static void insertCSRSaves(MachineBasicBlock &SaveBlock, |
609 | ArrayRef<CalleeSavedInfo> CSI) { |
610 | MachineFunction &MF = *SaveBlock.getParent(); |
611 | const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo(); |
612 | const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering(); |
613 | const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); |
614 | |
615 | MachineBasicBlock::iterator I = SaveBlock.begin(); |
616 | if (!TFI->spillCalleeSavedRegisters(MBB&: SaveBlock, MI: I, CSI, TRI)) { |
617 | for (const CalleeSavedInfo &CS : CSI) { |
618 | // Insert the spill to the stack frame. |
619 | unsigned Reg = CS.getReg(); |
620 | |
621 | if (CS.isSpilledToReg()) { |
622 | BuildMI(BB&: SaveBlock, I, MIMD: DebugLoc(), |
623 | MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: CS.getDstReg()) |
624 | .addReg(RegNo: Reg, flags: getKillRegState(B: true)); |
625 | } else { |
626 | const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg); |
627 | TII.storeRegToStackSlot(MBB&: SaveBlock, MI: I, SrcReg: Reg, isKill: true, FrameIndex: CS.getFrameIdx(), RC, |
628 | TRI, VReg: Register()); |
629 | } |
630 | } |
631 | } |
632 | } |
633 | |
634 | /// Insert restore code for the callee-saved registers used in the function. |
635 | static void insertCSRRestores(MachineBasicBlock &RestoreBlock, |
636 | std::vector<CalleeSavedInfo> &CSI) { |
637 | MachineFunction &MF = *RestoreBlock.getParent(); |
638 | const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo(); |
639 | const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering(); |
640 | const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); |
641 | |
642 | // Restore all registers immediately before the return and any |
643 | // terminators that precede it. |
644 | MachineBasicBlock::iterator I = RestoreBlock.getFirstTerminator(); |
645 | |
646 | if (!TFI->restoreCalleeSavedRegisters(MBB&: RestoreBlock, MI: I, CSI, TRI)) { |
647 | for (const CalleeSavedInfo &CI : reverse(C&: CSI)) { |
648 | unsigned Reg = CI.getReg(); |
649 | if (CI.isSpilledToReg()) { |
650 | BuildMI(BB&: RestoreBlock, I, MIMD: DebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: Reg) |
651 | .addReg(RegNo: CI.getDstReg(), flags: getKillRegState(B: true)); |
652 | } else { |
653 | const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg); |
654 | TII.loadRegFromStackSlot(MBB&: RestoreBlock, MI: I, DestReg: Reg, FrameIndex: CI.getFrameIdx(), RC, |
655 | TRI, VReg: Register()); |
656 | assert(I != RestoreBlock.begin() && |
657 | "loadRegFromStackSlot didn't insert any code!" ); |
658 | // Insert in reverse order. loadRegFromStackSlot can insert |
659 | // multiple instructions. |
660 | } |
661 | } |
662 | } |
663 | } |
664 | |
665 | void PEI::spillCalleeSavedRegs(MachineFunction &MF) { |
666 | // We can't list this requirement in getRequiredProperties because some |
667 | // targets (WebAssembly) use virtual registers past this point, and the pass |
668 | // pipeline is set up without giving the passes a chance to look at the |
669 | // TargetMachine. |
670 | // FIXME: Find a way to express this in getRequiredProperties. |
671 | assert(MF.getProperties().hasProperty( |
672 | MachineFunctionProperties::Property::NoVRegs)); |
673 | |
674 | const Function &F = MF.getFunction(); |
675 | const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering(); |
676 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
677 | MinCSFrameIndex = std::numeric_limits<unsigned>::max(); |
678 | MaxCSFrameIndex = 0; |
679 | |
680 | // Determine which of the registers in the callee save list should be saved. |
681 | BitVector SavedRegs; |
682 | TFI->determineCalleeSaves(MF, SavedRegs, RS); |
683 | |
684 | // Assign stack slots for any callee-saved registers that must be spilled. |
685 | assignCalleeSavedSpillSlots(F&: MF, SavedRegs, MinCSFrameIndex, MaxCSFrameIndex); |
686 | |
687 | // Add the code to save and restore the callee saved registers. |
688 | if (!F.hasFnAttribute(Attribute::Naked)) { |
689 | MFI.setCalleeSavedInfoValid(true); |
690 | |
691 | std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo(); |
692 | if (!CSI.empty()) { |
693 | if (!MFI.hasCalls()) |
694 | NumLeafFuncWithSpills++; |
695 | |
696 | for (MachineBasicBlock *SaveBlock : SaveBlocks) |
697 | insertCSRSaves(SaveBlock&: *SaveBlock, CSI); |
698 | |
699 | // Update the live-in information of all the blocks up to the save point. |
700 | updateLiveness(MF); |
701 | |
702 | for (MachineBasicBlock *RestoreBlock : RestoreBlocks) |
703 | insertCSRRestores(RestoreBlock&: *RestoreBlock, CSI); |
704 | } |
705 | } |
706 | } |
707 | |
708 | /// AdjustStackOffset - Helper function used to adjust the stack frame offset. |
709 | static inline void AdjustStackOffset(MachineFrameInfo &MFI, int FrameIdx, |
710 | bool StackGrowsDown, int64_t &Offset, |
711 | Align &MaxAlign) { |
712 | // If the stack grows down, add the object size to find the lowest address. |
713 | if (StackGrowsDown) |
714 | Offset += MFI.getObjectSize(ObjectIdx: FrameIdx); |
715 | |
716 | Align Alignment = MFI.getObjectAlign(ObjectIdx: FrameIdx); |
717 | |
718 | // If the alignment of this object is greater than that of the stack, then |
719 | // increase the stack alignment to match. |
720 | MaxAlign = std::max(a: MaxAlign, b: Alignment); |
721 | |
722 | // Adjust to alignment boundary. |
723 | Offset = alignTo(Size: Offset, A: Alignment); |
724 | |
725 | if (StackGrowsDown) { |
726 | LLVM_DEBUG(dbgs() << "alloc FI(" << FrameIdx << ") at SP[" << -Offset |
727 | << "]\n" ); |
728 | MFI.setObjectOffset(ObjectIdx: FrameIdx, SPOffset: -Offset); // Set the computed offset |
729 | } else { |
730 | LLVM_DEBUG(dbgs() << "alloc FI(" << FrameIdx << ") at SP[" << Offset |
731 | << "]\n" ); |
732 | MFI.setObjectOffset(ObjectIdx: FrameIdx, SPOffset: Offset); |
733 | Offset += MFI.getObjectSize(ObjectIdx: FrameIdx); |
734 | } |
735 | } |
736 | |
737 | /// Compute which bytes of fixed and callee-save stack area are unused and keep |
738 | /// track of them in StackBytesFree. |
739 | static inline void |
740 | computeFreeStackSlots(MachineFrameInfo &MFI, bool StackGrowsDown, |
741 | unsigned MinCSFrameIndex, unsigned MaxCSFrameIndex, |
742 | int64_t FixedCSEnd, BitVector &StackBytesFree) { |
743 | // Avoid undefined int64_t -> int conversion below in extreme case. |
744 | if (FixedCSEnd > std::numeric_limits<int>::max()) |
745 | return; |
746 | |
747 | StackBytesFree.resize(N: FixedCSEnd, t: true); |
748 | |
749 | SmallVector<int, 16> AllocatedFrameSlots; |
750 | // Add fixed objects. |
751 | for (int i = MFI.getObjectIndexBegin(); i != 0; ++i) |
752 | // StackSlot scavenging is only implemented for the default stack. |
753 | if (MFI.getStackID(ObjectIdx: i) == TargetStackID::Default) |
754 | AllocatedFrameSlots.push_back(Elt: i); |
755 | // Add callee-save objects if there are any. |
756 | if (MinCSFrameIndex <= MaxCSFrameIndex) { |
757 | for (int i = MinCSFrameIndex; i <= (int)MaxCSFrameIndex; ++i) |
758 | if (MFI.getStackID(ObjectIdx: i) == TargetStackID::Default) |
759 | AllocatedFrameSlots.push_back(Elt: i); |
760 | } |
761 | |
762 | for (int i : AllocatedFrameSlots) { |
763 | // These are converted from int64_t, but they should always fit in int |
764 | // because of the FixedCSEnd check above. |
765 | int ObjOffset = MFI.getObjectOffset(ObjectIdx: i); |
766 | int ObjSize = MFI.getObjectSize(ObjectIdx: i); |
767 | int ObjStart, ObjEnd; |
768 | if (StackGrowsDown) { |
769 | // ObjOffset is negative when StackGrowsDown is true. |
770 | ObjStart = -ObjOffset - ObjSize; |
771 | ObjEnd = -ObjOffset; |
772 | } else { |
773 | ObjStart = ObjOffset; |
774 | ObjEnd = ObjOffset + ObjSize; |
775 | } |
776 | // Ignore fixed holes that are in the previous stack frame. |
777 | if (ObjEnd > 0) |
778 | StackBytesFree.reset(I: ObjStart, E: ObjEnd); |
779 | } |
780 | } |
781 | |
782 | /// Assign frame object to an unused portion of the stack in the fixed stack |
783 | /// object range. Return true if the allocation was successful. |
784 | static inline bool scavengeStackSlot(MachineFrameInfo &MFI, int FrameIdx, |
785 | bool StackGrowsDown, Align MaxAlign, |
786 | BitVector &StackBytesFree) { |
787 | if (MFI.isVariableSizedObjectIndex(ObjectIdx: FrameIdx)) |
788 | return false; |
789 | |
790 | if (StackBytesFree.none()) { |
791 | // clear it to speed up later scavengeStackSlot calls to |
792 | // StackBytesFree.none() |
793 | StackBytesFree.clear(); |
794 | return false; |
795 | } |
796 | |
797 | Align ObjAlign = MFI.getObjectAlign(ObjectIdx: FrameIdx); |
798 | if (ObjAlign > MaxAlign) |
799 | return false; |
800 | |
801 | int64_t ObjSize = MFI.getObjectSize(ObjectIdx: FrameIdx); |
802 | int FreeStart; |
803 | for (FreeStart = StackBytesFree.find_first(); FreeStart != -1; |
804 | FreeStart = StackBytesFree.find_next(Prev: FreeStart)) { |
805 | |
806 | // Check that free space has suitable alignment. |
807 | unsigned ObjStart = StackGrowsDown ? FreeStart + ObjSize : FreeStart; |
808 | if (alignTo(Size: ObjStart, A: ObjAlign) != ObjStart) |
809 | continue; |
810 | |
811 | if (FreeStart + ObjSize > StackBytesFree.size()) |
812 | return false; |
813 | |
814 | bool AllBytesFree = true; |
815 | for (unsigned Byte = 0; Byte < ObjSize; ++Byte) |
816 | if (!StackBytesFree.test(Idx: FreeStart + Byte)) { |
817 | AllBytesFree = false; |
818 | break; |
819 | } |
820 | if (AllBytesFree) |
821 | break; |
822 | } |
823 | |
824 | if (FreeStart == -1) |
825 | return false; |
826 | |
827 | if (StackGrowsDown) { |
828 | int ObjStart = -(FreeStart + ObjSize); |
829 | LLVM_DEBUG(dbgs() << "alloc FI(" << FrameIdx << ") scavenged at SP[" |
830 | << ObjStart << "]\n" ); |
831 | MFI.setObjectOffset(ObjectIdx: FrameIdx, SPOffset: ObjStart); |
832 | } else { |
833 | LLVM_DEBUG(dbgs() << "alloc FI(" << FrameIdx << ") scavenged at SP[" |
834 | << FreeStart << "]\n" ); |
835 | MFI.setObjectOffset(ObjectIdx: FrameIdx, SPOffset: FreeStart); |
836 | } |
837 | |
838 | StackBytesFree.reset(I: FreeStart, E: FreeStart + ObjSize); |
839 | return true; |
840 | } |
841 | |
842 | /// AssignProtectedObjSet - Helper function to assign large stack objects (i.e., |
843 | /// those required to be close to the Stack Protector) to stack offsets. |
844 | static void AssignProtectedObjSet(const StackObjSet &UnassignedObjs, |
845 | SmallSet<int, 16> &ProtectedObjs, |
846 | MachineFrameInfo &MFI, bool StackGrowsDown, |
847 | int64_t &Offset, Align &MaxAlign) { |
848 | |
849 | for (int i : UnassignedObjs) { |
850 | AdjustStackOffset(MFI, FrameIdx: i, StackGrowsDown, Offset, MaxAlign); |
851 | ProtectedObjs.insert(V: i); |
852 | } |
853 | } |
854 | |
855 | /// calculateFrameObjectOffsets - Calculate actual frame offsets for all of the |
856 | /// abstract stack objects. |
857 | void PEI::calculateFrameObjectOffsets(MachineFunction &MF) { |
858 | const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering(); |
859 | |
860 | bool StackGrowsDown = |
861 | TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown; |
862 | |
863 | // Loop over all of the stack objects, assigning sequential addresses... |
864 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
865 | |
866 | // Start at the beginning of the local area. |
867 | // The Offset is the distance from the stack top in the direction |
868 | // of stack growth -- so it's always nonnegative. |
869 | int LocalAreaOffset = TFI.getOffsetOfLocalArea(); |
870 | if (StackGrowsDown) |
871 | LocalAreaOffset = -LocalAreaOffset; |
872 | assert(LocalAreaOffset >= 0 |
873 | && "Local area offset should be in direction of stack growth" ); |
874 | int64_t Offset = LocalAreaOffset; |
875 | |
876 | #ifdef EXPENSIVE_CHECKS |
877 | for (unsigned i = 0, e = MFI.getObjectIndexEnd(); i != e; ++i) |
878 | if (!MFI.isDeadObjectIndex(i) && |
879 | MFI.getStackID(i) == TargetStackID::Default) |
880 | assert(MFI.getObjectAlign(i) <= MFI.getMaxAlign() && |
881 | "MaxAlignment is invalid" ); |
882 | #endif |
883 | |
884 | // If there are fixed sized objects that are preallocated in the local area, |
885 | // non-fixed objects can't be allocated right at the start of local area. |
886 | // Adjust 'Offset' to point to the end of last fixed sized preallocated |
887 | // object. |
888 | for (int i = MFI.getObjectIndexBegin(); i != 0; ++i) { |
889 | // Only allocate objects on the default stack. |
890 | if (MFI.getStackID(ObjectIdx: i) != TargetStackID::Default) |
891 | continue; |
892 | |
893 | int64_t FixedOff; |
894 | if (StackGrowsDown) { |
895 | // The maximum distance from the stack pointer is at lower address of |
896 | // the object -- which is given by offset. For down growing stack |
897 | // the offset is negative, so we negate the offset to get the distance. |
898 | FixedOff = -MFI.getObjectOffset(ObjectIdx: i); |
899 | } else { |
900 | // The maximum distance from the start pointer is at the upper |
901 | // address of the object. |
902 | FixedOff = MFI.getObjectOffset(ObjectIdx: i) + MFI.getObjectSize(ObjectIdx: i); |
903 | } |
904 | if (FixedOff > Offset) Offset = FixedOff; |
905 | } |
906 | |
907 | Align MaxAlign = MFI.getMaxAlign(); |
908 | // First assign frame offsets to stack objects that are used to spill |
909 | // callee saved registers. |
910 | if (MaxCSFrameIndex >= MinCSFrameIndex) { |
911 | for (unsigned i = 0; i <= MaxCSFrameIndex - MinCSFrameIndex; ++i) { |
912 | unsigned FrameIndex = |
913 | StackGrowsDown ? MinCSFrameIndex + i : MaxCSFrameIndex - i; |
914 | |
915 | // Only allocate objects on the default stack. |
916 | if (MFI.getStackID(ObjectIdx: FrameIndex) != TargetStackID::Default) |
917 | continue; |
918 | |
919 | // TODO: should this just be if (MFI.isDeadObjectIndex(FrameIndex)) |
920 | if (!StackGrowsDown && MFI.isDeadObjectIndex(ObjectIdx: FrameIndex)) |
921 | continue; |
922 | |
923 | AdjustStackOffset(MFI, FrameIdx: FrameIndex, StackGrowsDown, Offset, MaxAlign); |
924 | } |
925 | } |
926 | |
927 | assert(MaxAlign == MFI.getMaxAlign() && |
928 | "MFI.getMaxAlign should already account for all callee-saved " |
929 | "registers without a fixed stack slot" ); |
930 | |
931 | // FixedCSEnd is the stack offset to the end of the fixed and callee-save |
932 | // stack area. |
933 | int64_t FixedCSEnd = Offset; |
934 | |
935 | // Make sure the special register scavenging spill slot is closest to the |
936 | // incoming stack pointer if a frame pointer is required and is closer |
937 | // to the incoming rather than the final stack pointer. |
938 | const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo(); |
939 | bool EarlyScavengingSlots = TFI.allocateScavengingFrameIndexesNearIncomingSP(MF); |
940 | if (RS && EarlyScavengingSlots) { |
941 | SmallVector<int, 2> SFIs; |
942 | RS->getScavengingFrameIndices(A&: SFIs); |
943 | for (int SFI : SFIs) |
944 | AdjustStackOffset(MFI, FrameIdx: SFI, StackGrowsDown, Offset, MaxAlign); |
945 | } |
946 | |
947 | // FIXME: Once this is working, then enable flag will change to a target |
948 | // check for whether the frame is large enough to want to use virtual |
949 | // frame index registers. Functions which don't want/need this optimization |
950 | // will continue to use the existing code path. |
951 | if (MFI.getUseLocalStackAllocationBlock()) { |
952 | Align Alignment = MFI.getLocalFrameMaxAlign(); |
953 | |
954 | // Adjust to alignment boundary. |
955 | Offset = alignTo(Size: Offset, A: Alignment); |
956 | |
957 | LLVM_DEBUG(dbgs() << "Local frame base offset: " << Offset << "\n" ); |
958 | |
959 | // Resolve offsets for objects in the local block. |
960 | for (unsigned i = 0, e = MFI.getLocalFrameObjectCount(); i != e; ++i) { |
961 | std::pair<int, int64_t> Entry = MFI.getLocalFrameObjectMap(i); |
962 | int64_t FIOffset = (StackGrowsDown ? -Offset : Offset) + Entry.second; |
963 | LLVM_DEBUG(dbgs() << "alloc FI(" << Entry.first << ") at SP[" << FIOffset |
964 | << "]\n" ); |
965 | MFI.setObjectOffset(ObjectIdx: Entry.first, SPOffset: FIOffset); |
966 | } |
967 | // Allocate the local block |
968 | Offset += MFI.getLocalFrameSize(); |
969 | |
970 | MaxAlign = std::max(a: Alignment, b: MaxAlign); |
971 | } |
972 | |
973 | // Retrieve the Exception Handler registration node. |
974 | int EHRegNodeFrameIndex = std::numeric_limits<int>::max(); |
975 | if (const WinEHFuncInfo *FuncInfo = MF.getWinEHFuncInfo()) |
976 | EHRegNodeFrameIndex = FuncInfo->EHRegNodeFrameIndex; |
977 | |
978 | // Make sure that the stack protector comes before the local variables on the |
979 | // stack. |
980 | SmallSet<int, 16> ProtectedObjs; |
981 | if (MFI.hasStackProtectorIndex()) { |
982 | int StackProtectorFI = MFI.getStackProtectorIndex(); |
983 | StackObjSet LargeArrayObjs; |
984 | StackObjSet SmallArrayObjs; |
985 | StackObjSet AddrOfObjs; |
986 | |
987 | // If we need a stack protector, we need to make sure that |
988 | // LocalStackSlotPass didn't already allocate a slot for it. |
989 | // If we are told to use the LocalStackAllocationBlock, the stack protector |
990 | // is expected to be already pre-allocated. |
991 | if (MFI.getStackID(ObjectIdx: StackProtectorFI) != TargetStackID::Default) { |
992 | // If the stack protector isn't on the default stack then it's up to the |
993 | // target to set the stack offset. |
994 | assert(MFI.getObjectOffset(StackProtectorFI) != 0 && |
995 | "Offset of stack protector on non-default stack expected to be " |
996 | "already set." ); |
997 | assert(!MFI.isObjectPreAllocated(MFI.getStackProtectorIndex()) && |
998 | "Stack protector on non-default stack expected to not be " |
999 | "pre-allocated by LocalStackSlotPass." ); |
1000 | } else if (!MFI.getUseLocalStackAllocationBlock()) { |
1001 | AdjustStackOffset(MFI, FrameIdx: StackProtectorFI, StackGrowsDown, Offset, |
1002 | MaxAlign); |
1003 | } else if (!MFI.isObjectPreAllocated(ObjectIdx: MFI.getStackProtectorIndex())) { |
1004 | llvm_unreachable( |
1005 | "Stack protector not pre-allocated by LocalStackSlotPass." ); |
1006 | } |
1007 | |
1008 | // Assign large stack objects first. |
1009 | for (unsigned i = 0, e = MFI.getObjectIndexEnd(); i != e; ++i) { |
1010 | if (MFI.isObjectPreAllocated(ObjectIdx: i) && MFI.getUseLocalStackAllocationBlock()) |
1011 | continue; |
1012 | if (i >= MinCSFrameIndex && i <= MaxCSFrameIndex) |
1013 | continue; |
1014 | if (RS && RS->isScavengingFrameIndex(FI: (int)i)) |
1015 | continue; |
1016 | if (MFI.isDeadObjectIndex(ObjectIdx: i)) |
1017 | continue; |
1018 | if (StackProtectorFI == (int)i || EHRegNodeFrameIndex == (int)i) |
1019 | continue; |
1020 | // Only allocate objects on the default stack. |
1021 | if (MFI.getStackID(ObjectIdx: i) != TargetStackID::Default) |
1022 | continue; |
1023 | |
1024 | switch (MFI.getObjectSSPLayout(ObjectIdx: i)) { |
1025 | case MachineFrameInfo::SSPLK_None: |
1026 | continue; |
1027 | case MachineFrameInfo::SSPLK_SmallArray: |
1028 | SmallArrayObjs.insert(X: i); |
1029 | continue; |
1030 | case MachineFrameInfo::SSPLK_AddrOf: |
1031 | AddrOfObjs.insert(X: i); |
1032 | continue; |
1033 | case MachineFrameInfo::SSPLK_LargeArray: |
1034 | LargeArrayObjs.insert(X: i); |
1035 | continue; |
1036 | } |
1037 | llvm_unreachable("Unexpected SSPLayoutKind." ); |
1038 | } |
1039 | |
1040 | // We expect **all** the protected stack objects to be pre-allocated by |
1041 | // LocalStackSlotPass. If it turns out that PEI still has to allocate some |
1042 | // of them, we may end up messing up the expected order of the objects. |
1043 | if (MFI.getUseLocalStackAllocationBlock() && |
1044 | !(LargeArrayObjs.empty() && SmallArrayObjs.empty() && |
1045 | AddrOfObjs.empty())) |
1046 | llvm_unreachable("Found protected stack objects not pre-allocated by " |
1047 | "LocalStackSlotPass." ); |
1048 | |
1049 | AssignProtectedObjSet(UnassignedObjs: LargeArrayObjs, ProtectedObjs, MFI, StackGrowsDown, |
1050 | Offset, MaxAlign); |
1051 | AssignProtectedObjSet(UnassignedObjs: SmallArrayObjs, ProtectedObjs, MFI, StackGrowsDown, |
1052 | Offset, MaxAlign); |
1053 | AssignProtectedObjSet(UnassignedObjs: AddrOfObjs, ProtectedObjs, MFI, StackGrowsDown, |
1054 | Offset, MaxAlign); |
1055 | } |
1056 | |
1057 | SmallVector<int, 8> ObjectsToAllocate; |
1058 | |
1059 | // Then prepare to assign frame offsets to stack objects that are not used to |
1060 | // spill callee saved registers. |
1061 | for (unsigned i = 0, e = MFI.getObjectIndexEnd(); i != e; ++i) { |
1062 | if (MFI.isObjectPreAllocated(ObjectIdx: i) && MFI.getUseLocalStackAllocationBlock()) |
1063 | continue; |
1064 | if (i >= MinCSFrameIndex && i <= MaxCSFrameIndex) |
1065 | continue; |
1066 | if (RS && RS->isScavengingFrameIndex(FI: (int)i)) |
1067 | continue; |
1068 | if (MFI.isDeadObjectIndex(ObjectIdx: i)) |
1069 | continue; |
1070 | if (MFI.getStackProtectorIndex() == (int)i || EHRegNodeFrameIndex == (int)i) |
1071 | continue; |
1072 | if (ProtectedObjs.count(V: i)) |
1073 | continue; |
1074 | // Only allocate objects on the default stack. |
1075 | if (MFI.getStackID(ObjectIdx: i) != TargetStackID::Default) |
1076 | continue; |
1077 | |
1078 | // Add the objects that we need to allocate to our working set. |
1079 | ObjectsToAllocate.push_back(Elt: i); |
1080 | } |
1081 | |
1082 | // Allocate the EH registration node first if one is present. |
1083 | if (EHRegNodeFrameIndex != std::numeric_limits<int>::max()) |
1084 | AdjustStackOffset(MFI, FrameIdx: EHRegNodeFrameIndex, StackGrowsDown, Offset, |
1085 | MaxAlign); |
1086 | |
1087 | // Give the targets a chance to order the objects the way they like it. |
1088 | if (MF.getTarget().getOptLevel() != CodeGenOptLevel::None && |
1089 | MF.getTarget().Options.StackSymbolOrdering) |
1090 | TFI.orderFrameObjects(MF, objectsToAllocate&: ObjectsToAllocate); |
1091 | |
1092 | // Keep track of which bytes in the fixed and callee-save range are used so we |
1093 | // can use the holes when allocating later stack objects. Only do this if |
1094 | // stack protector isn't being used and the target requests it and we're |
1095 | // optimizing. |
1096 | BitVector StackBytesFree; |
1097 | if (!ObjectsToAllocate.empty() && |
1098 | MF.getTarget().getOptLevel() != CodeGenOptLevel::None && |
1099 | MFI.getStackProtectorIndex() < 0 && TFI.enableStackSlotScavenging(MF)) |
1100 | computeFreeStackSlots(MFI, StackGrowsDown, MinCSFrameIndex, MaxCSFrameIndex, |
1101 | FixedCSEnd, StackBytesFree); |
1102 | |
1103 | // Now walk the objects and actually assign base offsets to them. |
1104 | for (auto &Object : ObjectsToAllocate) |
1105 | if (!scavengeStackSlot(MFI, FrameIdx: Object, StackGrowsDown, MaxAlign, |
1106 | StackBytesFree)) |
1107 | AdjustStackOffset(MFI, FrameIdx: Object, StackGrowsDown, Offset, MaxAlign); |
1108 | |
1109 | // Make sure the special register scavenging spill slot is closest to the |
1110 | // stack pointer. |
1111 | if (RS && !EarlyScavengingSlots) { |
1112 | SmallVector<int, 2> SFIs; |
1113 | RS->getScavengingFrameIndices(A&: SFIs); |
1114 | for (int SFI : SFIs) |
1115 | AdjustStackOffset(MFI, FrameIdx: SFI, StackGrowsDown, Offset, MaxAlign); |
1116 | } |
1117 | |
1118 | if (!TFI.targetHandlesStackFrameRounding()) { |
1119 | // If we have reserved argument space for call sites in the function |
1120 | // immediately on entry to the current function, count it as part of the |
1121 | // overall stack size. |
1122 | if (MFI.adjustsStack() && TFI.hasReservedCallFrame(MF)) |
1123 | Offset += MFI.getMaxCallFrameSize(); |
1124 | |
1125 | // Round up the size to a multiple of the alignment. If the function has |
1126 | // any calls or alloca's, align to the target's StackAlignment value to |
1127 | // ensure that the callee's frame or the alloca data is suitably aligned; |
1128 | // otherwise, for leaf functions, align to the TransientStackAlignment |
1129 | // value. |
1130 | Align StackAlign; |
1131 | if (MFI.adjustsStack() || MFI.hasVarSizedObjects() || |
1132 | (RegInfo->hasStackRealignment(MF) && MFI.getObjectIndexEnd() != 0)) |
1133 | StackAlign = TFI.getStackAlign(); |
1134 | else |
1135 | StackAlign = TFI.getTransientStackAlign(); |
1136 | |
1137 | // If the frame pointer is eliminated, all frame offsets will be relative to |
1138 | // SP not FP. Align to MaxAlign so this works. |
1139 | StackAlign = std::max(a: StackAlign, b: MaxAlign); |
1140 | int64_t OffsetBeforeAlignment = Offset; |
1141 | Offset = alignTo(Size: Offset, A: StackAlign); |
1142 | |
1143 | // If we have increased the offset to fulfill the alignment constrants, |
1144 | // then the scavenging spill slots may become harder to reach from the |
1145 | // stack pointer, float them so they stay close. |
1146 | if (StackGrowsDown && OffsetBeforeAlignment != Offset && RS && |
1147 | !EarlyScavengingSlots) { |
1148 | SmallVector<int, 2> SFIs; |
1149 | RS->getScavengingFrameIndices(A&: SFIs); |
1150 | LLVM_DEBUG(if (!SFIs.empty()) llvm::dbgs() |
1151 | << "Adjusting emergency spill slots!\n" ;); |
1152 | int64_t Delta = Offset - OffsetBeforeAlignment; |
1153 | for (int SFI : SFIs) { |
1154 | LLVM_DEBUG(llvm::dbgs() |
1155 | << "Adjusting offset of emergency spill slot #" << SFI |
1156 | << " from " << MFI.getObjectOffset(SFI);); |
1157 | MFI.setObjectOffset(ObjectIdx: SFI, SPOffset: MFI.getObjectOffset(ObjectIdx: SFI) - Delta); |
1158 | LLVM_DEBUG(llvm::dbgs() << " to " << MFI.getObjectOffset(SFI) << "\n" ;); |
1159 | } |
1160 | } |
1161 | } |
1162 | |
1163 | // Update frame info to pretend that this is part of the stack... |
1164 | int64_t StackSize = Offset - LocalAreaOffset; |
1165 | MFI.setStackSize(StackSize); |
1166 | NumBytesStackSpace += StackSize; |
1167 | } |
1168 | |
1169 | /// insertPrologEpilogCode - Scan the function for modified callee saved |
1170 | /// registers, insert spill code for these callee saved registers, then add |
1171 | /// prolog and epilog code to the function. |
1172 | void PEI::insertPrologEpilogCode(MachineFunction &MF) { |
1173 | const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering(); |
1174 | |
1175 | // Add prologue to the function... |
1176 | for (MachineBasicBlock *SaveBlock : SaveBlocks) |
1177 | TFI.emitPrologue(MF, MBB&: *SaveBlock); |
1178 | |
1179 | // Add epilogue to restore the callee-save registers in each exiting block. |
1180 | for (MachineBasicBlock *RestoreBlock : RestoreBlocks) |
1181 | TFI.emitEpilogue(MF, MBB&: *RestoreBlock); |
1182 | |
1183 | // Zero call used registers before restoring callee-saved registers. |
1184 | insertZeroCallUsedRegs(MF); |
1185 | |
1186 | for (MachineBasicBlock *SaveBlock : SaveBlocks) |
1187 | TFI.inlineStackProbe(MF, PrologueMBB&: *SaveBlock); |
1188 | |
1189 | // Emit additional code that is required to support segmented stacks, if |
1190 | // we've been asked for it. This, when linked with a runtime with support |
1191 | // for segmented stacks (libgcc is one), will result in allocating stack |
1192 | // space in small chunks instead of one large contiguous block. |
1193 | if (MF.shouldSplitStack()) { |
1194 | for (MachineBasicBlock *SaveBlock : SaveBlocks) |
1195 | TFI.adjustForSegmentedStacks(MF, PrologueMBB&: *SaveBlock); |
1196 | } |
1197 | |
1198 | // Emit additional code that is required to explicitly handle the stack in |
1199 | // HiPE native code (if needed) when loaded in the Erlang/OTP runtime. The |
1200 | // approach is rather similar to that of Segmented Stacks, but it uses a |
1201 | // different conditional check and another BIF for allocating more stack |
1202 | // space. |
1203 | if (MF.getFunction().getCallingConv() == CallingConv::HiPE) |
1204 | for (MachineBasicBlock *SaveBlock : SaveBlocks) |
1205 | TFI.adjustForHiPEPrologue(MF, PrologueMBB&: *SaveBlock); |
1206 | } |
1207 | |
1208 | /// insertZeroCallUsedRegs - Zero out call used registers. |
1209 | void PEI::insertZeroCallUsedRegs(MachineFunction &MF) { |
1210 | const Function &F = MF.getFunction(); |
1211 | |
1212 | if (!F.hasFnAttribute(Kind: "zero-call-used-regs" )) |
1213 | return; |
1214 | |
1215 | using namespace ZeroCallUsedRegs; |
1216 | |
1217 | ZeroCallUsedRegsKind ZeroRegsKind = |
1218 | StringSwitch<ZeroCallUsedRegsKind>( |
1219 | F.getFnAttribute(Kind: "zero-call-used-regs" ).getValueAsString()) |
1220 | .Case(S: "skip" , Value: ZeroCallUsedRegsKind::Skip) |
1221 | .Case(S: "used-gpr-arg" , Value: ZeroCallUsedRegsKind::UsedGPRArg) |
1222 | .Case(S: "used-gpr" , Value: ZeroCallUsedRegsKind::UsedGPR) |
1223 | .Case(S: "used-arg" , Value: ZeroCallUsedRegsKind::UsedArg) |
1224 | .Case(S: "used" , Value: ZeroCallUsedRegsKind::Used) |
1225 | .Case(S: "all-gpr-arg" , Value: ZeroCallUsedRegsKind::AllGPRArg) |
1226 | .Case(S: "all-gpr" , Value: ZeroCallUsedRegsKind::AllGPR) |
1227 | .Case(S: "all-arg" , Value: ZeroCallUsedRegsKind::AllArg) |
1228 | .Case(S: "all" , Value: ZeroCallUsedRegsKind::All); |
1229 | |
1230 | if (ZeroRegsKind == ZeroCallUsedRegsKind::Skip) |
1231 | return; |
1232 | |
1233 | const bool OnlyGPR = static_cast<unsigned>(ZeroRegsKind) & ONLY_GPR; |
1234 | const bool OnlyUsed = static_cast<unsigned>(ZeroRegsKind) & ONLY_USED; |
1235 | const bool OnlyArg = static_cast<unsigned>(ZeroRegsKind) & ONLY_ARG; |
1236 | |
1237 | const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); |
1238 | const BitVector AllocatableSet(TRI.getAllocatableSet(MF)); |
1239 | |
1240 | // Mark all used registers. |
1241 | BitVector UsedRegs(TRI.getNumRegs()); |
1242 | if (OnlyUsed) |
1243 | for (const MachineBasicBlock &MBB : MF) |
1244 | for (const MachineInstr &MI : MBB) { |
1245 | // skip debug instructions |
1246 | if (MI.isDebugInstr()) |
1247 | continue; |
1248 | |
1249 | for (const MachineOperand &MO : MI.operands()) { |
1250 | if (!MO.isReg()) |
1251 | continue; |
1252 | |
1253 | MCRegister Reg = MO.getReg(); |
1254 | if (AllocatableSet[Reg] && !MO.isImplicit() && |
1255 | (MO.isDef() || MO.isUse())) |
1256 | UsedRegs.set(Reg); |
1257 | } |
1258 | } |
1259 | |
1260 | // Get a list of registers that are used. |
1261 | BitVector LiveIns(TRI.getNumRegs()); |
1262 | for (const MachineBasicBlock::RegisterMaskPair &LI : MF.front().liveins()) |
1263 | LiveIns.set(LI.PhysReg); |
1264 | |
1265 | BitVector RegsToZero(TRI.getNumRegs()); |
1266 | for (MCRegister Reg : AllocatableSet.set_bits()) { |
1267 | // Skip over fixed registers. |
1268 | if (TRI.isFixedRegister(MF, PhysReg: Reg)) |
1269 | continue; |
1270 | |
1271 | // Want only general purpose registers. |
1272 | if (OnlyGPR && !TRI.isGeneralPurposeRegister(MF, PhysReg: Reg)) |
1273 | continue; |
1274 | |
1275 | // Want only used registers. |
1276 | if (OnlyUsed && !UsedRegs[Reg]) |
1277 | continue; |
1278 | |
1279 | // Want only registers used for arguments. |
1280 | if (OnlyArg) { |
1281 | if (OnlyUsed) { |
1282 | if (!LiveIns[Reg]) |
1283 | continue; |
1284 | } else if (!TRI.isArgumentRegister(MF, PhysReg: Reg)) { |
1285 | continue; |
1286 | } |
1287 | } |
1288 | |
1289 | RegsToZero.set(Reg); |
1290 | } |
1291 | |
1292 | // Don't clear registers that are live when leaving the function. |
1293 | for (const MachineBasicBlock &MBB : MF) |
1294 | for (const MachineInstr &MI : MBB.terminators()) { |
1295 | if (!MI.isReturn()) |
1296 | continue; |
1297 | |
1298 | for (const auto &MO : MI.operands()) { |
1299 | if (!MO.isReg()) |
1300 | continue; |
1301 | |
1302 | MCRegister Reg = MO.getReg(); |
1303 | if (!Reg) |
1304 | continue; |
1305 | |
1306 | // This picks up sibling registers (e.q. %al -> %ah). |
1307 | for (MCRegUnit Unit : TRI.regunits(Reg)) |
1308 | RegsToZero.reset(Idx: Unit); |
1309 | |
1310 | for (MCPhysReg SReg : TRI.sub_and_superregs_inclusive(Reg)) |
1311 | RegsToZero.reset(Idx: SReg); |
1312 | } |
1313 | } |
1314 | |
1315 | // Don't need to clear registers that are used/clobbered by terminating |
1316 | // instructions. |
1317 | for (const MachineBasicBlock &MBB : MF) { |
1318 | if (!MBB.isReturnBlock()) |
1319 | continue; |
1320 | |
1321 | MachineBasicBlock::const_iterator MBBI = MBB.getFirstTerminator(); |
1322 | for (MachineBasicBlock::const_iterator I = MBBI, E = MBB.end(); I != E; |
1323 | ++I) { |
1324 | for (const MachineOperand &MO : I->operands()) { |
1325 | if (!MO.isReg()) |
1326 | continue; |
1327 | |
1328 | MCRegister Reg = MO.getReg(); |
1329 | if (!Reg) |
1330 | continue; |
1331 | |
1332 | for (const MCPhysReg Reg : TRI.sub_and_superregs_inclusive(Reg)) |
1333 | RegsToZero.reset(Idx: Reg); |
1334 | } |
1335 | } |
1336 | } |
1337 | |
1338 | // Don't clear registers that must be preserved. |
1339 | for (const MCPhysReg *CSRegs = TRI.getCalleeSavedRegs(MF: &MF); |
1340 | MCPhysReg CSReg = *CSRegs; ++CSRegs) |
1341 | for (MCRegister Reg : TRI.sub_and_superregs_inclusive(Reg: CSReg)) |
1342 | RegsToZero.reset(Idx: Reg); |
1343 | |
1344 | const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering(); |
1345 | for (MachineBasicBlock &MBB : MF) |
1346 | if (MBB.isReturnBlock()) |
1347 | TFI.emitZeroCallUsedRegs(RegsToZero, MBB); |
1348 | } |
1349 | |
1350 | /// Replace all FrameIndex operands with physical register references and actual |
1351 | /// offsets. |
1352 | void PEI::replaceFrameIndicesBackward(MachineFunction &MF) { |
1353 | const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering(); |
1354 | |
1355 | for (auto &MBB : MF) { |
1356 | int SPAdj = 0; |
1357 | if (!MBB.succ_empty()) { |
1358 | // Get the SP adjustment for the end of MBB from the start of any of its |
1359 | // successors. They should all be the same. |
1360 | assert(all_of(MBB.successors(), [&MBB](const MachineBasicBlock *Succ) { |
1361 | return Succ->getCallFrameSize() == |
1362 | (*MBB.succ_begin())->getCallFrameSize(); |
1363 | })); |
1364 | const MachineBasicBlock &FirstSucc = **MBB.succ_begin(); |
1365 | SPAdj = TFI.alignSPAdjust(SPAdj: FirstSucc.getCallFrameSize()); |
1366 | if (TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp) |
1367 | SPAdj = -SPAdj; |
1368 | } |
1369 | |
1370 | replaceFrameIndicesBackward(BB: &MBB, MF, SPAdj); |
1371 | |
1372 | // We can't track the call frame size after call frame pseudos have been |
1373 | // eliminated. Set it to zero everywhere to keep MachineVerifier happy. |
1374 | MBB.setCallFrameSize(0); |
1375 | } |
1376 | } |
1377 | |
1378 | /// replaceFrameIndices - Replace all MO_FrameIndex operands with physical |
1379 | /// register references and actual offsets. |
1380 | void PEI::replaceFrameIndices(MachineFunction &MF) { |
1381 | const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering(); |
1382 | |
1383 | for (auto &MBB : MF) { |
1384 | int SPAdj = TFI.alignSPAdjust(SPAdj: MBB.getCallFrameSize()); |
1385 | if (TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp) |
1386 | SPAdj = -SPAdj; |
1387 | |
1388 | replaceFrameIndices(BB: &MBB, MF, SPAdj); |
1389 | |
1390 | // We can't track the call frame size after call frame pseudos have been |
1391 | // eliminated. Set it to zero everywhere to keep MachineVerifier happy. |
1392 | MBB.setCallFrameSize(0); |
1393 | } |
1394 | } |
1395 | |
1396 | bool PEI::replaceFrameIndexDebugInstr(MachineFunction &MF, MachineInstr &MI, |
1397 | unsigned OpIdx, int SPAdj) { |
1398 | const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering(); |
1399 | const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); |
1400 | if (MI.isDebugValue()) { |
1401 | |
1402 | MachineOperand &Op = MI.getOperand(i: OpIdx); |
1403 | assert(MI.isDebugOperand(&Op) && |
1404 | "Frame indices can only appear as a debug operand in a DBG_VALUE*" |
1405 | " machine instruction" ); |
1406 | Register Reg; |
1407 | unsigned FrameIdx = Op.getIndex(); |
1408 | unsigned Size = MF.getFrameInfo().getObjectSize(ObjectIdx: FrameIdx); |
1409 | |
1410 | StackOffset Offset = TFI->getFrameIndexReference(MF, FI: FrameIdx, FrameReg&: Reg); |
1411 | Op.ChangeToRegister(Reg, isDef: false /*isDef*/); |
1412 | |
1413 | const DIExpression *DIExpr = MI.getDebugExpression(); |
1414 | |
1415 | // If we have a direct DBG_VALUE, and its location expression isn't |
1416 | // currently complex, then adding an offset will morph it into a |
1417 | // complex location that is interpreted as being a memory address. |
1418 | // This changes a pointer-valued variable to dereference that pointer, |
1419 | // which is incorrect. Fix by adding DW_OP_stack_value. |
1420 | |
1421 | if (MI.isNonListDebugValue()) { |
1422 | unsigned PrependFlags = DIExpression::ApplyOffset; |
1423 | if (!MI.isIndirectDebugValue() && !DIExpr->isComplex()) |
1424 | PrependFlags |= DIExpression::StackValue; |
1425 | |
1426 | // If we have DBG_VALUE that is indirect and has a Implicit location |
1427 | // expression need to insert a deref before prepending a Memory |
1428 | // location expression. Also after doing this we change the DBG_VALUE |
1429 | // to be direct. |
1430 | if (MI.isIndirectDebugValue() && DIExpr->isImplicit()) { |
1431 | SmallVector<uint64_t, 2> Ops = {dwarf::DW_OP_deref_size, Size}; |
1432 | bool WithStackValue = true; |
1433 | DIExpr = DIExpression::prependOpcodes(Expr: DIExpr, Ops, StackValue: WithStackValue); |
1434 | // Make the DBG_VALUE direct. |
1435 | MI.getDebugOffset().ChangeToRegister(Reg: 0, isDef: false); |
1436 | } |
1437 | DIExpr = TRI.prependOffsetExpression(Expr: DIExpr, PrependFlags, Offset); |
1438 | } else { |
1439 | // The debug operand at DebugOpIndex was a frame index at offset |
1440 | // `Offset`; now the operand has been replaced with the frame |
1441 | // register, we must add Offset with `register x, plus Offset`. |
1442 | unsigned DebugOpIndex = MI.getDebugOperandIndex(Op: &Op); |
1443 | SmallVector<uint64_t, 3> Ops; |
1444 | TRI.getOffsetOpcodes(Offset, Ops); |
1445 | DIExpr = DIExpression::appendOpsToArg(Expr: DIExpr, Ops, ArgNo: DebugOpIndex); |
1446 | } |
1447 | MI.getDebugExpressionOp().setMetadata(DIExpr); |
1448 | return true; |
1449 | } |
1450 | |
1451 | if (MI.isDebugPHI()) { |
1452 | // Allow stack ref to continue onwards. |
1453 | return true; |
1454 | } |
1455 | |
1456 | // TODO: This code should be commoned with the code for |
1457 | // PATCHPOINT. There's no good reason for the difference in |
1458 | // implementation other than historical accident. The only |
1459 | // remaining difference is the unconditional use of the stack |
1460 | // pointer as the base register. |
1461 | if (MI.getOpcode() == TargetOpcode::STATEPOINT) { |
1462 | assert((!MI.isDebugValue() || OpIdx == 0) && |
1463 | "Frame indicies can only appear as the first operand of a " |
1464 | "DBG_VALUE machine instruction" ); |
1465 | Register Reg; |
1466 | MachineOperand &Offset = MI.getOperand(i: OpIdx + 1); |
1467 | StackOffset refOffset = TFI->getFrameIndexReferencePreferSP( |
1468 | MF, FI: MI.getOperand(i: OpIdx).getIndex(), FrameReg&: Reg, /*IgnoreSPUpdates*/ false); |
1469 | assert(!refOffset.getScalable() && |
1470 | "Frame offsets with a scalable component are not supported" ); |
1471 | Offset.setImm(Offset.getImm() + refOffset.getFixed() + SPAdj); |
1472 | MI.getOperand(i: OpIdx).ChangeToRegister(Reg, isDef: false /*isDef*/); |
1473 | return true; |
1474 | } |
1475 | return false; |
1476 | } |
1477 | |
1478 | void PEI::replaceFrameIndicesBackward(MachineBasicBlock *BB, |
1479 | MachineFunction &MF, int &SPAdj) { |
1480 | assert(MF.getSubtarget().getRegisterInfo() && |
1481 | "getRegisterInfo() must be implemented!" ); |
1482 | |
1483 | const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo(); |
1484 | const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); |
1485 | const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering(); |
1486 | |
1487 | RegScavenger *LocalRS = FrameIndexEliminationScavenging ? RS : nullptr; |
1488 | if (LocalRS) |
1489 | LocalRS->enterBasicBlockEnd(MBB&: *BB); |
1490 | |
1491 | for (MachineBasicBlock::iterator I = BB->end(); I != BB->begin();) { |
1492 | MachineInstr &MI = *std::prev(x: I); |
1493 | |
1494 | if (TII.isFrameInstr(I: MI)) { |
1495 | SPAdj -= TII.getSPAdjust(MI); |
1496 | TFI.eliminateCallFramePseudoInstr(MF, MBB&: *BB, MI: &MI); |
1497 | continue; |
1498 | } |
1499 | |
1500 | // Step backwards to get the liveness state at (immedately after) MI. |
1501 | if (LocalRS) |
1502 | LocalRS->backward(I); |
1503 | |
1504 | bool RemovedMI = false; |
1505 | for (const auto &[Idx, Op] : enumerate(First: MI.operands())) { |
1506 | if (!Op.isFI()) |
1507 | continue; |
1508 | |
1509 | if (replaceFrameIndexDebugInstr(MF, MI, OpIdx: Idx, SPAdj)) |
1510 | continue; |
1511 | |
1512 | // Eliminate this FrameIndex operand. |
1513 | RemovedMI = TRI.eliminateFrameIndex(MI, SPAdj, FIOperandNum: Idx, RS: LocalRS); |
1514 | if (RemovedMI) |
1515 | break; |
1516 | } |
1517 | |
1518 | if (!RemovedMI) |
1519 | --I; |
1520 | } |
1521 | } |
1522 | |
1523 | void PEI::replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &MF, |
1524 | int &SPAdj) { |
1525 | assert(MF.getSubtarget().getRegisterInfo() && |
1526 | "getRegisterInfo() must be implemented!" ); |
1527 | const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo(); |
1528 | const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); |
1529 | const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering(); |
1530 | |
1531 | bool InsideCallSequence = false; |
1532 | |
1533 | for (MachineBasicBlock::iterator I = BB->begin(); I != BB->end(); ) { |
1534 | if (TII.isFrameInstr(I: *I)) { |
1535 | InsideCallSequence = TII.isFrameSetup(I: *I); |
1536 | SPAdj += TII.getSPAdjust(MI: *I); |
1537 | I = TFI->eliminateCallFramePseudoInstr(MF, MBB&: *BB, MI: I); |
1538 | continue; |
1539 | } |
1540 | |
1541 | MachineInstr &MI = *I; |
1542 | bool DoIncr = true; |
1543 | bool DidFinishLoop = true; |
1544 | for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { |
1545 | if (!MI.getOperand(i).isFI()) |
1546 | continue; |
1547 | |
1548 | if (replaceFrameIndexDebugInstr(MF, MI, OpIdx: i, SPAdj)) |
1549 | continue; |
1550 | |
1551 | // Some instructions (e.g. inline asm instructions) can have |
1552 | // multiple frame indices and/or cause eliminateFrameIndex |
1553 | // to insert more than one instruction. We need the register |
1554 | // scavenger to go through all of these instructions so that |
1555 | // it can update its register information. We keep the |
1556 | // iterator at the point before insertion so that we can |
1557 | // revisit them in full. |
1558 | bool AtBeginning = (I == BB->begin()); |
1559 | if (!AtBeginning) --I; |
1560 | |
1561 | // If this instruction has a FrameIndex operand, we need to |
1562 | // use that target machine register info object to eliminate |
1563 | // it. |
1564 | TRI.eliminateFrameIndex(MI, SPAdj, FIOperandNum: i); |
1565 | |
1566 | // Reset the iterator if we were at the beginning of the BB. |
1567 | if (AtBeginning) { |
1568 | I = BB->begin(); |
1569 | DoIncr = false; |
1570 | } |
1571 | |
1572 | DidFinishLoop = false; |
1573 | break; |
1574 | } |
1575 | |
1576 | // If we are looking at a call sequence, we need to keep track of |
1577 | // the SP adjustment made by each instruction in the sequence. |
1578 | // This includes both the frame setup/destroy pseudos (handled above), |
1579 | // as well as other instructions that have side effects w.r.t the SP. |
1580 | // Note that this must come after eliminateFrameIndex, because |
1581 | // if I itself referred to a frame index, we shouldn't count its own |
1582 | // adjustment. |
1583 | if (DidFinishLoop && InsideCallSequence) |
1584 | SPAdj += TII.getSPAdjust(MI); |
1585 | |
1586 | if (DoIncr && I != BB->end()) |
1587 | ++I; |
1588 | } |
1589 | } |
1590 | |