1//===- PrologEpilogInserter.cpp - Insert Prolog/Epilog code in function ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass is responsible for finalizing the functions frame layout, saving
10// callee saved registers, and for emitting prolog & epilog code for the
11// function.
12//
13// This pass must be run after register allocation. After this pass is
14// executed, it is illegal to construct MO_FrameIndex operands.
15//
16//===----------------------------------------------------------------------===//
17
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/BitVector.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/SetVector.h"
22#include "llvm/ADT/SmallPtrSet.h"
23#include "llvm/ADT/SmallSet.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/Statistic.h"
26#include "llvm/Analysis/OptimizationRemarkEmitter.h"
27#include "llvm/CodeGen/MachineBasicBlock.h"
28#include "llvm/CodeGen/MachineDominators.h"
29#include "llvm/CodeGen/MachineFrameInfo.h"
30#include "llvm/CodeGen/MachineFunction.h"
31#include "llvm/CodeGen/MachineFunctionPass.h"
32#include "llvm/CodeGen/MachineInstr.h"
33#include "llvm/CodeGen/MachineInstrBuilder.h"
34#include "llvm/CodeGen/MachineLoopInfo.h"
35#include "llvm/CodeGen/MachineModuleInfo.h"
36#include "llvm/CodeGen/MachineOperand.h"
37#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
38#include "llvm/CodeGen/MachineRegisterInfo.h"
39#include "llvm/CodeGen/RegisterScavenging.h"
40#include "llvm/CodeGen/TargetFrameLowering.h"
41#include "llvm/CodeGen/TargetInstrInfo.h"
42#include "llvm/CodeGen/TargetOpcodes.h"
43#include "llvm/CodeGen/TargetRegisterInfo.h"
44#include "llvm/CodeGen/TargetSubtargetInfo.h"
45#include "llvm/CodeGen/WinEHFuncInfo.h"
46#include "llvm/IR/Attributes.h"
47#include "llvm/IR/CallingConv.h"
48#include "llvm/IR/DebugInfoMetadata.h"
49#include "llvm/IR/DiagnosticInfo.h"
50#include "llvm/IR/Function.h"
51#include "llvm/IR/InlineAsm.h"
52#include "llvm/IR/LLVMContext.h"
53#include "llvm/InitializePasses.h"
54#include "llvm/MC/MCRegisterInfo.h"
55#include "llvm/Pass.h"
56#include "llvm/Support/CodeGen.h"
57#include "llvm/Support/Debug.h"
58#include "llvm/Support/ErrorHandling.h"
59#include "llvm/Support/FormatVariadic.h"
60#include "llvm/Support/raw_ostream.h"
61#include "llvm/Target/TargetMachine.h"
62#include "llvm/Target/TargetOptions.h"
63#include <algorithm>
64#include <cassert>
65#include <cstdint>
66#include <functional>
67#include <limits>
68#include <utility>
69#include <vector>
70
71using namespace llvm;
72
73#define DEBUG_TYPE "prologepilog"
74
75using MBBVector = SmallVector<MachineBasicBlock *, 4>;
76
77STATISTIC(NumLeafFuncWithSpills, "Number of leaf functions with CSRs");
78STATISTIC(NumFuncSeen, "Number of functions seen in PEI");
79
80
81namespace {
82
83class PEI : public MachineFunctionPass {
84public:
85 static char ID;
86
87 PEI() : MachineFunctionPass(ID) {
88 initializePEIPass(*PassRegistry::getPassRegistry());
89 }
90
91 void getAnalysisUsage(AnalysisUsage &AU) const override;
92
93 /// runOnMachineFunction - Insert prolog/epilog code and replace abstract
94 /// frame indexes with appropriate references.
95 bool runOnMachineFunction(MachineFunction &MF) override;
96
97private:
98 RegScavenger *RS = nullptr;
99
100 // MinCSFrameIndex, MaxCSFrameIndex - Keeps the range of callee saved
101 // stack frame indexes.
102 unsigned MinCSFrameIndex = std::numeric_limits<unsigned>::max();
103 unsigned MaxCSFrameIndex = 0;
104
105 // Save and Restore blocks of the current function. Typically there is a
106 // single save block, unless Windows EH funclets are involved.
107 MBBVector SaveBlocks;
108 MBBVector RestoreBlocks;
109
110 // Flag to control whether to use the register scavenger to resolve
111 // frame index materialization registers. Set according to
112 // TRI->requiresFrameIndexScavenging() for the current function.
113 bool FrameIndexVirtualScavenging = false;
114
115 // Flag to control whether the scavenger should be passed even though
116 // FrameIndexVirtualScavenging is used.
117 bool FrameIndexEliminationScavenging = false;
118
119 // Emit remarks.
120 MachineOptimizationRemarkEmitter *ORE = nullptr;
121
122 void calculateCallFrameInfo(MachineFunction &MF);
123 void calculateSaveRestoreBlocks(MachineFunction &MF);
124 void spillCalleeSavedRegs(MachineFunction &MF);
125
126 void calculateFrameObjectOffsets(MachineFunction &MF);
127 void replaceFrameIndices(MachineFunction &MF);
128 void replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &MF,
129 int &SPAdj);
130 // Frame indices in debug values are encoded in a target independent
131 // way with simply the frame index and offset rather than any
132 // target-specific addressing mode.
133 bool replaceFrameIndexDebugInstr(MachineFunction &MF, MachineInstr &MI,
134 unsigned OpIdx, int SPAdj = 0);
135 // Does same as replaceFrameIndices but using the backward MIR walk and
136 // backward register scavenger walk.
137 void replaceFrameIndicesBackward(MachineFunction &MF);
138 void replaceFrameIndicesBackward(MachineBasicBlock *BB, MachineFunction &MF,
139 int &SPAdj);
140
141 void insertPrologEpilogCode(MachineFunction &MF);
142 void insertZeroCallUsedRegs(MachineFunction &MF);
143};
144
145} // end anonymous namespace
146
147char PEI::ID = 0;
148
149char &llvm::PrologEpilogCodeInserterID = PEI::ID;
150
151INITIALIZE_PASS_BEGIN(PEI, DEBUG_TYPE, "Prologue/Epilogue Insertion", false,
152 false)
153INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo)
154INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
155INITIALIZE_PASS_DEPENDENCY(MachineOptimizationRemarkEmitterPass)
156INITIALIZE_PASS_END(PEI, DEBUG_TYPE,
157 "Prologue/Epilogue Insertion & Frame Finalization", false,
158 false)
159
160MachineFunctionPass *llvm::createPrologEpilogInserterPass() {
161 return new PEI();
162}
163
164STATISTIC(NumBytesStackSpace,
165 "Number of bytes used for stack in all functions");
166
167void PEI::getAnalysisUsage(AnalysisUsage &AU) const {
168 AU.setPreservesCFG();
169 AU.addPreserved<MachineLoopInfo>();
170 AU.addPreserved<MachineDominatorTree>();
171 AU.addRequired<MachineOptimizationRemarkEmitterPass>();
172 MachineFunctionPass::getAnalysisUsage(AU);
173}
174
175/// StackObjSet - A set of stack object indexes
176using StackObjSet = SmallSetVector<int, 8>;
177
178using SavedDbgValuesMap =
179 SmallDenseMap<MachineBasicBlock *, SmallVector<MachineInstr *, 4>, 4>;
180
181/// Stash DBG_VALUEs that describe parameters and which are placed at the start
182/// of the block. Later on, after the prologue code has been emitted, the
183/// stashed DBG_VALUEs will be reinserted at the start of the block.
184static void stashEntryDbgValues(MachineBasicBlock &MBB,
185 SavedDbgValuesMap &EntryDbgValues) {
186 SmallVector<const MachineInstr *, 4> FrameIndexValues;
187
188 for (auto &MI : MBB) {
189 if (!MI.isDebugInstr())
190 break;
191 if (!MI.isDebugValue() || !MI.getDebugVariable()->isParameter())
192 continue;
193 if (any_of(Range: MI.debug_operands(),
194 P: [](const MachineOperand &MO) { return MO.isFI(); })) {
195 // We can only emit valid locations for frame indices after the frame
196 // setup, so do not stash away them.
197 FrameIndexValues.push_back(Elt: &MI);
198 continue;
199 }
200 const DILocalVariable *Var = MI.getDebugVariable();
201 const DIExpression *Expr = MI.getDebugExpression();
202 auto Overlaps = [Var, Expr](const MachineInstr *DV) {
203 return Var == DV->getDebugVariable() &&
204 Expr->fragmentsOverlap(Other: DV->getDebugExpression());
205 };
206 // See if the debug value overlaps with any preceding debug value that will
207 // not be stashed. If that is the case, then we can't stash this value, as
208 // we would then reorder the values at reinsertion.
209 if (llvm::none_of(Range&: FrameIndexValues, P: Overlaps))
210 EntryDbgValues[&MBB].push_back(Elt: &MI);
211 }
212
213 // Remove stashed debug values from the block.
214 if (EntryDbgValues.count(Val: &MBB))
215 for (auto *MI : EntryDbgValues[&MBB])
216 MI->removeFromParent();
217}
218
219/// runOnMachineFunction - Insert prolog/epilog code and replace abstract
220/// frame indexes with appropriate references.
221bool PEI::runOnMachineFunction(MachineFunction &MF) {
222 NumFuncSeen++;
223 const Function &F = MF.getFunction();
224 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
225 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
226
227 RS = TRI->requiresRegisterScavenging(MF) ? new RegScavenger() : nullptr;
228 FrameIndexVirtualScavenging = TRI->requiresFrameIndexScavenging(MF);
229 ORE = &getAnalysis<MachineOptimizationRemarkEmitterPass>().getORE();
230
231 // Calculate the MaxCallFrameSize and AdjustsStack variables for the
232 // function's frame information. Also eliminates call frame pseudo
233 // instructions.
234 calculateCallFrameInfo(MF);
235
236 // Determine placement of CSR spill/restore code and prolog/epilog code:
237 // place all spills in the entry block, all restores in return blocks.
238 calculateSaveRestoreBlocks(MF);
239
240 // Stash away DBG_VALUEs that should not be moved by insertion of prolog code.
241 SavedDbgValuesMap EntryDbgValues;
242 for (MachineBasicBlock *SaveBlock : SaveBlocks)
243 stashEntryDbgValues(MBB&: *SaveBlock, EntryDbgValues);
244
245 // Handle CSR spilling and restoring, for targets that need it.
246 if (MF.getTarget().usesPhysRegsForValues())
247 spillCalleeSavedRegs(MF);
248
249 // Allow the target machine to make final modifications to the function
250 // before the frame layout is finalized.
251 TFI->processFunctionBeforeFrameFinalized(MF, RS);
252
253 // Calculate actual frame offsets for all abstract stack objects...
254 calculateFrameObjectOffsets(MF);
255
256 // Add prolog and epilog code to the function. This function is required
257 // to align the stack frame as necessary for any stack variables or
258 // called functions. Because of this, calculateCalleeSavedRegisters()
259 // must be called before this function in order to set the AdjustsStack
260 // and MaxCallFrameSize variables.
261 if (!F.hasFnAttribute(Attribute::Naked))
262 insertPrologEpilogCode(MF);
263
264 // Reinsert stashed debug values at the start of the entry blocks.
265 for (auto &I : EntryDbgValues)
266 I.first->insert(I: I.first->begin(), S: I.second.begin(), E: I.second.end());
267
268 // Allow the target machine to make final modifications to the function
269 // before the frame layout is finalized.
270 TFI->processFunctionBeforeFrameIndicesReplaced(MF, RS);
271
272 // Replace all MO_FrameIndex operands with physical register references
273 // and actual offsets.
274 if (TFI->needsFrameIndexResolution(MF)) {
275 // Allow the target to determine this after knowing the frame size.
276 FrameIndexEliminationScavenging =
277 (RS && !FrameIndexVirtualScavenging) ||
278 TRI->requiresFrameIndexReplacementScavenging(MF);
279
280 if (TRI->eliminateFrameIndicesBackwards())
281 replaceFrameIndicesBackward(MF);
282 else
283 replaceFrameIndices(MF);
284 }
285
286 // If register scavenging is needed, as we've enabled doing it as a
287 // post-pass, scavenge the virtual registers that frame index elimination
288 // inserted.
289 if (TRI->requiresRegisterScavenging(MF) && FrameIndexVirtualScavenging)
290 scavengeFrameVirtualRegs(MF, RS&: *RS);
291
292 // Warn on stack size when we exceeds the given limit.
293 MachineFrameInfo &MFI = MF.getFrameInfo();
294 uint64_t StackSize = MFI.getStackSize();
295
296 uint64_t Threshold = TFI->getStackThreshold();
297 if (MF.getFunction().hasFnAttribute(Kind: "warn-stack-size")) {
298 bool Failed = MF.getFunction()
299 .getFnAttribute(Kind: "warn-stack-size")
300 .getValueAsString()
301 .getAsInteger(Radix: 10, Result&: Threshold);
302 // Verifier should have caught this.
303 assert(!Failed && "Invalid warn-stack-size fn attr value");
304 (void)Failed;
305 }
306 uint64_t UnsafeStackSize = MFI.getUnsafeStackSize();
307 if (MF.getFunction().hasFnAttribute(Attribute::SafeStack))
308 StackSize += UnsafeStackSize;
309
310 if (StackSize > Threshold) {
311 DiagnosticInfoStackSize DiagStackSize(F, StackSize, Threshold, DS_Warning);
312 F.getContext().diagnose(DI: DiagStackSize);
313 int64_t SpillSize = 0;
314 for (int Idx = MFI.getObjectIndexBegin(), End = MFI.getObjectIndexEnd();
315 Idx != End; ++Idx) {
316 if (MFI.isSpillSlotObjectIndex(ObjectIdx: Idx))
317 SpillSize += MFI.getObjectSize(ObjectIdx: Idx);
318 }
319
320 [[maybe_unused]] float SpillPct =
321 static_cast<float>(SpillSize) / static_cast<float>(StackSize);
322 LLVM_DEBUG(
323 dbgs() << formatv("{0}/{1} ({3:P}) spills, {2}/{1} ({4:P}) variables",
324 SpillSize, StackSize, StackSize - SpillSize, SpillPct,
325 1.0f - SpillPct));
326 if (UnsafeStackSize != 0) {
327 LLVM_DEBUG(dbgs() << formatv(", {0}/{2} ({1:P}) unsafe stack",
328 UnsafeStackSize,
329 static_cast<float>(UnsafeStackSize) /
330 static_cast<float>(StackSize),
331 StackSize));
332 }
333 LLVM_DEBUG(dbgs() << "\n");
334 }
335
336 ORE->emit(RemarkBuilder: [&]() {
337 return MachineOptimizationRemarkAnalysis(DEBUG_TYPE, "StackSize",
338 MF.getFunction().getSubprogram(),
339 &MF.front())
340 << ore::NV("NumStackBytes", StackSize)
341 << " stack bytes in function '"
342 << ore::NV("Function", MF.getFunction().getName()) << "'";
343 });
344
345 delete RS;
346 SaveBlocks.clear();
347 RestoreBlocks.clear();
348 MFI.setSavePoint(nullptr);
349 MFI.setRestorePoint(nullptr);
350 return true;
351}
352
353/// Calculate the MaxCallFrameSize and AdjustsStack
354/// variables for the function's frame information and eliminate call frame
355/// pseudo instructions.
356void PEI::calculateCallFrameInfo(MachineFunction &MF) {
357 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
358 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
359 MachineFrameInfo &MFI = MF.getFrameInfo();
360
361 unsigned MaxCallFrameSize = 0;
362 bool AdjustsStack = MFI.adjustsStack();
363
364 // Get the function call frame set-up and tear-down instruction opcode
365 unsigned FrameSetupOpcode = TII.getCallFrameSetupOpcode();
366 unsigned FrameDestroyOpcode = TII.getCallFrameDestroyOpcode();
367
368 // Early exit for targets which have no call frame setup/destroy pseudo
369 // instructions.
370 if (FrameSetupOpcode == ~0u && FrameDestroyOpcode == ~0u)
371 return;
372
373 std::vector<MachineBasicBlock::iterator> FrameSDOps;
374 for (MachineBasicBlock &BB : MF)
375 for (MachineBasicBlock::iterator I = BB.begin(); I != BB.end(); ++I)
376 if (TII.isFrameInstr(I: *I)) {
377 unsigned Size = TII.getFrameSize(I: *I);
378 if (Size > MaxCallFrameSize) MaxCallFrameSize = Size;
379 AdjustsStack = true;
380 FrameSDOps.push_back(x: I);
381 } else if (I->isInlineAsm()) {
382 // Some inline asm's need a stack frame, as indicated by operand 1.
383 unsigned ExtraInfo = I->getOperand(i: InlineAsm::MIOp_ExtraInfo).getImm();
384 if (ExtraInfo & InlineAsm::Extra_IsAlignStack)
385 AdjustsStack = true;
386 }
387
388 assert(!MFI.isMaxCallFrameSizeComputed() ||
389 (MFI.getMaxCallFrameSize() >= MaxCallFrameSize &&
390 !(AdjustsStack && !MFI.adjustsStack())));
391 MFI.setAdjustsStack(AdjustsStack);
392 MFI.setMaxCallFrameSize(MaxCallFrameSize);
393
394 if (TFI->canSimplifyCallFramePseudos(MF)) {
395 // If call frames are not being included as part of the stack frame, and
396 // the target doesn't indicate otherwise, remove the call frame pseudos
397 // here. The sub/add sp instruction pairs are still inserted, but we don't
398 // need to track the SP adjustment for frame index elimination.
399 for (MachineBasicBlock::iterator I : FrameSDOps)
400 TFI->eliminateCallFramePseudoInstr(MF, MBB&: *I->getParent(), MI: I);
401
402 // We can't track the call frame size after call frame pseudos have been
403 // eliminated. Set it to zero everywhere to keep MachineVerifier happy.
404 for (MachineBasicBlock &MBB : MF)
405 MBB.setCallFrameSize(0);
406 }
407}
408
409/// Compute the sets of entry and return blocks for saving and restoring
410/// callee-saved registers, and placing prolog and epilog code.
411void PEI::calculateSaveRestoreBlocks(MachineFunction &MF) {
412 const MachineFrameInfo &MFI = MF.getFrameInfo();
413
414 // Even when we do not change any CSR, we still want to insert the
415 // prologue and epilogue of the function.
416 // So set the save points for those.
417
418 // Use the points found by shrink-wrapping, if any.
419 if (MFI.getSavePoint()) {
420 SaveBlocks.push_back(Elt: MFI.getSavePoint());
421 assert(MFI.getRestorePoint() && "Both restore and save must be set");
422 MachineBasicBlock *RestoreBlock = MFI.getRestorePoint();
423 // If RestoreBlock does not have any successor and is not a return block
424 // then the end point is unreachable and we do not need to insert any
425 // epilogue.
426 if (!RestoreBlock->succ_empty() || RestoreBlock->isReturnBlock())
427 RestoreBlocks.push_back(Elt: RestoreBlock);
428 return;
429 }
430
431 // Save refs to entry and return blocks.
432 SaveBlocks.push_back(Elt: &MF.front());
433 for (MachineBasicBlock &MBB : MF) {
434 if (MBB.isEHFuncletEntry())
435 SaveBlocks.push_back(Elt: &MBB);
436 if (MBB.isReturnBlock())
437 RestoreBlocks.push_back(Elt: &MBB);
438 }
439}
440
441static void assignCalleeSavedSpillSlots(MachineFunction &F,
442 const BitVector &SavedRegs,
443 unsigned &MinCSFrameIndex,
444 unsigned &MaxCSFrameIndex) {
445 if (SavedRegs.empty())
446 return;
447
448 const TargetRegisterInfo *RegInfo = F.getSubtarget().getRegisterInfo();
449 const MCPhysReg *CSRegs = F.getRegInfo().getCalleeSavedRegs();
450 BitVector CSMask(SavedRegs.size());
451
452 for (unsigned i = 0; CSRegs[i]; ++i)
453 CSMask.set(CSRegs[i]);
454
455 std::vector<CalleeSavedInfo> CSI;
456 for (unsigned i = 0; CSRegs[i]; ++i) {
457 unsigned Reg = CSRegs[i];
458 if (SavedRegs.test(Idx: Reg)) {
459 bool SavedSuper = false;
460 for (const MCPhysReg &SuperReg : RegInfo->superregs(Reg)) {
461 // Some backends set all aliases for some registers as saved, such as
462 // Mips's $fp, so they appear in SavedRegs but not CSRegs.
463 if (SavedRegs.test(Idx: SuperReg) && CSMask.test(Idx: SuperReg)) {
464 SavedSuper = true;
465 break;
466 }
467 }
468
469 if (!SavedSuper)
470 CSI.push_back(x: CalleeSavedInfo(Reg));
471 }
472 }
473
474 const TargetFrameLowering *TFI = F.getSubtarget().getFrameLowering();
475 MachineFrameInfo &MFI = F.getFrameInfo();
476 if (!TFI->assignCalleeSavedSpillSlots(MF&: F, TRI: RegInfo, CSI, MinCSFrameIndex,
477 MaxCSFrameIndex)) {
478 // If target doesn't implement this, use generic code.
479
480 if (CSI.empty())
481 return; // Early exit if no callee saved registers are modified!
482
483 unsigned NumFixedSpillSlots;
484 const TargetFrameLowering::SpillSlot *FixedSpillSlots =
485 TFI->getCalleeSavedSpillSlots(NumEntries&: NumFixedSpillSlots);
486
487 // Now that we know which registers need to be saved and restored, allocate
488 // stack slots for them.
489 for (auto &CS : CSI) {
490 // If the target has spilled this register to another register, we don't
491 // need to allocate a stack slot.
492 if (CS.isSpilledToReg())
493 continue;
494
495 unsigned Reg = CS.getReg();
496 const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg);
497
498 int FrameIdx;
499 if (RegInfo->hasReservedSpillSlot(MF: F, Reg, FrameIdx)) {
500 CS.setFrameIdx(FrameIdx);
501 continue;
502 }
503
504 // Check to see if this physreg must be spilled to a particular stack slot
505 // on this target.
506 const TargetFrameLowering::SpillSlot *FixedSlot = FixedSpillSlots;
507 while (FixedSlot != FixedSpillSlots + NumFixedSpillSlots &&
508 FixedSlot->Reg != Reg)
509 ++FixedSlot;
510
511 unsigned Size = RegInfo->getSpillSize(RC: *RC);
512 if (FixedSlot == FixedSpillSlots + NumFixedSpillSlots) {
513 // Nope, just spill it anywhere convenient.
514 Align Alignment = RegInfo->getSpillAlign(RC: *RC);
515 // We may not be able to satisfy the desired alignment specification of
516 // the TargetRegisterClass if the stack alignment is smaller. Use the
517 // min.
518 Alignment = std::min(a: Alignment, b: TFI->getStackAlign());
519 FrameIdx = MFI.CreateStackObject(Size, Alignment, isSpillSlot: true);
520 if ((unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx;
521 if ((unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx;
522 } else {
523 // Spill it to the stack where we must.
524 FrameIdx = MFI.CreateFixedSpillStackObject(Size, SPOffset: FixedSlot->Offset);
525 }
526
527 CS.setFrameIdx(FrameIdx);
528 }
529 }
530
531 MFI.setCalleeSavedInfo(CSI);
532}
533
534/// Helper function to update the liveness information for the callee-saved
535/// registers.
536static void updateLiveness(MachineFunction &MF) {
537 MachineFrameInfo &MFI = MF.getFrameInfo();
538 // Visited will contain all the basic blocks that are in the region
539 // where the callee saved registers are alive:
540 // - Anything that is not Save or Restore -> LiveThrough.
541 // - Save -> LiveIn.
542 // - Restore -> LiveOut.
543 // The live-out is not attached to the block, so no need to keep
544 // Restore in this set.
545 SmallPtrSet<MachineBasicBlock *, 8> Visited;
546 SmallVector<MachineBasicBlock *, 8> WorkList;
547 MachineBasicBlock *Entry = &MF.front();
548 MachineBasicBlock *Save = MFI.getSavePoint();
549
550 if (!Save)
551 Save = Entry;
552
553 if (Entry != Save) {
554 WorkList.push_back(Elt: Entry);
555 Visited.insert(Ptr: Entry);
556 }
557 Visited.insert(Ptr: Save);
558
559 MachineBasicBlock *Restore = MFI.getRestorePoint();
560 if (Restore)
561 // By construction Restore cannot be visited, otherwise it
562 // means there exists a path to Restore that does not go
563 // through Save.
564 WorkList.push_back(Elt: Restore);
565
566 while (!WorkList.empty()) {
567 const MachineBasicBlock *CurBB = WorkList.pop_back_val();
568 // By construction, the region that is after the save point is
569 // dominated by the Save and post-dominated by the Restore.
570 if (CurBB == Save && Save != Restore)
571 continue;
572 // Enqueue all the successors not already visited.
573 // Those are by construction either before Save or after Restore.
574 for (MachineBasicBlock *SuccBB : CurBB->successors())
575 if (Visited.insert(Ptr: SuccBB).second)
576 WorkList.push_back(Elt: SuccBB);
577 }
578
579 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
580
581 MachineRegisterInfo &MRI = MF.getRegInfo();
582 for (const CalleeSavedInfo &I : CSI) {
583 for (MachineBasicBlock *MBB : Visited) {
584 MCPhysReg Reg = I.getReg();
585 // Add the callee-saved register as live-in.
586 // It's killed at the spill.
587 if (!MRI.isReserved(PhysReg: Reg) && !MBB->isLiveIn(Reg))
588 MBB->addLiveIn(PhysReg: Reg);
589 }
590 // If callee-saved register is spilled to another register rather than
591 // spilling to stack, the destination register has to be marked as live for
592 // each MBB between the prologue and epilogue so that it is not clobbered
593 // before it is reloaded in the epilogue. The Visited set contains all
594 // blocks outside of the region delimited by prologue/epilogue.
595 if (I.isSpilledToReg()) {
596 for (MachineBasicBlock &MBB : MF) {
597 if (Visited.count(Ptr: &MBB))
598 continue;
599 MCPhysReg DstReg = I.getDstReg();
600 if (!MBB.isLiveIn(Reg: DstReg))
601 MBB.addLiveIn(PhysReg: DstReg);
602 }
603 }
604 }
605}
606
607/// Insert spill code for the callee-saved registers used in the function.
608static void insertCSRSaves(MachineBasicBlock &SaveBlock,
609 ArrayRef<CalleeSavedInfo> CSI) {
610 MachineFunction &MF = *SaveBlock.getParent();
611 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
612 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
613 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
614
615 MachineBasicBlock::iterator I = SaveBlock.begin();
616 if (!TFI->spillCalleeSavedRegisters(MBB&: SaveBlock, MI: I, CSI, TRI)) {
617 for (const CalleeSavedInfo &CS : CSI) {
618 // Insert the spill to the stack frame.
619 unsigned Reg = CS.getReg();
620
621 if (CS.isSpilledToReg()) {
622 BuildMI(BB&: SaveBlock, I, MIMD: DebugLoc(),
623 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: CS.getDstReg())
624 .addReg(RegNo: Reg, flags: getKillRegState(B: true));
625 } else {
626 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
627 TII.storeRegToStackSlot(MBB&: SaveBlock, MI: I, SrcReg: Reg, isKill: true, FrameIndex: CS.getFrameIdx(), RC,
628 TRI, VReg: Register());
629 }
630 }
631 }
632}
633
634/// Insert restore code for the callee-saved registers used in the function.
635static void insertCSRRestores(MachineBasicBlock &RestoreBlock,
636 std::vector<CalleeSavedInfo> &CSI) {
637 MachineFunction &MF = *RestoreBlock.getParent();
638 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
639 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
640 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
641
642 // Restore all registers immediately before the return and any
643 // terminators that precede it.
644 MachineBasicBlock::iterator I = RestoreBlock.getFirstTerminator();
645
646 if (!TFI->restoreCalleeSavedRegisters(MBB&: RestoreBlock, MI: I, CSI, TRI)) {
647 for (const CalleeSavedInfo &CI : reverse(C&: CSI)) {
648 unsigned Reg = CI.getReg();
649 if (CI.isSpilledToReg()) {
650 BuildMI(BB&: RestoreBlock, I, MIMD: DebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: Reg)
651 .addReg(RegNo: CI.getDstReg(), flags: getKillRegState(B: true));
652 } else {
653 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
654 TII.loadRegFromStackSlot(MBB&: RestoreBlock, MI: I, DestReg: Reg, FrameIndex: CI.getFrameIdx(), RC,
655 TRI, VReg: Register());
656 assert(I != RestoreBlock.begin() &&
657 "loadRegFromStackSlot didn't insert any code!");
658 // Insert in reverse order. loadRegFromStackSlot can insert
659 // multiple instructions.
660 }
661 }
662 }
663}
664
665void PEI::spillCalleeSavedRegs(MachineFunction &MF) {
666 // We can't list this requirement in getRequiredProperties because some
667 // targets (WebAssembly) use virtual registers past this point, and the pass
668 // pipeline is set up without giving the passes a chance to look at the
669 // TargetMachine.
670 // FIXME: Find a way to express this in getRequiredProperties.
671 assert(MF.getProperties().hasProperty(
672 MachineFunctionProperties::Property::NoVRegs));
673
674 const Function &F = MF.getFunction();
675 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
676 MachineFrameInfo &MFI = MF.getFrameInfo();
677 MinCSFrameIndex = std::numeric_limits<unsigned>::max();
678 MaxCSFrameIndex = 0;
679
680 // Determine which of the registers in the callee save list should be saved.
681 BitVector SavedRegs;
682 TFI->determineCalleeSaves(MF, SavedRegs, RS);
683
684 // Assign stack slots for any callee-saved registers that must be spilled.
685 assignCalleeSavedSpillSlots(F&: MF, SavedRegs, MinCSFrameIndex, MaxCSFrameIndex);
686
687 // Add the code to save and restore the callee saved registers.
688 if (!F.hasFnAttribute(Attribute::Naked)) {
689 MFI.setCalleeSavedInfoValid(true);
690
691 std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
692 if (!CSI.empty()) {
693 if (!MFI.hasCalls())
694 NumLeafFuncWithSpills++;
695
696 for (MachineBasicBlock *SaveBlock : SaveBlocks)
697 insertCSRSaves(SaveBlock&: *SaveBlock, CSI);
698
699 // Update the live-in information of all the blocks up to the save point.
700 updateLiveness(MF);
701
702 for (MachineBasicBlock *RestoreBlock : RestoreBlocks)
703 insertCSRRestores(RestoreBlock&: *RestoreBlock, CSI);
704 }
705 }
706}
707
708/// AdjustStackOffset - Helper function used to adjust the stack frame offset.
709static inline void AdjustStackOffset(MachineFrameInfo &MFI, int FrameIdx,
710 bool StackGrowsDown, int64_t &Offset,
711 Align &MaxAlign) {
712 // If the stack grows down, add the object size to find the lowest address.
713 if (StackGrowsDown)
714 Offset += MFI.getObjectSize(ObjectIdx: FrameIdx);
715
716 Align Alignment = MFI.getObjectAlign(ObjectIdx: FrameIdx);
717
718 // If the alignment of this object is greater than that of the stack, then
719 // increase the stack alignment to match.
720 MaxAlign = std::max(a: MaxAlign, b: Alignment);
721
722 // Adjust to alignment boundary.
723 Offset = alignTo(Size: Offset, A: Alignment);
724
725 if (StackGrowsDown) {
726 LLVM_DEBUG(dbgs() << "alloc FI(" << FrameIdx << ") at SP[" << -Offset
727 << "]\n");
728 MFI.setObjectOffset(ObjectIdx: FrameIdx, SPOffset: -Offset); // Set the computed offset
729 } else {
730 LLVM_DEBUG(dbgs() << "alloc FI(" << FrameIdx << ") at SP[" << Offset
731 << "]\n");
732 MFI.setObjectOffset(ObjectIdx: FrameIdx, SPOffset: Offset);
733 Offset += MFI.getObjectSize(ObjectIdx: FrameIdx);
734 }
735}
736
737/// Compute which bytes of fixed and callee-save stack area are unused and keep
738/// track of them in StackBytesFree.
739static inline void
740computeFreeStackSlots(MachineFrameInfo &MFI, bool StackGrowsDown,
741 unsigned MinCSFrameIndex, unsigned MaxCSFrameIndex,
742 int64_t FixedCSEnd, BitVector &StackBytesFree) {
743 // Avoid undefined int64_t -> int conversion below in extreme case.
744 if (FixedCSEnd > std::numeric_limits<int>::max())
745 return;
746
747 StackBytesFree.resize(N: FixedCSEnd, t: true);
748
749 SmallVector<int, 16> AllocatedFrameSlots;
750 // Add fixed objects.
751 for (int i = MFI.getObjectIndexBegin(); i != 0; ++i)
752 // StackSlot scavenging is only implemented for the default stack.
753 if (MFI.getStackID(ObjectIdx: i) == TargetStackID::Default)
754 AllocatedFrameSlots.push_back(Elt: i);
755 // Add callee-save objects if there are any.
756 if (MinCSFrameIndex <= MaxCSFrameIndex) {
757 for (int i = MinCSFrameIndex; i <= (int)MaxCSFrameIndex; ++i)
758 if (MFI.getStackID(ObjectIdx: i) == TargetStackID::Default)
759 AllocatedFrameSlots.push_back(Elt: i);
760 }
761
762 for (int i : AllocatedFrameSlots) {
763 // These are converted from int64_t, but they should always fit in int
764 // because of the FixedCSEnd check above.
765 int ObjOffset = MFI.getObjectOffset(ObjectIdx: i);
766 int ObjSize = MFI.getObjectSize(ObjectIdx: i);
767 int ObjStart, ObjEnd;
768 if (StackGrowsDown) {
769 // ObjOffset is negative when StackGrowsDown is true.
770 ObjStart = -ObjOffset - ObjSize;
771 ObjEnd = -ObjOffset;
772 } else {
773 ObjStart = ObjOffset;
774 ObjEnd = ObjOffset + ObjSize;
775 }
776 // Ignore fixed holes that are in the previous stack frame.
777 if (ObjEnd > 0)
778 StackBytesFree.reset(I: ObjStart, E: ObjEnd);
779 }
780}
781
782/// Assign frame object to an unused portion of the stack in the fixed stack
783/// object range. Return true if the allocation was successful.
784static inline bool scavengeStackSlot(MachineFrameInfo &MFI, int FrameIdx,
785 bool StackGrowsDown, Align MaxAlign,
786 BitVector &StackBytesFree) {
787 if (MFI.isVariableSizedObjectIndex(ObjectIdx: FrameIdx))
788 return false;
789
790 if (StackBytesFree.none()) {
791 // clear it to speed up later scavengeStackSlot calls to
792 // StackBytesFree.none()
793 StackBytesFree.clear();
794 return false;
795 }
796
797 Align ObjAlign = MFI.getObjectAlign(ObjectIdx: FrameIdx);
798 if (ObjAlign > MaxAlign)
799 return false;
800
801 int64_t ObjSize = MFI.getObjectSize(ObjectIdx: FrameIdx);
802 int FreeStart;
803 for (FreeStart = StackBytesFree.find_first(); FreeStart != -1;
804 FreeStart = StackBytesFree.find_next(Prev: FreeStart)) {
805
806 // Check that free space has suitable alignment.
807 unsigned ObjStart = StackGrowsDown ? FreeStart + ObjSize : FreeStart;
808 if (alignTo(Size: ObjStart, A: ObjAlign) != ObjStart)
809 continue;
810
811 if (FreeStart + ObjSize > StackBytesFree.size())
812 return false;
813
814 bool AllBytesFree = true;
815 for (unsigned Byte = 0; Byte < ObjSize; ++Byte)
816 if (!StackBytesFree.test(Idx: FreeStart + Byte)) {
817 AllBytesFree = false;
818 break;
819 }
820 if (AllBytesFree)
821 break;
822 }
823
824 if (FreeStart == -1)
825 return false;
826
827 if (StackGrowsDown) {
828 int ObjStart = -(FreeStart + ObjSize);
829 LLVM_DEBUG(dbgs() << "alloc FI(" << FrameIdx << ") scavenged at SP["
830 << ObjStart << "]\n");
831 MFI.setObjectOffset(ObjectIdx: FrameIdx, SPOffset: ObjStart);
832 } else {
833 LLVM_DEBUG(dbgs() << "alloc FI(" << FrameIdx << ") scavenged at SP["
834 << FreeStart << "]\n");
835 MFI.setObjectOffset(ObjectIdx: FrameIdx, SPOffset: FreeStart);
836 }
837
838 StackBytesFree.reset(I: FreeStart, E: FreeStart + ObjSize);
839 return true;
840}
841
842/// AssignProtectedObjSet - Helper function to assign large stack objects (i.e.,
843/// those required to be close to the Stack Protector) to stack offsets.
844static void AssignProtectedObjSet(const StackObjSet &UnassignedObjs,
845 SmallSet<int, 16> &ProtectedObjs,
846 MachineFrameInfo &MFI, bool StackGrowsDown,
847 int64_t &Offset, Align &MaxAlign) {
848
849 for (int i : UnassignedObjs) {
850 AdjustStackOffset(MFI, FrameIdx: i, StackGrowsDown, Offset, MaxAlign);
851 ProtectedObjs.insert(V: i);
852 }
853}
854
855/// calculateFrameObjectOffsets - Calculate actual frame offsets for all of the
856/// abstract stack objects.
857void PEI::calculateFrameObjectOffsets(MachineFunction &MF) {
858 const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering();
859
860 bool StackGrowsDown =
861 TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown;
862
863 // Loop over all of the stack objects, assigning sequential addresses...
864 MachineFrameInfo &MFI = MF.getFrameInfo();
865
866 // Start at the beginning of the local area.
867 // The Offset is the distance from the stack top in the direction
868 // of stack growth -- so it's always nonnegative.
869 int LocalAreaOffset = TFI.getOffsetOfLocalArea();
870 if (StackGrowsDown)
871 LocalAreaOffset = -LocalAreaOffset;
872 assert(LocalAreaOffset >= 0
873 && "Local area offset should be in direction of stack growth");
874 int64_t Offset = LocalAreaOffset;
875
876#ifdef EXPENSIVE_CHECKS
877 for (unsigned i = 0, e = MFI.getObjectIndexEnd(); i != e; ++i)
878 if (!MFI.isDeadObjectIndex(i) &&
879 MFI.getStackID(i) == TargetStackID::Default)
880 assert(MFI.getObjectAlign(i) <= MFI.getMaxAlign() &&
881 "MaxAlignment is invalid");
882#endif
883
884 // If there are fixed sized objects that are preallocated in the local area,
885 // non-fixed objects can't be allocated right at the start of local area.
886 // Adjust 'Offset' to point to the end of last fixed sized preallocated
887 // object.
888 for (int i = MFI.getObjectIndexBegin(); i != 0; ++i) {
889 // Only allocate objects on the default stack.
890 if (MFI.getStackID(ObjectIdx: i) != TargetStackID::Default)
891 continue;
892
893 int64_t FixedOff;
894 if (StackGrowsDown) {
895 // The maximum distance from the stack pointer is at lower address of
896 // the object -- which is given by offset. For down growing stack
897 // the offset is negative, so we negate the offset to get the distance.
898 FixedOff = -MFI.getObjectOffset(ObjectIdx: i);
899 } else {
900 // The maximum distance from the start pointer is at the upper
901 // address of the object.
902 FixedOff = MFI.getObjectOffset(ObjectIdx: i) + MFI.getObjectSize(ObjectIdx: i);
903 }
904 if (FixedOff > Offset) Offset = FixedOff;
905 }
906
907 Align MaxAlign = MFI.getMaxAlign();
908 // First assign frame offsets to stack objects that are used to spill
909 // callee saved registers.
910 if (MaxCSFrameIndex >= MinCSFrameIndex) {
911 for (unsigned i = 0; i <= MaxCSFrameIndex - MinCSFrameIndex; ++i) {
912 unsigned FrameIndex =
913 StackGrowsDown ? MinCSFrameIndex + i : MaxCSFrameIndex - i;
914
915 // Only allocate objects on the default stack.
916 if (MFI.getStackID(ObjectIdx: FrameIndex) != TargetStackID::Default)
917 continue;
918
919 // TODO: should this just be if (MFI.isDeadObjectIndex(FrameIndex))
920 if (!StackGrowsDown && MFI.isDeadObjectIndex(ObjectIdx: FrameIndex))
921 continue;
922
923 AdjustStackOffset(MFI, FrameIdx: FrameIndex, StackGrowsDown, Offset, MaxAlign);
924 }
925 }
926
927 assert(MaxAlign == MFI.getMaxAlign() &&
928 "MFI.getMaxAlign should already account for all callee-saved "
929 "registers without a fixed stack slot");
930
931 // FixedCSEnd is the stack offset to the end of the fixed and callee-save
932 // stack area.
933 int64_t FixedCSEnd = Offset;
934
935 // Make sure the special register scavenging spill slot is closest to the
936 // incoming stack pointer if a frame pointer is required and is closer
937 // to the incoming rather than the final stack pointer.
938 const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
939 bool EarlyScavengingSlots = TFI.allocateScavengingFrameIndexesNearIncomingSP(MF);
940 if (RS && EarlyScavengingSlots) {
941 SmallVector<int, 2> SFIs;
942 RS->getScavengingFrameIndices(A&: SFIs);
943 for (int SFI : SFIs)
944 AdjustStackOffset(MFI, FrameIdx: SFI, StackGrowsDown, Offset, MaxAlign);
945 }
946
947 // FIXME: Once this is working, then enable flag will change to a target
948 // check for whether the frame is large enough to want to use virtual
949 // frame index registers. Functions which don't want/need this optimization
950 // will continue to use the existing code path.
951 if (MFI.getUseLocalStackAllocationBlock()) {
952 Align Alignment = MFI.getLocalFrameMaxAlign();
953
954 // Adjust to alignment boundary.
955 Offset = alignTo(Size: Offset, A: Alignment);
956
957 LLVM_DEBUG(dbgs() << "Local frame base offset: " << Offset << "\n");
958
959 // Resolve offsets for objects in the local block.
960 for (unsigned i = 0, e = MFI.getLocalFrameObjectCount(); i != e; ++i) {
961 std::pair<int, int64_t> Entry = MFI.getLocalFrameObjectMap(i);
962 int64_t FIOffset = (StackGrowsDown ? -Offset : Offset) + Entry.second;
963 LLVM_DEBUG(dbgs() << "alloc FI(" << Entry.first << ") at SP[" << FIOffset
964 << "]\n");
965 MFI.setObjectOffset(ObjectIdx: Entry.first, SPOffset: FIOffset);
966 }
967 // Allocate the local block
968 Offset += MFI.getLocalFrameSize();
969
970 MaxAlign = std::max(a: Alignment, b: MaxAlign);
971 }
972
973 // Retrieve the Exception Handler registration node.
974 int EHRegNodeFrameIndex = std::numeric_limits<int>::max();
975 if (const WinEHFuncInfo *FuncInfo = MF.getWinEHFuncInfo())
976 EHRegNodeFrameIndex = FuncInfo->EHRegNodeFrameIndex;
977
978 // Make sure that the stack protector comes before the local variables on the
979 // stack.
980 SmallSet<int, 16> ProtectedObjs;
981 if (MFI.hasStackProtectorIndex()) {
982 int StackProtectorFI = MFI.getStackProtectorIndex();
983 StackObjSet LargeArrayObjs;
984 StackObjSet SmallArrayObjs;
985 StackObjSet AddrOfObjs;
986
987 // If we need a stack protector, we need to make sure that
988 // LocalStackSlotPass didn't already allocate a slot for it.
989 // If we are told to use the LocalStackAllocationBlock, the stack protector
990 // is expected to be already pre-allocated.
991 if (MFI.getStackID(ObjectIdx: StackProtectorFI) != TargetStackID::Default) {
992 // If the stack protector isn't on the default stack then it's up to the
993 // target to set the stack offset.
994 assert(MFI.getObjectOffset(StackProtectorFI) != 0 &&
995 "Offset of stack protector on non-default stack expected to be "
996 "already set.");
997 assert(!MFI.isObjectPreAllocated(MFI.getStackProtectorIndex()) &&
998 "Stack protector on non-default stack expected to not be "
999 "pre-allocated by LocalStackSlotPass.");
1000 } else if (!MFI.getUseLocalStackAllocationBlock()) {
1001 AdjustStackOffset(MFI, FrameIdx: StackProtectorFI, StackGrowsDown, Offset,
1002 MaxAlign);
1003 } else if (!MFI.isObjectPreAllocated(ObjectIdx: MFI.getStackProtectorIndex())) {
1004 llvm_unreachable(
1005 "Stack protector not pre-allocated by LocalStackSlotPass.");
1006 }
1007
1008 // Assign large stack objects first.
1009 for (unsigned i = 0, e = MFI.getObjectIndexEnd(); i != e; ++i) {
1010 if (MFI.isObjectPreAllocated(ObjectIdx: i) && MFI.getUseLocalStackAllocationBlock())
1011 continue;
1012 if (i >= MinCSFrameIndex && i <= MaxCSFrameIndex)
1013 continue;
1014 if (RS && RS->isScavengingFrameIndex(FI: (int)i))
1015 continue;
1016 if (MFI.isDeadObjectIndex(ObjectIdx: i))
1017 continue;
1018 if (StackProtectorFI == (int)i || EHRegNodeFrameIndex == (int)i)
1019 continue;
1020 // Only allocate objects on the default stack.
1021 if (MFI.getStackID(ObjectIdx: i) != TargetStackID::Default)
1022 continue;
1023
1024 switch (MFI.getObjectSSPLayout(ObjectIdx: i)) {
1025 case MachineFrameInfo::SSPLK_None:
1026 continue;
1027 case MachineFrameInfo::SSPLK_SmallArray:
1028 SmallArrayObjs.insert(X: i);
1029 continue;
1030 case MachineFrameInfo::SSPLK_AddrOf:
1031 AddrOfObjs.insert(X: i);
1032 continue;
1033 case MachineFrameInfo::SSPLK_LargeArray:
1034 LargeArrayObjs.insert(X: i);
1035 continue;
1036 }
1037 llvm_unreachable("Unexpected SSPLayoutKind.");
1038 }
1039
1040 // We expect **all** the protected stack objects to be pre-allocated by
1041 // LocalStackSlotPass. If it turns out that PEI still has to allocate some
1042 // of them, we may end up messing up the expected order of the objects.
1043 if (MFI.getUseLocalStackAllocationBlock() &&
1044 !(LargeArrayObjs.empty() && SmallArrayObjs.empty() &&
1045 AddrOfObjs.empty()))
1046 llvm_unreachable("Found protected stack objects not pre-allocated by "
1047 "LocalStackSlotPass.");
1048
1049 AssignProtectedObjSet(UnassignedObjs: LargeArrayObjs, ProtectedObjs, MFI, StackGrowsDown,
1050 Offset, MaxAlign);
1051 AssignProtectedObjSet(UnassignedObjs: SmallArrayObjs, ProtectedObjs, MFI, StackGrowsDown,
1052 Offset, MaxAlign);
1053 AssignProtectedObjSet(UnassignedObjs: AddrOfObjs, ProtectedObjs, MFI, StackGrowsDown,
1054 Offset, MaxAlign);
1055 }
1056
1057 SmallVector<int, 8> ObjectsToAllocate;
1058
1059 // Then prepare to assign frame offsets to stack objects that are not used to
1060 // spill callee saved registers.
1061 for (unsigned i = 0, e = MFI.getObjectIndexEnd(); i != e; ++i) {
1062 if (MFI.isObjectPreAllocated(ObjectIdx: i) && MFI.getUseLocalStackAllocationBlock())
1063 continue;
1064 if (i >= MinCSFrameIndex && i <= MaxCSFrameIndex)
1065 continue;
1066 if (RS && RS->isScavengingFrameIndex(FI: (int)i))
1067 continue;
1068 if (MFI.isDeadObjectIndex(ObjectIdx: i))
1069 continue;
1070 if (MFI.getStackProtectorIndex() == (int)i || EHRegNodeFrameIndex == (int)i)
1071 continue;
1072 if (ProtectedObjs.count(V: i))
1073 continue;
1074 // Only allocate objects on the default stack.
1075 if (MFI.getStackID(ObjectIdx: i) != TargetStackID::Default)
1076 continue;
1077
1078 // Add the objects that we need to allocate to our working set.
1079 ObjectsToAllocate.push_back(Elt: i);
1080 }
1081
1082 // Allocate the EH registration node first if one is present.
1083 if (EHRegNodeFrameIndex != std::numeric_limits<int>::max())
1084 AdjustStackOffset(MFI, FrameIdx: EHRegNodeFrameIndex, StackGrowsDown, Offset,
1085 MaxAlign);
1086
1087 // Give the targets a chance to order the objects the way they like it.
1088 if (MF.getTarget().getOptLevel() != CodeGenOptLevel::None &&
1089 MF.getTarget().Options.StackSymbolOrdering)
1090 TFI.orderFrameObjects(MF, objectsToAllocate&: ObjectsToAllocate);
1091
1092 // Keep track of which bytes in the fixed and callee-save range are used so we
1093 // can use the holes when allocating later stack objects. Only do this if
1094 // stack protector isn't being used and the target requests it and we're
1095 // optimizing.
1096 BitVector StackBytesFree;
1097 if (!ObjectsToAllocate.empty() &&
1098 MF.getTarget().getOptLevel() != CodeGenOptLevel::None &&
1099 MFI.getStackProtectorIndex() < 0 && TFI.enableStackSlotScavenging(MF))
1100 computeFreeStackSlots(MFI, StackGrowsDown, MinCSFrameIndex, MaxCSFrameIndex,
1101 FixedCSEnd, StackBytesFree);
1102
1103 // Now walk the objects and actually assign base offsets to them.
1104 for (auto &Object : ObjectsToAllocate)
1105 if (!scavengeStackSlot(MFI, FrameIdx: Object, StackGrowsDown, MaxAlign,
1106 StackBytesFree))
1107 AdjustStackOffset(MFI, FrameIdx: Object, StackGrowsDown, Offset, MaxAlign);
1108
1109 // Make sure the special register scavenging spill slot is closest to the
1110 // stack pointer.
1111 if (RS && !EarlyScavengingSlots) {
1112 SmallVector<int, 2> SFIs;
1113 RS->getScavengingFrameIndices(A&: SFIs);
1114 for (int SFI : SFIs)
1115 AdjustStackOffset(MFI, FrameIdx: SFI, StackGrowsDown, Offset, MaxAlign);
1116 }
1117
1118 if (!TFI.targetHandlesStackFrameRounding()) {
1119 // If we have reserved argument space for call sites in the function
1120 // immediately on entry to the current function, count it as part of the
1121 // overall stack size.
1122 if (MFI.adjustsStack() && TFI.hasReservedCallFrame(MF))
1123 Offset += MFI.getMaxCallFrameSize();
1124
1125 // Round up the size to a multiple of the alignment. If the function has
1126 // any calls or alloca's, align to the target's StackAlignment value to
1127 // ensure that the callee's frame or the alloca data is suitably aligned;
1128 // otherwise, for leaf functions, align to the TransientStackAlignment
1129 // value.
1130 Align StackAlign;
1131 if (MFI.adjustsStack() || MFI.hasVarSizedObjects() ||
1132 (RegInfo->hasStackRealignment(MF) && MFI.getObjectIndexEnd() != 0))
1133 StackAlign = TFI.getStackAlign();
1134 else
1135 StackAlign = TFI.getTransientStackAlign();
1136
1137 // If the frame pointer is eliminated, all frame offsets will be relative to
1138 // SP not FP. Align to MaxAlign so this works.
1139 StackAlign = std::max(a: StackAlign, b: MaxAlign);
1140 int64_t OffsetBeforeAlignment = Offset;
1141 Offset = alignTo(Size: Offset, A: StackAlign);
1142
1143 // If we have increased the offset to fulfill the alignment constrants,
1144 // then the scavenging spill slots may become harder to reach from the
1145 // stack pointer, float them so they stay close.
1146 if (StackGrowsDown && OffsetBeforeAlignment != Offset && RS &&
1147 !EarlyScavengingSlots) {
1148 SmallVector<int, 2> SFIs;
1149 RS->getScavengingFrameIndices(A&: SFIs);
1150 LLVM_DEBUG(if (!SFIs.empty()) llvm::dbgs()
1151 << "Adjusting emergency spill slots!\n";);
1152 int64_t Delta = Offset - OffsetBeforeAlignment;
1153 for (int SFI : SFIs) {
1154 LLVM_DEBUG(llvm::dbgs()
1155 << "Adjusting offset of emergency spill slot #" << SFI
1156 << " from " << MFI.getObjectOffset(SFI););
1157 MFI.setObjectOffset(ObjectIdx: SFI, SPOffset: MFI.getObjectOffset(ObjectIdx: SFI) - Delta);
1158 LLVM_DEBUG(llvm::dbgs() << " to " << MFI.getObjectOffset(SFI) << "\n";);
1159 }
1160 }
1161 }
1162
1163 // Update frame info to pretend that this is part of the stack...
1164 int64_t StackSize = Offset - LocalAreaOffset;
1165 MFI.setStackSize(StackSize);
1166 NumBytesStackSpace += StackSize;
1167}
1168
1169/// insertPrologEpilogCode - Scan the function for modified callee saved
1170/// registers, insert spill code for these callee saved registers, then add
1171/// prolog and epilog code to the function.
1172void PEI::insertPrologEpilogCode(MachineFunction &MF) {
1173 const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering();
1174
1175 // Add prologue to the function...
1176 for (MachineBasicBlock *SaveBlock : SaveBlocks)
1177 TFI.emitPrologue(MF, MBB&: *SaveBlock);
1178
1179 // Add epilogue to restore the callee-save registers in each exiting block.
1180 for (MachineBasicBlock *RestoreBlock : RestoreBlocks)
1181 TFI.emitEpilogue(MF, MBB&: *RestoreBlock);
1182
1183 // Zero call used registers before restoring callee-saved registers.
1184 insertZeroCallUsedRegs(MF);
1185
1186 for (MachineBasicBlock *SaveBlock : SaveBlocks)
1187 TFI.inlineStackProbe(MF, PrologueMBB&: *SaveBlock);
1188
1189 // Emit additional code that is required to support segmented stacks, if
1190 // we've been asked for it. This, when linked with a runtime with support
1191 // for segmented stacks (libgcc is one), will result in allocating stack
1192 // space in small chunks instead of one large contiguous block.
1193 if (MF.shouldSplitStack()) {
1194 for (MachineBasicBlock *SaveBlock : SaveBlocks)
1195 TFI.adjustForSegmentedStacks(MF, PrologueMBB&: *SaveBlock);
1196 }
1197
1198 // Emit additional code that is required to explicitly handle the stack in
1199 // HiPE native code (if needed) when loaded in the Erlang/OTP runtime. The
1200 // approach is rather similar to that of Segmented Stacks, but it uses a
1201 // different conditional check and another BIF for allocating more stack
1202 // space.
1203 if (MF.getFunction().getCallingConv() == CallingConv::HiPE)
1204 for (MachineBasicBlock *SaveBlock : SaveBlocks)
1205 TFI.adjustForHiPEPrologue(MF, PrologueMBB&: *SaveBlock);
1206}
1207
1208/// insertZeroCallUsedRegs - Zero out call used registers.
1209void PEI::insertZeroCallUsedRegs(MachineFunction &MF) {
1210 const Function &F = MF.getFunction();
1211
1212 if (!F.hasFnAttribute(Kind: "zero-call-used-regs"))
1213 return;
1214
1215 using namespace ZeroCallUsedRegs;
1216
1217 ZeroCallUsedRegsKind ZeroRegsKind =
1218 StringSwitch<ZeroCallUsedRegsKind>(
1219 F.getFnAttribute(Kind: "zero-call-used-regs").getValueAsString())
1220 .Case(S: "skip", Value: ZeroCallUsedRegsKind::Skip)
1221 .Case(S: "used-gpr-arg", Value: ZeroCallUsedRegsKind::UsedGPRArg)
1222 .Case(S: "used-gpr", Value: ZeroCallUsedRegsKind::UsedGPR)
1223 .Case(S: "used-arg", Value: ZeroCallUsedRegsKind::UsedArg)
1224 .Case(S: "used", Value: ZeroCallUsedRegsKind::Used)
1225 .Case(S: "all-gpr-arg", Value: ZeroCallUsedRegsKind::AllGPRArg)
1226 .Case(S: "all-gpr", Value: ZeroCallUsedRegsKind::AllGPR)
1227 .Case(S: "all-arg", Value: ZeroCallUsedRegsKind::AllArg)
1228 .Case(S: "all", Value: ZeroCallUsedRegsKind::All);
1229
1230 if (ZeroRegsKind == ZeroCallUsedRegsKind::Skip)
1231 return;
1232
1233 const bool OnlyGPR = static_cast<unsigned>(ZeroRegsKind) & ONLY_GPR;
1234 const bool OnlyUsed = static_cast<unsigned>(ZeroRegsKind) & ONLY_USED;
1235 const bool OnlyArg = static_cast<unsigned>(ZeroRegsKind) & ONLY_ARG;
1236
1237 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
1238 const BitVector AllocatableSet(TRI.getAllocatableSet(MF));
1239
1240 // Mark all used registers.
1241 BitVector UsedRegs(TRI.getNumRegs());
1242 if (OnlyUsed)
1243 for (const MachineBasicBlock &MBB : MF)
1244 for (const MachineInstr &MI : MBB) {
1245 // skip debug instructions
1246 if (MI.isDebugInstr())
1247 continue;
1248
1249 for (const MachineOperand &MO : MI.operands()) {
1250 if (!MO.isReg())
1251 continue;
1252
1253 MCRegister Reg = MO.getReg();
1254 if (AllocatableSet[Reg] && !MO.isImplicit() &&
1255 (MO.isDef() || MO.isUse()))
1256 UsedRegs.set(Reg);
1257 }
1258 }
1259
1260 // Get a list of registers that are used.
1261 BitVector LiveIns(TRI.getNumRegs());
1262 for (const MachineBasicBlock::RegisterMaskPair &LI : MF.front().liveins())
1263 LiveIns.set(LI.PhysReg);
1264
1265 BitVector RegsToZero(TRI.getNumRegs());
1266 for (MCRegister Reg : AllocatableSet.set_bits()) {
1267 // Skip over fixed registers.
1268 if (TRI.isFixedRegister(MF, PhysReg: Reg))
1269 continue;
1270
1271 // Want only general purpose registers.
1272 if (OnlyGPR && !TRI.isGeneralPurposeRegister(MF, PhysReg: Reg))
1273 continue;
1274
1275 // Want only used registers.
1276 if (OnlyUsed && !UsedRegs[Reg])
1277 continue;
1278
1279 // Want only registers used for arguments.
1280 if (OnlyArg) {
1281 if (OnlyUsed) {
1282 if (!LiveIns[Reg])
1283 continue;
1284 } else if (!TRI.isArgumentRegister(MF, PhysReg: Reg)) {
1285 continue;
1286 }
1287 }
1288
1289 RegsToZero.set(Reg);
1290 }
1291
1292 // Don't clear registers that are live when leaving the function.
1293 for (const MachineBasicBlock &MBB : MF)
1294 for (const MachineInstr &MI : MBB.terminators()) {
1295 if (!MI.isReturn())
1296 continue;
1297
1298 for (const auto &MO : MI.operands()) {
1299 if (!MO.isReg())
1300 continue;
1301
1302 MCRegister Reg = MO.getReg();
1303 if (!Reg)
1304 continue;
1305
1306 // This picks up sibling registers (e.q. %al -> %ah).
1307 for (MCRegUnit Unit : TRI.regunits(Reg))
1308 RegsToZero.reset(Idx: Unit);
1309
1310 for (MCPhysReg SReg : TRI.sub_and_superregs_inclusive(Reg))
1311 RegsToZero.reset(Idx: SReg);
1312 }
1313 }
1314
1315 // Don't need to clear registers that are used/clobbered by terminating
1316 // instructions.
1317 for (const MachineBasicBlock &MBB : MF) {
1318 if (!MBB.isReturnBlock())
1319 continue;
1320
1321 MachineBasicBlock::const_iterator MBBI = MBB.getFirstTerminator();
1322 for (MachineBasicBlock::const_iterator I = MBBI, E = MBB.end(); I != E;
1323 ++I) {
1324 for (const MachineOperand &MO : I->operands()) {
1325 if (!MO.isReg())
1326 continue;
1327
1328 MCRegister Reg = MO.getReg();
1329 if (!Reg)
1330 continue;
1331
1332 for (const MCPhysReg Reg : TRI.sub_and_superregs_inclusive(Reg))
1333 RegsToZero.reset(Idx: Reg);
1334 }
1335 }
1336 }
1337
1338 // Don't clear registers that must be preserved.
1339 for (const MCPhysReg *CSRegs = TRI.getCalleeSavedRegs(MF: &MF);
1340 MCPhysReg CSReg = *CSRegs; ++CSRegs)
1341 for (MCRegister Reg : TRI.sub_and_superregs_inclusive(Reg: CSReg))
1342 RegsToZero.reset(Idx: Reg);
1343
1344 const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering();
1345 for (MachineBasicBlock &MBB : MF)
1346 if (MBB.isReturnBlock())
1347 TFI.emitZeroCallUsedRegs(RegsToZero, MBB);
1348}
1349
1350/// Replace all FrameIndex operands with physical register references and actual
1351/// offsets.
1352void PEI::replaceFrameIndicesBackward(MachineFunction &MF) {
1353 const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering();
1354
1355 for (auto &MBB : MF) {
1356 int SPAdj = 0;
1357 if (!MBB.succ_empty()) {
1358 // Get the SP adjustment for the end of MBB from the start of any of its
1359 // successors. They should all be the same.
1360 assert(all_of(MBB.successors(), [&MBB](const MachineBasicBlock *Succ) {
1361 return Succ->getCallFrameSize() ==
1362 (*MBB.succ_begin())->getCallFrameSize();
1363 }));
1364 const MachineBasicBlock &FirstSucc = **MBB.succ_begin();
1365 SPAdj = TFI.alignSPAdjust(SPAdj: FirstSucc.getCallFrameSize());
1366 if (TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp)
1367 SPAdj = -SPAdj;
1368 }
1369
1370 replaceFrameIndicesBackward(BB: &MBB, MF, SPAdj);
1371
1372 // We can't track the call frame size after call frame pseudos have been
1373 // eliminated. Set it to zero everywhere to keep MachineVerifier happy.
1374 MBB.setCallFrameSize(0);
1375 }
1376}
1377
1378/// replaceFrameIndices - Replace all MO_FrameIndex operands with physical
1379/// register references and actual offsets.
1380void PEI::replaceFrameIndices(MachineFunction &MF) {
1381 const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering();
1382
1383 for (auto &MBB : MF) {
1384 int SPAdj = TFI.alignSPAdjust(SPAdj: MBB.getCallFrameSize());
1385 if (TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp)
1386 SPAdj = -SPAdj;
1387
1388 replaceFrameIndices(BB: &MBB, MF, SPAdj);
1389
1390 // We can't track the call frame size after call frame pseudos have been
1391 // eliminated. Set it to zero everywhere to keep MachineVerifier happy.
1392 MBB.setCallFrameSize(0);
1393 }
1394}
1395
1396bool PEI::replaceFrameIndexDebugInstr(MachineFunction &MF, MachineInstr &MI,
1397 unsigned OpIdx, int SPAdj) {
1398 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
1399 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
1400 if (MI.isDebugValue()) {
1401
1402 MachineOperand &Op = MI.getOperand(i: OpIdx);
1403 assert(MI.isDebugOperand(&Op) &&
1404 "Frame indices can only appear as a debug operand in a DBG_VALUE*"
1405 " machine instruction");
1406 Register Reg;
1407 unsigned FrameIdx = Op.getIndex();
1408 unsigned Size = MF.getFrameInfo().getObjectSize(ObjectIdx: FrameIdx);
1409
1410 StackOffset Offset = TFI->getFrameIndexReference(MF, FI: FrameIdx, FrameReg&: Reg);
1411 Op.ChangeToRegister(Reg, isDef: false /*isDef*/);
1412
1413 const DIExpression *DIExpr = MI.getDebugExpression();
1414
1415 // If we have a direct DBG_VALUE, and its location expression isn't
1416 // currently complex, then adding an offset will morph it into a
1417 // complex location that is interpreted as being a memory address.
1418 // This changes a pointer-valued variable to dereference that pointer,
1419 // which is incorrect. Fix by adding DW_OP_stack_value.
1420
1421 if (MI.isNonListDebugValue()) {
1422 unsigned PrependFlags = DIExpression::ApplyOffset;
1423 if (!MI.isIndirectDebugValue() && !DIExpr->isComplex())
1424 PrependFlags |= DIExpression::StackValue;
1425
1426 // If we have DBG_VALUE that is indirect and has a Implicit location
1427 // expression need to insert a deref before prepending a Memory
1428 // location expression. Also after doing this we change the DBG_VALUE
1429 // to be direct.
1430 if (MI.isIndirectDebugValue() && DIExpr->isImplicit()) {
1431 SmallVector<uint64_t, 2> Ops = {dwarf::DW_OP_deref_size, Size};
1432 bool WithStackValue = true;
1433 DIExpr = DIExpression::prependOpcodes(Expr: DIExpr, Ops, StackValue: WithStackValue);
1434 // Make the DBG_VALUE direct.
1435 MI.getDebugOffset().ChangeToRegister(Reg: 0, isDef: false);
1436 }
1437 DIExpr = TRI.prependOffsetExpression(Expr: DIExpr, PrependFlags, Offset);
1438 } else {
1439 // The debug operand at DebugOpIndex was a frame index at offset
1440 // `Offset`; now the operand has been replaced with the frame
1441 // register, we must add Offset with `register x, plus Offset`.
1442 unsigned DebugOpIndex = MI.getDebugOperandIndex(Op: &Op);
1443 SmallVector<uint64_t, 3> Ops;
1444 TRI.getOffsetOpcodes(Offset, Ops);
1445 DIExpr = DIExpression::appendOpsToArg(Expr: DIExpr, Ops, ArgNo: DebugOpIndex);
1446 }
1447 MI.getDebugExpressionOp().setMetadata(DIExpr);
1448 return true;
1449 }
1450
1451 if (MI.isDebugPHI()) {
1452 // Allow stack ref to continue onwards.
1453 return true;
1454 }
1455
1456 // TODO: This code should be commoned with the code for
1457 // PATCHPOINT. There's no good reason for the difference in
1458 // implementation other than historical accident. The only
1459 // remaining difference is the unconditional use of the stack
1460 // pointer as the base register.
1461 if (MI.getOpcode() == TargetOpcode::STATEPOINT) {
1462 assert((!MI.isDebugValue() || OpIdx == 0) &&
1463 "Frame indicies can only appear as the first operand of a "
1464 "DBG_VALUE machine instruction");
1465 Register Reg;
1466 MachineOperand &Offset = MI.getOperand(i: OpIdx + 1);
1467 StackOffset refOffset = TFI->getFrameIndexReferencePreferSP(
1468 MF, FI: MI.getOperand(i: OpIdx).getIndex(), FrameReg&: Reg, /*IgnoreSPUpdates*/ false);
1469 assert(!refOffset.getScalable() &&
1470 "Frame offsets with a scalable component are not supported");
1471 Offset.setImm(Offset.getImm() + refOffset.getFixed() + SPAdj);
1472 MI.getOperand(i: OpIdx).ChangeToRegister(Reg, isDef: false /*isDef*/);
1473 return true;
1474 }
1475 return false;
1476}
1477
1478void PEI::replaceFrameIndicesBackward(MachineBasicBlock *BB,
1479 MachineFunction &MF, int &SPAdj) {
1480 assert(MF.getSubtarget().getRegisterInfo() &&
1481 "getRegisterInfo() must be implemented!");
1482
1483 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1484 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
1485 const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering();
1486
1487 RegScavenger *LocalRS = FrameIndexEliminationScavenging ? RS : nullptr;
1488 if (LocalRS)
1489 LocalRS->enterBasicBlockEnd(MBB&: *BB);
1490
1491 for (MachineBasicBlock::iterator I = BB->end(); I != BB->begin();) {
1492 MachineInstr &MI = *std::prev(x: I);
1493
1494 if (TII.isFrameInstr(I: MI)) {
1495 SPAdj -= TII.getSPAdjust(MI);
1496 TFI.eliminateCallFramePseudoInstr(MF, MBB&: *BB, MI: &MI);
1497 continue;
1498 }
1499
1500 // Step backwards to get the liveness state at (immedately after) MI.
1501 if (LocalRS)
1502 LocalRS->backward(I);
1503
1504 bool RemovedMI = false;
1505 for (const auto &[Idx, Op] : enumerate(First: MI.operands())) {
1506 if (!Op.isFI())
1507 continue;
1508
1509 if (replaceFrameIndexDebugInstr(MF, MI, OpIdx: Idx, SPAdj))
1510 continue;
1511
1512 // Eliminate this FrameIndex operand.
1513 RemovedMI = TRI.eliminateFrameIndex(MI, SPAdj, FIOperandNum: Idx, RS: LocalRS);
1514 if (RemovedMI)
1515 break;
1516 }
1517
1518 if (!RemovedMI)
1519 --I;
1520 }
1521}
1522
1523void PEI::replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &MF,
1524 int &SPAdj) {
1525 assert(MF.getSubtarget().getRegisterInfo() &&
1526 "getRegisterInfo() must be implemented!");
1527 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1528 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
1529 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
1530
1531 bool InsideCallSequence = false;
1532
1533 for (MachineBasicBlock::iterator I = BB->begin(); I != BB->end(); ) {
1534 if (TII.isFrameInstr(I: *I)) {
1535 InsideCallSequence = TII.isFrameSetup(I: *I);
1536 SPAdj += TII.getSPAdjust(MI: *I);
1537 I = TFI->eliminateCallFramePseudoInstr(MF, MBB&: *BB, MI: I);
1538 continue;
1539 }
1540
1541 MachineInstr &MI = *I;
1542 bool DoIncr = true;
1543 bool DidFinishLoop = true;
1544 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
1545 if (!MI.getOperand(i).isFI())
1546 continue;
1547
1548 if (replaceFrameIndexDebugInstr(MF, MI, OpIdx: i, SPAdj))
1549 continue;
1550
1551 // Some instructions (e.g. inline asm instructions) can have
1552 // multiple frame indices and/or cause eliminateFrameIndex
1553 // to insert more than one instruction. We need the register
1554 // scavenger to go through all of these instructions so that
1555 // it can update its register information. We keep the
1556 // iterator at the point before insertion so that we can
1557 // revisit them in full.
1558 bool AtBeginning = (I == BB->begin());
1559 if (!AtBeginning) --I;
1560
1561 // If this instruction has a FrameIndex operand, we need to
1562 // use that target machine register info object to eliminate
1563 // it.
1564 TRI.eliminateFrameIndex(MI, SPAdj, FIOperandNum: i);
1565
1566 // Reset the iterator if we were at the beginning of the BB.
1567 if (AtBeginning) {
1568 I = BB->begin();
1569 DoIncr = false;
1570 }
1571
1572 DidFinishLoop = false;
1573 break;
1574 }
1575
1576 // If we are looking at a call sequence, we need to keep track of
1577 // the SP adjustment made by each instruction in the sequence.
1578 // This includes both the frame setup/destroy pseudos (handled above),
1579 // as well as other instructions that have side effects w.r.t the SP.
1580 // Note that this must come after eliminateFrameIndex, because
1581 // if I itself referred to a frame index, we shouldn't count its own
1582 // adjustment.
1583 if (DidFinishLoop && InsideCallSequence)
1584 SPAdj += TII.getSPAdjust(MI);
1585
1586 if (DoIncr && I != BB->end())
1587 ++I;
1588 }
1589}
1590

source code of llvm/lib/CodeGen/PrologEpilogInserter.cpp