1 | //===-- RISCVFrameLowering.cpp - RISC-V Frame Information -----------------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file contains the RISC-V implementation of TargetFrameLowering class. |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | |
13 | #include "RISCVFrameLowering.h" |
14 | #include "RISCVMachineFunctionInfo.h" |
15 | #include "RISCVSubtarget.h" |
16 | #include "llvm/BinaryFormat/Dwarf.h" |
17 | #include "llvm/CodeGen/MachineFrameInfo.h" |
18 | #include "llvm/CodeGen/MachineFunction.h" |
19 | #include "llvm/CodeGen/MachineInstrBuilder.h" |
20 | #include "llvm/CodeGen/MachineRegisterInfo.h" |
21 | #include "llvm/CodeGen/RegisterScavenging.h" |
22 | #include "llvm/IR/DiagnosticInfo.h" |
23 | #include "llvm/MC/MCDwarf.h" |
24 | #include "llvm/Support/LEB128.h" |
25 | |
26 | #include <algorithm> |
27 | |
28 | using namespace llvm; |
29 | |
30 | static Align getABIStackAlignment(RISCVABI::ABI ABI) { |
31 | if (ABI == RISCVABI::ABI_ILP32E) |
32 | return Align(4); |
33 | if (ABI == RISCVABI::ABI_LP64E) |
34 | return Align(8); |
35 | return Align(16); |
36 | } |
37 | |
38 | RISCVFrameLowering::RISCVFrameLowering(const RISCVSubtarget &STI) |
39 | : TargetFrameLowering( |
40 | StackGrowsDown, getABIStackAlignment(ABI: STI.getTargetABI()), |
41 | /*LocalAreaOffset=*/0, |
42 | /*TransientStackAlignment=*/getABIStackAlignment(ABI: STI.getTargetABI())), |
43 | STI(STI) {} |
44 | |
45 | static const MCPhysReg AllPopRegs[] = { |
46 | RISCV::X1, RISCV::X8, RISCV::X9, RISCV::X18, RISCV::X19, |
47 | RISCV::X20, RISCV::X21, RISCV::X22, RISCV::X23, RISCV::X24, |
48 | RISCV::X25, RISCV::X26, RISCV::X27}; |
49 | |
50 | // For now we use x3, a.k.a gp, as pointer to shadow call stack. |
51 | // User should not use x3 in their asm. |
52 | static void emitSCSPrologue(MachineFunction &MF, MachineBasicBlock &MBB, |
53 | MachineBasicBlock::iterator MI, |
54 | const DebugLoc &DL) { |
55 | if (!MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack)) |
56 | return; |
57 | |
58 | const auto &STI = MF.getSubtarget<RISCVSubtarget>(); |
59 | const llvm::RISCVRegisterInfo *TRI = STI.getRegisterInfo(); |
60 | Register RAReg = TRI->getRARegister(); |
61 | |
62 | // Do not save RA to the SCS if it's not saved to the regular stack, |
63 | // i.e. RA is not at risk of being overwritten. |
64 | std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo(); |
65 | if (llvm::none_of( |
66 | Range&: CSI, P: [&](CalleeSavedInfo &CSR) { return CSR.getReg() == RAReg; })) |
67 | return; |
68 | |
69 | const RISCVInstrInfo *TII = STI.getInstrInfo(); |
70 | if (!STI.hasForcedSWShadowStack() && STI.hasStdExtZicfiss()) { |
71 | BuildMI(MBB, MI, DL, TII->get(RISCV::SSPUSH)).addReg(RAReg); |
72 | return; |
73 | } |
74 | |
75 | Register SCSPReg = RISCVABI::getSCSPReg(); |
76 | |
77 | bool IsRV64 = STI.hasFeature(RISCV::Feature64Bit); |
78 | int64_t SlotSize = STI.getXLen() / 8; |
79 | // Store return address to shadow call stack |
80 | // addi gp, gp, [4|8] |
81 | // s[w|d] ra, -[4|8](gp) |
82 | BuildMI(MBB, MI, DL, TII->get(RISCV::ADDI)) |
83 | .addReg(SCSPReg, RegState::Define) |
84 | .addReg(SCSPReg) |
85 | .addImm(SlotSize) |
86 | .setMIFlag(MachineInstr::FrameSetup); |
87 | BuildMI(MBB, MI, DL, TII->get(IsRV64 ? RISCV::SD : RISCV::SW)) |
88 | .addReg(RAReg) |
89 | .addReg(SCSPReg) |
90 | .addImm(-SlotSize) |
91 | .setMIFlag(MachineInstr::FrameSetup); |
92 | |
93 | // Emit a CFI instruction that causes SlotSize to be subtracted from the value |
94 | // of the shadow stack pointer when unwinding past this frame. |
95 | char DwarfSCSReg = TRI->getDwarfRegNum(SCSPReg, /*IsEH*/ true); |
96 | assert(DwarfSCSReg < 32 && "SCS Register should be < 32 (X3)." ); |
97 | |
98 | char Offset = static_cast<char>(-SlotSize) & 0x7f; |
99 | const char CFIInst[] = { |
100 | dwarf::DW_CFA_val_expression, |
101 | DwarfSCSReg, // register |
102 | 2, // length |
103 | static_cast<char>(unsigned(dwarf::DW_OP_breg0 + DwarfSCSReg)), |
104 | Offset, // addend (sleb128) |
105 | }; |
106 | |
107 | unsigned CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::createEscape( |
108 | L: nullptr, Vals: StringRef(CFIInst, sizeof(CFIInst)))); |
109 | BuildMI(MBB, MI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION)) |
110 | .addCFIIndex(CFIIndex) |
111 | .setMIFlag(MachineInstr::FrameSetup); |
112 | } |
113 | |
114 | static void emitSCSEpilogue(MachineFunction &MF, MachineBasicBlock &MBB, |
115 | MachineBasicBlock::iterator MI, |
116 | const DebugLoc &DL) { |
117 | if (!MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack)) |
118 | return; |
119 | |
120 | const auto &STI = MF.getSubtarget<RISCVSubtarget>(); |
121 | Register RAReg = STI.getRegisterInfo()->getRARegister(); |
122 | |
123 | // See emitSCSPrologue() above. |
124 | std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo(); |
125 | if (llvm::none_of( |
126 | Range&: CSI, P: [&](CalleeSavedInfo &CSR) { return CSR.getReg() == RAReg; })) |
127 | return; |
128 | |
129 | const RISCVInstrInfo *TII = STI.getInstrInfo(); |
130 | if (!STI.hasForcedSWShadowStack() && STI.hasStdExtZicfiss()) { |
131 | BuildMI(MBB, MI, DL, TII->get(RISCV::SSPOPCHK)).addReg(RAReg); |
132 | return; |
133 | } |
134 | |
135 | Register SCSPReg = RISCVABI::getSCSPReg(); |
136 | |
137 | bool IsRV64 = STI.hasFeature(RISCV::Feature64Bit); |
138 | int64_t SlotSize = STI.getXLen() / 8; |
139 | // Load return address from shadow call stack |
140 | // l[w|d] ra, -[4|8](gp) |
141 | // addi gp, gp, -[4|8] |
142 | BuildMI(MBB, MI, DL, TII->get(IsRV64 ? RISCV::LD : RISCV::LW)) |
143 | .addReg(RAReg, RegState::Define) |
144 | .addReg(SCSPReg) |
145 | .addImm(-SlotSize) |
146 | .setMIFlag(MachineInstr::FrameDestroy); |
147 | BuildMI(MBB, MI, DL, TII->get(RISCV::ADDI)) |
148 | .addReg(SCSPReg, RegState::Define) |
149 | .addReg(SCSPReg) |
150 | .addImm(-SlotSize) |
151 | .setMIFlag(MachineInstr::FrameDestroy); |
152 | // Restore the SCS pointer |
153 | unsigned CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::createRestore( |
154 | L: nullptr, Register: STI.getRegisterInfo()->getDwarfRegNum(SCSPReg, /*IsEH*/ true))); |
155 | BuildMI(MBB, MI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION)) |
156 | .addCFIIndex(CFIIndex) |
157 | .setMIFlags(MachineInstr::FrameDestroy); |
158 | } |
159 | |
160 | // Get the ID of the libcall used for spilling and restoring callee saved |
161 | // registers. The ID is representative of the number of registers saved or |
162 | // restored by the libcall, except it is zero-indexed - ID 0 corresponds to a |
163 | // single register. |
164 | static int getLibCallID(const MachineFunction &MF, |
165 | const std::vector<CalleeSavedInfo> &CSI) { |
166 | const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>(); |
167 | |
168 | if (CSI.empty() || !RVFI->useSaveRestoreLibCalls(MF)) |
169 | return -1; |
170 | |
171 | Register MaxReg = RISCV::NoRegister; |
172 | for (auto &CS : CSI) |
173 | // RISCVRegisterInfo::hasReservedSpillSlot assigns negative frame indexes to |
174 | // registers which can be saved by libcall. |
175 | if (CS.getFrameIdx() < 0) |
176 | MaxReg = std::max(a: MaxReg.id(), b: CS.getReg().id()); |
177 | |
178 | if (MaxReg == RISCV::NoRegister) |
179 | return -1; |
180 | |
181 | switch (MaxReg) { |
182 | default: |
183 | llvm_unreachable("Something has gone wrong!" ); |
184 | case /*s11*/ RISCV::X27: return 12; |
185 | case /*s10*/ RISCV::X26: return 11; |
186 | case /*s9*/ RISCV::X25: return 10; |
187 | case /*s8*/ RISCV::X24: return 9; |
188 | case /*s7*/ RISCV::X23: return 8; |
189 | case /*s6*/ RISCV::X22: return 7; |
190 | case /*s5*/ RISCV::X21: return 6; |
191 | case /*s4*/ RISCV::X20: return 5; |
192 | case /*s3*/ RISCV::X19: return 4; |
193 | case /*s2*/ RISCV::X18: return 3; |
194 | case /*s1*/ RISCV::X9: return 2; |
195 | case /*s0*/ RISCV::X8: return 1; |
196 | case /*ra*/ RISCV::X1: return 0; |
197 | } |
198 | } |
199 | |
200 | // Get the name of the libcall used for spilling callee saved registers. |
201 | // If this function will not use save/restore libcalls, then return a nullptr. |
202 | static const char * |
203 | getSpillLibCallName(const MachineFunction &MF, |
204 | const std::vector<CalleeSavedInfo> &CSI) { |
205 | static const char *const SpillLibCalls[] = { |
206 | "__riscv_save_0" , |
207 | "__riscv_save_1" , |
208 | "__riscv_save_2" , |
209 | "__riscv_save_3" , |
210 | "__riscv_save_4" , |
211 | "__riscv_save_5" , |
212 | "__riscv_save_6" , |
213 | "__riscv_save_7" , |
214 | "__riscv_save_8" , |
215 | "__riscv_save_9" , |
216 | "__riscv_save_10" , |
217 | "__riscv_save_11" , |
218 | "__riscv_save_12" |
219 | }; |
220 | |
221 | int LibCallID = getLibCallID(MF, CSI); |
222 | if (LibCallID == -1) |
223 | return nullptr; |
224 | return SpillLibCalls[LibCallID]; |
225 | } |
226 | |
227 | // Get the name of the libcall used for restoring callee saved registers. |
228 | // If this function will not use save/restore libcalls, then return a nullptr. |
229 | static const char * |
230 | getRestoreLibCallName(const MachineFunction &MF, |
231 | const std::vector<CalleeSavedInfo> &CSI) { |
232 | static const char *const RestoreLibCalls[] = { |
233 | "__riscv_restore_0" , |
234 | "__riscv_restore_1" , |
235 | "__riscv_restore_2" , |
236 | "__riscv_restore_3" , |
237 | "__riscv_restore_4" , |
238 | "__riscv_restore_5" , |
239 | "__riscv_restore_6" , |
240 | "__riscv_restore_7" , |
241 | "__riscv_restore_8" , |
242 | "__riscv_restore_9" , |
243 | "__riscv_restore_10" , |
244 | "__riscv_restore_11" , |
245 | "__riscv_restore_12" |
246 | }; |
247 | |
248 | int LibCallID = getLibCallID(MF, CSI); |
249 | if (LibCallID == -1) |
250 | return nullptr; |
251 | return RestoreLibCalls[LibCallID]; |
252 | } |
253 | |
254 | // Return encoded value and register count for PUSH/POP instruction, |
255 | // representing registers to store/load. |
256 | static std::pair<unsigned, unsigned> |
257 | getPushPopEncodingAndNum(const Register MaxReg) { |
258 | switch (MaxReg) { |
259 | default: |
260 | llvm_unreachable("Unexpected Reg for Push/Pop Inst" ); |
261 | case RISCV::X27: /*s11*/ |
262 | case RISCV::X26: /*s10*/ |
263 | return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S11, y: 13); |
264 | case RISCV::X25: /*s9*/ |
265 | return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S9, y: 11); |
266 | case RISCV::X24: /*s8*/ |
267 | return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S8, y: 10); |
268 | case RISCV::X23: /*s7*/ |
269 | return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S7, y: 9); |
270 | case RISCV::X22: /*s6*/ |
271 | return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S6, y: 8); |
272 | case RISCV::X21: /*s5*/ |
273 | return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S5, y: 7); |
274 | case RISCV::X20: /*s4*/ |
275 | return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S4, y: 6); |
276 | case RISCV::X19: /*s3*/ |
277 | return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S3, y: 5); |
278 | case RISCV::X18: /*s2*/ |
279 | return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S2, y: 4); |
280 | case RISCV::X9: /*s1*/ |
281 | return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S1, y: 3); |
282 | case RISCV::X8: /*s0*/ |
283 | return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0, y: 2); |
284 | case RISCV::X1: /*ra*/ |
285 | return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA, y: 1); |
286 | } |
287 | } |
288 | |
289 | // Get the max reg of Push/Pop for restoring callee saved registers. |
290 | static Register getMaxPushPopReg(const MachineFunction &MF, |
291 | const std::vector<CalleeSavedInfo> &CSI) { |
292 | Register MaxPushPopReg = RISCV::NoRegister; |
293 | for (auto &CS : CSI) { |
294 | if (llvm::is_contained(Range: AllPopRegs, Element: CS.getReg().id())) |
295 | MaxPushPopReg = std::max(a: MaxPushPopReg.id(), b: CS.getReg().id()); |
296 | } |
297 | // if rlist is {rs, s0-s10}, then s11 will also be included |
298 | if (MaxPushPopReg == RISCV::X26) |
299 | MaxPushPopReg = RISCV::X27; |
300 | return MaxPushPopReg; |
301 | } |
302 | |
303 | // Return true if the specified function should have a dedicated frame |
304 | // pointer register. This is true if frame pointer elimination is |
305 | // disabled, if it needs dynamic stack realignment, if the function has |
306 | // variable sized allocas, or if the frame address is taken. |
307 | bool RISCVFrameLowering::hasFP(const MachineFunction &MF) const { |
308 | const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo(); |
309 | |
310 | const MachineFrameInfo &MFI = MF.getFrameInfo(); |
311 | return MF.getTarget().Options.DisableFramePointerElim(MF) || |
312 | RegInfo->hasStackRealignment(MF) || MFI.hasVarSizedObjects() || |
313 | MFI.isFrameAddressTaken(); |
314 | } |
315 | |
316 | bool RISCVFrameLowering::hasBP(const MachineFunction &MF) const { |
317 | const MachineFrameInfo &MFI = MF.getFrameInfo(); |
318 | const TargetRegisterInfo *TRI = STI.getRegisterInfo(); |
319 | |
320 | // If we do not reserve stack space for outgoing arguments in prologue, |
321 | // we will adjust the stack pointer before call instruction. After the |
322 | // adjustment, we can not use SP to access the stack objects for the |
323 | // arguments. Instead, use BP to access these stack objects. |
324 | return (MFI.hasVarSizedObjects() || |
325 | (!hasReservedCallFrame(MF) && (!MFI.isMaxCallFrameSizeComputed() || |
326 | MFI.getMaxCallFrameSize() != 0))) && |
327 | TRI->hasStackRealignment(MF); |
328 | } |
329 | |
330 | // Determines the size of the frame and maximum call frame size. |
331 | void RISCVFrameLowering::determineFrameLayout(MachineFunction &MF) const { |
332 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
333 | auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>(); |
334 | |
335 | // Get the number of bytes to allocate from the FrameInfo. |
336 | uint64_t FrameSize = MFI.getStackSize(); |
337 | |
338 | // Get the alignment. |
339 | Align StackAlign = getStackAlign(); |
340 | |
341 | // Make sure the frame is aligned. |
342 | FrameSize = alignTo(Size: FrameSize, A: StackAlign); |
343 | |
344 | // Update frame info. |
345 | MFI.setStackSize(FrameSize); |
346 | |
347 | // When using SP or BP to access stack objects, we may require extra padding |
348 | // to ensure the bottom of the RVV stack is correctly aligned within the main |
349 | // stack. We calculate this as the amount required to align the scalar local |
350 | // variable section up to the RVV alignment. |
351 | const TargetRegisterInfo *TRI = STI.getRegisterInfo(); |
352 | if (RVFI->getRVVStackSize() && (!hasFP(MF) || TRI->hasStackRealignment(MF))) { |
353 | int ScalarLocalVarSize = FrameSize - RVFI->getCalleeSavedStackSize() - |
354 | RVFI->getVarArgsSaveSize(); |
355 | if (auto RVVPadding = |
356 | offsetToAlignment(Value: ScalarLocalVarSize, Alignment: RVFI->getRVVStackAlign())) |
357 | RVFI->setRVVPadding(RVVPadding); |
358 | } |
359 | } |
360 | |
361 | // Returns the stack size including RVV padding (when required), rounded back |
362 | // up to the required stack alignment. |
363 | uint64_t RISCVFrameLowering::getStackSizeWithRVVPadding( |
364 | const MachineFunction &MF) const { |
365 | const MachineFrameInfo &MFI = MF.getFrameInfo(); |
366 | auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>(); |
367 | return alignTo(Size: MFI.getStackSize() + RVFI->getRVVPadding(), A: getStackAlign()); |
368 | } |
369 | |
370 | // Returns the register used to hold the frame pointer. |
371 | static Register getFPReg(const RISCVSubtarget &STI) { return RISCV::X8; } |
372 | |
373 | // Returns the register used to hold the stack pointer. |
374 | static Register getSPReg(const RISCVSubtarget &STI) { return RISCV::X2; } |
375 | |
376 | static SmallVector<CalleeSavedInfo, 8> |
377 | getUnmanagedCSI(const MachineFunction &MF, |
378 | const std::vector<CalleeSavedInfo> &CSI) { |
379 | const MachineFrameInfo &MFI = MF.getFrameInfo(); |
380 | SmallVector<CalleeSavedInfo, 8> NonLibcallCSI; |
381 | |
382 | for (auto &CS : CSI) { |
383 | int FI = CS.getFrameIdx(); |
384 | if (FI >= 0 && MFI.getStackID(ObjectIdx: FI) == TargetStackID::Default) |
385 | NonLibcallCSI.push_back(Elt: CS); |
386 | } |
387 | |
388 | return NonLibcallCSI; |
389 | } |
390 | |
391 | static SmallVector<CalleeSavedInfo, 8> |
392 | getRVVCalleeSavedInfo(const MachineFunction &MF, |
393 | const std::vector<CalleeSavedInfo> &CSI) { |
394 | const MachineFrameInfo &MFI = MF.getFrameInfo(); |
395 | SmallVector<CalleeSavedInfo, 8> RVVCSI; |
396 | |
397 | for (auto &CS : CSI) { |
398 | int FI = CS.getFrameIdx(); |
399 | if (FI >= 0 && MFI.getStackID(ObjectIdx: FI) == TargetStackID::ScalableVector) |
400 | RVVCSI.push_back(Elt: CS); |
401 | } |
402 | |
403 | return RVVCSI; |
404 | } |
405 | |
406 | void RISCVFrameLowering::adjustStackForRVV(MachineFunction &MF, |
407 | MachineBasicBlock &MBB, |
408 | MachineBasicBlock::iterator MBBI, |
409 | const DebugLoc &DL, int64_t Amount, |
410 | MachineInstr::MIFlag Flag) const { |
411 | assert(Amount != 0 && "Did not need to adjust stack pointer for RVV." ); |
412 | |
413 | const Register SPReg = getSPReg(STI); |
414 | |
415 | // Optimize compile time offset case |
416 | StackOffset Offset = StackOffset::getScalable(Scalable: Amount); |
417 | if (auto VLEN = STI.getRealVLen()) { |
418 | // 1. Multiply the number of v-slots by the (constant) length of register |
419 | const int64_t VLENB = *VLEN / 8; |
420 | assert(Amount % 8 == 0 && |
421 | "Reserve the stack by the multiple of one vector size." ); |
422 | const int64_t NumOfVReg = Amount / 8; |
423 | const int64_t FixedOffset = NumOfVReg * VLENB; |
424 | if (!isInt<32>(x: FixedOffset)) { |
425 | report_fatal_error( |
426 | reason: "Frame size outside of the signed 32-bit range not supported" ); |
427 | } |
428 | Offset = StackOffset::getFixed(Fixed: FixedOffset); |
429 | } |
430 | |
431 | const RISCVRegisterInfo &RI = *STI.getRegisterInfo(); |
432 | // We must keep the stack pointer aligned through any intermediate |
433 | // updates. |
434 | RI.adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg, Offset, |
435 | Flag, RequiredAlign: getStackAlign()); |
436 | } |
437 | |
438 | static void appendScalableVectorExpression(const TargetRegisterInfo &TRI, |
439 | SmallVectorImpl<char> &Expr, |
440 | int FixedOffset, int ScalableOffset, |
441 | llvm::raw_string_ostream &) { |
442 | unsigned DwarfVLenB = TRI.getDwarfRegNum(RISCV::VLENB, true); |
443 | uint8_t Buffer[16]; |
444 | if (FixedOffset) { |
445 | Expr.push_back(Elt: dwarf::DW_OP_consts); |
446 | Expr.append(in_start: Buffer, in_end: Buffer + encodeSLEB128(Value: FixedOffset, p: Buffer)); |
447 | Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_plus); |
448 | Comment << (FixedOffset < 0 ? " - " : " + " ) << std::abs(x: FixedOffset); |
449 | } |
450 | |
451 | Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_consts); |
452 | Expr.append(in_start: Buffer, in_end: Buffer + encodeSLEB128(Value: ScalableOffset, p: Buffer)); |
453 | |
454 | Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_bregx); |
455 | Expr.append(in_start: Buffer, in_end: Buffer + encodeULEB128(Value: DwarfVLenB, p: Buffer)); |
456 | Expr.push_back(Elt: 0); |
457 | |
458 | Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_mul); |
459 | Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_plus); |
460 | |
461 | Comment << (ScalableOffset < 0 ? " - " : " + " ) << std::abs(x: ScalableOffset) |
462 | << " * vlenb" ; |
463 | } |
464 | |
465 | static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI, |
466 | Register Reg, |
467 | uint64_t FixedOffset, |
468 | uint64_t ScalableOffset) { |
469 | assert(ScalableOffset != 0 && "Did not need to adjust CFA for RVV" ); |
470 | SmallString<64> Expr; |
471 | std::string ; |
472 | llvm::raw_string_ostream (CommentBuffer); |
473 | // Build up the expression (Reg + FixedOffset + ScalableOffset * VLENB). |
474 | unsigned DwarfReg = TRI.getDwarfRegNum(RegNum: Reg, isEH: true); |
475 | Expr.push_back(Elt: (uint8_t)(dwarf::DW_OP_breg0 + DwarfReg)); |
476 | Expr.push_back(Elt: 0); |
477 | if (Reg == RISCV::X2) |
478 | Comment << "sp" ; |
479 | else |
480 | Comment << printReg(Reg, TRI: &TRI); |
481 | |
482 | appendScalableVectorExpression(TRI, Expr, FixedOffset, ScalableOffset, |
483 | Comment); |
484 | |
485 | SmallString<64> DefCfaExpr; |
486 | uint8_t Buffer[16]; |
487 | DefCfaExpr.push_back(Elt: dwarf::DW_CFA_def_cfa_expression); |
488 | DefCfaExpr.append(in_start: Buffer, in_end: Buffer + encodeULEB128(Value: Expr.size(), p: Buffer)); |
489 | DefCfaExpr.append(RHS: Expr.str()); |
490 | |
491 | return MCCFIInstruction::createEscape(L: nullptr, Vals: DefCfaExpr.str(), Loc: SMLoc(), |
492 | Comment: Comment.str()); |
493 | } |
494 | |
495 | static MCCFIInstruction createDefCFAOffset(const TargetRegisterInfo &TRI, |
496 | Register Reg, uint64_t FixedOffset, |
497 | uint64_t ScalableOffset) { |
498 | assert(ScalableOffset != 0 && "Did not need to adjust CFA for RVV" ); |
499 | SmallString<64> Expr; |
500 | std::string ; |
501 | llvm::raw_string_ostream (CommentBuffer); |
502 | Comment << printReg(Reg, TRI: &TRI) << " @ cfa" ; |
503 | |
504 | // Build up the expression (FixedOffset + ScalableOffset * VLENB). |
505 | appendScalableVectorExpression(TRI, Expr, FixedOffset, ScalableOffset, |
506 | Comment); |
507 | |
508 | SmallString<64> DefCfaExpr; |
509 | uint8_t Buffer[16]; |
510 | unsigned DwarfReg = TRI.getDwarfRegNum(RegNum: Reg, isEH: true); |
511 | DefCfaExpr.push_back(Elt: dwarf::DW_CFA_expression); |
512 | DefCfaExpr.append(in_start: Buffer, in_end: Buffer + encodeULEB128(Value: DwarfReg, p: Buffer)); |
513 | DefCfaExpr.append(in_start: Buffer, in_end: Buffer + encodeULEB128(Value: Expr.size(), p: Buffer)); |
514 | DefCfaExpr.append(RHS: Expr.str()); |
515 | |
516 | return MCCFIInstruction::createEscape(L: nullptr, Vals: DefCfaExpr.str(), Loc: SMLoc(), |
517 | Comment: Comment.str()); |
518 | } |
519 | |
520 | void RISCVFrameLowering::emitPrologue(MachineFunction &MF, |
521 | MachineBasicBlock &MBB) const { |
522 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
523 | auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>(); |
524 | const RISCVRegisterInfo *RI = STI.getRegisterInfo(); |
525 | const RISCVInstrInfo *TII = STI.getInstrInfo(); |
526 | MachineBasicBlock::iterator MBBI = MBB.begin(); |
527 | |
528 | Register FPReg = getFPReg(STI); |
529 | Register SPReg = getSPReg(STI); |
530 | Register BPReg = RISCVABI::getBPReg(); |
531 | |
532 | // Debug location must be unknown since the first debug location is used |
533 | // to determine the end of the prologue. |
534 | DebugLoc DL; |
535 | |
536 | // All calls are tail calls in GHC calling conv, and functions have no |
537 | // prologue/epilogue. |
538 | if (MF.getFunction().getCallingConv() == CallingConv::GHC) |
539 | return; |
540 | |
541 | // Emit prologue for shadow call stack. |
542 | emitSCSPrologue(MF, MBB, MI: MBBI, DL); |
543 | |
544 | auto FirstFrameSetup = MBBI; |
545 | |
546 | // Since spillCalleeSavedRegisters may have inserted a libcall, skip past |
547 | // any instructions marked as FrameSetup |
548 | while (MBBI != MBB.end() && MBBI->getFlag(Flag: MachineInstr::FrameSetup)) |
549 | ++MBBI; |
550 | |
551 | // Determine the correct frame layout |
552 | determineFrameLayout(MF); |
553 | |
554 | // If libcalls are used to spill and restore callee-saved registers, the frame |
555 | // has two sections; the opaque section managed by the libcalls, and the |
556 | // section managed by MachineFrameInfo which can also hold callee saved |
557 | // registers in fixed stack slots, both of which have negative frame indices. |
558 | // This gets even more complicated when incoming arguments are passed via the |
559 | // stack, as these too have negative frame indices. An example is detailed |
560 | // below: |
561 | // |
562 | // | incoming arg | <- FI[-3] |
563 | // | libcallspill | |
564 | // | calleespill | <- FI[-2] |
565 | // | calleespill | <- FI[-1] |
566 | // | this_frame | <- FI[0] |
567 | // |
568 | // For negative frame indices, the offset from the frame pointer will differ |
569 | // depending on which of these groups the frame index applies to. |
570 | // The following calculates the correct offset knowing the number of callee |
571 | // saved registers spilt by the two methods. |
572 | if (int LibCallRegs = getLibCallID(MF, CSI: MFI.getCalleeSavedInfo()) + 1) { |
573 | // Calculate the size of the frame managed by the libcall. The stack |
574 | // alignment of these libcalls should be the same as how we set it in |
575 | // getABIStackAlignment. |
576 | unsigned LibCallFrameSize = |
577 | alignTo(Size: (STI.getXLen() / 8) * LibCallRegs, A: getStackAlign()); |
578 | RVFI->setLibCallStackSize(LibCallFrameSize); |
579 | } |
580 | |
581 | // FIXME (note copied from Lanai): This appears to be overallocating. Needs |
582 | // investigation. Get the number of bytes to allocate from the FrameInfo. |
583 | uint64_t RealStackSize = getStackSizeWithRVVPadding(MF); |
584 | uint64_t StackSize = RealStackSize - RVFI->getReservedSpillsSize(); |
585 | uint64_t RVVStackSize = RVFI->getRVVStackSize(); |
586 | |
587 | // Early exit if there is no need to allocate on the stack |
588 | if (RealStackSize == 0 && !MFI.adjustsStack() && RVVStackSize == 0) |
589 | return; |
590 | |
591 | // If the stack pointer has been marked as reserved, then produce an error if |
592 | // the frame requires stack allocation |
593 | if (STI.isRegisterReservedByUser(i: SPReg)) |
594 | MF.getFunction().getContext().diagnose(DI: DiagnosticInfoUnsupported{ |
595 | MF.getFunction(), "Stack pointer required, but has been reserved." }); |
596 | |
597 | uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF); |
598 | // Split the SP adjustment to reduce the offsets of callee saved spill. |
599 | if (FirstSPAdjustAmount) { |
600 | StackSize = FirstSPAdjustAmount; |
601 | RealStackSize = FirstSPAdjustAmount; |
602 | } |
603 | |
604 | if (RVFI->isPushable(MF) && FirstFrameSetup != MBB.end() && |
605 | FirstFrameSetup->getOpcode() == RISCV::CM_PUSH) { |
606 | // Use available stack adjustment in push instruction to allocate additional |
607 | // stack space. Align the stack size down to a multiple of 16. This is |
608 | // needed for RVE. |
609 | // FIXME: Can we increase the stack size to a multiple of 16 instead? |
610 | uint64_t Spimm = std::min(a: alignDown(Value: StackSize, Align: 16), b: (uint64_t)48); |
611 | FirstFrameSetup->getOperand(i: 1).setImm(Spimm); |
612 | StackSize -= Spimm; |
613 | } |
614 | |
615 | if (StackSize != 0) { |
616 | // Allocate space on the stack if necessary. |
617 | RI->adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg, |
618 | Offset: StackOffset::getFixed(Fixed: -StackSize), Flag: MachineInstr::FrameSetup, |
619 | RequiredAlign: getStackAlign()); |
620 | } |
621 | |
622 | // Emit ".cfi_def_cfa_offset RealStackSize" |
623 | unsigned CFIIndex = MF.addFrameInst( |
624 | Inst: MCCFIInstruction::cfiDefCfaOffset(L: nullptr, Offset: RealStackSize)); |
625 | BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION)) |
626 | .addCFIIndex(CFIIndex) |
627 | .setMIFlag(MachineInstr::FrameSetup); |
628 | |
629 | const auto &CSI = MFI.getCalleeSavedInfo(); |
630 | |
631 | // The frame pointer is callee-saved, and code has been generated for us to |
632 | // save it to the stack. We need to skip over the storing of callee-saved |
633 | // registers as the frame pointer must be modified after it has been saved |
634 | // to the stack, not before. |
635 | // FIXME: assumes exactly one instruction is used to save each callee-saved |
636 | // register. |
637 | std::advance(i&: MBBI, n: getUnmanagedCSI(MF, CSI).size()); |
638 | |
639 | // Iterate over list of callee-saved registers and emit .cfi_offset |
640 | // directives. |
641 | for (const auto &Entry : CSI) { |
642 | int FrameIdx = Entry.getFrameIdx(); |
643 | if (FrameIdx >= 0 && |
644 | MFI.getStackID(ObjectIdx: FrameIdx) == TargetStackID::ScalableVector) |
645 | continue; |
646 | |
647 | int64_t Offset = MFI.getObjectOffset(ObjectIdx: FrameIdx); |
648 | Register Reg = Entry.getReg(); |
649 | unsigned CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::createOffset( |
650 | L: nullptr, Register: RI->getDwarfRegNum(Reg, true), Offset)); |
651 | BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION)) |
652 | .addCFIIndex(CFIIndex) |
653 | .setMIFlag(MachineInstr::FrameSetup); |
654 | } |
655 | |
656 | // Generate new FP. |
657 | if (hasFP(MF)) { |
658 | if (STI.isRegisterReservedByUser(i: FPReg)) |
659 | MF.getFunction().getContext().diagnose(DI: DiagnosticInfoUnsupported{ |
660 | MF.getFunction(), "Frame pointer required, but has been reserved." }); |
661 | // The frame pointer does need to be reserved from register allocation. |
662 | assert(MF.getRegInfo().isReserved(FPReg) && "FP not reserved" ); |
663 | |
664 | RI->adjustReg(MBB, II: MBBI, DL, DestReg: FPReg, SrcReg: SPReg, |
665 | Offset: StackOffset::getFixed(Fixed: RealStackSize - RVFI->getVarArgsSaveSize()), |
666 | Flag: MachineInstr::FrameSetup, RequiredAlign: getStackAlign()); |
667 | |
668 | // Emit ".cfi_def_cfa $fp, RVFI->getVarArgsSaveSize()" |
669 | unsigned CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::cfiDefCfa( |
670 | L: nullptr, Register: RI->getDwarfRegNum(FPReg, true), Offset: RVFI->getVarArgsSaveSize())); |
671 | BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION)) |
672 | .addCFIIndex(CFIIndex) |
673 | .setMIFlag(MachineInstr::FrameSetup); |
674 | } |
675 | |
676 | // Emit the second SP adjustment after saving callee saved registers. |
677 | if (FirstSPAdjustAmount) { |
678 | uint64_t SecondSPAdjustAmount = |
679 | getStackSizeWithRVVPadding(MF) - FirstSPAdjustAmount; |
680 | assert(SecondSPAdjustAmount > 0 && |
681 | "SecondSPAdjustAmount should be greater than zero" ); |
682 | RI->adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg, |
683 | Offset: StackOffset::getFixed(Fixed: -SecondSPAdjustAmount), |
684 | Flag: MachineInstr::FrameSetup, RequiredAlign: getStackAlign()); |
685 | |
686 | // If we are using a frame-pointer, and thus emitted ".cfi_def_cfa fp, 0", |
687 | // don't emit an sp-based .cfi_def_cfa_offset |
688 | if (!hasFP(MF)) { |
689 | // Emit ".cfi_def_cfa_offset StackSize" |
690 | unsigned CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::cfiDefCfaOffset( |
691 | L: nullptr, Offset: getStackSizeWithRVVPadding(MF))); |
692 | BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION)) |
693 | .addCFIIndex(CFIIndex) |
694 | .setMIFlag(MachineInstr::FrameSetup); |
695 | } |
696 | } |
697 | |
698 | if (RVVStackSize) { |
699 | adjustStackForRVV(MF, MBB, MBBI, DL, Amount: -RVVStackSize, |
700 | Flag: MachineInstr::FrameSetup); |
701 | if (!hasFP(MF)) { |
702 | // Emit .cfi_def_cfa_expression "sp + StackSize + RVVStackSize * vlenb". |
703 | unsigned CFIIndex = MF.addFrameInst(Inst: createDefCFAExpression( |
704 | *RI, SPReg, getStackSizeWithRVVPadding(MF), RVVStackSize / 8)); |
705 | BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION)) |
706 | .addCFIIndex(CFIIndex) |
707 | .setMIFlag(MachineInstr::FrameSetup); |
708 | } |
709 | |
710 | std::advance(i&: MBBI, n: getRVVCalleeSavedInfo(MF, CSI).size()); |
711 | emitCalleeSavedRVVPrologCFI(MBB, MI: MBBI, HasFP: hasFP(MF)); |
712 | } |
713 | |
714 | if (hasFP(MF)) { |
715 | // Realign Stack |
716 | const RISCVRegisterInfo *RI = STI.getRegisterInfo(); |
717 | if (RI->hasStackRealignment(MF)) { |
718 | Align MaxAlignment = MFI.getMaxAlign(); |
719 | |
720 | const RISCVInstrInfo *TII = STI.getInstrInfo(); |
721 | if (isInt<12>(x: -(int)MaxAlignment.value())) { |
722 | BuildMI(MBB, MBBI, DL, TII->get(RISCV::ANDI), SPReg) |
723 | .addReg(SPReg) |
724 | .addImm(-(int)MaxAlignment.value()) |
725 | .setMIFlag(MachineInstr::FrameSetup); |
726 | } else { |
727 | unsigned ShiftAmount = Log2(A: MaxAlignment); |
728 | Register VR = |
729 | MF.getRegInfo().createVirtualRegister(&RISCV::GPRRegClass); |
730 | BuildMI(MBB, MBBI, DL, TII->get(RISCV::SRLI), VR) |
731 | .addReg(SPReg) |
732 | .addImm(ShiftAmount) |
733 | .setMIFlag(MachineInstr::FrameSetup); |
734 | BuildMI(MBB, MBBI, DL, TII->get(RISCV::SLLI), SPReg) |
735 | .addReg(VR) |
736 | .addImm(ShiftAmount) |
737 | .setMIFlag(MachineInstr::FrameSetup); |
738 | } |
739 | // FP will be used to restore the frame in the epilogue, so we need |
740 | // another base register BP to record SP after re-alignment. SP will |
741 | // track the current stack after allocating variable sized objects. |
742 | if (hasBP(MF)) { |
743 | // move BP, SP |
744 | BuildMI(MBB, MBBI, DL, TII->get(RISCV::ADDI), BPReg) |
745 | .addReg(SPReg) |
746 | .addImm(0) |
747 | .setMIFlag(MachineInstr::FrameSetup); |
748 | } |
749 | } |
750 | } |
751 | } |
752 | |
753 | void RISCVFrameLowering::emitEpilogue(MachineFunction &MF, |
754 | MachineBasicBlock &MBB) const { |
755 | const RISCVRegisterInfo *RI = STI.getRegisterInfo(); |
756 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
757 | auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>(); |
758 | Register FPReg = getFPReg(STI); |
759 | Register SPReg = getSPReg(STI); |
760 | |
761 | // All calls are tail calls in GHC calling conv, and functions have no |
762 | // prologue/epilogue. |
763 | if (MF.getFunction().getCallingConv() == CallingConv::GHC) |
764 | return; |
765 | |
766 | // Get the insert location for the epilogue. If there were no terminators in |
767 | // the block, get the last instruction. |
768 | MachineBasicBlock::iterator MBBI = MBB.end(); |
769 | DebugLoc DL; |
770 | if (!MBB.empty()) { |
771 | MBBI = MBB.getLastNonDebugInstr(); |
772 | if (MBBI != MBB.end()) |
773 | DL = MBBI->getDebugLoc(); |
774 | |
775 | MBBI = MBB.getFirstTerminator(); |
776 | |
777 | // If callee-saved registers are saved via libcall, place stack adjustment |
778 | // before this call. |
779 | while (MBBI != MBB.begin() && |
780 | std::prev(x: MBBI)->getFlag(Flag: MachineInstr::FrameDestroy)) |
781 | --MBBI; |
782 | } |
783 | |
784 | const auto &CSI = getUnmanagedCSI(MF, CSI: MFI.getCalleeSavedInfo()); |
785 | |
786 | // Skip to before the restores of scalar callee-saved registers |
787 | // FIXME: assumes exactly one instruction is used to restore each |
788 | // callee-saved register. |
789 | auto LastFrameDestroy = MBBI; |
790 | if (!CSI.empty()) |
791 | LastFrameDestroy = std::prev(x: MBBI, n: CSI.size()); |
792 | |
793 | uint64_t RealStackSize = getStackSizeWithRVVPadding(MF); |
794 | uint64_t StackSize = RealStackSize - RVFI->getReservedSpillsSize(); |
795 | uint64_t FPOffset = RealStackSize - RVFI->getVarArgsSaveSize(); |
796 | uint64_t RVVStackSize = RVFI->getRVVStackSize(); |
797 | |
798 | // Restore the stack pointer using the value of the frame pointer. Only |
799 | // necessary if the stack pointer was modified, meaning the stack size is |
800 | // unknown. |
801 | // |
802 | // In order to make sure the stack point is right through the EH region, |
803 | // we also need to restore stack pointer from the frame pointer if we |
804 | // don't preserve stack space within prologue/epilogue for outgoing variables, |
805 | // normally it's just checking the variable sized object is present or not |
806 | // is enough, but we also don't preserve that at prologue/epilogue when |
807 | // have vector objects in stack. |
808 | if (RI->hasStackRealignment(MF) || MFI.hasVarSizedObjects() || |
809 | !hasReservedCallFrame(MF)) { |
810 | assert(hasFP(MF) && "frame pointer should not have been eliminated" ); |
811 | RI->adjustReg(MBB, II: LastFrameDestroy, DL, DestReg: SPReg, SrcReg: FPReg, |
812 | Offset: StackOffset::getFixed(Fixed: -FPOffset), |
813 | Flag: MachineInstr::FrameDestroy, RequiredAlign: getStackAlign()); |
814 | } else { |
815 | if (RVVStackSize) |
816 | adjustStackForRVV(MF, MBB, MBBI: LastFrameDestroy, DL, Amount: RVVStackSize, |
817 | Flag: MachineInstr::FrameDestroy); |
818 | } |
819 | |
820 | uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF); |
821 | if (FirstSPAdjustAmount) { |
822 | uint64_t SecondSPAdjustAmount = |
823 | getStackSizeWithRVVPadding(MF) - FirstSPAdjustAmount; |
824 | assert(SecondSPAdjustAmount > 0 && |
825 | "SecondSPAdjustAmount should be greater than zero" ); |
826 | |
827 | RI->adjustReg(MBB, II: LastFrameDestroy, DL, DestReg: SPReg, SrcReg: SPReg, |
828 | Offset: StackOffset::getFixed(Fixed: SecondSPAdjustAmount), |
829 | Flag: MachineInstr::FrameDestroy, RequiredAlign: getStackAlign()); |
830 | } |
831 | |
832 | if (FirstSPAdjustAmount) |
833 | StackSize = FirstSPAdjustAmount; |
834 | |
835 | if (RVFI->isPushable(MF) && MBBI != MBB.end() && |
836 | MBBI->getOpcode() == RISCV::CM_POP) { |
837 | // Use available stack adjustment in pop instruction to deallocate stack |
838 | // space. Align the stack size down to a multiple of 16. This is needed for |
839 | // RVE. |
840 | // FIXME: Can we increase the stack size to a multiple of 16 instead? |
841 | uint64_t Spimm = std::min(a: alignDown(Value: StackSize, Align: 16), b: (uint64_t)48); |
842 | MBBI->getOperand(i: 1).setImm(Spimm); |
843 | StackSize -= Spimm; |
844 | } |
845 | |
846 | // Deallocate stack |
847 | if (StackSize != 0) { |
848 | RI->adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg, Offset: StackOffset::getFixed(Fixed: StackSize), |
849 | Flag: MachineInstr::FrameDestroy, RequiredAlign: getStackAlign()); |
850 | } |
851 | |
852 | // Emit epilogue for shadow call stack. |
853 | emitSCSEpilogue(MF, MBB, MI: MBBI, DL); |
854 | } |
855 | |
856 | StackOffset |
857 | RISCVFrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI, |
858 | Register &FrameReg) const { |
859 | const MachineFrameInfo &MFI = MF.getFrameInfo(); |
860 | const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo(); |
861 | const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>(); |
862 | |
863 | // Callee-saved registers should be referenced relative to the stack |
864 | // pointer (positive offset), otherwise use the frame pointer (negative |
865 | // offset). |
866 | const auto &CSI = getUnmanagedCSI(MF, CSI: MFI.getCalleeSavedInfo()); |
867 | int MinCSFI = 0; |
868 | int MaxCSFI = -1; |
869 | StackOffset Offset; |
870 | auto StackID = MFI.getStackID(ObjectIdx: FI); |
871 | |
872 | assert((StackID == TargetStackID::Default || |
873 | StackID == TargetStackID::ScalableVector) && |
874 | "Unexpected stack ID for the frame object." ); |
875 | if (StackID == TargetStackID::Default) { |
876 | Offset = |
877 | StackOffset::getFixed(Fixed: MFI.getObjectOffset(ObjectIdx: FI) - getOffsetOfLocalArea() + |
878 | MFI.getOffsetAdjustment()); |
879 | } else if (StackID == TargetStackID::ScalableVector) { |
880 | Offset = StackOffset::getScalable(Scalable: MFI.getObjectOffset(ObjectIdx: FI)); |
881 | } |
882 | |
883 | uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF); |
884 | |
885 | if (CSI.size()) { |
886 | MinCSFI = CSI[0].getFrameIdx(); |
887 | MaxCSFI = CSI[CSI.size() - 1].getFrameIdx(); |
888 | } |
889 | |
890 | if (FI >= MinCSFI && FI <= MaxCSFI) { |
891 | FrameReg = RISCV::X2; |
892 | |
893 | if (FirstSPAdjustAmount) |
894 | Offset += StackOffset::getFixed(Fixed: FirstSPAdjustAmount); |
895 | else |
896 | Offset += StackOffset::getFixed(Fixed: getStackSizeWithRVVPadding(MF)); |
897 | return Offset; |
898 | } |
899 | |
900 | if (RI->hasStackRealignment(MF) && !MFI.isFixedObjectIndex(ObjectIdx: FI)) { |
901 | // If the stack was realigned, the frame pointer is set in order to allow |
902 | // SP to be restored, so we need another base register to record the stack |
903 | // after realignment. |
904 | // |--------------------------| -- <-- FP |
905 | // | callee-allocated save | | <----| |
906 | // | area for register varargs| | | |
907 | // |--------------------------| | | |
908 | // | callee-saved registers | | | |
909 | // |--------------------------| -- | |
910 | // | realignment (the size of | | | |
911 | // | this area is not counted | | | |
912 | // | in MFI.getStackSize()) | | | |
913 | // |--------------------------| -- |-- MFI.getStackSize() |
914 | // | RVV alignment padding | | | |
915 | // | (not counted in | | | |
916 | // | MFI.getStackSize() but | | | |
917 | // | counted in | | | |
918 | // | RVFI.getRVVStackSize()) | | | |
919 | // |--------------------------| -- | |
920 | // | RVV objects | | | |
921 | // | (not counted in | | | |
922 | // | MFI.getStackSize()) | | | |
923 | // |--------------------------| -- | |
924 | // | padding before RVV | | | |
925 | // | (not counted in | | | |
926 | // | MFI.getStackSize() or in | | | |
927 | // | RVFI.getRVVStackSize()) | | | |
928 | // |--------------------------| -- | |
929 | // | scalar local variables | | <----' |
930 | // |--------------------------| -- <-- BP (if var sized objects present) |
931 | // | VarSize objects | | |
932 | // |--------------------------| -- <-- SP |
933 | if (hasBP(MF)) { |
934 | FrameReg = RISCVABI::getBPReg(); |
935 | } else { |
936 | // VarSize objects must be empty in this case! |
937 | assert(!MFI.hasVarSizedObjects()); |
938 | FrameReg = RISCV::X2; |
939 | } |
940 | } else { |
941 | FrameReg = RI->getFrameRegister(MF); |
942 | } |
943 | |
944 | if (FrameReg == getFPReg(STI)) { |
945 | Offset += StackOffset::getFixed(Fixed: RVFI->getVarArgsSaveSize()); |
946 | // When using FP to access scalable vector objects, we need to minus |
947 | // the frame size. |
948 | // |
949 | // |--------------------------| -- <-- FP |
950 | // | callee-allocated save | | |
951 | // | area for register varargs| | |
952 | // |--------------------------| | |
953 | // | callee-saved registers | | |
954 | // |--------------------------| | MFI.getStackSize() |
955 | // | scalar local variables | | |
956 | // |--------------------------| -- (Offset of RVV objects is from here.) |
957 | // | RVV objects | |
958 | // |--------------------------| |
959 | // | VarSize objects | |
960 | // |--------------------------| <-- SP |
961 | if (MFI.getStackID(ObjectIdx: FI) == TargetStackID::ScalableVector) { |
962 | assert(!RI->hasStackRealignment(MF) && |
963 | "Can't index across variable sized realign" ); |
964 | // We don't expect any extra RVV alignment padding, as the stack size |
965 | // and RVV object sections should be correct aligned in their own |
966 | // right. |
967 | assert(MFI.getStackSize() == getStackSizeWithRVVPadding(MF) && |
968 | "Inconsistent stack layout" ); |
969 | Offset -= StackOffset::getFixed(Fixed: MFI.getStackSize()); |
970 | } |
971 | return Offset; |
972 | } |
973 | |
974 | // This case handles indexing off both SP and BP. |
975 | // If indexing off SP, there must not be any var sized objects |
976 | assert(FrameReg == RISCVABI::getBPReg() || !MFI.hasVarSizedObjects()); |
977 | |
978 | // When using SP to access frame objects, we need to add RVV stack size. |
979 | // |
980 | // |--------------------------| -- <-- FP |
981 | // | callee-allocated save | | <----| |
982 | // | area for register varargs| | | |
983 | // |--------------------------| | | |
984 | // | callee-saved registers | | | |
985 | // |--------------------------| -- | |
986 | // | RVV alignment padding | | | |
987 | // | (not counted in | | | |
988 | // | MFI.getStackSize() but | | | |
989 | // | counted in | | | |
990 | // | RVFI.getRVVStackSize()) | | | |
991 | // |--------------------------| -- | |
992 | // | RVV objects | | |-- MFI.getStackSize() |
993 | // | (not counted in | | | |
994 | // | MFI.getStackSize()) | | | |
995 | // |--------------------------| -- | |
996 | // | padding before RVV | | | |
997 | // | (not counted in | | | |
998 | // | MFI.getStackSize()) | | | |
999 | // |--------------------------| -- | |
1000 | // | scalar local variables | | <----' |
1001 | // |--------------------------| -- <-- BP (if var sized objects present) |
1002 | // | VarSize objects | | |
1003 | // |--------------------------| -- <-- SP |
1004 | // |
1005 | // The total amount of padding surrounding RVV objects is described by |
1006 | // RVV->getRVVPadding() and it can be zero. It allows us to align the RVV |
1007 | // objects to the required alignment. |
1008 | if (MFI.getStackID(ObjectIdx: FI) == TargetStackID::Default) { |
1009 | if (MFI.isFixedObjectIndex(ObjectIdx: FI)) { |
1010 | assert(!RI->hasStackRealignment(MF) && |
1011 | "Can't index across variable sized realign" ); |
1012 | Offset += StackOffset::get(Fixed: getStackSizeWithRVVPadding(MF), |
1013 | Scalable: RVFI->getRVVStackSize()); |
1014 | } else { |
1015 | Offset += StackOffset::getFixed(Fixed: MFI.getStackSize()); |
1016 | } |
1017 | } else if (MFI.getStackID(ObjectIdx: FI) == TargetStackID::ScalableVector) { |
1018 | // Ensure the base of the RVV stack is correctly aligned: add on the |
1019 | // alignment padding. |
1020 | int ScalarLocalVarSize = MFI.getStackSize() - |
1021 | RVFI->getCalleeSavedStackSize() - |
1022 | RVFI->getRVPushStackSize() - |
1023 | RVFI->getVarArgsSaveSize() + RVFI->getRVVPadding(); |
1024 | Offset += StackOffset::get(Fixed: ScalarLocalVarSize, Scalable: RVFI->getRVVStackSize()); |
1025 | } |
1026 | return Offset; |
1027 | } |
1028 | |
1029 | void RISCVFrameLowering::determineCalleeSaves(MachineFunction &MF, |
1030 | BitVector &SavedRegs, |
1031 | RegScavenger *RS) const { |
1032 | TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS); |
1033 | // Unconditionally spill RA and FP only if the function uses a frame |
1034 | // pointer. |
1035 | if (hasFP(MF)) { |
1036 | SavedRegs.set(RISCV::X1); |
1037 | SavedRegs.set(RISCV::X8); |
1038 | } |
1039 | // Mark BP as used if function has dedicated base pointer. |
1040 | if (hasBP(MF)) |
1041 | SavedRegs.set(RISCVABI::getBPReg()); |
1042 | } |
1043 | |
1044 | std::pair<int64_t, Align> |
1045 | RISCVFrameLowering::assignRVVStackObjectOffsets(MachineFunction &MF) const { |
1046 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
1047 | // Create a buffer of RVV objects to allocate. |
1048 | SmallVector<int, 8> ObjectsToAllocate; |
1049 | auto pushRVVObjects = [&](int FIBegin, int FIEnd) { |
1050 | for (int I = FIBegin, E = FIEnd; I != E; ++I) { |
1051 | unsigned StackID = MFI.getStackID(ObjectIdx: I); |
1052 | if (StackID != TargetStackID::ScalableVector) |
1053 | continue; |
1054 | if (MFI.isDeadObjectIndex(ObjectIdx: I)) |
1055 | continue; |
1056 | |
1057 | ObjectsToAllocate.push_back(Elt: I); |
1058 | } |
1059 | }; |
1060 | // First push RVV Callee Saved object, then push RVV stack object |
1061 | std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo(); |
1062 | const auto &RVVCSI = getRVVCalleeSavedInfo(MF, CSI); |
1063 | if (!RVVCSI.empty()) |
1064 | pushRVVObjects(RVVCSI[0].getFrameIdx(), |
1065 | RVVCSI[RVVCSI.size() - 1].getFrameIdx() + 1); |
1066 | pushRVVObjects(0, MFI.getObjectIndexEnd() - RVVCSI.size()); |
1067 | |
1068 | // The minimum alignment is 16 bytes. |
1069 | Align RVVStackAlign(16); |
1070 | const auto &ST = MF.getSubtarget<RISCVSubtarget>(); |
1071 | |
1072 | if (!ST.hasVInstructions()) { |
1073 | assert(ObjectsToAllocate.empty() && |
1074 | "Can't allocate scalable-vector objects without V instructions" ); |
1075 | return std::make_pair(x: 0, y&: RVVStackAlign); |
1076 | } |
1077 | |
1078 | // Allocate all RVV locals and spills |
1079 | int64_t Offset = 0; |
1080 | for (int FI : ObjectsToAllocate) { |
1081 | // ObjectSize in bytes. |
1082 | int64_t ObjectSize = MFI.getObjectSize(ObjectIdx: FI); |
1083 | auto ObjectAlign = std::max(a: Align(8), b: MFI.getObjectAlign(ObjectIdx: FI)); |
1084 | // If the data type is the fractional vector type, reserve one vector |
1085 | // register for it. |
1086 | if (ObjectSize < 8) |
1087 | ObjectSize = 8; |
1088 | Offset = alignTo(Size: Offset + ObjectSize, A: ObjectAlign); |
1089 | MFI.setObjectOffset(ObjectIdx: FI, SPOffset: -Offset); |
1090 | // Update the maximum alignment of the RVV stack section |
1091 | RVVStackAlign = std::max(a: RVVStackAlign, b: ObjectAlign); |
1092 | } |
1093 | |
1094 | // Ensure the alignment of the RVV stack. Since we want the most-aligned |
1095 | // object right at the bottom (i.e., any padding at the top of the frame), |
1096 | // readjust all RVV objects down by the alignment padding. |
1097 | uint64_t StackSize = Offset; |
1098 | if (auto AlignmentPadding = offsetToAlignment(Value: StackSize, Alignment: RVVStackAlign)) { |
1099 | StackSize += AlignmentPadding; |
1100 | for (int FI : ObjectsToAllocate) |
1101 | MFI.setObjectOffset(ObjectIdx: FI, SPOffset: MFI.getObjectOffset(ObjectIdx: FI) - AlignmentPadding); |
1102 | } |
1103 | |
1104 | return std::make_pair(x&: StackSize, y&: RVVStackAlign); |
1105 | } |
1106 | |
1107 | static unsigned getScavSlotsNumForRVV(MachineFunction &MF) { |
1108 | // For RVV spill, scalable stack offsets computing requires up to two scratch |
1109 | // registers |
1110 | static constexpr unsigned ScavSlotsNumRVVSpillScalableObject = 2; |
1111 | |
1112 | // For RVV spill, non-scalable stack offsets computing requires up to one |
1113 | // scratch register. |
1114 | static constexpr unsigned ScavSlotsNumRVVSpillNonScalableObject = 1; |
1115 | |
1116 | // ADDI instruction's destination register can be used for computing |
1117 | // offsets. So Scalable stack offsets require up to one scratch register. |
1118 | static constexpr unsigned ScavSlotsADDIScalableObject = 1; |
1119 | |
1120 | static constexpr unsigned MaxScavSlotsNumKnown = |
1121 | std::max(l: {ScavSlotsADDIScalableObject, ScavSlotsNumRVVSpillScalableObject, |
1122 | ScavSlotsNumRVVSpillNonScalableObject}); |
1123 | |
1124 | unsigned MaxScavSlotsNum = 0; |
1125 | if (!MF.getSubtarget<RISCVSubtarget>().hasVInstructions()) |
1126 | return false; |
1127 | for (const MachineBasicBlock &MBB : MF) |
1128 | for (const MachineInstr &MI : MBB) { |
1129 | bool IsRVVSpill = RISCV::isRVVSpill(MI); |
1130 | for (auto &MO : MI.operands()) { |
1131 | if (!MO.isFI()) |
1132 | continue; |
1133 | bool IsScalableVectorID = MF.getFrameInfo().getStackID(ObjectIdx: MO.getIndex()) == |
1134 | TargetStackID::ScalableVector; |
1135 | if (IsRVVSpill) { |
1136 | MaxScavSlotsNum = std::max( |
1137 | a: MaxScavSlotsNum, b: IsScalableVectorID |
1138 | ? ScavSlotsNumRVVSpillScalableObject |
1139 | : ScavSlotsNumRVVSpillNonScalableObject); |
1140 | } else if (MI.getOpcode() == RISCV::ADDI && IsScalableVectorID) { |
1141 | MaxScavSlotsNum = |
1142 | std::max(a: MaxScavSlotsNum, b: ScavSlotsADDIScalableObject); |
1143 | } |
1144 | } |
1145 | if (MaxScavSlotsNum == MaxScavSlotsNumKnown) |
1146 | return MaxScavSlotsNumKnown; |
1147 | } |
1148 | return MaxScavSlotsNum; |
1149 | } |
1150 | |
1151 | static bool hasRVVFrameObject(const MachineFunction &MF) { |
1152 | // Originally, the function will scan all the stack objects to check whether |
1153 | // if there is any scalable vector object on the stack or not. However, it |
1154 | // causes errors in the register allocator. In issue 53016, it returns false |
1155 | // before RA because there is no RVV stack objects. After RA, it returns true |
1156 | // because there are spilling slots for RVV values during RA. It will not |
1157 | // reserve BP during register allocation and generate BP access in the PEI |
1158 | // pass due to the inconsistent behavior of the function. |
1159 | // |
1160 | // The function is changed to use hasVInstructions() as the return value. It |
1161 | // is not precise, but it can make the register allocation correct. |
1162 | // |
1163 | // FIXME: Find a better way to make the decision or revisit the solution in |
1164 | // D103622. |
1165 | // |
1166 | // Refer to https://github.com/llvm/llvm-project/issues/53016. |
1167 | return MF.getSubtarget<RISCVSubtarget>().hasVInstructions(); |
1168 | } |
1169 | |
1170 | static unsigned estimateFunctionSizeInBytes(const MachineFunction &MF, |
1171 | const RISCVInstrInfo &TII) { |
1172 | unsigned FnSize = 0; |
1173 | for (auto &MBB : MF) { |
1174 | for (auto &MI : MBB) { |
1175 | // Far branches over 20-bit offset will be relaxed in branch relaxation |
1176 | // pass. In the worst case, conditional branches will be relaxed into |
1177 | // the following instruction sequence. Unconditional branches are |
1178 | // relaxed in the same way, with the exception that there is no first |
1179 | // branch instruction. |
1180 | // |
1181 | // foo |
1182 | // bne t5, t6, .rev_cond # `TII->getInstSizeInBytes(MI)` bytes |
1183 | // sd s11, 0(sp) # 4 bytes, or 2 bytes in RVC |
1184 | // jump .restore, s11 # 8 bytes |
1185 | // .rev_cond |
1186 | // bar |
1187 | // j .dest_bb # 4 bytes, or 2 bytes in RVC |
1188 | // .restore: |
1189 | // ld s11, 0(sp) # 4 bytes, or 2 bytes in RVC |
1190 | // .dest: |
1191 | // baz |
1192 | if (MI.isConditionalBranch()) |
1193 | FnSize += TII.getInstSizeInBytes(MI); |
1194 | if (MI.isConditionalBranch() || MI.isUnconditionalBranch()) { |
1195 | if (MF.getSubtarget<RISCVSubtarget>().hasStdExtCOrZca()) |
1196 | FnSize += 2 + 8 + 2 + 2; |
1197 | else |
1198 | FnSize += 4 + 8 + 4 + 4; |
1199 | continue; |
1200 | } |
1201 | |
1202 | FnSize += TII.getInstSizeInBytes(MI); |
1203 | } |
1204 | } |
1205 | return FnSize; |
1206 | } |
1207 | |
1208 | void RISCVFrameLowering::processFunctionBeforeFrameFinalized( |
1209 | MachineFunction &MF, RegScavenger *RS) const { |
1210 | const RISCVRegisterInfo *RegInfo = |
1211 | MF.getSubtarget<RISCVSubtarget>().getRegisterInfo(); |
1212 | const RISCVInstrInfo *TII = MF.getSubtarget<RISCVSubtarget>().getInstrInfo(); |
1213 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
1214 | const TargetRegisterClass *RC = &RISCV::GPRRegClass; |
1215 | auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>(); |
1216 | |
1217 | int64_t RVVStackSize; |
1218 | Align RVVStackAlign; |
1219 | std::tie(args&: RVVStackSize, args&: RVVStackAlign) = assignRVVStackObjectOffsets(MF); |
1220 | |
1221 | RVFI->setRVVStackSize(RVVStackSize); |
1222 | RVFI->setRVVStackAlign(RVVStackAlign); |
1223 | |
1224 | if (hasRVVFrameObject(MF)) { |
1225 | // Ensure the entire stack is aligned to at least the RVV requirement: some |
1226 | // scalable-vector object alignments are not considered by the |
1227 | // target-independent code. |
1228 | MFI.ensureMaxAlignment(Alignment: RVVStackAlign); |
1229 | } |
1230 | |
1231 | unsigned ScavSlotsNum = 0; |
1232 | |
1233 | // estimateStackSize has been observed to under-estimate the final stack |
1234 | // size, so give ourselves wiggle-room by checking for stack size |
1235 | // representable an 11-bit signed field rather than 12-bits. |
1236 | if (!isInt<11>(x: MFI.estimateStackSize(MF))) |
1237 | ScavSlotsNum = 1; |
1238 | |
1239 | // Far branches over 20-bit offset require a spill slot for scratch register. |
1240 | bool IsLargeFunction = !isInt<20>(x: estimateFunctionSizeInBytes(MF, TII: *TII)); |
1241 | if (IsLargeFunction) |
1242 | ScavSlotsNum = std::max(a: ScavSlotsNum, b: 1u); |
1243 | |
1244 | // RVV loads & stores have no capacity to hold the immediate address offsets |
1245 | // so we must always reserve an emergency spill slot if the MachineFunction |
1246 | // contains any RVV spills. |
1247 | ScavSlotsNum = std::max(a: ScavSlotsNum, b: getScavSlotsNumForRVV(MF)); |
1248 | |
1249 | for (unsigned I = 0; I < ScavSlotsNum; I++) { |
1250 | int FI = MFI.CreateStackObject(Size: RegInfo->getSpillSize(*RC), |
1251 | Alignment: RegInfo->getSpillAlign(*RC), isSpillSlot: false); |
1252 | RS->addScavengingFrameIndex(FI); |
1253 | |
1254 | if (IsLargeFunction && RVFI->getBranchRelaxationScratchFrameIndex() == -1) |
1255 | RVFI->setBranchRelaxationScratchFrameIndex(FI); |
1256 | } |
1257 | |
1258 | unsigned Size = RVFI->getReservedSpillsSize(); |
1259 | for (const auto &Info : MFI.getCalleeSavedInfo()) { |
1260 | int FrameIdx = Info.getFrameIdx(); |
1261 | if (FrameIdx < 0 || MFI.getStackID(ObjectIdx: FrameIdx) != TargetStackID::Default) |
1262 | continue; |
1263 | |
1264 | Size += MFI.getObjectSize(ObjectIdx: FrameIdx); |
1265 | } |
1266 | RVFI->setCalleeSavedStackSize(Size); |
1267 | } |
1268 | |
1269 | // Not preserve stack space within prologue for outgoing variables when the |
1270 | // function contains variable size objects or there are vector objects accessed |
1271 | // by the frame pointer. |
1272 | // Let eliminateCallFramePseudoInstr preserve stack space for it. |
1273 | bool RISCVFrameLowering::hasReservedCallFrame(const MachineFunction &MF) const { |
1274 | return !MF.getFrameInfo().hasVarSizedObjects() && |
1275 | !(hasFP(MF) && hasRVVFrameObject(MF)); |
1276 | } |
1277 | |
1278 | // Eliminate ADJCALLSTACKDOWN, ADJCALLSTACKUP pseudo instructions. |
1279 | MachineBasicBlock::iterator RISCVFrameLowering::eliminateCallFramePseudoInstr( |
1280 | MachineFunction &MF, MachineBasicBlock &MBB, |
1281 | MachineBasicBlock::iterator MI) const { |
1282 | Register SPReg = RISCV::X2; |
1283 | DebugLoc DL = MI->getDebugLoc(); |
1284 | |
1285 | if (!hasReservedCallFrame(MF)) { |
1286 | // If space has not been reserved for a call frame, ADJCALLSTACKDOWN and |
1287 | // ADJCALLSTACKUP must be converted to instructions manipulating the stack |
1288 | // pointer. This is necessary when there is a variable length stack |
1289 | // allocation (e.g. alloca), which means it's not possible to allocate |
1290 | // space for outgoing arguments from within the function prologue. |
1291 | int64_t Amount = MI->getOperand(i: 0).getImm(); |
1292 | |
1293 | if (Amount != 0) { |
1294 | // Ensure the stack remains aligned after adjustment. |
1295 | Amount = alignSPAdjust(SPAdj: Amount); |
1296 | |
1297 | if (MI->getOpcode() == RISCV::ADJCALLSTACKDOWN) |
1298 | Amount = -Amount; |
1299 | |
1300 | const RISCVRegisterInfo &RI = *STI.getRegisterInfo(); |
1301 | RI.adjustReg(MBB, II: MI, DL, DestReg: SPReg, SrcReg: SPReg, Offset: StackOffset::getFixed(Fixed: Amount), |
1302 | Flag: MachineInstr::NoFlags, RequiredAlign: getStackAlign()); |
1303 | } |
1304 | } |
1305 | |
1306 | return MBB.erase(I: MI); |
1307 | } |
1308 | |
1309 | // We would like to split the SP adjustment to reduce prologue/epilogue |
1310 | // as following instructions. In this way, the offset of the callee saved |
1311 | // register could fit in a single store. Supposed that the first sp adjust |
1312 | // amount is 2032. |
1313 | // add sp,sp,-2032 |
1314 | // sw ra,2028(sp) |
1315 | // sw s0,2024(sp) |
1316 | // sw s1,2020(sp) |
1317 | // sw s3,2012(sp) |
1318 | // sw s4,2008(sp) |
1319 | // add sp,sp,-64 |
1320 | uint64_t |
1321 | RISCVFrameLowering::getFirstSPAdjustAmount(const MachineFunction &MF) const { |
1322 | const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>(); |
1323 | const MachineFrameInfo &MFI = MF.getFrameInfo(); |
1324 | const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo(); |
1325 | uint64_t StackSize = getStackSizeWithRVVPadding(MF); |
1326 | |
1327 | // Disable SplitSPAdjust if save-restore libcall is used. The callee-saved |
1328 | // registers will be pushed by the save-restore libcalls, so we don't have to |
1329 | // split the SP adjustment in this case. |
1330 | if (RVFI->getReservedSpillsSize()) |
1331 | return 0; |
1332 | |
1333 | // Return the FirstSPAdjustAmount if the StackSize can not fit in a signed |
1334 | // 12-bit and there exists a callee-saved register needing to be pushed. |
1335 | if (!isInt<12>(x: StackSize) && (CSI.size() > 0)) { |
1336 | // FirstSPAdjustAmount is chosen at most as (2048 - StackAlign) because |
1337 | // 2048 will cause sp = sp + 2048 in the epilogue to be split into multiple |
1338 | // instructions. Offsets smaller than 2048 can fit in a single load/store |
1339 | // instruction, and we have to stick with the stack alignment. 2048 has |
1340 | // 16-byte alignment. The stack alignment for RV32 and RV64 is 16 and for |
1341 | // RV32E it is 4. So (2048 - StackAlign) will satisfy the stack alignment. |
1342 | const uint64_t StackAlign = getStackAlign().value(); |
1343 | |
1344 | // Amount of (2048 - StackAlign) will prevent callee saved and restored |
1345 | // instructions be compressed, so try to adjust the amount to the largest |
1346 | // offset that stack compression instructions accept when target supports |
1347 | // compression instructions. |
1348 | if (STI.hasStdExtCOrZca()) { |
1349 | // The compression extensions may support the following instructions: |
1350 | // riscv32: c.lwsp rd, offset[7:2] => 2^(6 + 2) |
1351 | // c.swsp rs2, offset[7:2] => 2^(6 + 2) |
1352 | // c.flwsp rd, offset[7:2] => 2^(6 + 2) |
1353 | // c.fswsp rs2, offset[7:2] => 2^(6 + 2) |
1354 | // riscv64: c.ldsp rd, offset[8:3] => 2^(6 + 3) |
1355 | // c.sdsp rs2, offset[8:3] => 2^(6 + 3) |
1356 | // c.fldsp rd, offset[8:3] => 2^(6 + 3) |
1357 | // c.fsdsp rs2, offset[8:3] => 2^(6 + 3) |
1358 | const uint64_t RVCompressLen = STI.getXLen() * 8; |
1359 | // Compared with amount (2048 - StackAlign), StackSize needs to |
1360 | // satisfy the following conditions to avoid using more instructions |
1361 | // to adjust the sp after adjusting the amount, such as |
1362 | // StackSize meets the condition (StackSize <= 2048 + RVCompressLen), |
1363 | // case1: Amount is 2048 - StackAlign: use addi + addi to adjust sp. |
1364 | // case2: Amount is RVCompressLen: use addi + addi to adjust sp. |
1365 | auto CanCompress = [&](uint64_t CompressLen) -> bool { |
1366 | if (StackSize <= 2047 + CompressLen || |
1367 | (StackSize > 2048 * 2 - StackAlign && |
1368 | StackSize <= 2047 * 2 + CompressLen) || |
1369 | StackSize > 2048 * 3 - StackAlign) |
1370 | return true; |
1371 | |
1372 | return false; |
1373 | }; |
1374 | // In the epilogue, addi sp, sp, 496 is used to recover the sp and it |
1375 | // can be compressed(C.ADDI16SP, offset can be [-512, 496]), but |
1376 | // addi sp, sp, 512 can not be compressed. So try to use 496 first. |
1377 | const uint64_t ADDI16SPCompressLen = 496; |
1378 | if (STI.is64Bit() && CanCompress(ADDI16SPCompressLen)) |
1379 | return ADDI16SPCompressLen; |
1380 | if (CanCompress(RVCompressLen)) |
1381 | return RVCompressLen; |
1382 | } |
1383 | return 2048 - StackAlign; |
1384 | } |
1385 | return 0; |
1386 | } |
1387 | |
1388 | // Offsets which need to be scale by XLen representing locations of CSRs which |
1389 | // are given a fixed location by save/restore libcalls or Zcmp Push/Pop. |
1390 | static const std::pair<MCPhysReg, int8_t> FixedCSRFIMap[] = { |
1391 | {/*ra*/ RISCV::X1, -1}, {/*s0*/ RISCV::X8, -2}, |
1392 | {/*s1*/ RISCV::X9, -3}, {/*s2*/ RISCV::X18, -4}, |
1393 | {/*s3*/ RISCV::X19, -5}, {/*s4*/ RISCV::X20, -6}, |
1394 | {/*s5*/ RISCV::X21, -7}, {/*s6*/ RISCV::X22, -8}, |
1395 | {/*s7*/ RISCV::X23, -9}, {/*s8*/ RISCV::X24, -10}, |
1396 | {/*s9*/ RISCV::X25, -11}, {/*s10*/ RISCV::X26, -12}, |
1397 | {/*s11*/ RISCV::X27, -13}}; |
1398 | |
1399 | bool RISCVFrameLowering::assignCalleeSavedSpillSlots( |
1400 | MachineFunction &MF, const TargetRegisterInfo *TRI, |
1401 | std::vector<CalleeSavedInfo> &CSI, unsigned &MinCSFrameIndex, |
1402 | unsigned &MaxCSFrameIndex) const { |
1403 | // Early exit if no callee saved registers are modified! |
1404 | if (CSI.empty()) |
1405 | return true; |
1406 | |
1407 | auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>(); |
1408 | |
1409 | if (RVFI->isPushable(MF)) { |
1410 | // Determine how many GPRs we need to push and save it to RVFI. |
1411 | Register MaxReg = getMaxPushPopReg(MF, CSI); |
1412 | if (MaxReg != RISCV::NoRegister) { |
1413 | auto [RegEnc, PushedRegNum] = getPushPopEncodingAndNum(MaxReg); |
1414 | RVFI->setRVPushRegs(PushedRegNum); |
1415 | RVFI->setRVPushStackSize(alignTo(Value: (STI.getXLen() / 8) * PushedRegNum, Align: 16)); |
1416 | |
1417 | // Use encoded number to represent registers to spill. |
1418 | RVFI->setRVPushRlist(RegEnc); |
1419 | } |
1420 | } |
1421 | |
1422 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
1423 | const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo(); |
1424 | |
1425 | for (auto &CS : CSI) { |
1426 | unsigned Reg = CS.getReg(); |
1427 | const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg); |
1428 | unsigned Size = RegInfo->getSpillSize(RC: *RC); |
1429 | |
1430 | // This might need a fixed stack slot. |
1431 | if (RVFI->useSaveRestoreLibCalls(MF) || RVFI->isPushable(MF)) { |
1432 | const auto *FII = llvm::find_if( |
1433 | FixedCSRFIMap, [&](auto P) { return P.first == CS.getReg(); }); |
1434 | if (FII != std::end(FixedCSRFIMap)) { |
1435 | int64_t Offset; |
1436 | if (RVFI->isPushable(MF)) |
1437 | Offset = -((FII->second + RVFI->getRVPushRegs() + 1) * (int64_t)Size); |
1438 | else |
1439 | Offset = FII->second * (int64_t)Size; |
1440 | |
1441 | int FrameIdx = MFI.CreateFixedSpillStackObject(Size, SPOffset: Offset); |
1442 | assert(FrameIdx < 0); |
1443 | CS.setFrameIdx(FrameIdx); |
1444 | continue; |
1445 | } |
1446 | } |
1447 | |
1448 | // Not a fixed slot. |
1449 | Align Alignment = RegInfo->getSpillAlign(RC: *RC); |
1450 | // We may not be able to satisfy the desired alignment specification of |
1451 | // the TargetRegisterClass if the stack alignment is smaller. Use the |
1452 | // min. |
1453 | Alignment = std::min(a: Alignment, b: getStackAlign()); |
1454 | int FrameIdx = MFI.CreateStackObject(Size, Alignment, isSpillSlot: true); |
1455 | if ((unsigned)FrameIdx < MinCSFrameIndex) |
1456 | MinCSFrameIndex = FrameIdx; |
1457 | if ((unsigned)FrameIdx > MaxCSFrameIndex) |
1458 | MaxCSFrameIndex = FrameIdx; |
1459 | CS.setFrameIdx(FrameIdx); |
1460 | } |
1461 | |
1462 | // Allocate a fixed object that covers the full push or libcall size. |
1463 | if (RVFI->isPushable(MF)) { |
1464 | if (int64_t PushSize = RVFI->getRVPushStackSize()) |
1465 | MFI.CreateFixedSpillStackObject(Size: PushSize, SPOffset: -PushSize); |
1466 | } else if (int LibCallRegs = getLibCallID(MF, CSI) + 1) { |
1467 | int64_t LibCallFrameSize = |
1468 | alignTo(Size: (STI.getXLen() / 8) * LibCallRegs, A: getStackAlign()); |
1469 | MFI.CreateFixedSpillStackObject(Size: LibCallFrameSize, SPOffset: -LibCallFrameSize); |
1470 | } |
1471 | |
1472 | return true; |
1473 | } |
1474 | |
1475 | bool RISCVFrameLowering::spillCalleeSavedRegisters( |
1476 | MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, |
1477 | ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const { |
1478 | if (CSI.empty()) |
1479 | return true; |
1480 | |
1481 | MachineFunction *MF = MBB.getParent(); |
1482 | const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo(); |
1483 | DebugLoc DL; |
1484 | if (MI != MBB.end() && !MI->isDebugInstr()) |
1485 | DL = MI->getDebugLoc(); |
1486 | |
1487 | // Emit CM.PUSH with base SPimm & evaluate Push stack |
1488 | RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>(); |
1489 | if (RVFI->isPushable(MF: *MF)) { |
1490 | unsigned PushedRegNum = RVFI->getRVPushRegs(); |
1491 | if (PushedRegNum > 0) { |
1492 | // Use encoded number to represent registers to spill. |
1493 | int RegEnc = RVFI->getRVPushRlist(); |
1494 | MachineInstrBuilder PushBuilder = |
1495 | BuildMI(MBB, MI, DL, TII.get(RISCV::CM_PUSH)) |
1496 | .setMIFlag(MachineInstr::FrameSetup); |
1497 | PushBuilder.addImm(Val: (int64_t)RegEnc); |
1498 | PushBuilder.addImm(Val: 0); |
1499 | |
1500 | for (unsigned i = 0; i < PushedRegNum; i++) |
1501 | PushBuilder.addUse(RegNo: AllPopRegs[i], Flags: RegState::Implicit); |
1502 | } |
1503 | } else if (const char *SpillLibCall = getSpillLibCallName(MF: *MF, CSI)) { |
1504 | // Add spill libcall via non-callee-saved register t0. |
1505 | BuildMI(MBB, MI, DL, TII.get(RISCV::PseudoCALLReg), RISCV::X5) |
1506 | .addExternalSymbol(SpillLibCall, RISCVII::MO_CALL) |
1507 | .setMIFlag(MachineInstr::FrameSetup); |
1508 | |
1509 | // Add registers spilled in libcall as liveins. |
1510 | for (auto &CS : CSI) |
1511 | MBB.addLiveIn(PhysReg: CS.getReg()); |
1512 | } |
1513 | |
1514 | // Manually spill values not spilled by libcall & Push/Pop. |
1515 | const auto &UnmanagedCSI = getUnmanagedCSI(MF: *MF, CSI); |
1516 | const auto &RVVCSI = getRVVCalleeSavedInfo(MF: *MF, CSI); |
1517 | |
1518 | auto storeRegToStackSlot = [&](decltype(UnmanagedCSI) CSInfo) { |
1519 | for (auto &CS : CSInfo) { |
1520 | // Insert the spill to the stack frame. |
1521 | Register Reg = CS.getReg(); |
1522 | const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg); |
1523 | TII.storeRegToStackSlot(MBB, MI, SrcReg: Reg, isKill: !MBB.isLiveIn(Reg), |
1524 | FrameIndex: CS.getFrameIdx(), RC, TRI, VReg: Register()); |
1525 | } |
1526 | }; |
1527 | storeRegToStackSlot(UnmanagedCSI); |
1528 | storeRegToStackSlot(RVVCSI); |
1529 | |
1530 | return true; |
1531 | } |
1532 | |
1533 | void RISCVFrameLowering::emitCalleeSavedRVVPrologCFI( |
1534 | MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, bool HasFP) const { |
1535 | MachineFunction *MF = MBB.getParent(); |
1536 | const MachineFrameInfo &MFI = MF->getFrameInfo(); |
1537 | RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>(); |
1538 | const TargetInstrInfo &TII = *STI.getInstrInfo(); |
1539 | DebugLoc DL = MBB.findDebugLoc(MBBI: MI); |
1540 | |
1541 | const auto &RVVCSI = getRVVCalleeSavedInfo(MF: *MF, CSI: MFI.getCalleeSavedInfo()); |
1542 | if (RVVCSI.empty()) |
1543 | return; |
1544 | |
1545 | uint64_t FixedSize = getStackSizeWithRVVPadding(MF: *MF); |
1546 | if (!HasFP) { |
1547 | uint64_t ScalarLocalVarSize = |
1548 | MFI.getStackSize() - RVFI->getCalleeSavedStackSize() - |
1549 | RVFI->getRVPushStackSize() - RVFI->getVarArgsSaveSize() + |
1550 | RVFI->getRVVPadding(); |
1551 | FixedSize -= ScalarLocalVarSize; |
1552 | } |
1553 | |
1554 | for (auto &CS : RVVCSI) { |
1555 | // Insert the spill to the stack frame. |
1556 | int FI = CS.getFrameIdx(); |
1557 | if (FI >= 0 && MFI.getStackID(ObjectIdx: FI) == TargetStackID::ScalableVector) { |
1558 | unsigned CFIIndex = MF->addFrameInst( |
1559 | Inst: createDefCFAOffset(*STI.getRegisterInfo(), CS.getReg(), -FixedSize, |
1560 | MFI.getObjectOffset(ObjectIdx: FI) / 8)); |
1561 | BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::CFI_INSTRUCTION)) |
1562 | .addCFIIndex(CFIIndex) |
1563 | .setMIFlag(MachineInstr::FrameSetup); |
1564 | } |
1565 | } |
1566 | } |
1567 | |
1568 | bool RISCVFrameLowering::restoreCalleeSavedRegisters( |
1569 | MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, |
1570 | MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const { |
1571 | if (CSI.empty()) |
1572 | return true; |
1573 | |
1574 | MachineFunction *MF = MBB.getParent(); |
1575 | const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo(); |
1576 | DebugLoc DL; |
1577 | if (MI != MBB.end() && !MI->isDebugInstr()) |
1578 | DL = MI->getDebugLoc(); |
1579 | |
1580 | // Manually restore values not restored by libcall & Push/Pop. |
1581 | // Reverse the restore order in epilog. In addition, the return |
1582 | // address will be restored first in the epilogue. It increases |
1583 | // the opportunity to avoid the load-to-use data hazard between |
1584 | // loading RA and return by RA. loadRegFromStackSlot can insert |
1585 | // multiple instructions. |
1586 | const auto &UnmanagedCSI = getUnmanagedCSI(MF: *MF, CSI); |
1587 | const auto &RVVCSI = getRVVCalleeSavedInfo(MF: *MF, CSI); |
1588 | |
1589 | auto loadRegFromStackSlot = [&](decltype(UnmanagedCSI) CSInfo) { |
1590 | for (auto &CS : CSInfo) { |
1591 | Register Reg = CS.getReg(); |
1592 | const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg); |
1593 | TII.loadRegFromStackSlot(MBB, MI, DestReg: Reg, FrameIndex: CS.getFrameIdx(), RC, TRI, |
1594 | VReg: Register()); |
1595 | assert(MI != MBB.begin() && |
1596 | "loadRegFromStackSlot didn't insert any code!" ); |
1597 | } |
1598 | }; |
1599 | loadRegFromStackSlot(RVVCSI); |
1600 | loadRegFromStackSlot(UnmanagedCSI); |
1601 | |
1602 | RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>(); |
1603 | if (RVFI->isPushable(MF: *MF)) { |
1604 | int RegEnc = RVFI->getRVPushRlist(); |
1605 | if (RegEnc != llvm::RISCVZC::RLISTENCODE::INVALID_RLIST) { |
1606 | MachineInstrBuilder PopBuilder = |
1607 | BuildMI(MBB, MI, DL, TII.get(RISCV::CM_POP)) |
1608 | .setMIFlag(MachineInstr::FrameDestroy); |
1609 | // Use encoded number to represent registers to restore. |
1610 | PopBuilder.addImm(Val: RegEnc); |
1611 | PopBuilder.addImm(Val: 0); |
1612 | |
1613 | for (unsigned i = 0; i < RVFI->getRVPushRegs(); i++) |
1614 | PopBuilder.addDef(RegNo: AllPopRegs[i], Flags: RegState::ImplicitDefine); |
1615 | } |
1616 | } else { |
1617 | const char *RestoreLibCall = getRestoreLibCallName(MF: *MF, CSI); |
1618 | if (RestoreLibCall) { |
1619 | // Add restore libcall via tail call. |
1620 | MachineBasicBlock::iterator NewMI = |
1621 | BuildMI(MBB, MI, DL, TII.get(RISCV::PseudoTAIL)) |
1622 | .addExternalSymbol(RestoreLibCall, RISCVII::MO_CALL) |
1623 | .setMIFlag(MachineInstr::FrameDestroy); |
1624 | |
1625 | // Remove trailing returns, since the terminator is now a tail call to the |
1626 | // restore function. |
1627 | if (MI != MBB.end() && MI->getOpcode() == RISCV::PseudoRET) { |
1628 | NewMI->copyImplicitOps(MF&: *MF, MI: *MI); |
1629 | MI->eraseFromParent(); |
1630 | } |
1631 | } |
1632 | } |
1633 | return true; |
1634 | } |
1635 | |
1636 | bool RISCVFrameLowering::enableShrinkWrapping(const MachineFunction &MF) const { |
1637 | // Keep the conventional code flow when not optimizing. |
1638 | if (MF.getFunction().hasOptNone()) |
1639 | return false; |
1640 | |
1641 | return true; |
1642 | } |
1643 | |
1644 | bool RISCVFrameLowering::canUseAsPrologue(const MachineBasicBlock &MBB) const { |
1645 | MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB); |
1646 | const MachineFunction *MF = MBB.getParent(); |
1647 | const auto *RVFI = MF->getInfo<RISCVMachineFunctionInfo>(); |
1648 | |
1649 | if (!RVFI->useSaveRestoreLibCalls(MF: *MF)) |
1650 | return true; |
1651 | |
1652 | // Inserting a call to a __riscv_save libcall requires the use of the register |
1653 | // t0 (X5) to hold the return address. Therefore if this register is already |
1654 | // used we can't insert the call. |
1655 | |
1656 | RegScavenger RS; |
1657 | RS.enterBasicBlock(MBB&: *TmpMBB); |
1658 | return !RS.isRegUsed(RISCV::X5); |
1659 | } |
1660 | |
1661 | bool RISCVFrameLowering::canUseAsEpilogue(const MachineBasicBlock &MBB) const { |
1662 | const MachineFunction *MF = MBB.getParent(); |
1663 | MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB); |
1664 | const auto *RVFI = MF->getInfo<RISCVMachineFunctionInfo>(); |
1665 | |
1666 | if (!RVFI->useSaveRestoreLibCalls(MF: *MF)) |
1667 | return true; |
1668 | |
1669 | // Using the __riscv_restore libcalls to restore CSRs requires a tail call. |
1670 | // This means if we still need to continue executing code within this function |
1671 | // the restore cannot take place in this basic block. |
1672 | |
1673 | if (MBB.succ_size() > 1) |
1674 | return false; |
1675 | |
1676 | MachineBasicBlock *SuccMBB = |
1677 | MBB.succ_empty() ? TmpMBB->getFallThrough() : *MBB.succ_begin(); |
1678 | |
1679 | // Doing a tail call should be safe if there are no successors, because either |
1680 | // we have a returning block or the end of the block is unreachable, so the |
1681 | // restore will be eliminated regardless. |
1682 | if (!SuccMBB) |
1683 | return true; |
1684 | |
1685 | // The successor can only contain a return, since we would effectively be |
1686 | // replacing the successor with our own tail return at the end of our block. |
1687 | return SuccMBB->isReturnBlock() && SuccMBB->size() == 1; |
1688 | } |
1689 | |
1690 | bool RISCVFrameLowering::isSupportedStackID(TargetStackID::Value ID) const { |
1691 | switch (ID) { |
1692 | case TargetStackID::Default: |
1693 | case TargetStackID::ScalableVector: |
1694 | return true; |
1695 | case TargetStackID::NoAlloc: |
1696 | case TargetStackID::SGPRSpill: |
1697 | case TargetStackID::WasmLocal: |
1698 | return false; |
1699 | } |
1700 | llvm_unreachable("Invalid TargetStackID::Value" ); |
1701 | } |
1702 | |
1703 | TargetStackID::Value RISCVFrameLowering::getStackIDForScalableVectors() const { |
1704 | return TargetStackID::ScalableVector; |
1705 | } |
1706 | |