1 | //===-- lib/CodeGen/GlobalISel/CallLowering.cpp - Call lowering -----------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | /// |
9 | /// \file |
10 | /// This file implements some simple delegations needed for call lowering. |
11 | /// |
12 | //===----------------------------------------------------------------------===// |
13 | |
14 | #include "llvm/CodeGen/GlobalISel/CallLowering.h" |
15 | #include "llvm/CodeGen/Analysis.h" |
16 | #include "llvm/CodeGen/CallingConvLower.h" |
17 | #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" |
18 | #include "llvm/CodeGen/GlobalISel/Utils.h" |
19 | #include "llvm/CodeGen/MachineFrameInfo.h" |
20 | #include "llvm/CodeGen/MachineOperand.h" |
21 | #include "llvm/CodeGen/MachineRegisterInfo.h" |
22 | #include "llvm/CodeGen/TargetLowering.h" |
23 | #include "llvm/IR/DataLayout.h" |
24 | #include "llvm/IR/LLVMContext.h" |
25 | #include "llvm/IR/Module.h" |
26 | #include "llvm/Target/TargetMachine.h" |
27 | |
28 | #define DEBUG_TYPE "call-lowering" |
29 | |
30 | using namespace llvm; |
31 | |
32 | void CallLowering::anchor() {} |
33 | |
34 | /// Helper function which updates \p Flags when \p AttrFn returns true. |
35 | static void |
36 | addFlagsUsingAttrFn(ISD::ArgFlagsTy &Flags, |
37 | const std::function<bool(Attribute::AttrKind)> &AttrFn) { |
38 | if (AttrFn(Attribute::SExt)) |
39 | Flags.setSExt(); |
40 | if (AttrFn(Attribute::ZExt)) |
41 | Flags.setZExt(); |
42 | if (AttrFn(Attribute::InReg)) |
43 | Flags.setInReg(); |
44 | if (AttrFn(Attribute::StructRet)) |
45 | Flags.setSRet(); |
46 | if (AttrFn(Attribute::Nest)) |
47 | Flags.setNest(); |
48 | if (AttrFn(Attribute::ByVal)) |
49 | Flags.setByVal(); |
50 | if (AttrFn(Attribute::Preallocated)) |
51 | Flags.setPreallocated(); |
52 | if (AttrFn(Attribute::InAlloca)) |
53 | Flags.setInAlloca(); |
54 | if (AttrFn(Attribute::Returned)) |
55 | Flags.setReturned(); |
56 | if (AttrFn(Attribute::SwiftSelf)) |
57 | Flags.setSwiftSelf(); |
58 | if (AttrFn(Attribute::SwiftAsync)) |
59 | Flags.setSwiftAsync(); |
60 | if (AttrFn(Attribute::SwiftError)) |
61 | Flags.setSwiftError(); |
62 | } |
63 | |
64 | ISD::ArgFlagsTy CallLowering::getAttributesForArgIdx(const CallBase &Call, |
65 | unsigned ArgIdx) const { |
66 | ISD::ArgFlagsTy Flags; |
67 | addFlagsUsingAttrFn(Flags, AttrFn: [&Call, &ArgIdx](Attribute::AttrKind Attr) { |
68 | return Call.paramHasAttr(ArgNo: ArgIdx, Kind: Attr); |
69 | }); |
70 | return Flags; |
71 | } |
72 | |
73 | ISD::ArgFlagsTy |
74 | CallLowering::getAttributesForReturn(const CallBase &Call) const { |
75 | ISD::ArgFlagsTy Flags; |
76 | addFlagsUsingAttrFn(Flags, AttrFn: [&Call](Attribute::AttrKind Attr) { |
77 | return Call.hasRetAttr(Kind: Attr); |
78 | }); |
79 | return Flags; |
80 | } |
81 | |
82 | void CallLowering::addArgFlagsFromAttributes(ISD::ArgFlagsTy &Flags, |
83 | const AttributeList &Attrs, |
84 | unsigned OpIdx) const { |
85 | addFlagsUsingAttrFn(Flags, AttrFn: [&Attrs, &OpIdx](Attribute::AttrKind Attr) { |
86 | return Attrs.hasAttributeAtIndex(Index: OpIdx, Kind: Attr); |
87 | }); |
88 | } |
89 | |
90 | bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB, |
91 | ArrayRef<Register> ResRegs, |
92 | ArrayRef<ArrayRef<Register>> ArgRegs, |
93 | Register SwiftErrorVReg, |
94 | std::function<unsigned()> GetCalleeReg) const { |
95 | CallLoweringInfo Info; |
96 | const DataLayout &DL = MIRBuilder.getDataLayout(); |
97 | MachineFunction &MF = MIRBuilder.getMF(); |
98 | MachineRegisterInfo &MRI = MF.getRegInfo(); |
99 | bool CanBeTailCalled = CB.isTailCall() && |
100 | isInTailCallPosition(Call: CB, TM: MF.getTarget()) && |
101 | (MF.getFunction() |
102 | .getFnAttribute(Kind: "disable-tail-calls" ) |
103 | .getValueAsString() != "true" ); |
104 | |
105 | CallingConv::ID CallConv = CB.getCallingConv(); |
106 | Type *RetTy = CB.getType(); |
107 | bool IsVarArg = CB.getFunctionType()->isVarArg(); |
108 | |
109 | SmallVector<BaseArgInfo, 4> SplitArgs; |
110 | getReturnInfo(CallConv, RetTy, Attrs: CB.getAttributes(), Outs&: SplitArgs, DL); |
111 | Info.CanLowerReturn = canLowerReturn(MF, CallConv, Outs&: SplitArgs, IsVarArg); |
112 | |
113 | Info.IsConvergent = CB.isConvergent(); |
114 | |
115 | if (!Info.CanLowerReturn) { |
116 | // Callee requires sret demotion. |
117 | insertSRetOutgoingArgument(MIRBuilder, CB, Info); |
118 | |
119 | // The sret demotion isn't compatible with tail-calls, since the sret |
120 | // argument points into the caller's stack frame. |
121 | CanBeTailCalled = false; |
122 | } |
123 | |
124 | |
125 | // First step is to marshall all the function's parameters into the correct |
126 | // physregs and memory locations. Gather the sequence of argument types that |
127 | // we'll pass to the assigner function. |
128 | unsigned i = 0; |
129 | unsigned NumFixedArgs = CB.getFunctionType()->getNumParams(); |
130 | for (const auto &Arg : CB.args()) { |
131 | ArgInfo OrigArg{ArgRegs[i], *Arg.get(), i, getAttributesForArgIdx(Call: CB, ArgIdx: i), |
132 | i < NumFixedArgs}; |
133 | setArgFlags(Arg&: OrigArg, OpIdx: i + AttributeList::FirstArgIndex, DL, FuncInfo: CB); |
134 | |
135 | // If we have an explicit sret argument that is an Instruction, (i.e., it |
136 | // might point to function-local memory), we can't meaningfully tail-call. |
137 | if (OrigArg.Flags[0].isSRet() && isa<Instruction>(Val: &Arg)) |
138 | CanBeTailCalled = false; |
139 | |
140 | Info.OrigArgs.push_back(Elt: OrigArg); |
141 | ++i; |
142 | } |
143 | |
144 | // Try looking through a bitcast from one function type to another. |
145 | // Commonly happens with calls to objc_msgSend(). |
146 | const Value *CalleeV = CB.getCalledOperand()->stripPointerCasts(); |
147 | if (const Function *F = dyn_cast<Function>(Val: CalleeV)) |
148 | Info.Callee = MachineOperand::CreateGA(GV: F, Offset: 0); |
149 | else if (isa<GlobalIFunc>(Val: CalleeV) || isa<GlobalAlias>(Val: CalleeV)) { |
150 | // IR IFuncs and Aliases can't be forward declared (only defined), so the |
151 | // callee must be in the same TU and therefore we can direct-call it without |
152 | // worrying about it being out of range. |
153 | Info.Callee = MachineOperand::CreateGA(GV: cast<GlobalValue>(Val: CalleeV), Offset: 0); |
154 | } else |
155 | Info.Callee = MachineOperand::CreateReg(Reg: GetCalleeReg(), isDef: false); |
156 | |
157 | Register ReturnHintAlignReg; |
158 | Align ReturnHintAlign; |
159 | |
160 | Info.OrigRet = ArgInfo{ResRegs, RetTy, 0, getAttributesForReturn(Call: CB)}; |
161 | |
162 | if (!Info.OrigRet.Ty->isVoidTy()) { |
163 | setArgFlags(Arg&: Info.OrigRet, OpIdx: AttributeList::ReturnIndex, DL, FuncInfo: CB); |
164 | |
165 | if (MaybeAlign Alignment = CB.getRetAlign()) { |
166 | if (*Alignment > Align(1)) { |
167 | ReturnHintAlignReg = MRI.cloneVirtualRegister(VReg: ResRegs[0]); |
168 | Info.OrigRet.Regs[0] = ReturnHintAlignReg; |
169 | ReturnHintAlign = *Alignment; |
170 | } |
171 | } |
172 | } |
173 | |
174 | auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_kcfi); |
175 | if (Bundle && CB.isIndirectCall()) { |
176 | Info.CFIType = cast<ConstantInt>(Val: Bundle->Inputs[0]); |
177 | assert(Info.CFIType->getType()->isIntegerTy(32) && "Invalid CFI type" ); |
178 | } |
179 | |
180 | Info.CB = &CB; |
181 | Info.KnownCallees = CB.getMetadata(KindID: LLVMContext::MD_callees); |
182 | Info.CallConv = CallConv; |
183 | Info.SwiftErrorVReg = SwiftErrorVReg; |
184 | Info.IsMustTailCall = CB.isMustTailCall(); |
185 | Info.IsTailCall = CanBeTailCalled; |
186 | Info.IsVarArg = IsVarArg; |
187 | if (!lowerCall(MIRBuilder, Info)) |
188 | return false; |
189 | |
190 | if (ReturnHintAlignReg && !Info.IsTailCall) { |
191 | MIRBuilder.buildAssertAlign(Res: ResRegs[0], Op: ReturnHintAlignReg, |
192 | AlignVal: ReturnHintAlign); |
193 | } |
194 | |
195 | return true; |
196 | } |
197 | |
198 | template <typename FuncInfoTy> |
199 | void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx, |
200 | const DataLayout &DL, |
201 | const FuncInfoTy &FuncInfo) const { |
202 | auto &Flags = Arg.Flags[0]; |
203 | const AttributeList &Attrs = FuncInfo.getAttributes(); |
204 | addArgFlagsFromAttributes(Flags, Attrs, OpIdx); |
205 | |
206 | PointerType *PtrTy = dyn_cast<PointerType>(Val: Arg.Ty->getScalarType()); |
207 | if (PtrTy) { |
208 | Flags.setPointer(); |
209 | Flags.setPointerAddrSpace(PtrTy->getPointerAddressSpace()); |
210 | } |
211 | |
212 | Align MemAlign = DL.getABITypeAlign(Ty: Arg.Ty); |
213 | if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated()) { |
214 | assert(OpIdx >= AttributeList::FirstArgIndex); |
215 | unsigned ParamIdx = OpIdx - AttributeList::FirstArgIndex; |
216 | |
217 | Type *ElementTy = FuncInfo.getParamByValType(ParamIdx); |
218 | if (!ElementTy) |
219 | ElementTy = FuncInfo.getParamInAllocaType(ParamIdx); |
220 | if (!ElementTy) |
221 | ElementTy = FuncInfo.getParamPreallocatedType(ParamIdx); |
222 | assert(ElementTy && "Must have byval, inalloca or preallocated type" ); |
223 | Flags.setByValSize(DL.getTypeAllocSize(Ty: ElementTy)); |
224 | |
225 | // For ByVal, alignment should be passed from FE. BE will guess if |
226 | // this info is not there but there are cases it cannot get right. |
227 | if (auto ParamAlign = FuncInfo.getParamStackAlign(ParamIdx)) |
228 | MemAlign = *ParamAlign; |
229 | else if ((ParamAlign = FuncInfo.getParamAlign(ParamIdx))) |
230 | MemAlign = *ParamAlign; |
231 | else |
232 | MemAlign = Align(getTLI()->getByValTypeAlignment(Ty: ElementTy, DL)); |
233 | } else if (OpIdx >= AttributeList::FirstArgIndex) { |
234 | if (auto ParamAlign = |
235 | FuncInfo.getParamStackAlign(OpIdx - AttributeList::FirstArgIndex)) |
236 | MemAlign = *ParamAlign; |
237 | } |
238 | Flags.setMemAlign(MemAlign); |
239 | Flags.setOrigAlign(DL.getABITypeAlign(Ty: Arg.Ty)); |
240 | |
241 | // Don't try to use the returned attribute if the argument is marked as |
242 | // swiftself, since it won't be passed in x0. |
243 | if (Flags.isSwiftSelf()) |
244 | Flags.setReturned(false); |
245 | } |
246 | |
247 | template void |
248 | CallLowering::setArgFlags<Function>(CallLowering::ArgInfo &Arg, unsigned OpIdx, |
249 | const DataLayout &DL, |
250 | const Function &FuncInfo) const; |
251 | |
252 | template void |
253 | CallLowering::setArgFlags<CallBase>(CallLowering::ArgInfo &Arg, unsigned OpIdx, |
254 | const DataLayout &DL, |
255 | const CallBase &FuncInfo) const; |
256 | |
257 | void CallLowering::splitToValueTypes(const ArgInfo &OrigArg, |
258 | SmallVectorImpl<ArgInfo> &SplitArgs, |
259 | const DataLayout &DL, |
260 | CallingConv::ID CallConv, |
261 | SmallVectorImpl<uint64_t> *Offsets) const { |
262 | LLVMContext &Ctx = OrigArg.Ty->getContext(); |
263 | |
264 | SmallVector<EVT, 4> SplitVTs; |
265 | ComputeValueVTs(TLI: *TLI, DL, Ty: OrigArg.Ty, ValueVTs&: SplitVTs, FixedOffsets: Offsets, StartingOffset: 0); |
266 | |
267 | if (SplitVTs.size() == 0) |
268 | return; |
269 | |
270 | if (SplitVTs.size() == 1) { |
271 | // No splitting to do, but we want to replace the original type (e.g. [1 x |
272 | // double] -> double). |
273 | SplitArgs.emplace_back(Args: OrigArg.Regs[0], Args: SplitVTs[0].getTypeForEVT(Context&: Ctx), |
274 | Args: OrigArg.OrigArgIndex, Args: OrigArg.Flags[0], |
275 | Args: OrigArg.IsFixed, Args: OrigArg.OrigValue); |
276 | return; |
277 | } |
278 | |
279 | // Create one ArgInfo for each virtual register in the original ArgInfo. |
280 | assert(OrigArg.Regs.size() == SplitVTs.size() && "Regs / types mismatch" ); |
281 | |
282 | bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters( |
283 | Ty: OrigArg.Ty, CallConv, isVarArg: false, DL); |
284 | for (unsigned i = 0, e = SplitVTs.size(); i < e; ++i) { |
285 | Type *SplitTy = SplitVTs[i].getTypeForEVT(Context&: Ctx); |
286 | SplitArgs.emplace_back(Args: OrigArg.Regs[i], Args&: SplitTy, Args: OrigArg.OrigArgIndex, |
287 | Args: OrigArg.Flags[0], Args: OrigArg.IsFixed); |
288 | if (NeedsRegBlock) |
289 | SplitArgs.back().Flags[0].setInConsecutiveRegs(); |
290 | } |
291 | |
292 | SplitArgs.back().Flags[0].setInConsecutiveRegsLast(); |
293 | } |
294 | |
295 | /// Pack values \p SrcRegs to cover the vector type result \p DstRegs. |
296 | static MachineInstrBuilder |
297 | mergeVectorRegsToResultRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs, |
298 | ArrayRef<Register> SrcRegs) { |
299 | MachineRegisterInfo &MRI = *B.getMRI(); |
300 | LLT LLTy = MRI.getType(Reg: DstRegs[0]); |
301 | LLT PartLLT = MRI.getType(Reg: SrcRegs[0]); |
302 | |
303 | // Deal with v3s16 split into v2s16 |
304 | LLT LCMTy = getCoverTy(OrigTy: LLTy, TargetTy: PartLLT); |
305 | if (LCMTy == LLTy) { |
306 | // Common case where no padding is needed. |
307 | assert(DstRegs.size() == 1); |
308 | return B.buildConcatVectors(Res: DstRegs[0], Ops: SrcRegs); |
309 | } |
310 | |
311 | // We need to create an unmerge to the result registers, which may require |
312 | // widening the original value. |
313 | Register UnmergeSrcReg; |
314 | if (LCMTy != PartLLT) { |
315 | assert(DstRegs.size() == 1); |
316 | return B.buildDeleteTrailingVectorElements( |
317 | Res: DstRegs[0], Op0: B.buildMergeLikeInstr(Res: LCMTy, Ops: SrcRegs)); |
318 | } else { |
319 | // We don't need to widen anything if we're extracting a scalar which was |
320 | // promoted to a vector e.g. s8 -> v4s8 -> s8 |
321 | assert(SrcRegs.size() == 1); |
322 | UnmergeSrcReg = SrcRegs[0]; |
323 | } |
324 | |
325 | int NumDst = LCMTy.getSizeInBits() / LLTy.getSizeInBits(); |
326 | |
327 | SmallVector<Register, 8> PadDstRegs(NumDst); |
328 | std::copy(first: DstRegs.begin(), last: DstRegs.end(), result: PadDstRegs.begin()); |
329 | |
330 | // Create the excess dead defs for the unmerge. |
331 | for (int I = DstRegs.size(); I != NumDst; ++I) |
332 | PadDstRegs[I] = MRI.createGenericVirtualRegister(Ty: LLTy); |
333 | |
334 | if (PadDstRegs.size() == 1) |
335 | return B.buildDeleteTrailingVectorElements(Res: DstRegs[0], Op0: UnmergeSrcReg); |
336 | return B.buildUnmerge(Res: PadDstRegs, Op: UnmergeSrcReg); |
337 | } |
338 | |
339 | /// Create a sequence of instructions to combine pieces split into register |
340 | /// typed values to the original IR value. \p OrigRegs contains the destination |
341 | /// value registers of type \p LLTy, and \p Regs contains the legalized pieces |
342 | /// with type \p PartLLT. This is used for incoming values (physregs to vregs). |
343 | static void buildCopyFromRegs(MachineIRBuilder &B, ArrayRef<Register> OrigRegs, |
344 | ArrayRef<Register> Regs, LLT LLTy, LLT PartLLT, |
345 | const ISD::ArgFlagsTy Flags) { |
346 | MachineRegisterInfo &MRI = *B.getMRI(); |
347 | |
348 | if (PartLLT == LLTy) { |
349 | // We should have avoided introducing a new virtual register, and just |
350 | // directly assigned here. |
351 | assert(OrigRegs[0] == Regs[0]); |
352 | return; |
353 | } |
354 | |
355 | if (PartLLT.getSizeInBits() == LLTy.getSizeInBits() && OrigRegs.size() == 1 && |
356 | Regs.size() == 1) { |
357 | B.buildBitcast(Dst: OrigRegs[0], Src: Regs[0]); |
358 | return; |
359 | } |
360 | |
361 | // A vector PartLLT needs extending to LLTy's element size. |
362 | // E.g. <2 x s64> = G_SEXT <2 x s32>. |
363 | if (PartLLT.isVector() == LLTy.isVector() && |
364 | PartLLT.getScalarSizeInBits() > LLTy.getScalarSizeInBits() && |
365 | (!PartLLT.isVector() || |
366 | PartLLT.getElementCount() == LLTy.getElementCount()) && |
367 | OrigRegs.size() == 1 && Regs.size() == 1) { |
368 | Register SrcReg = Regs[0]; |
369 | |
370 | LLT LocTy = MRI.getType(Reg: SrcReg); |
371 | |
372 | if (Flags.isSExt()) { |
373 | SrcReg = B.buildAssertSExt(Res: LocTy, Op: SrcReg, Size: LLTy.getScalarSizeInBits()) |
374 | .getReg(Idx: 0); |
375 | } else if (Flags.isZExt()) { |
376 | SrcReg = B.buildAssertZExt(Res: LocTy, Op: SrcReg, Size: LLTy.getScalarSizeInBits()) |
377 | .getReg(Idx: 0); |
378 | } |
379 | |
380 | // Sometimes pointers are passed zero extended. |
381 | LLT OrigTy = MRI.getType(Reg: OrigRegs[0]); |
382 | if (OrigTy.isPointer()) { |
383 | LLT IntPtrTy = LLT::scalar(SizeInBits: OrigTy.getSizeInBits()); |
384 | B.buildIntToPtr(Dst: OrigRegs[0], Src: B.buildTrunc(Res: IntPtrTy, Op: SrcReg)); |
385 | return; |
386 | } |
387 | |
388 | B.buildTrunc(Res: OrigRegs[0], Op: SrcReg); |
389 | return; |
390 | } |
391 | |
392 | if (!LLTy.isVector() && !PartLLT.isVector()) { |
393 | assert(OrigRegs.size() == 1); |
394 | LLT OrigTy = MRI.getType(Reg: OrigRegs[0]); |
395 | |
396 | unsigned SrcSize = PartLLT.getSizeInBits().getFixedValue() * Regs.size(); |
397 | if (SrcSize == OrigTy.getSizeInBits()) |
398 | B.buildMergeValues(Res: OrigRegs[0], Ops: Regs); |
399 | else { |
400 | auto Widened = B.buildMergeLikeInstr(Res: LLT::scalar(SizeInBits: SrcSize), Ops: Regs); |
401 | B.buildTrunc(Res: OrigRegs[0], Op: Widened); |
402 | } |
403 | |
404 | return; |
405 | } |
406 | |
407 | if (PartLLT.isVector()) { |
408 | assert(OrigRegs.size() == 1); |
409 | SmallVector<Register> CastRegs(Regs.begin(), Regs.end()); |
410 | |
411 | // If PartLLT is a mismatched vector in both number of elements and element |
412 | // size, e.g. PartLLT == v2s64 and LLTy is v3s32, then first coerce it to |
413 | // have the same elt type, i.e. v4s32. |
414 | // TODO: Extend this coersion to element multiples other than just 2. |
415 | if (TypeSize::isKnownGT(LHS: PartLLT.getSizeInBits(), RHS: LLTy.getSizeInBits()) && |
416 | PartLLT.getScalarSizeInBits() == LLTy.getScalarSizeInBits() * 2 && |
417 | Regs.size() == 1) { |
418 | LLT NewTy = PartLLT.changeElementType(NewEltTy: LLTy.getElementType()) |
419 | .changeElementCount(EC: PartLLT.getElementCount() * 2); |
420 | CastRegs[0] = B.buildBitcast(Dst: NewTy, Src: Regs[0]).getReg(Idx: 0); |
421 | PartLLT = NewTy; |
422 | } |
423 | |
424 | if (LLTy.getScalarType() == PartLLT.getElementType()) { |
425 | mergeVectorRegsToResultRegs(B, DstRegs: OrigRegs, SrcRegs: CastRegs); |
426 | } else { |
427 | unsigned I = 0; |
428 | LLT GCDTy = getGCDType(OrigTy: LLTy, TargetTy: PartLLT); |
429 | |
430 | // We are both splitting a vector, and bitcasting its element types. Cast |
431 | // the source pieces into the appropriate number of pieces with the result |
432 | // element type. |
433 | for (Register SrcReg : CastRegs) |
434 | CastRegs[I++] = B.buildBitcast(Dst: GCDTy, Src: SrcReg).getReg(Idx: 0); |
435 | mergeVectorRegsToResultRegs(B, DstRegs: OrigRegs, SrcRegs: CastRegs); |
436 | } |
437 | |
438 | return; |
439 | } |
440 | |
441 | assert(LLTy.isVector() && !PartLLT.isVector()); |
442 | |
443 | LLT DstEltTy = LLTy.getElementType(); |
444 | |
445 | // Pointer information was discarded. We'll need to coerce some register types |
446 | // to avoid violating type constraints. |
447 | LLT RealDstEltTy = MRI.getType(Reg: OrigRegs[0]).getElementType(); |
448 | |
449 | assert(DstEltTy.getSizeInBits() == RealDstEltTy.getSizeInBits()); |
450 | |
451 | if (DstEltTy == PartLLT) { |
452 | // Vector was trivially scalarized. |
453 | |
454 | if (RealDstEltTy.isPointer()) { |
455 | for (Register Reg : Regs) |
456 | MRI.setType(VReg: Reg, Ty: RealDstEltTy); |
457 | } |
458 | |
459 | B.buildBuildVector(Res: OrigRegs[0], Ops: Regs); |
460 | } else if (DstEltTy.getSizeInBits() > PartLLT.getSizeInBits()) { |
461 | // Deal with vector with 64-bit elements decomposed to 32-bit |
462 | // registers. Need to create intermediate 64-bit elements. |
463 | SmallVector<Register, 8> EltMerges; |
464 | int PartsPerElt = DstEltTy.getSizeInBits() / PartLLT.getSizeInBits(); |
465 | |
466 | assert(DstEltTy.getSizeInBits() % PartLLT.getSizeInBits() == 0); |
467 | |
468 | for (int I = 0, NumElts = LLTy.getNumElements(); I != NumElts; ++I) { |
469 | auto Merge = |
470 | B.buildMergeLikeInstr(Res: RealDstEltTy, Ops: Regs.take_front(N: PartsPerElt)); |
471 | // Fix the type in case this is really a vector of pointers. |
472 | MRI.setType(VReg: Merge.getReg(Idx: 0), Ty: RealDstEltTy); |
473 | EltMerges.push_back(Elt: Merge.getReg(Idx: 0)); |
474 | Regs = Regs.drop_front(N: PartsPerElt); |
475 | } |
476 | |
477 | B.buildBuildVector(Res: OrigRegs[0], Ops: EltMerges); |
478 | } else { |
479 | // Vector was split, and elements promoted to a wider type. |
480 | // FIXME: Should handle floating point promotions. |
481 | unsigned NumElts = LLTy.getNumElements(); |
482 | LLT BVType = LLT::fixed_vector(NumElements: NumElts, ScalarTy: PartLLT); |
483 | |
484 | Register BuildVec; |
485 | if (NumElts == Regs.size()) |
486 | BuildVec = B.buildBuildVector(Res: BVType, Ops: Regs).getReg(Idx: 0); |
487 | else { |
488 | // Vector elements are packed in the inputs. |
489 | // e.g. we have a <4 x s16> but 2 x s32 in regs. |
490 | assert(NumElts > Regs.size()); |
491 | LLT SrcEltTy = MRI.getType(Reg: Regs[0]); |
492 | |
493 | LLT OriginalEltTy = MRI.getType(Reg: OrigRegs[0]).getElementType(); |
494 | |
495 | // Input registers contain packed elements. |
496 | // Determine how many elements per reg. |
497 | assert((SrcEltTy.getSizeInBits() % OriginalEltTy.getSizeInBits()) == 0); |
498 | unsigned EltPerReg = |
499 | (SrcEltTy.getSizeInBits() / OriginalEltTy.getSizeInBits()); |
500 | |
501 | SmallVector<Register, 0> BVRegs; |
502 | BVRegs.reserve(N: Regs.size() * EltPerReg); |
503 | for (Register R : Regs) { |
504 | auto Unmerge = B.buildUnmerge(Res: OriginalEltTy, Op: R); |
505 | for (unsigned K = 0; K < EltPerReg; ++K) |
506 | BVRegs.push_back(Elt: B.buildAnyExt(Res: PartLLT, Op: Unmerge.getReg(Idx: K)).getReg(Idx: 0)); |
507 | } |
508 | |
509 | // We may have some more elements in BVRegs, e.g. if we have 2 s32 pieces |
510 | // for a <3 x s16> vector. We should have less than EltPerReg extra items. |
511 | if (BVRegs.size() > NumElts) { |
512 | assert((BVRegs.size() - NumElts) < EltPerReg); |
513 | BVRegs.truncate(N: NumElts); |
514 | } |
515 | BuildVec = B.buildBuildVector(Res: BVType, Ops: BVRegs).getReg(Idx: 0); |
516 | } |
517 | B.buildTrunc(Res: OrigRegs[0], Op: BuildVec); |
518 | } |
519 | } |
520 | |
521 | /// Create a sequence of instructions to expand the value in \p SrcReg (of type |
522 | /// \p SrcTy) to the types in \p DstRegs (of type \p PartTy). \p ExtendOp should |
523 | /// contain the type of scalar value extension if necessary. |
524 | /// |
525 | /// This is used for outgoing values (vregs to physregs) |
526 | static void buildCopyToRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs, |
527 | Register SrcReg, LLT SrcTy, LLT PartTy, |
528 | unsigned ExtendOp = TargetOpcode::G_ANYEXT) { |
529 | // We could just insert a regular copy, but this is unreachable at the moment. |
530 | assert(SrcTy != PartTy && "identical part types shouldn't reach here" ); |
531 | |
532 | const TypeSize PartSize = PartTy.getSizeInBits(); |
533 | |
534 | if (PartTy.isVector() == SrcTy.isVector() && |
535 | PartTy.getScalarSizeInBits() > SrcTy.getScalarSizeInBits()) { |
536 | assert(DstRegs.size() == 1); |
537 | B.buildInstr(Opc: ExtendOp, DstOps: {DstRegs[0]}, SrcOps: {SrcReg}); |
538 | return; |
539 | } |
540 | |
541 | if (SrcTy.isVector() && !PartTy.isVector() && |
542 | TypeSize::isKnownGT(LHS: PartSize, RHS: SrcTy.getElementType().getSizeInBits())) { |
543 | // Vector was scalarized, and the elements extended. |
544 | auto UnmergeToEltTy = B.buildUnmerge(Res: SrcTy.getElementType(), Op: SrcReg); |
545 | for (int i = 0, e = DstRegs.size(); i != e; ++i) |
546 | B.buildAnyExt(Res: DstRegs[i], Op: UnmergeToEltTy.getReg(Idx: i)); |
547 | return; |
548 | } |
549 | |
550 | if (SrcTy.isVector() && PartTy.isVector() && |
551 | PartTy.getSizeInBits() == SrcTy.getSizeInBits() && |
552 | ElementCount::isKnownLT(LHS: SrcTy.getElementCount(), |
553 | RHS: PartTy.getElementCount())) { |
554 | // A coercion like: v2f32 -> v4f32 or nxv2f32 -> nxv4f32 |
555 | Register DstReg = DstRegs.front(); |
556 | B.buildPadVectorWithUndefElements(Res: DstReg, Op0: SrcReg); |
557 | return; |
558 | } |
559 | |
560 | LLT GCDTy = getGCDType(OrigTy: SrcTy, TargetTy: PartTy); |
561 | if (GCDTy == PartTy) { |
562 | // If this already evenly divisible, we can create a simple unmerge. |
563 | B.buildUnmerge(Res: DstRegs, Op: SrcReg); |
564 | return; |
565 | } |
566 | |
567 | MachineRegisterInfo &MRI = *B.getMRI(); |
568 | LLT DstTy = MRI.getType(Reg: DstRegs[0]); |
569 | LLT LCMTy = getCoverTy(OrigTy: SrcTy, TargetTy: PartTy); |
570 | |
571 | if (PartTy.isVector() && LCMTy == PartTy) { |
572 | assert(DstRegs.size() == 1); |
573 | B.buildPadVectorWithUndefElements(Res: DstRegs[0], Op0: SrcReg); |
574 | return; |
575 | } |
576 | |
577 | const unsigned DstSize = DstTy.getSizeInBits(); |
578 | const unsigned SrcSize = SrcTy.getSizeInBits(); |
579 | unsigned CoveringSize = LCMTy.getSizeInBits(); |
580 | |
581 | Register UnmergeSrc = SrcReg; |
582 | |
583 | if (!LCMTy.isVector() && CoveringSize != SrcSize) { |
584 | // For scalars, it's common to be able to use a simple extension. |
585 | if (SrcTy.isScalar() && DstTy.isScalar()) { |
586 | CoveringSize = alignTo(Value: SrcSize, Align: DstSize); |
587 | LLT CoverTy = LLT::scalar(SizeInBits: CoveringSize); |
588 | UnmergeSrc = B.buildInstr(Opc: ExtendOp, DstOps: {CoverTy}, SrcOps: {SrcReg}).getReg(Idx: 0); |
589 | } else { |
590 | // Widen to the common type. |
591 | // FIXME: This should respect the extend type |
592 | Register Undef = B.buildUndef(Res: SrcTy).getReg(Idx: 0); |
593 | SmallVector<Register, 8> MergeParts(1, SrcReg); |
594 | for (unsigned Size = SrcSize; Size != CoveringSize; Size += SrcSize) |
595 | MergeParts.push_back(Elt: Undef); |
596 | UnmergeSrc = B.buildMergeLikeInstr(Res: LCMTy, Ops: MergeParts).getReg(Idx: 0); |
597 | } |
598 | } |
599 | |
600 | if (LCMTy.isVector() && CoveringSize != SrcSize) |
601 | UnmergeSrc = B.buildPadVectorWithUndefElements(Res: LCMTy, Op0: SrcReg).getReg(Idx: 0); |
602 | |
603 | B.buildUnmerge(Res: DstRegs, Op: UnmergeSrc); |
604 | } |
605 | |
606 | bool CallLowering::determineAndHandleAssignments( |
607 | ValueHandler &Handler, ValueAssigner &Assigner, |
608 | SmallVectorImpl<ArgInfo> &Args, MachineIRBuilder &MIRBuilder, |
609 | CallingConv::ID CallConv, bool IsVarArg, |
610 | ArrayRef<Register> ThisReturnRegs) const { |
611 | MachineFunction &MF = MIRBuilder.getMF(); |
612 | const Function &F = MF.getFunction(); |
613 | SmallVector<CCValAssign, 16> ArgLocs; |
614 | |
615 | CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, F.getContext()); |
616 | if (!determineAssignments(Assigner, Args, CCInfo)) |
617 | return false; |
618 | |
619 | return handleAssignments(Handler, Args, CCState&: CCInfo, ArgLocs, MIRBuilder, |
620 | ThisReturnRegs); |
621 | } |
622 | |
623 | static unsigned extendOpFromFlags(llvm::ISD::ArgFlagsTy Flags) { |
624 | if (Flags.isSExt()) |
625 | return TargetOpcode::G_SEXT; |
626 | if (Flags.isZExt()) |
627 | return TargetOpcode::G_ZEXT; |
628 | return TargetOpcode::G_ANYEXT; |
629 | } |
630 | |
631 | bool CallLowering::determineAssignments(ValueAssigner &Assigner, |
632 | SmallVectorImpl<ArgInfo> &Args, |
633 | CCState &CCInfo) const { |
634 | LLVMContext &Ctx = CCInfo.getContext(); |
635 | const CallingConv::ID CallConv = CCInfo.getCallingConv(); |
636 | |
637 | unsigned NumArgs = Args.size(); |
638 | for (unsigned i = 0; i != NumArgs; ++i) { |
639 | EVT CurVT = EVT::getEVT(Ty: Args[i].Ty); |
640 | |
641 | MVT NewVT = TLI->getRegisterTypeForCallingConv(Context&: Ctx, CC: CallConv, VT: CurVT); |
642 | |
643 | // If we need to split the type over multiple regs, check it's a scenario |
644 | // we currently support. |
645 | unsigned NumParts = |
646 | TLI->getNumRegistersForCallingConv(Context&: Ctx, CC: CallConv, VT: CurVT); |
647 | |
648 | if (NumParts == 1) { |
649 | // Try to use the register type if we couldn't assign the VT. |
650 | if (Assigner.assignArg(ValNo: i, OrigVT: CurVT, ValVT: NewVT, LocVT: NewVT, LocInfo: CCValAssign::Full, Info: Args[i], |
651 | Flags: Args[i].Flags[0], State&: CCInfo)) |
652 | return false; |
653 | continue; |
654 | } |
655 | |
656 | // For incoming arguments (physregs to vregs), we could have values in |
657 | // physregs (or memlocs) which we want to extract and copy to vregs. |
658 | // During this, we might have to deal with the LLT being split across |
659 | // multiple regs, so we have to record this information for later. |
660 | // |
661 | // If we have outgoing args, then we have the opposite case. We have a |
662 | // vreg with an LLT which we want to assign to a physical location, and |
663 | // we might have to record that the value has to be split later. |
664 | |
665 | // We're handling an incoming arg which is split over multiple regs. |
666 | // E.g. passing an s128 on AArch64. |
667 | ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0]; |
668 | Args[i].Flags.clear(); |
669 | |
670 | for (unsigned Part = 0; Part < NumParts; ++Part) { |
671 | ISD::ArgFlagsTy Flags = OrigFlags; |
672 | if (Part == 0) { |
673 | Flags.setSplit(); |
674 | } else { |
675 | Flags.setOrigAlign(Align(1)); |
676 | if (Part == NumParts - 1) |
677 | Flags.setSplitEnd(); |
678 | } |
679 | |
680 | Args[i].Flags.push_back(Elt: Flags); |
681 | if (Assigner.assignArg(ValNo: i, OrigVT: CurVT, ValVT: NewVT, LocVT: NewVT, LocInfo: CCValAssign::Full, Info: Args[i], |
682 | Flags: Args[i].Flags[Part], State&: CCInfo)) { |
683 | // Still couldn't assign this smaller part type for some reason. |
684 | return false; |
685 | } |
686 | } |
687 | } |
688 | |
689 | return true; |
690 | } |
691 | |
692 | bool CallLowering::handleAssignments(ValueHandler &Handler, |
693 | SmallVectorImpl<ArgInfo> &Args, |
694 | CCState &CCInfo, |
695 | SmallVectorImpl<CCValAssign> &ArgLocs, |
696 | MachineIRBuilder &MIRBuilder, |
697 | ArrayRef<Register> ThisReturnRegs) const { |
698 | MachineFunction &MF = MIRBuilder.getMF(); |
699 | MachineRegisterInfo &MRI = MF.getRegInfo(); |
700 | const Function &F = MF.getFunction(); |
701 | const DataLayout &DL = F.getParent()->getDataLayout(); |
702 | |
703 | const unsigned NumArgs = Args.size(); |
704 | |
705 | // Stores thunks for outgoing register assignments. This is used so we delay |
706 | // generating register copies until mem loc assignments are done. We do this |
707 | // so that if the target is using the delayed stack protector feature, we can |
708 | // find the split point of the block accurately. E.g. if we have: |
709 | // G_STORE %val, %memloc |
710 | // $x0 = COPY %foo |
711 | // $x1 = COPY %bar |
712 | // CALL func |
713 | // ... then the split point for the block will correctly be at, and including, |
714 | // the copy to $x0. If instead the G_STORE instruction immediately precedes |
715 | // the CALL, then we'd prematurely choose the CALL as the split point, thus |
716 | // generating a split block with a CALL that uses undefined physregs. |
717 | SmallVector<std::function<void()>> DelayedOutgoingRegAssignments; |
718 | |
719 | for (unsigned i = 0, j = 0; i != NumArgs; ++i, ++j) { |
720 | assert(j < ArgLocs.size() && "Skipped too many arg locs" ); |
721 | CCValAssign &VA = ArgLocs[j]; |
722 | assert(VA.getValNo() == i && "Location doesn't correspond to current arg" ); |
723 | |
724 | if (VA.needsCustom()) { |
725 | std::function<void()> Thunk; |
726 | unsigned NumArgRegs = Handler.assignCustomValue( |
727 | Arg&: Args[i], VAs: ArrayRef(ArgLocs).slice(N: j), Thunk: &Thunk); |
728 | if (Thunk) |
729 | DelayedOutgoingRegAssignments.emplace_back(Args&: Thunk); |
730 | if (!NumArgRegs) |
731 | return false; |
732 | j += (NumArgRegs - 1); |
733 | continue; |
734 | } |
735 | |
736 | const MVT ValVT = VA.getValVT(); |
737 | const MVT LocVT = VA.getLocVT(); |
738 | |
739 | const LLT LocTy(LocVT); |
740 | const LLT ValTy(ValVT); |
741 | const LLT NewLLT = Handler.isIncomingArgumentHandler() ? LocTy : ValTy; |
742 | const EVT OrigVT = EVT::getEVT(Ty: Args[i].Ty); |
743 | const LLT OrigTy = getLLTForType(Ty&: *Args[i].Ty, DL); |
744 | |
745 | // Expected to be multiple regs for a single incoming arg. |
746 | // There should be Regs.size() ArgLocs per argument. |
747 | // This should be the same as getNumRegistersForCallingConv |
748 | const unsigned NumParts = Args[i].Flags.size(); |
749 | |
750 | // Now split the registers into the assigned types. |
751 | Args[i].OrigRegs.assign(in_start: Args[i].Regs.begin(), in_end: Args[i].Regs.end()); |
752 | |
753 | if (NumParts != 1 || NewLLT != OrigTy) { |
754 | // If we can't directly assign the register, we need one or more |
755 | // intermediate values. |
756 | Args[i].Regs.resize(N: NumParts); |
757 | |
758 | // For each split register, create and assign a vreg that will store |
759 | // the incoming component of the larger value. These will later be |
760 | // merged to form the final vreg. |
761 | for (unsigned Part = 0; Part < NumParts; ++Part) |
762 | Args[i].Regs[Part] = MRI.createGenericVirtualRegister(Ty: NewLLT); |
763 | } |
764 | |
765 | assert((j + (NumParts - 1)) < ArgLocs.size() && |
766 | "Too many regs for number of args" ); |
767 | |
768 | // Coerce into outgoing value types before register assignment. |
769 | if (!Handler.isIncomingArgumentHandler() && OrigTy != ValTy) { |
770 | assert(Args[i].OrigRegs.size() == 1); |
771 | buildCopyToRegs(B&: MIRBuilder, DstRegs: Args[i].Regs, SrcReg: Args[i].OrigRegs[0], SrcTy: OrigTy, |
772 | PartTy: ValTy, ExtendOp: extendOpFromFlags(Flags: Args[i].Flags[0])); |
773 | } |
774 | |
775 | bool BigEndianPartOrdering = TLI->hasBigEndianPartOrdering(VT: OrigVT, DL); |
776 | for (unsigned Part = 0; Part < NumParts; ++Part) { |
777 | Register ArgReg = Args[i].Regs[Part]; |
778 | // There should be Regs.size() ArgLocs per argument. |
779 | unsigned Idx = BigEndianPartOrdering ? NumParts - 1 - Part : Part; |
780 | CCValAssign &VA = ArgLocs[j + Idx]; |
781 | const ISD::ArgFlagsTy Flags = Args[i].Flags[Part]; |
782 | |
783 | if (VA.isMemLoc() && !Flags.isByVal()) { |
784 | // Individual pieces may have been spilled to the stack and others |
785 | // passed in registers. |
786 | |
787 | // TODO: The memory size may be larger than the value we need to |
788 | // store. We may need to adjust the offset for big endian targets. |
789 | LLT MemTy = Handler.getStackValueStoreType(DL, VA, Flags); |
790 | |
791 | MachinePointerInfo MPO; |
792 | Register StackAddr = Handler.getStackAddress( |
793 | MemSize: MemTy.getSizeInBytes(), Offset: VA.getLocMemOffset(), MPO, Flags); |
794 | |
795 | Handler.assignValueToAddress(Arg: Args[i], ValRegIndex: Part, Addr: StackAddr, MemTy, MPO, VA); |
796 | continue; |
797 | } |
798 | |
799 | if (VA.isMemLoc() && Flags.isByVal()) { |
800 | assert(Args[i].Regs.size() == 1 && |
801 | "didn't expect split byval pointer" ); |
802 | |
803 | if (Handler.isIncomingArgumentHandler()) { |
804 | // We just need to copy the frame index value to the pointer. |
805 | MachinePointerInfo MPO; |
806 | Register StackAddr = Handler.getStackAddress( |
807 | MemSize: Flags.getByValSize(), Offset: VA.getLocMemOffset(), MPO, Flags); |
808 | MIRBuilder.buildCopy(Res: Args[i].Regs[0], Op: StackAddr); |
809 | } else { |
810 | // For outgoing byval arguments, insert the implicit copy byval |
811 | // implies, such that writes in the callee do not modify the caller's |
812 | // value. |
813 | uint64_t MemSize = Flags.getByValSize(); |
814 | int64_t Offset = VA.getLocMemOffset(); |
815 | |
816 | MachinePointerInfo DstMPO; |
817 | Register StackAddr = |
818 | Handler.getStackAddress(MemSize, Offset, MPO&: DstMPO, Flags); |
819 | |
820 | MachinePointerInfo SrcMPO(Args[i].OrigValue); |
821 | if (!Args[i].OrigValue) { |
822 | // We still need to accurately track the stack address space if we |
823 | // don't know the underlying value. |
824 | const LLT PtrTy = MRI.getType(Reg: StackAddr); |
825 | SrcMPO = MachinePointerInfo(PtrTy.getAddressSpace()); |
826 | } |
827 | |
828 | Align DstAlign = std::max(a: Flags.getNonZeroByValAlign(), |
829 | b: inferAlignFromPtrInfo(MF, MPO: DstMPO)); |
830 | |
831 | Align SrcAlign = std::max(a: Flags.getNonZeroByValAlign(), |
832 | b: inferAlignFromPtrInfo(MF, MPO: SrcMPO)); |
833 | |
834 | Handler.copyArgumentMemory(Arg: Args[i], DstPtr: StackAddr, SrcPtr: Args[i].Regs[0], |
835 | DstPtrInfo: DstMPO, DstAlign, SrcPtrInfo: SrcMPO, SrcAlign, |
836 | MemSize, VA); |
837 | } |
838 | continue; |
839 | } |
840 | |
841 | assert(!VA.needsCustom() && "custom loc should have been handled already" ); |
842 | |
843 | if (i == 0 && !ThisReturnRegs.empty() && |
844 | Handler.isIncomingArgumentHandler() && |
845 | isTypeIsValidForThisReturn(Ty: ValVT)) { |
846 | Handler.assignValueToReg(ValVReg: ArgReg, PhysReg: ThisReturnRegs[Part], VA); |
847 | continue; |
848 | } |
849 | |
850 | if (Handler.isIncomingArgumentHandler()) |
851 | Handler.assignValueToReg(ValVReg: ArgReg, PhysReg: VA.getLocReg(), VA); |
852 | else { |
853 | DelayedOutgoingRegAssignments.emplace_back(Args: [=, &Handler]() { |
854 | Handler.assignValueToReg(ValVReg: ArgReg, PhysReg: VA.getLocReg(), VA); |
855 | }); |
856 | } |
857 | } |
858 | |
859 | // Now that all pieces have been assigned, re-pack the register typed values |
860 | // into the original value typed registers. |
861 | if (Handler.isIncomingArgumentHandler() && OrigVT != LocVT) { |
862 | // Merge the split registers into the expected larger result vregs of |
863 | // the original call. |
864 | buildCopyFromRegs(B&: MIRBuilder, OrigRegs: Args[i].OrigRegs, Regs: Args[i].Regs, LLTy: OrigTy, |
865 | PartLLT: LocTy, Flags: Args[i].Flags[0]); |
866 | } |
867 | |
868 | j += NumParts - 1; |
869 | } |
870 | for (auto &Fn : DelayedOutgoingRegAssignments) |
871 | Fn(); |
872 | |
873 | return true; |
874 | } |
875 | |
876 | void CallLowering::insertSRetLoads(MachineIRBuilder &MIRBuilder, Type *RetTy, |
877 | ArrayRef<Register> VRegs, Register DemoteReg, |
878 | int FI) const { |
879 | MachineFunction &MF = MIRBuilder.getMF(); |
880 | MachineRegisterInfo &MRI = MF.getRegInfo(); |
881 | const DataLayout &DL = MF.getDataLayout(); |
882 | |
883 | SmallVector<EVT, 4> SplitVTs; |
884 | SmallVector<uint64_t, 4> Offsets; |
885 | ComputeValueVTs(TLI: *TLI, DL, Ty: RetTy, ValueVTs&: SplitVTs, FixedOffsets: &Offsets, StartingOffset: 0); |
886 | |
887 | assert(VRegs.size() == SplitVTs.size()); |
888 | |
889 | unsigned NumValues = SplitVTs.size(); |
890 | Align BaseAlign = DL.getPrefTypeAlign(Ty: RetTy); |
891 | Type *RetPtrTy = |
892 | PointerType::get(C&: RetTy->getContext(), AddressSpace: DL.getAllocaAddrSpace()); |
893 | LLT OffsetLLTy = getLLTForType(Ty&: *DL.getIndexType(PtrTy: RetPtrTy), DL); |
894 | |
895 | MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI); |
896 | |
897 | for (unsigned I = 0; I < NumValues; ++I) { |
898 | Register Addr; |
899 | MIRBuilder.materializePtrAdd(Res&: Addr, Op0: DemoteReg, ValueTy: OffsetLLTy, Value: Offsets[I]); |
900 | auto *MMO = MF.getMachineMemOperand(PtrInfo, f: MachineMemOperand::MOLoad, |
901 | MemTy: MRI.getType(Reg: VRegs[I]), |
902 | base_alignment: commonAlignment(A: BaseAlign, Offset: Offsets[I])); |
903 | MIRBuilder.buildLoad(Res: VRegs[I], Addr, MMO&: *MMO); |
904 | } |
905 | } |
906 | |
907 | void CallLowering::insertSRetStores(MachineIRBuilder &MIRBuilder, Type *RetTy, |
908 | ArrayRef<Register> VRegs, |
909 | Register DemoteReg) const { |
910 | MachineFunction &MF = MIRBuilder.getMF(); |
911 | MachineRegisterInfo &MRI = MF.getRegInfo(); |
912 | const DataLayout &DL = MF.getDataLayout(); |
913 | |
914 | SmallVector<EVT, 4> SplitVTs; |
915 | SmallVector<uint64_t, 4> Offsets; |
916 | ComputeValueVTs(TLI: *TLI, DL, Ty: RetTy, ValueVTs&: SplitVTs, FixedOffsets: &Offsets, StartingOffset: 0); |
917 | |
918 | assert(VRegs.size() == SplitVTs.size()); |
919 | |
920 | unsigned NumValues = SplitVTs.size(); |
921 | Align BaseAlign = DL.getPrefTypeAlign(Ty: RetTy); |
922 | unsigned AS = DL.getAllocaAddrSpace(); |
923 | LLT OffsetLLTy = getLLTForType(Ty&: *DL.getIndexType(PtrTy: RetTy->getPointerTo(AddrSpace: AS)), DL); |
924 | |
925 | MachinePointerInfo PtrInfo(AS); |
926 | |
927 | for (unsigned I = 0; I < NumValues; ++I) { |
928 | Register Addr; |
929 | MIRBuilder.materializePtrAdd(Res&: Addr, Op0: DemoteReg, ValueTy: OffsetLLTy, Value: Offsets[I]); |
930 | auto *MMO = MF.getMachineMemOperand(PtrInfo, f: MachineMemOperand::MOStore, |
931 | MemTy: MRI.getType(Reg: VRegs[I]), |
932 | base_alignment: commonAlignment(A: BaseAlign, Offset: Offsets[I])); |
933 | MIRBuilder.buildStore(Val: VRegs[I], Addr, MMO&: *MMO); |
934 | } |
935 | } |
936 | |
937 | void CallLowering::insertSRetIncomingArgument( |
938 | const Function &F, SmallVectorImpl<ArgInfo> &SplitArgs, Register &DemoteReg, |
939 | MachineRegisterInfo &MRI, const DataLayout &DL) const { |
940 | unsigned AS = DL.getAllocaAddrSpace(); |
941 | DemoteReg = MRI.createGenericVirtualRegister( |
942 | Ty: LLT::pointer(AddressSpace: AS, SizeInBits: DL.getPointerSizeInBits(AS))); |
943 | |
944 | Type *PtrTy = PointerType::get(ElementType: F.getReturnType(), AddressSpace: AS); |
945 | |
946 | SmallVector<EVT, 1> ValueVTs; |
947 | ComputeValueVTs(TLI: *TLI, DL, Ty: PtrTy, ValueVTs); |
948 | |
949 | // NOTE: Assume that a pointer won't get split into more than one VT. |
950 | assert(ValueVTs.size() == 1); |
951 | |
952 | ArgInfo DemoteArg(DemoteReg, ValueVTs[0].getTypeForEVT(Context&: PtrTy->getContext()), |
953 | ArgInfo::NoArgIndex); |
954 | setArgFlags(Arg&: DemoteArg, OpIdx: AttributeList::ReturnIndex, DL, FuncInfo: F); |
955 | DemoteArg.Flags[0].setSRet(); |
956 | SplitArgs.insert(I: SplitArgs.begin(), Elt: DemoteArg); |
957 | } |
958 | |
959 | void CallLowering::insertSRetOutgoingArgument(MachineIRBuilder &MIRBuilder, |
960 | const CallBase &CB, |
961 | CallLoweringInfo &Info) const { |
962 | const DataLayout &DL = MIRBuilder.getDataLayout(); |
963 | Type *RetTy = CB.getType(); |
964 | unsigned AS = DL.getAllocaAddrSpace(); |
965 | LLT FramePtrTy = LLT::pointer(AddressSpace: AS, SizeInBits: DL.getPointerSizeInBits(AS)); |
966 | |
967 | int FI = MIRBuilder.getMF().getFrameInfo().CreateStackObject( |
968 | Size: DL.getTypeAllocSize(Ty: RetTy), Alignment: DL.getPrefTypeAlign(Ty: RetTy), isSpillSlot: false); |
969 | |
970 | Register DemoteReg = MIRBuilder.buildFrameIndex(Res: FramePtrTy, Idx: FI).getReg(Idx: 0); |
971 | ArgInfo DemoteArg(DemoteReg, PointerType::get(ElementType: RetTy, AddressSpace: AS), |
972 | ArgInfo::NoArgIndex); |
973 | setArgFlags(Arg&: DemoteArg, OpIdx: AttributeList::ReturnIndex, DL, FuncInfo: CB); |
974 | DemoteArg.Flags[0].setSRet(); |
975 | |
976 | Info.OrigArgs.insert(I: Info.OrigArgs.begin(), Elt: DemoteArg); |
977 | Info.DemoteStackIndex = FI; |
978 | Info.DemoteRegister = DemoteReg; |
979 | } |
980 | |
981 | bool CallLowering::checkReturn(CCState &CCInfo, |
982 | SmallVectorImpl<BaseArgInfo> &Outs, |
983 | CCAssignFn *Fn) const { |
984 | for (unsigned I = 0, E = Outs.size(); I < E; ++I) { |
985 | MVT VT = MVT::getVT(Ty: Outs[I].Ty); |
986 | if (Fn(I, VT, VT, CCValAssign::Full, Outs[I].Flags[0], CCInfo)) |
987 | return false; |
988 | } |
989 | return true; |
990 | } |
991 | |
992 | void CallLowering::getReturnInfo(CallingConv::ID CallConv, Type *RetTy, |
993 | AttributeList Attrs, |
994 | SmallVectorImpl<BaseArgInfo> &Outs, |
995 | const DataLayout &DL) const { |
996 | LLVMContext &Context = RetTy->getContext(); |
997 | ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); |
998 | |
999 | SmallVector<EVT, 4> SplitVTs; |
1000 | ComputeValueVTs(TLI: *TLI, DL, Ty: RetTy, ValueVTs&: SplitVTs); |
1001 | addArgFlagsFromAttributes(Flags, Attrs, OpIdx: AttributeList::ReturnIndex); |
1002 | |
1003 | for (EVT VT : SplitVTs) { |
1004 | unsigned NumParts = |
1005 | TLI->getNumRegistersForCallingConv(Context, CC: CallConv, VT); |
1006 | MVT RegVT = TLI->getRegisterTypeForCallingConv(Context, CC: CallConv, VT); |
1007 | Type *PartTy = EVT(RegVT).getTypeForEVT(Context); |
1008 | |
1009 | for (unsigned I = 0; I < NumParts; ++I) { |
1010 | Outs.emplace_back(Args&: PartTy, Args&: Flags); |
1011 | } |
1012 | } |
1013 | } |
1014 | |
1015 | bool CallLowering::checkReturnTypeForCallConv(MachineFunction &MF) const { |
1016 | const auto &F = MF.getFunction(); |
1017 | Type *ReturnType = F.getReturnType(); |
1018 | CallingConv::ID CallConv = F.getCallingConv(); |
1019 | |
1020 | SmallVector<BaseArgInfo, 4> SplitArgs; |
1021 | getReturnInfo(CallConv, RetTy: ReturnType, Attrs: F.getAttributes(), Outs&: SplitArgs, |
1022 | DL: MF.getDataLayout()); |
1023 | return canLowerReturn(MF, CallConv, Outs&: SplitArgs, IsVarArg: F.isVarArg()); |
1024 | } |
1025 | |
1026 | bool CallLowering::parametersInCSRMatch( |
1027 | const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, |
1028 | const SmallVectorImpl<CCValAssign> &OutLocs, |
1029 | const SmallVectorImpl<ArgInfo> &OutArgs) const { |
1030 | for (unsigned i = 0; i < OutLocs.size(); ++i) { |
1031 | const auto &ArgLoc = OutLocs[i]; |
1032 | // If it's not a register, it's fine. |
1033 | if (!ArgLoc.isRegLoc()) |
1034 | continue; |
1035 | |
1036 | MCRegister PhysReg = ArgLoc.getLocReg(); |
1037 | |
1038 | // Only look at callee-saved registers. |
1039 | if (MachineOperand::clobbersPhysReg(RegMask: CallerPreservedMask, PhysReg)) |
1040 | continue; |
1041 | |
1042 | LLVM_DEBUG( |
1043 | dbgs() |
1044 | << "... Call has an argument passed in a callee-saved register.\n" ); |
1045 | |
1046 | // Check if it was copied from. |
1047 | const ArgInfo &OutInfo = OutArgs[i]; |
1048 | |
1049 | if (OutInfo.Regs.size() > 1) { |
1050 | LLVM_DEBUG( |
1051 | dbgs() << "... Cannot handle arguments in multiple registers.\n" ); |
1052 | return false; |
1053 | } |
1054 | |
1055 | // Check if we copy the register, walking through copies from virtual |
1056 | // registers. Note that getDefIgnoringCopies does not ignore copies from |
1057 | // physical registers. |
1058 | MachineInstr *RegDef = getDefIgnoringCopies(Reg: OutInfo.Regs[0], MRI); |
1059 | if (!RegDef || RegDef->getOpcode() != TargetOpcode::COPY) { |
1060 | LLVM_DEBUG( |
1061 | dbgs() |
1062 | << "... Parameter was not copied into a VReg, cannot tail call.\n" ); |
1063 | return false; |
1064 | } |
1065 | |
1066 | // Got a copy. Verify that it's the same as the register we want. |
1067 | Register CopyRHS = RegDef->getOperand(i: 1).getReg(); |
1068 | if (CopyRHS != PhysReg) { |
1069 | LLVM_DEBUG(dbgs() << "... Callee-saved register was not copied into " |
1070 | "VReg, cannot tail call.\n" ); |
1071 | return false; |
1072 | } |
1073 | } |
1074 | |
1075 | return true; |
1076 | } |
1077 | |
1078 | bool CallLowering::resultsCompatible(CallLoweringInfo &Info, |
1079 | MachineFunction &MF, |
1080 | SmallVectorImpl<ArgInfo> &InArgs, |
1081 | ValueAssigner &CalleeAssigner, |
1082 | ValueAssigner &CallerAssigner) const { |
1083 | const Function &F = MF.getFunction(); |
1084 | CallingConv::ID CalleeCC = Info.CallConv; |
1085 | CallingConv::ID CallerCC = F.getCallingConv(); |
1086 | |
1087 | if (CallerCC == CalleeCC) |
1088 | return true; |
1089 | |
1090 | SmallVector<CCValAssign, 16> ArgLocs1; |
1091 | CCState CCInfo1(CalleeCC, Info.IsVarArg, MF, ArgLocs1, F.getContext()); |
1092 | if (!determineAssignments(Assigner&: CalleeAssigner, Args&: InArgs, CCInfo&: CCInfo1)) |
1093 | return false; |
1094 | |
1095 | SmallVector<CCValAssign, 16> ArgLocs2; |
1096 | CCState CCInfo2(CallerCC, F.isVarArg(), MF, ArgLocs2, F.getContext()); |
1097 | if (!determineAssignments(Assigner&: CallerAssigner, Args&: InArgs, CCInfo&: CCInfo2)) |
1098 | return false; |
1099 | |
1100 | // We need the argument locations to match up exactly. If there's more in |
1101 | // one than the other, then we are done. |
1102 | if (ArgLocs1.size() != ArgLocs2.size()) |
1103 | return false; |
1104 | |
1105 | // Make sure that each location is passed in exactly the same way. |
1106 | for (unsigned i = 0, e = ArgLocs1.size(); i < e; ++i) { |
1107 | const CCValAssign &Loc1 = ArgLocs1[i]; |
1108 | const CCValAssign &Loc2 = ArgLocs2[i]; |
1109 | |
1110 | // We need both of them to be the same. So if one is a register and one |
1111 | // isn't, we're done. |
1112 | if (Loc1.isRegLoc() != Loc2.isRegLoc()) |
1113 | return false; |
1114 | |
1115 | if (Loc1.isRegLoc()) { |
1116 | // If they don't have the same register location, we're done. |
1117 | if (Loc1.getLocReg() != Loc2.getLocReg()) |
1118 | return false; |
1119 | |
1120 | // They matched, so we can move to the next ArgLoc. |
1121 | continue; |
1122 | } |
1123 | |
1124 | // Loc1 wasn't a RegLoc, so they both must be MemLocs. Check if they match. |
1125 | if (Loc1.getLocMemOffset() != Loc2.getLocMemOffset()) |
1126 | return false; |
1127 | } |
1128 | |
1129 | return true; |
1130 | } |
1131 | |
1132 | LLT CallLowering::ValueHandler::getStackValueStoreType( |
1133 | const DataLayout &DL, const CCValAssign &VA, ISD::ArgFlagsTy Flags) const { |
1134 | const MVT ValVT = VA.getValVT(); |
1135 | if (ValVT != MVT::iPTR) { |
1136 | LLT ValTy(ValVT); |
1137 | |
1138 | // We lost the pointeriness going through CCValAssign, so try to restore it |
1139 | // based on the flags. |
1140 | if (Flags.isPointer()) { |
1141 | LLT PtrTy = LLT::pointer(AddressSpace: Flags.getPointerAddrSpace(), |
1142 | SizeInBits: ValTy.getScalarSizeInBits()); |
1143 | if (ValVT.isVector()) |
1144 | return LLT::vector(EC: ValTy.getElementCount(), ScalarTy: PtrTy); |
1145 | return PtrTy; |
1146 | } |
1147 | |
1148 | return ValTy; |
1149 | } |
1150 | |
1151 | unsigned AddrSpace = Flags.getPointerAddrSpace(); |
1152 | return LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL.getPointerSize(AS: AddrSpace)); |
1153 | } |
1154 | |
1155 | void CallLowering::ValueHandler::copyArgumentMemory( |
1156 | const ArgInfo &Arg, Register DstPtr, Register SrcPtr, |
1157 | const MachinePointerInfo &DstPtrInfo, Align DstAlign, |
1158 | const MachinePointerInfo &SrcPtrInfo, Align SrcAlign, uint64_t MemSize, |
1159 | CCValAssign &VA) const { |
1160 | MachineFunction &MF = MIRBuilder.getMF(); |
1161 | MachineMemOperand *SrcMMO = MF.getMachineMemOperand( |
1162 | PtrInfo: SrcPtrInfo, |
1163 | f: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable, s: MemSize, |
1164 | base_alignment: SrcAlign); |
1165 | |
1166 | MachineMemOperand *DstMMO = MF.getMachineMemOperand( |
1167 | PtrInfo: DstPtrInfo, |
1168 | f: MachineMemOperand::MOStore | MachineMemOperand::MODereferenceable, |
1169 | s: MemSize, base_alignment: DstAlign); |
1170 | |
1171 | const LLT PtrTy = MRI.getType(Reg: DstPtr); |
1172 | const LLT SizeTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits()); |
1173 | |
1174 | auto SizeConst = MIRBuilder.buildConstant(Res: SizeTy, Val: MemSize); |
1175 | MIRBuilder.buildMemCpy(DstPtr, SrcPtr, Size: SizeConst, DstMMO&: *DstMMO, SrcMMO&: *SrcMMO); |
1176 | } |
1177 | |
1178 | Register CallLowering::ValueHandler::extendRegister(Register ValReg, |
1179 | const CCValAssign &VA, |
1180 | unsigned MaxSizeBits) { |
1181 | LLT LocTy{VA.getLocVT()}; |
1182 | LLT ValTy{VA.getValVT()}; |
1183 | |
1184 | if (LocTy.getSizeInBits() == ValTy.getSizeInBits()) |
1185 | return ValReg; |
1186 | |
1187 | if (LocTy.isScalar() && MaxSizeBits && MaxSizeBits < LocTy.getSizeInBits()) { |
1188 | if (MaxSizeBits <= ValTy.getSizeInBits()) |
1189 | return ValReg; |
1190 | LocTy = LLT::scalar(SizeInBits: MaxSizeBits); |
1191 | } |
1192 | |
1193 | const LLT ValRegTy = MRI.getType(Reg: ValReg); |
1194 | if (ValRegTy.isPointer()) { |
1195 | // The x32 ABI wants to zero extend 32-bit pointers to 64-bit registers, so |
1196 | // we have to cast to do the extension. |
1197 | LLT IntPtrTy = LLT::scalar(SizeInBits: ValRegTy.getSizeInBits()); |
1198 | ValReg = MIRBuilder.buildPtrToInt(Dst: IntPtrTy, Src: ValReg).getReg(Idx: 0); |
1199 | } |
1200 | |
1201 | switch (VA.getLocInfo()) { |
1202 | default: break; |
1203 | case CCValAssign::Full: |
1204 | case CCValAssign::BCvt: |
1205 | // FIXME: bitconverting between vector types may or may not be a |
1206 | // nop in big-endian situations. |
1207 | return ValReg; |
1208 | case CCValAssign::AExt: { |
1209 | auto MIB = MIRBuilder.buildAnyExt(Res: LocTy, Op: ValReg); |
1210 | return MIB.getReg(Idx: 0); |
1211 | } |
1212 | case CCValAssign::SExt: { |
1213 | Register NewReg = MRI.createGenericVirtualRegister(Ty: LocTy); |
1214 | MIRBuilder.buildSExt(Res: NewReg, Op: ValReg); |
1215 | return NewReg; |
1216 | } |
1217 | case CCValAssign::ZExt: { |
1218 | Register NewReg = MRI.createGenericVirtualRegister(Ty: LocTy); |
1219 | MIRBuilder.buildZExt(Res: NewReg, Op: ValReg); |
1220 | return NewReg; |
1221 | } |
1222 | } |
1223 | llvm_unreachable("unable to extend register" ); |
1224 | } |
1225 | |
1226 | void CallLowering::ValueAssigner::anchor() {} |
1227 | |
1228 | Register CallLowering::IncomingValueHandler::buildExtensionHint( |
1229 | const CCValAssign &VA, Register SrcReg, LLT NarrowTy) { |
1230 | switch (VA.getLocInfo()) { |
1231 | case CCValAssign::LocInfo::ZExt: { |
1232 | return MIRBuilder |
1233 | .buildAssertZExt(Res: MRI.cloneVirtualRegister(VReg: SrcReg), Op: SrcReg, |
1234 | Size: NarrowTy.getScalarSizeInBits()) |
1235 | .getReg(Idx: 0); |
1236 | } |
1237 | case CCValAssign::LocInfo::SExt: { |
1238 | return MIRBuilder |
1239 | .buildAssertSExt(Res: MRI.cloneVirtualRegister(VReg: SrcReg), Op: SrcReg, |
1240 | Size: NarrowTy.getScalarSizeInBits()) |
1241 | .getReg(Idx: 0); |
1242 | break; |
1243 | } |
1244 | default: |
1245 | return SrcReg; |
1246 | } |
1247 | } |
1248 | |
1249 | /// Check if we can use a basic COPY instruction between the two types. |
1250 | /// |
1251 | /// We're currently building on top of the infrastructure using MVT, which loses |
1252 | /// pointer information in the CCValAssign. We accept copies from physical |
1253 | /// registers that have been reported as integers if it's to an equivalent sized |
1254 | /// pointer LLT. |
1255 | static bool isCopyCompatibleType(LLT SrcTy, LLT DstTy) { |
1256 | if (SrcTy == DstTy) |
1257 | return true; |
1258 | |
1259 | if (SrcTy.getSizeInBits() != DstTy.getSizeInBits()) |
1260 | return false; |
1261 | |
1262 | SrcTy = SrcTy.getScalarType(); |
1263 | DstTy = DstTy.getScalarType(); |
1264 | |
1265 | return (SrcTy.isPointer() && DstTy.isScalar()) || |
1266 | (DstTy.isPointer() && SrcTy.isScalar()); |
1267 | } |
1268 | |
1269 | void CallLowering::IncomingValueHandler::assignValueToReg( |
1270 | Register ValVReg, Register PhysReg, const CCValAssign &VA) { |
1271 | const MVT LocVT = VA.getLocVT(); |
1272 | const LLT LocTy(LocVT); |
1273 | const LLT RegTy = MRI.getType(Reg: ValVReg); |
1274 | |
1275 | if (isCopyCompatibleType(SrcTy: RegTy, DstTy: LocTy)) { |
1276 | MIRBuilder.buildCopy(Res: ValVReg, Op: PhysReg); |
1277 | return; |
1278 | } |
1279 | |
1280 | auto Copy = MIRBuilder.buildCopy(Res: LocTy, Op: PhysReg); |
1281 | auto Hint = buildExtensionHint(VA, SrcReg: Copy.getReg(Idx: 0), NarrowTy: RegTy); |
1282 | MIRBuilder.buildTrunc(Res: ValVReg, Op: Hint); |
1283 | } |
1284 | |