1//===-- lib/CodeGen/GlobalISel/CallLowering.cpp - Call lowering -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements some simple delegations needed for call lowering.
11///
12//===----------------------------------------------------------------------===//
13
14#include "llvm/CodeGen/GlobalISel/CallLowering.h"
15#include "llvm/CodeGen/Analysis.h"
16#include "llvm/CodeGen/CallingConvLower.h"
17#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
18#include "llvm/CodeGen/GlobalISel/Utils.h"
19#include "llvm/CodeGen/MachineFrameInfo.h"
20#include "llvm/CodeGen/MachineOperand.h"
21#include "llvm/CodeGen/MachineRegisterInfo.h"
22#include "llvm/CodeGen/TargetLowering.h"
23#include "llvm/IR/DataLayout.h"
24#include "llvm/IR/LLVMContext.h"
25#include "llvm/IR/Module.h"
26#include "llvm/Target/TargetMachine.h"
27
28#define DEBUG_TYPE "call-lowering"
29
30using namespace llvm;
31
32void CallLowering::anchor() {}
33
34/// Helper function which updates \p Flags when \p AttrFn returns true.
35static void
36addFlagsUsingAttrFn(ISD::ArgFlagsTy &Flags,
37 const std::function<bool(Attribute::AttrKind)> &AttrFn) {
38 if (AttrFn(Attribute::SExt))
39 Flags.setSExt();
40 if (AttrFn(Attribute::ZExt))
41 Flags.setZExt();
42 if (AttrFn(Attribute::InReg))
43 Flags.setInReg();
44 if (AttrFn(Attribute::StructRet))
45 Flags.setSRet();
46 if (AttrFn(Attribute::Nest))
47 Flags.setNest();
48 if (AttrFn(Attribute::ByVal))
49 Flags.setByVal();
50 if (AttrFn(Attribute::Preallocated))
51 Flags.setPreallocated();
52 if (AttrFn(Attribute::InAlloca))
53 Flags.setInAlloca();
54 if (AttrFn(Attribute::Returned))
55 Flags.setReturned();
56 if (AttrFn(Attribute::SwiftSelf))
57 Flags.setSwiftSelf();
58 if (AttrFn(Attribute::SwiftAsync))
59 Flags.setSwiftAsync();
60 if (AttrFn(Attribute::SwiftError))
61 Flags.setSwiftError();
62}
63
64ISD::ArgFlagsTy CallLowering::getAttributesForArgIdx(const CallBase &Call,
65 unsigned ArgIdx) const {
66 ISD::ArgFlagsTy Flags;
67 addFlagsUsingAttrFn(Flags, AttrFn: [&Call, &ArgIdx](Attribute::AttrKind Attr) {
68 return Call.paramHasAttr(ArgNo: ArgIdx, Kind: Attr);
69 });
70 return Flags;
71}
72
73ISD::ArgFlagsTy
74CallLowering::getAttributesForReturn(const CallBase &Call) const {
75 ISD::ArgFlagsTy Flags;
76 addFlagsUsingAttrFn(Flags, AttrFn: [&Call](Attribute::AttrKind Attr) {
77 return Call.hasRetAttr(Kind: Attr);
78 });
79 return Flags;
80}
81
82void CallLowering::addArgFlagsFromAttributes(ISD::ArgFlagsTy &Flags,
83 const AttributeList &Attrs,
84 unsigned OpIdx) const {
85 addFlagsUsingAttrFn(Flags, AttrFn: [&Attrs, &OpIdx](Attribute::AttrKind Attr) {
86 return Attrs.hasAttributeAtIndex(Index: OpIdx, Kind: Attr);
87 });
88}
89
90bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB,
91 ArrayRef<Register> ResRegs,
92 ArrayRef<ArrayRef<Register>> ArgRegs,
93 Register SwiftErrorVReg,
94 std::function<unsigned()> GetCalleeReg) const {
95 CallLoweringInfo Info;
96 const DataLayout &DL = MIRBuilder.getDataLayout();
97 MachineFunction &MF = MIRBuilder.getMF();
98 MachineRegisterInfo &MRI = MF.getRegInfo();
99 bool CanBeTailCalled = CB.isTailCall() &&
100 isInTailCallPosition(Call: CB, TM: MF.getTarget()) &&
101 (MF.getFunction()
102 .getFnAttribute(Kind: "disable-tail-calls")
103 .getValueAsString() != "true");
104
105 CallingConv::ID CallConv = CB.getCallingConv();
106 Type *RetTy = CB.getType();
107 bool IsVarArg = CB.getFunctionType()->isVarArg();
108
109 SmallVector<BaseArgInfo, 4> SplitArgs;
110 getReturnInfo(CallConv, RetTy, Attrs: CB.getAttributes(), Outs&: SplitArgs, DL);
111 Info.CanLowerReturn = canLowerReturn(MF, CallConv, Outs&: SplitArgs, IsVarArg);
112
113 Info.IsConvergent = CB.isConvergent();
114
115 if (!Info.CanLowerReturn) {
116 // Callee requires sret demotion.
117 insertSRetOutgoingArgument(MIRBuilder, CB, Info);
118
119 // The sret demotion isn't compatible with tail-calls, since the sret
120 // argument points into the caller's stack frame.
121 CanBeTailCalled = false;
122 }
123
124
125 // First step is to marshall all the function's parameters into the correct
126 // physregs and memory locations. Gather the sequence of argument types that
127 // we'll pass to the assigner function.
128 unsigned i = 0;
129 unsigned NumFixedArgs = CB.getFunctionType()->getNumParams();
130 for (const auto &Arg : CB.args()) {
131 ArgInfo OrigArg{ArgRegs[i], *Arg.get(), i, getAttributesForArgIdx(Call: CB, ArgIdx: i),
132 i < NumFixedArgs};
133 setArgFlags(Arg&: OrigArg, OpIdx: i + AttributeList::FirstArgIndex, DL, FuncInfo: CB);
134
135 // If we have an explicit sret argument that is an Instruction, (i.e., it
136 // might point to function-local memory), we can't meaningfully tail-call.
137 if (OrigArg.Flags[0].isSRet() && isa<Instruction>(Val: &Arg))
138 CanBeTailCalled = false;
139
140 Info.OrigArgs.push_back(Elt: OrigArg);
141 ++i;
142 }
143
144 // Try looking through a bitcast from one function type to another.
145 // Commonly happens with calls to objc_msgSend().
146 const Value *CalleeV = CB.getCalledOperand()->stripPointerCasts();
147 if (const Function *F = dyn_cast<Function>(Val: CalleeV))
148 Info.Callee = MachineOperand::CreateGA(GV: F, Offset: 0);
149 else if (isa<GlobalIFunc>(Val: CalleeV) || isa<GlobalAlias>(Val: CalleeV)) {
150 // IR IFuncs and Aliases can't be forward declared (only defined), so the
151 // callee must be in the same TU and therefore we can direct-call it without
152 // worrying about it being out of range.
153 Info.Callee = MachineOperand::CreateGA(GV: cast<GlobalValue>(Val: CalleeV), Offset: 0);
154 } else
155 Info.Callee = MachineOperand::CreateReg(Reg: GetCalleeReg(), isDef: false);
156
157 Register ReturnHintAlignReg;
158 Align ReturnHintAlign;
159
160 Info.OrigRet = ArgInfo{ResRegs, RetTy, 0, getAttributesForReturn(Call: CB)};
161
162 if (!Info.OrigRet.Ty->isVoidTy()) {
163 setArgFlags(Arg&: Info.OrigRet, OpIdx: AttributeList::ReturnIndex, DL, FuncInfo: CB);
164
165 if (MaybeAlign Alignment = CB.getRetAlign()) {
166 if (*Alignment > Align(1)) {
167 ReturnHintAlignReg = MRI.cloneVirtualRegister(VReg: ResRegs[0]);
168 Info.OrigRet.Regs[0] = ReturnHintAlignReg;
169 ReturnHintAlign = *Alignment;
170 }
171 }
172 }
173
174 auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_kcfi);
175 if (Bundle && CB.isIndirectCall()) {
176 Info.CFIType = cast<ConstantInt>(Val: Bundle->Inputs[0]);
177 assert(Info.CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");
178 }
179
180 Info.CB = &CB;
181 Info.KnownCallees = CB.getMetadata(KindID: LLVMContext::MD_callees);
182 Info.CallConv = CallConv;
183 Info.SwiftErrorVReg = SwiftErrorVReg;
184 Info.IsMustTailCall = CB.isMustTailCall();
185 Info.IsTailCall = CanBeTailCalled;
186 Info.IsVarArg = IsVarArg;
187 if (!lowerCall(MIRBuilder, Info))
188 return false;
189
190 if (ReturnHintAlignReg && !Info.IsTailCall) {
191 MIRBuilder.buildAssertAlign(Res: ResRegs[0], Op: ReturnHintAlignReg,
192 AlignVal: ReturnHintAlign);
193 }
194
195 return true;
196}
197
198template <typename FuncInfoTy>
199void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx,
200 const DataLayout &DL,
201 const FuncInfoTy &FuncInfo) const {
202 auto &Flags = Arg.Flags[0];
203 const AttributeList &Attrs = FuncInfo.getAttributes();
204 addArgFlagsFromAttributes(Flags, Attrs, OpIdx);
205
206 PointerType *PtrTy = dyn_cast<PointerType>(Val: Arg.Ty->getScalarType());
207 if (PtrTy) {
208 Flags.setPointer();
209 Flags.setPointerAddrSpace(PtrTy->getPointerAddressSpace());
210 }
211
212 Align MemAlign = DL.getABITypeAlign(Ty: Arg.Ty);
213 if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated()) {
214 assert(OpIdx >= AttributeList::FirstArgIndex);
215 unsigned ParamIdx = OpIdx - AttributeList::FirstArgIndex;
216
217 Type *ElementTy = FuncInfo.getParamByValType(ParamIdx);
218 if (!ElementTy)
219 ElementTy = FuncInfo.getParamInAllocaType(ParamIdx);
220 if (!ElementTy)
221 ElementTy = FuncInfo.getParamPreallocatedType(ParamIdx);
222 assert(ElementTy && "Must have byval, inalloca or preallocated type");
223 Flags.setByValSize(DL.getTypeAllocSize(Ty: ElementTy));
224
225 // For ByVal, alignment should be passed from FE. BE will guess if
226 // this info is not there but there are cases it cannot get right.
227 if (auto ParamAlign = FuncInfo.getParamStackAlign(ParamIdx))
228 MemAlign = *ParamAlign;
229 else if ((ParamAlign = FuncInfo.getParamAlign(ParamIdx)))
230 MemAlign = *ParamAlign;
231 else
232 MemAlign = Align(getTLI()->getByValTypeAlignment(Ty: ElementTy, DL));
233 } else if (OpIdx >= AttributeList::FirstArgIndex) {
234 if (auto ParamAlign =
235 FuncInfo.getParamStackAlign(OpIdx - AttributeList::FirstArgIndex))
236 MemAlign = *ParamAlign;
237 }
238 Flags.setMemAlign(MemAlign);
239 Flags.setOrigAlign(DL.getABITypeAlign(Ty: Arg.Ty));
240
241 // Don't try to use the returned attribute if the argument is marked as
242 // swiftself, since it won't be passed in x0.
243 if (Flags.isSwiftSelf())
244 Flags.setReturned(false);
245}
246
247template void
248CallLowering::setArgFlags<Function>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
249 const DataLayout &DL,
250 const Function &FuncInfo) const;
251
252template void
253CallLowering::setArgFlags<CallBase>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
254 const DataLayout &DL,
255 const CallBase &FuncInfo) const;
256
257void CallLowering::splitToValueTypes(const ArgInfo &OrigArg,
258 SmallVectorImpl<ArgInfo> &SplitArgs,
259 const DataLayout &DL,
260 CallingConv::ID CallConv,
261 SmallVectorImpl<uint64_t> *Offsets) const {
262 LLVMContext &Ctx = OrigArg.Ty->getContext();
263
264 SmallVector<EVT, 4> SplitVTs;
265 ComputeValueVTs(TLI: *TLI, DL, Ty: OrigArg.Ty, ValueVTs&: SplitVTs, FixedOffsets: Offsets, StartingOffset: 0);
266
267 if (SplitVTs.size() == 0)
268 return;
269
270 if (SplitVTs.size() == 1) {
271 // No splitting to do, but we want to replace the original type (e.g. [1 x
272 // double] -> double).
273 SplitArgs.emplace_back(Args: OrigArg.Regs[0], Args: SplitVTs[0].getTypeForEVT(Context&: Ctx),
274 Args: OrigArg.OrigArgIndex, Args: OrigArg.Flags[0],
275 Args: OrigArg.IsFixed, Args: OrigArg.OrigValue);
276 return;
277 }
278
279 // Create one ArgInfo for each virtual register in the original ArgInfo.
280 assert(OrigArg.Regs.size() == SplitVTs.size() && "Regs / types mismatch");
281
282 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
283 Ty: OrigArg.Ty, CallConv, isVarArg: false, DL);
284 for (unsigned i = 0, e = SplitVTs.size(); i < e; ++i) {
285 Type *SplitTy = SplitVTs[i].getTypeForEVT(Context&: Ctx);
286 SplitArgs.emplace_back(Args: OrigArg.Regs[i], Args&: SplitTy, Args: OrigArg.OrigArgIndex,
287 Args: OrigArg.Flags[0], Args: OrigArg.IsFixed);
288 if (NeedsRegBlock)
289 SplitArgs.back().Flags[0].setInConsecutiveRegs();
290 }
291
292 SplitArgs.back().Flags[0].setInConsecutiveRegsLast();
293}
294
295/// Pack values \p SrcRegs to cover the vector type result \p DstRegs.
296static MachineInstrBuilder
297mergeVectorRegsToResultRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
298 ArrayRef<Register> SrcRegs) {
299 MachineRegisterInfo &MRI = *B.getMRI();
300 LLT LLTy = MRI.getType(Reg: DstRegs[0]);
301 LLT PartLLT = MRI.getType(Reg: SrcRegs[0]);
302
303 // Deal with v3s16 split into v2s16
304 LLT LCMTy = getCoverTy(OrigTy: LLTy, TargetTy: PartLLT);
305 if (LCMTy == LLTy) {
306 // Common case where no padding is needed.
307 assert(DstRegs.size() == 1);
308 return B.buildConcatVectors(Res: DstRegs[0], Ops: SrcRegs);
309 }
310
311 // We need to create an unmerge to the result registers, which may require
312 // widening the original value.
313 Register UnmergeSrcReg;
314 if (LCMTy != PartLLT) {
315 assert(DstRegs.size() == 1);
316 return B.buildDeleteTrailingVectorElements(
317 Res: DstRegs[0], Op0: B.buildMergeLikeInstr(Res: LCMTy, Ops: SrcRegs));
318 } else {
319 // We don't need to widen anything if we're extracting a scalar which was
320 // promoted to a vector e.g. s8 -> v4s8 -> s8
321 assert(SrcRegs.size() == 1);
322 UnmergeSrcReg = SrcRegs[0];
323 }
324
325 int NumDst = LCMTy.getSizeInBits() / LLTy.getSizeInBits();
326
327 SmallVector<Register, 8> PadDstRegs(NumDst);
328 std::copy(first: DstRegs.begin(), last: DstRegs.end(), result: PadDstRegs.begin());
329
330 // Create the excess dead defs for the unmerge.
331 for (int I = DstRegs.size(); I != NumDst; ++I)
332 PadDstRegs[I] = MRI.createGenericVirtualRegister(Ty: LLTy);
333
334 if (PadDstRegs.size() == 1)
335 return B.buildDeleteTrailingVectorElements(Res: DstRegs[0], Op0: UnmergeSrcReg);
336 return B.buildUnmerge(Res: PadDstRegs, Op: UnmergeSrcReg);
337}
338
339/// Create a sequence of instructions to combine pieces split into register
340/// typed values to the original IR value. \p OrigRegs contains the destination
341/// value registers of type \p LLTy, and \p Regs contains the legalized pieces
342/// with type \p PartLLT. This is used for incoming values (physregs to vregs).
343static void buildCopyFromRegs(MachineIRBuilder &B, ArrayRef<Register> OrigRegs,
344 ArrayRef<Register> Regs, LLT LLTy, LLT PartLLT,
345 const ISD::ArgFlagsTy Flags) {
346 MachineRegisterInfo &MRI = *B.getMRI();
347
348 if (PartLLT == LLTy) {
349 // We should have avoided introducing a new virtual register, and just
350 // directly assigned here.
351 assert(OrigRegs[0] == Regs[0]);
352 return;
353 }
354
355 if (PartLLT.getSizeInBits() == LLTy.getSizeInBits() && OrigRegs.size() == 1 &&
356 Regs.size() == 1) {
357 B.buildBitcast(Dst: OrigRegs[0], Src: Regs[0]);
358 return;
359 }
360
361 // A vector PartLLT needs extending to LLTy's element size.
362 // E.g. <2 x s64> = G_SEXT <2 x s32>.
363 if (PartLLT.isVector() == LLTy.isVector() &&
364 PartLLT.getScalarSizeInBits() > LLTy.getScalarSizeInBits() &&
365 (!PartLLT.isVector() ||
366 PartLLT.getElementCount() == LLTy.getElementCount()) &&
367 OrigRegs.size() == 1 && Regs.size() == 1) {
368 Register SrcReg = Regs[0];
369
370 LLT LocTy = MRI.getType(Reg: SrcReg);
371
372 if (Flags.isSExt()) {
373 SrcReg = B.buildAssertSExt(Res: LocTy, Op: SrcReg, Size: LLTy.getScalarSizeInBits())
374 .getReg(Idx: 0);
375 } else if (Flags.isZExt()) {
376 SrcReg = B.buildAssertZExt(Res: LocTy, Op: SrcReg, Size: LLTy.getScalarSizeInBits())
377 .getReg(Idx: 0);
378 }
379
380 // Sometimes pointers are passed zero extended.
381 LLT OrigTy = MRI.getType(Reg: OrigRegs[0]);
382 if (OrigTy.isPointer()) {
383 LLT IntPtrTy = LLT::scalar(SizeInBits: OrigTy.getSizeInBits());
384 B.buildIntToPtr(Dst: OrigRegs[0], Src: B.buildTrunc(Res: IntPtrTy, Op: SrcReg));
385 return;
386 }
387
388 B.buildTrunc(Res: OrigRegs[0], Op: SrcReg);
389 return;
390 }
391
392 if (!LLTy.isVector() && !PartLLT.isVector()) {
393 assert(OrigRegs.size() == 1);
394 LLT OrigTy = MRI.getType(Reg: OrigRegs[0]);
395
396 unsigned SrcSize = PartLLT.getSizeInBits().getFixedValue() * Regs.size();
397 if (SrcSize == OrigTy.getSizeInBits())
398 B.buildMergeValues(Res: OrigRegs[0], Ops: Regs);
399 else {
400 auto Widened = B.buildMergeLikeInstr(Res: LLT::scalar(SizeInBits: SrcSize), Ops: Regs);
401 B.buildTrunc(Res: OrigRegs[0], Op: Widened);
402 }
403
404 return;
405 }
406
407 if (PartLLT.isVector()) {
408 assert(OrigRegs.size() == 1);
409 SmallVector<Register> CastRegs(Regs.begin(), Regs.end());
410
411 // If PartLLT is a mismatched vector in both number of elements and element
412 // size, e.g. PartLLT == v2s64 and LLTy is v3s32, then first coerce it to
413 // have the same elt type, i.e. v4s32.
414 // TODO: Extend this coersion to element multiples other than just 2.
415 if (TypeSize::isKnownGT(LHS: PartLLT.getSizeInBits(), RHS: LLTy.getSizeInBits()) &&
416 PartLLT.getScalarSizeInBits() == LLTy.getScalarSizeInBits() * 2 &&
417 Regs.size() == 1) {
418 LLT NewTy = PartLLT.changeElementType(NewEltTy: LLTy.getElementType())
419 .changeElementCount(EC: PartLLT.getElementCount() * 2);
420 CastRegs[0] = B.buildBitcast(Dst: NewTy, Src: Regs[0]).getReg(Idx: 0);
421 PartLLT = NewTy;
422 }
423
424 if (LLTy.getScalarType() == PartLLT.getElementType()) {
425 mergeVectorRegsToResultRegs(B, DstRegs: OrigRegs, SrcRegs: CastRegs);
426 } else {
427 unsigned I = 0;
428 LLT GCDTy = getGCDType(OrigTy: LLTy, TargetTy: PartLLT);
429
430 // We are both splitting a vector, and bitcasting its element types. Cast
431 // the source pieces into the appropriate number of pieces with the result
432 // element type.
433 for (Register SrcReg : CastRegs)
434 CastRegs[I++] = B.buildBitcast(Dst: GCDTy, Src: SrcReg).getReg(Idx: 0);
435 mergeVectorRegsToResultRegs(B, DstRegs: OrigRegs, SrcRegs: CastRegs);
436 }
437
438 return;
439 }
440
441 assert(LLTy.isVector() && !PartLLT.isVector());
442
443 LLT DstEltTy = LLTy.getElementType();
444
445 // Pointer information was discarded. We'll need to coerce some register types
446 // to avoid violating type constraints.
447 LLT RealDstEltTy = MRI.getType(Reg: OrigRegs[0]).getElementType();
448
449 assert(DstEltTy.getSizeInBits() == RealDstEltTy.getSizeInBits());
450
451 if (DstEltTy == PartLLT) {
452 // Vector was trivially scalarized.
453
454 if (RealDstEltTy.isPointer()) {
455 for (Register Reg : Regs)
456 MRI.setType(VReg: Reg, Ty: RealDstEltTy);
457 }
458
459 B.buildBuildVector(Res: OrigRegs[0], Ops: Regs);
460 } else if (DstEltTy.getSizeInBits() > PartLLT.getSizeInBits()) {
461 // Deal with vector with 64-bit elements decomposed to 32-bit
462 // registers. Need to create intermediate 64-bit elements.
463 SmallVector<Register, 8> EltMerges;
464 int PartsPerElt = DstEltTy.getSizeInBits() / PartLLT.getSizeInBits();
465
466 assert(DstEltTy.getSizeInBits() % PartLLT.getSizeInBits() == 0);
467
468 for (int I = 0, NumElts = LLTy.getNumElements(); I != NumElts; ++I) {
469 auto Merge =
470 B.buildMergeLikeInstr(Res: RealDstEltTy, Ops: Regs.take_front(N: PartsPerElt));
471 // Fix the type in case this is really a vector of pointers.
472 MRI.setType(VReg: Merge.getReg(Idx: 0), Ty: RealDstEltTy);
473 EltMerges.push_back(Elt: Merge.getReg(Idx: 0));
474 Regs = Regs.drop_front(N: PartsPerElt);
475 }
476
477 B.buildBuildVector(Res: OrigRegs[0], Ops: EltMerges);
478 } else {
479 // Vector was split, and elements promoted to a wider type.
480 // FIXME: Should handle floating point promotions.
481 unsigned NumElts = LLTy.getNumElements();
482 LLT BVType = LLT::fixed_vector(NumElements: NumElts, ScalarTy: PartLLT);
483
484 Register BuildVec;
485 if (NumElts == Regs.size())
486 BuildVec = B.buildBuildVector(Res: BVType, Ops: Regs).getReg(Idx: 0);
487 else {
488 // Vector elements are packed in the inputs.
489 // e.g. we have a <4 x s16> but 2 x s32 in regs.
490 assert(NumElts > Regs.size());
491 LLT SrcEltTy = MRI.getType(Reg: Regs[0]);
492
493 LLT OriginalEltTy = MRI.getType(Reg: OrigRegs[0]).getElementType();
494
495 // Input registers contain packed elements.
496 // Determine how many elements per reg.
497 assert((SrcEltTy.getSizeInBits() % OriginalEltTy.getSizeInBits()) == 0);
498 unsigned EltPerReg =
499 (SrcEltTy.getSizeInBits() / OriginalEltTy.getSizeInBits());
500
501 SmallVector<Register, 0> BVRegs;
502 BVRegs.reserve(N: Regs.size() * EltPerReg);
503 for (Register R : Regs) {
504 auto Unmerge = B.buildUnmerge(Res: OriginalEltTy, Op: R);
505 for (unsigned K = 0; K < EltPerReg; ++K)
506 BVRegs.push_back(Elt: B.buildAnyExt(Res: PartLLT, Op: Unmerge.getReg(Idx: K)).getReg(Idx: 0));
507 }
508
509 // We may have some more elements in BVRegs, e.g. if we have 2 s32 pieces
510 // for a <3 x s16> vector. We should have less than EltPerReg extra items.
511 if (BVRegs.size() > NumElts) {
512 assert((BVRegs.size() - NumElts) < EltPerReg);
513 BVRegs.truncate(N: NumElts);
514 }
515 BuildVec = B.buildBuildVector(Res: BVType, Ops: BVRegs).getReg(Idx: 0);
516 }
517 B.buildTrunc(Res: OrigRegs[0], Op: BuildVec);
518 }
519}
520
521/// Create a sequence of instructions to expand the value in \p SrcReg (of type
522/// \p SrcTy) to the types in \p DstRegs (of type \p PartTy). \p ExtendOp should
523/// contain the type of scalar value extension if necessary.
524///
525/// This is used for outgoing values (vregs to physregs)
526static void buildCopyToRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
527 Register SrcReg, LLT SrcTy, LLT PartTy,
528 unsigned ExtendOp = TargetOpcode::G_ANYEXT) {
529 // We could just insert a regular copy, but this is unreachable at the moment.
530 assert(SrcTy != PartTy && "identical part types shouldn't reach here");
531
532 const TypeSize PartSize = PartTy.getSizeInBits();
533
534 if (PartTy.isVector() == SrcTy.isVector() &&
535 PartTy.getScalarSizeInBits() > SrcTy.getScalarSizeInBits()) {
536 assert(DstRegs.size() == 1);
537 B.buildInstr(Opc: ExtendOp, DstOps: {DstRegs[0]}, SrcOps: {SrcReg});
538 return;
539 }
540
541 if (SrcTy.isVector() && !PartTy.isVector() &&
542 TypeSize::isKnownGT(LHS: PartSize, RHS: SrcTy.getElementType().getSizeInBits())) {
543 // Vector was scalarized, and the elements extended.
544 auto UnmergeToEltTy = B.buildUnmerge(Res: SrcTy.getElementType(), Op: SrcReg);
545 for (int i = 0, e = DstRegs.size(); i != e; ++i)
546 B.buildAnyExt(Res: DstRegs[i], Op: UnmergeToEltTy.getReg(Idx: i));
547 return;
548 }
549
550 if (SrcTy.isVector() && PartTy.isVector() &&
551 PartTy.getSizeInBits() == SrcTy.getSizeInBits() &&
552 ElementCount::isKnownLT(LHS: SrcTy.getElementCount(),
553 RHS: PartTy.getElementCount())) {
554 // A coercion like: v2f32 -> v4f32 or nxv2f32 -> nxv4f32
555 Register DstReg = DstRegs.front();
556 B.buildPadVectorWithUndefElements(Res: DstReg, Op0: SrcReg);
557 return;
558 }
559
560 LLT GCDTy = getGCDType(OrigTy: SrcTy, TargetTy: PartTy);
561 if (GCDTy == PartTy) {
562 // If this already evenly divisible, we can create a simple unmerge.
563 B.buildUnmerge(Res: DstRegs, Op: SrcReg);
564 return;
565 }
566
567 MachineRegisterInfo &MRI = *B.getMRI();
568 LLT DstTy = MRI.getType(Reg: DstRegs[0]);
569 LLT LCMTy = getCoverTy(OrigTy: SrcTy, TargetTy: PartTy);
570
571 if (PartTy.isVector() && LCMTy == PartTy) {
572 assert(DstRegs.size() == 1);
573 B.buildPadVectorWithUndefElements(Res: DstRegs[0], Op0: SrcReg);
574 return;
575 }
576
577 const unsigned DstSize = DstTy.getSizeInBits();
578 const unsigned SrcSize = SrcTy.getSizeInBits();
579 unsigned CoveringSize = LCMTy.getSizeInBits();
580
581 Register UnmergeSrc = SrcReg;
582
583 if (!LCMTy.isVector() && CoveringSize != SrcSize) {
584 // For scalars, it's common to be able to use a simple extension.
585 if (SrcTy.isScalar() && DstTy.isScalar()) {
586 CoveringSize = alignTo(Value: SrcSize, Align: DstSize);
587 LLT CoverTy = LLT::scalar(SizeInBits: CoveringSize);
588 UnmergeSrc = B.buildInstr(Opc: ExtendOp, DstOps: {CoverTy}, SrcOps: {SrcReg}).getReg(Idx: 0);
589 } else {
590 // Widen to the common type.
591 // FIXME: This should respect the extend type
592 Register Undef = B.buildUndef(Res: SrcTy).getReg(Idx: 0);
593 SmallVector<Register, 8> MergeParts(1, SrcReg);
594 for (unsigned Size = SrcSize; Size != CoveringSize; Size += SrcSize)
595 MergeParts.push_back(Elt: Undef);
596 UnmergeSrc = B.buildMergeLikeInstr(Res: LCMTy, Ops: MergeParts).getReg(Idx: 0);
597 }
598 }
599
600 if (LCMTy.isVector() && CoveringSize != SrcSize)
601 UnmergeSrc = B.buildPadVectorWithUndefElements(Res: LCMTy, Op0: SrcReg).getReg(Idx: 0);
602
603 B.buildUnmerge(Res: DstRegs, Op: UnmergeSrc);
604}
605
606bool CallLowering::determineAndHandleAssignments(
607 ValueHandler &Handler, ValueAssigner &Assigner,
608 SmallVectorImpl<ArgInfo> &Args, MachineIRBuilder &MIRBuilder,
609 CallingConv::ID CallConv, bool IsVarArg,
610 ArrayRef<Register> ThisReturnRegs) const {
611 MachineFunction &MF = MIRBuilder.getMF();
612 const Function &F = MF.getFunction();
613 SmallVector<CCValAssign, 16> ArgLocs;
614
615 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, F.getContext());
616 if (!determineAssignments(Assigner, Args, CCInfo))
617 return false;
618
619 return handleAssignments(Handler, Args, CCState&: CCInfo, ArgLocs, MIRBuilder,
620 ThisReturnRegs);
621}
622
623static unsigned extendOpFromFlags(llvm::ISD::ArgFlagsTy Flags) {
624 if (Flags.isSExt())
625 return TargetOpcode::G_SEXT;
626 if (Flags.isZExt())
627 return TargetOpcode::G_ZEXT;
628 return TargetOpcode::G_ANYEXT;
629}
630
631bool CallLowering::determineAssignments(ValueAssigner &Assigner,
632 SmallVectorImpl<ArgInfo> &Args,
633 CCState &CCInfo) const {
634 LLVMContext &Ctx = CCInfo.getContext();
635 const CallingConv::ID CallConv = CCInfo.getCallingConv();
636
637 unsigned NumArgs = Args.size();
638 for (unsigned i = 0; i != NumArgs; ++i) {
639 EVT CurVT = EVT::getEVT(Ty: Args[i].Ty);
640
641 MVT NewVT = TLI->getRegisterTypeForCallingConv(Context&: Ctx, CC: CallConv, VT: CurVT);
642
643 // If we need to split the type over multiple regs, check it's a scenario
644 // we currently support.
645 unsigned NumParts =
646 TLI->getNumRegistersForCallingConv(Context&: Ctx, CC: CallConv, VT: CurVT);
647
648 if (NumParts == 1) {
649 // Try to use the register type if we couldn't assign the VT.
650 if (Assigner.assignArg(ValNo: i, OrigVT: CurVT, ValVT: NewVT, LocVT: NewVT, LocInfo: CCValAssign::Full, Info: Args[i],
651 Flags: Args[i].Flags[0], State&: CCInfo))
652 return false;
653 continue;
654 }
655
656 // For incoming arguments (physregs to vregs), we could have values in
657 // physregs (or memlocs) which we want to extract and copy to vregs.
658 // During this, we might have to deal with the LLT being split across
659 // multiple regs, so we have to record this information for later.
660 //
661 // If we have outgoing args, then we have the opposite case. We have a
662 // vreg with an LLT which we want to assign to a physical location, and
663 // we might have to record that the value has to be split later.
664
665 // We're handling an incoming arg which is split over multiple regs.
666 // E.g. passing an s128 on AArch64.
667 ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0];
668 Args[i].Flags.clear();
669
670 for (unsigned Part = 0; Part < NumParts; ++Part) {
671 ISD::ArgFlagsTy Flags = OrigFlags;
672 if (Part == 0) {
673 Flags.setSplit();
674 } else {
675 Flags.setOrigAlign(Align(1));
676 if (Part == NumParts - 1)
677 Flags.setSplitEnd();
678 }
679
680 Args[i].Flags.push_back(Elt: Flags);
681 if (Assigner.assignArg(ValNo: i, OrigVT: CurVT, ValVT: NewVT, LocVT: NewVT, LocInfo: CCValAssign::Full, Info: Args[i],
682 Flags: Args[i].Flags[Part], State&: CCInfo)) {
683 // Still couldn't assign this smaller part type for some reason.
684 return false;
685 }
686 }
687 }
688
689 return true;
690}
691
692bool CallLowering::handleAssignments(ValueHandler &Handler,
693 SmallVectorImpl<ArgInfo> &Args,
694 CCState &CCInfo,
695 SmallVectorImpl<CCValAssign> &ArgLocs,
696 MachineIRBuilder &MIRBuilder,
697 ArrayRef<Register> ThisReturnRegs) const {
698 MachineFunction &MF = MIRBuilder.getMF();
699 MachineRegisterInfo &MRI = MF.getRegInfo();
700 const Function &F = MF.getFunction();
701 const DataLayout &DL = F.getParent()->getDataLayout();
702
703 const unsigned NumArgs = Args.size();
704
705 // Stores thunks for outgoing register assignments. This is used so we delay
706 // generating register copies until mem loc assignments are done. We do this
707 // so that if the target is using the delayed stack protector feature, we can
708 // find the split point of the block accurately. E.g. if we have:
709 // G_STORE %val, %memloc
710 // $x0 = COPY %foo
711 // $x1 = COPY %bar
712 // CALL func
713 // ... then the split point for the block will correctly be at, and including,
714 // the copy to $x0. If instead the G_STORE instruction immediately precedes
715 // the CALL, then we'd prematurely choose the CALL as the split point, thus
716 // generating a split block with a CALL that uses undefined physregs.
717 SmallVector<std::function<void()>> DelayedOutgoingRegAssignments;
718
719 for (unsigned i = 0, j = 0; i != NumArgs; ++i, ++j) {
720 assert(j < ArgLocs.size() && "Skipped too many arg locs");
721 CCValAssign &VA = ArgLocs[j];
722 assert(VA.getValNo() == i && "Location doesn't correspond to current arg");
723
724 if (VA.needsCustom()) {
725 std::function<void()> Thunk;
726 unsigned NumArgRegs = Handler.assignCustomValue(
727 Arg&: Args[i], VAs: ArrayRef(ArgLocs).slice(N: j), Thunk: &Thunk);
728 if (Thunk)
729 DelayedOutgoingRegAssignments.emplace_back(Args&: Thunk);
730 if (!NumArgRegs)
731 return false;
732 j += (NumArgRegs - 1);
733 continue;
734 }
735
736 const MVT ValVT = VA.getValVT();
737 const MVT LocVT = VA.getLocVT();
738
739 const LLT LocTy(LocVT);
740 const LLT ValTy(ValVT);
741 const LLT NewLLT = Handler.isIncomingArgumentHandler() ? LocTy : ValTy;
742 const EVT OrigVT = EVT::getEVT(Ty: Args[i].Ty);
743 const LLT OrigTy = getLLTForType(Ty&: *Args[i].Ty, DL);
744
745 // Expected to be multiple regs for a single incoming arg.
746 // There should be Regs.size() ArgLocs per argument.
747 // This should be the same as getNumRegistersForCallingConv
748 const unsigned NumParts = Args[i].Flags.size();
749
750 // Now split the registers into the assigned types.
751 Args[i].OrigRegs.assign(in_start: Args[i].Regs.begin(), in_end: Args[i].Regs.end());
752
753 if (NumParts != 1 || NewLLT != OrigTy) {
754 // If we can't directly assign the register, we need one or more
755 // intermediate values.
756 Args[i].Regs.resize(N: NumParts);
757
758 // For each split register, create and assign a vreg that will store
759 // the incoming component of the larger value. These will later be
760 // merged to form the final vreg.
761 for (unsigned Part = 0; Part < NumParts; ++Part)
762 Args[i].Regs[Part] = MRI.createGenericVirtualRegister(Ty: NewLLT);
763 }
764
765 assert((j + (NumParts - 1)) < ArgLocs.size() &&
766 "Too many regs for number of args");
767
768 // Coerce into outgoing value types before register assignment.
769 if (!Handler.isIncomingArgumentHandler() && OrigTy != ValTy) {
770 assert(Args[i].OrigRegs.size() == 1);
771 buildCopyToRegs(B&: MIRBuilder, DstRegs: Args[i].Regs, SrcReg: Args[i].OrigRegs[0], SrcTy: OrigTy,
772 PartTy: ValTy, ExtendOp: extendOpFromFlags(Flags: Args[i].Flags[0]));
773 }
774
775 bool BigEndianPartOrdering = TLI->hasBigEndianPartOrdering(VT: OrigVT, DL);
776 for (unsigned Part = 0; Part < NumParts; ++Part) {
777 Register ArgReg = Args[i].Regs[Part];
778 // There should be Regs.size() ArgLocs per argument.
779 unsigned Idx = BigEndianPartOrdering ? NumParts - 1 - Part : Part;
780 CCValAssign &VA = ArgLocs[j + Idx];
781 const ISD::ArgFlagsTy Flags = Args[i].Flags[Part];
782
783 if (VA.isMemLoc() && !Flags.isByVal()) {
784 // Individual pieces may have been spilled to the stack and others
785 // passed in registers.
786
787 // TODO: The memory size may be larger than the value we need to
788 // store. We may need to adjust the offset for big endian targets.
789 LLT MemTy = Handler.getStackValueStoreType(DL, VA, Flags);
790
791 MachinePointerInfo MPO;
792 Register StackAddr = Handler.getStackAddress(
793 MemSize: MemTy.getSizeInBytes(), Offset: VA.getLocMemOffset(), MPO, Flags);
794
795 Handler.assignValueToAddress(Arg: Args[i], ValRegIndex: Part, Addr: StackAddr, MemTy, MPO, VA);
796 continue;
797 }
798
799 if (VA.isMemLoc() && Flags.isByVal()) {
800 assert(Args[i].Regs.size() == 1 &&
801 "didn't expect split byval pointer");
802
803 if (Handler.isIncomingArgumentHandler()) {
804 // We just need to copy the frame index value to the pointer.
805 MachinePointerInfo MPO;
806 Register StackAddr = Handler.getStackAddress(
807 MemSize: Flags.getByValSize(), Offset: VA.getLocMemOffset(), MPO, Flags);
808 MIRBuilder.buildCopy(Res: Args[i].Regs[0], Op: StackAddr);
809 } else {
810 // For outgoing byval arguments, insert the implicit copy byval
811 // implies, such that writes in the callee do not modify the caller's
812 // value.
813 uint64_t MemSize = Flags.getByValSize();
814 int64_t Offset = VA.getLocMemOffset();
815
816 MachinePointerInfo DstMPO;
817 Register StackAddr =
818 Handler.getStackAddress(MemSize, Offset, MPO&: DstMPO, Flags);
819
820 MachinePointerInfo SrcMPO(Args[i].OrigValue);
821 if (!Args[i].OrigValue) {
822 // We still need to accurately track the stack address space if we
823 // don't know the underlying value.
824 const LLT PtrTy = MRI.getType(Reg: StackAddr);
825 SrcMPO = MachinePointerInfo(PtrTy.getAddressSpace());
826 }
827
828 Align DstAlign = std::max(a: Flags.getNonZeroByValAlign(),
829 b: inferAlignFromPtrInfo(MF, MPO: DstMPO));
830
831 Align SrcAlign = std::max(a: Flags.getNonZeroByValAlign(),
832 b: inferAlignFromPtrInfo(MF, MPO: SrcMPO));
833
834 Handler.copyArgumentMemory(Arg: Args[i], DstPtr: StackAddr, SrcPtr: Args[i].Regs[0],
835 DstPtrInfo: DstMPO, DstAlign, SrcPtrInfo: SrcMPO, SrcAlign,
836 MemSize, VA);
837 }
838 continue;
839 }
840
841 assert(!VA.needsCustom() && "custom loc should have been handled already");
842
843 if (i == 0 && !ThisReturnRegs.empty() &&
844 Handler.isIncomingArgumentHandler() &&
845 isTypeIsValidForThisReturn(Ty: ValVT)) {
846 Handler.assignValueToReg(ValVReg: ArgReg, PhysReg: ThisReturnRegs[Part], VA);
847 continue;
848 }
849
850 if (Handler.isIncomingArgumentHandler())
851 Handler.assignValueToReg(ValVReg: ArgReg, PhysReg: VA.getLocReg(), VA);
852 else {
853 DelayedOutgoingRegAssignments.emplace_back(Args: [=, &Handler]() {
854 Handler.assignValueToReg(ValVReg: ArgReg, PhysReg: VA.getLocReg(), VA);
855 });
856 }
857 }
858
859 // Now that all pieces have been assigned, re-pack the register typed values
860 // into the original value typed registers.
861 if (Handler.isIncomingArgumentHandler() && OrigVT != LocVT) {
862 // Merge the split registers into the expected larger result vregs of
863 // the original call.
864 buildCopyFromRegs(B&: MIRBuilder, OrigRegs: Args[i].OrigRegs, Regs: Args[i].Regs, LLTy: OrigTy,
865 PartLLT: LocTy, Flags: Args[i].Flags[0]);
866 }
867
868 j += NumParts - 1;
869 }
870 for (auto &Fn : DelayedOutgoingRegAssignments)
871 Fn();
872
873 return true;
874}
875
876void CallLowering::insertSRetLoads(MachineIRBuilder &MIRBuilder, Type *RetTy,
877 ArrayRef<Register> VRegs, Register DemoteReg,
878 int FI) const {
879 MachineFunction &MF = MIRBuilder.getMF();
880 MachineRegisterInfo &MRI = MF.getRegInfo();
881 const DataLayout &DL = MF.getDataLayout();
882
883 SmallVector<EVT, 4> SplitVTs;
884 SmallVector<uint64_t, 4> Offsets;
885 ComputeValueVTs(TLI: *TLI, DL, Ty: RetTy, ValueVTs&: SplitVTs, FixedOffsets: &Offsets, StartingOffset: 0);
886
887 assert(VRegs.size() == SplitVTs.size());
888
889 unsigned NumValues = SplitVTs.size();
890 Align BaseAlign = DL.getPrefTypeAlign(Ty: RetTy);
891 Type *RetPtrTy =
892 PointerType::get(C&: RetTy->getContext(), AddressSpace: DL.getAllocaAddrSpace());
893 LLT OffsetLLTy = getLLTForType(Ty&: *DL.getIndexType(PtrTy: RetPtrTy), DL);
894
895 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
896
897 for (unsigned I = 0; I < NumValues; ++I) {
898 Register Addr;
899 MIRBuilder.materializePtrAdd(Res&: Addr, Op0: DemoteReg, ValueTy: OffsetLLTy, Value: Offsets[I]);
900 auto *MMO = MF.getMachineMemOperand(PtrInfo, f: MachineMemOperand::MOLoad,
901 MemTy: MRI.getType(Reg: VRegs[I]),
902 base_alignment: commonAlignment(A: BaseAlign, Offset: Offsets[I]));
903 MIRBuilder.buildLoad(Res: VRegs[I], Addr, MMO&: *MMO);
904 }
905}
906
907void CallLowering::insertSRetStores(MachineIRBuilder &MIRBuilder, Type *RetTy,
908 ArrayRef<Register> VRegs,
909 Register DemoteReg) const {
910 MachineFunction &MF = MIRBuilder.getMF();
911 MachineRegisterInfo &MRI = MF.getRegInfo();
912 const DataLayout &DL = MF.getDataLayout();
913
914 SmallVector<EVT, 4> SplitVTs;
915 SmallVector<uint64_t, 4> Offsets;
916 ComputeValueVTs(TLI: *TLI, DL, Ty: RetTy, ValueVTs&: SplitVTs, FixedOffsets: &Offsets, StartingOffset: 0);
917
918 assert(VRegs.size() == SplitVTs.size());
919
920 unsigned NumValues = SplitVTs.size();
921 Align BaseAlign = DL.getPrefTypeAlign(Ty: RetTy);
922 unsigned AS = DL.getAllocaAddrSpace();
923 LLT OffsetLLTy = getLLTForType(Ty&: *DL.getIndexType(PtrTy: RetTy->getPointerTo(AddrSpace: AS)), DL);
924
925 MachinePointerInfo PtrInfo(AS);
926
927 for (unsigned I = 0; I < NumValues; ++I) {
928 Register Addr;
929 MIRBuilder.materializePtrAdd(Res&: Addr, Op0: DemoteReg, ValueTy: OffsetLLTy, Value: Offsets[I]);
930 auto *MMO = MF.getMachineMemOperand(PtrInfo, f: MachineMemOperand::MOStore,
931 MemTy: MRI.getType(Reg: VRegs[I]),
932 base_alignment: commonAlignment(A: BaseAlign, Offset: Offsets[I]));
933 MIRBuilder.buildStore(Val: VRegs[I], Addr, MMO&: *MMO);
934 }
935}
936
937void CallLowering::insertSRetIncomingArgument(
938 const Function &F, SmallVectorImpl<ArgInfo> &SplitArgs, Register &DemoteReg,
939 MachineRegisterInfo &MRI, const DataLayout &DL) const {
940 unsigned AS = DL.getAllocaAddrSpace();
941 DemoteReg = MRI.createGenericVirtualRegister(
942 Ty: LLT::pointer(AddressSpace: AS, SizeInBits: DL.getPointerSizeInBits(AS)));
943
944 Type *PtrTy = PointerType::get(ElementType: F.getReturnType(), AddressSpace: AS);
945
946 SmallVector<EVT, 1> ValueVTs;
947 ComputeValueVTs(TLI: *TLI, DL, Ty: PtrTy, ValueVTs);
948
949 // NOTE: Assume that a pointer won't get split into more than one VT.
950 assert(ValueVTs.size() == 1);
951
952 ArgInfo DemoteArg(DemoteReg, ValueVTs[0].getTypeForEVT(Context&: PtrTy->getContext()),
953 ArgInfo::NoArgIndex);
954 setArgFlags(Arg&: DemoteArg, OpIdx: AttributeList::ReturnIndex, DL, FuncInfo: F);
955 DemoteArg.Flags[0].setSRet();
956 SplitArgs.insert(I: SplitArgs.begin(), Elt: DemoteArg);
957}
958
959void CallLowering::insertSRetOutgoingArgument(MachineIRBuilder &MIRBuilder,
960 const CallBase &CB,
961 CallLoweringInfo &Info) const {
962 const DataLayout &DL = MIRBuilder.getDataLayout();
963 Type *RetTy = CB.getType();
964 unsigned AS = DL.getAllocaAddrSpace();
965 LLT FramePtrTy = LLT::pointer(AddressSpace: AS, SizeInBits: DL.getPointerSizeInBits(AS));
966
967 int FI = MIRBuilder.getMF().getFrameInfo().CreateStackObject(
968 Size: DL.getTypeAllocSize(Ty: RetTy), Alignment: DL.getPrefTypeAlign(Ty: RetTy), isSpillSlot: false);
969
970 Register DemoteReg = MIRBuilder.buildFrameIndex(Res: FramePtrTy, Idx: FI).getReg(Idx: 0);
971 ArgInfo DemoteArg(DemoteReg, PointerType::get(ElementType: RetTy, AddressSpace: AS),
972 ArgInfo::NoArgIndex);
973 setArgFlags(Arg&: DemoteArg, OpIdx: AttributeList::ReturnIndex, DL, FuncInfo: CB);
974 DemoteArg.Flags[0].setSRet();
975
976 Info.OrigArgs.insert(I: Info.OrigArgs.begin(), Elt: DemoteArg);
977 Info.DemoteStackIndex = FI;
978 Info.DemoteRegister = DemoteReg;
979}
980
981bool CallLowering::checkReturn(CCState &CCInfo,
982 SmallVectorImpl<BaseArgInfo> &Outs,
983 CCAssignFn *Fn) const {
984 for (unsigned I = 0, E = Outs.size(); I < E; ++I) {
985 MVT VT = MVT::getVT(Ty: Outs[I].Ty);
986 if (Fn(I, VT, VT, CCValAssign::Full, Outs[I].Flags[0], CCInfo))
987 return false;
988 }
989 return true;
990}
991
992void CallLowering::getReturnInfo(CallingConv::ID CallConv, Type *RetTy,
993 AttributeList Attrs,
994 SmallVectorImpl<BaseArgInfo> &Outs,
995 const DataLayout &DL) const {
996 LLVMContext &Context = RetTy->getContext();
997 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
998
999 SmallVector<EVT, 4> SplitVTs;
1000 ComputeValueVTs(TLI: *TLI, DL, Ty: RetTy, ValueVTs&: SplitVTs);
1001 addArgFlagsFromAttributes(Flags, Attrs, OpIdx: AttributeList::ReturnIndex);
1002
1003 for (EVT VT : SplitVTs) {
1004 unsigned NumParts =
1005 TLI->getNumRegistersForCallingConv(Context, CC: CallConv, VT);
1006 MVT RegVT = TLI->getRegisterTypeForCallingConv(Context, CC: CallConv, VT);
1007 Type *PartTy = EVT(RegVT).getTypeForEVT(Context);
1008
1009 for (unsigned I = 0; I < NumParts; ++I) {
1010 Outs.emplace_back(Args&: PartTy, Args&: Flags);
1011 }
1012 }
1013}
1014
1015bool CallLowering::checkReturnTypeForCallConv(MachineFunction &MF) const {
1016 const auto &F = MF.getFunction();
1017 Type *ReturnType = F.getReturnType();
1018 CallingConv::ID CallConv = F.getCallingConv();
1019
1020 SmallVector<BaseArgInfo, 4> SplitArgs;
1021 getReturnInfo(CallConv, RetTy: ReturnType, Attrs: F.getAttributes(), Outs&: SplitArgs,
1022 DL: MF.getDataLayout());
1023 return canLowerReturn(MF, CallConv, Outs&: SplitArgs, IsVarArg: F.isVarArg());
1024}
1025
1026bool CallLowering::parametersInCSRMatch(
1027 const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask,
1028 const SmallVectorImpl<CCValAssign> &OutLocs,
1029 const SmallVectorImpl<ArgInfo> &OutArgs) const {
1030 for (unsigned i = 0; i < OutLocs.size(); ++i) {
1031 const auto &ArgLoc = OutLocs[i];
1032 // If it's not a register, it's fine.
1033 if (!ArgLoc.isRegLoc())
1034 continue;
1035
1036 MCRegister PhysReg = ArgLoc.getLocReg();
1037
1038 // Only look at callee-saved registers.
1039 if (MachineOperand::clobbersPhysReg(RegMask: CallerPreservedMask, PhysReg))
1040 continue;
1041
1042 LLVM_DEBUG(
1043 dbgs()
1044 << "... Call has an argument passed in a callee-saved register.\n");
1045
1046 // Check if it was copied from.
1047 const ArgInfo &OutInfo = OutArgs[i];
1048
1049 if (OutInfo.Regs.size() > 1) {
1050 LLVM_DEBUG(
1051 dbgs() << "... Cannot handle arguments in multiple registers.\n");
1052 return false;
1053 }
1054
1055 // Check if we copy the register, walking through copies from virtual
1056 // registers. Note that getDefIgnoringCopies does not ignore copies from
1057 // physical registers.
1058 MachineInstr *RegDef = getDefIgnoringCopies(Reg: OutInfo.Regs[0], MRI);
1059 if (!RegDef || RegDef->getOpcode() != TargetOpcode::COPY) {
1060 LLVM_DEBUG(
1061 dbgs()
1062 << "... Parameter was not copied into a VReg, cannot tail call.\n");
1063 return false;
1064 }
1065
1066 // Got a copy. Verify that it's the same as the register we want.
1067 Register CopyRHS = RegDef->getOperand(i: 1).getReg();
1068 if (CopyRHS != PhysReg) {
1069 LLVM_DEBUG(dbgs() << "... Callee-saved register was not copied into "
1070 "VReg, cannot tail call.\n");
1071 return false;
1072 }
1073 }
1074
1075 return true;
1076}
1077
1078bool CallLowering::resultsCompatible(CallLoweringInfo &Info,
1079 MachineFunction &MF,
1080 SmallVectorImpl<ArgInfo> &InArgs,
1081 ValueAssigner &CalleeAssigner,
1082 ValueAssigner &CallerAssigner) const {
1083 const Function &F = MF.getFunction();
1084 CallingConv::ID CalleeCC = Info.CallConv;
1085 CallingConv::ID CallerCC = F.getCallingConv();
1086
1087 if (CallerCC == CalleeCC)
1088 return true;
1089
1090 SmallVector<CCValAssign, 16> ArgLocs1;
1091 CCState CCInfo1(CalleeCC, Info.IsVarArg, MF, ArgLocs1, F.getContext());
1092 if (!determineAssignments(Assigner&: CalleeAssigner, Args&: InArgs, CCInfo&: CCInfo1))
1093 return false;
1094
1095 SmallVector<CCValAssign, 16> ArgLocs2;
1096 CCState CCInfo2(CallerCC, F.isVarArg(), MF, ArgLocs2, F.getContext());
1097 if (!determineAssignments(Assigner&: CallerAssigner, Args&: InArgs, CCInfo&: CCInfo2))
1098 return false;
1099
1100 // We need the argument locations to match up exactly. If there's more in
1101 // one than the other, then we are done.
1102 if (ArgLocs1.size() != ArgLocs2.size())
1103 return false;
1104
1105 // Make sure that each location is passed in exactly the same way.
1106 for (unsigned i = 0, e = ArgLocs1.size(); i < e; ++i) {
1107 const CCValAssign &Loc1 = ArgLocs1[i];
1108 const CCValAssign &Loc2 = ArgLocs2[i];
1109
1110 // We need both of them to be the same. So if one is a register and one
1111 // isn't, we're done.
1112 if (Loc1.isRegLoc() != Loc2.isRegLoc())
1113 return false;
1114
1115 if (Loc1.isRegLoc()) {
1116 // If they don't have the same register location, we're done.
1117 if (Loc1.getLocReg() != Loc2.getLocReg())
1118 return false;
1119
1120 // They matched, so we can move to the next ArgLoc.
1121 continue;
1122 }
1123
1124 // Loc1 wasn't a RegLoc, so they both must be MemLocs. Check if they match.
1125 if (Loc1.getLocMemOffset() != Loc2.getLocMemOffset())
1126 return false;
1127 }
1128
1129 return true;
1130}
1131
1132LLT CallLowering::ValueHandler::getStackValueStoreType(
1133 const DataLayout &DL, const CCValAssign &VA, ISD::ArgFlagsTy Flags) const {
1134 const MVT ValVT = VA.getValVT();
1135 if (ValVT != MVT::iPTR) {
1136 LLT ValTy(ValVT);
1137
1138 // We lost the pointeriness going through CCValAssign, so try to restore it
1139 // based on the flags.
1140 if (Flags.isPointer()) {
1141 LLT PtrTy = LLT::pointer(AddressSpace: Flags.getPointerAddrSpace(),
1142 SizeInBits: ValTy.getScalarSizeInBits());
1143 if (ValVT.isVector())
1144 return LLT::vector(EC: ValTy.getElementCount(), ScalarTy: PtrTy);
1145 return PtrTy;
1146 }
1147
1148 return ValTy;
1149 }
1150
1151 unsigned AddrSpace = Flags.getPointerAddrSpace();
1152 return LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL.getPointerSize(AS: AddrSpace));
1153}
1154
1155void CallLowering::ValueHandler::copyArgumentMemory(
1156 const ArgInfo &Arg, Register DstPtr, Register SrcPtr,
1157 const MachinePointerInfo &DstPtrInfo, Align DstAlign,
1158 const MachinePointerInfo &SrcPtrInfo, Align SrcAlign, uint64_t MemSize,
1159 CCValAssign &VA) const {
1160 MachineFunction &MF = MIRBuilder.getMF();
1161 MachineMemOperand *SrcMMO = MF.getMachineMemOperand(
1162 PtrInfo: SrcPtrInfo,
1163 f: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable, s: MemSize,
1164 base_alignment: SrcAlign);
1165
1166 MachineMemOperand *DstMMO = MF.getMachineMemOperand(
1167 PtrInfo: DstPtrInfo,
1168 f: MachineMemOperand::MOStore | MachineMemOperand::MODereferenceable,
1169 s: MemSize, base_alignment: DstAlign);
1170
1171 const LLT PtrTy = MRI.getType(Reg: DstPtr);
1172 const LLT SizeTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
1173
1174 auto SizeConst = MIRBuilder.buildConstant(Res: SizeTy, Val: MemSize);
1175 MIRBuilder.buildMemCpy(DstPtr, SrcPtr, Size: SizeConst, DstMMO&: *DstMMO, SrcMMO&: *SrcMMO);
1176}
1177
1178Register CallLowering::ValueHandler::extendRegister(Register ValReg,
1179 const CCValAssign &VA,
1180 unsigned MaxSizeBits) {
1181 LLT LocTy{VA.getLocVT()};
1182 LLT ValTy{VA.getValVT()};
1183
1184 if (LocTy.getSizeInBits() == ValTy.getSizeInBits())
1185 return ValReg;
1186
1187 if (LocTy.isScalar() && MaxSizeBits && MaxSizeBits < LocTy.getSizeInBits()) {
1188 if (MaxSizeBits <= ValTy.getSizeInBits())
1189 return ValReg;
1190 LocTy = LLT::scalar(SizeInBits: MaxSizeBits);
1191 }
1192
1193 const LLT ValRegTy = MRI.getType(Reg: ValReg);
1194 if (ValRegTy.isPointer()) {
1195 // The x32 ABI wants to zero extend 32-bit pointers to 64-bit registers, so
1196 // we have to cast to do the extension.
1197 LLT IntPtrTy = LLT::scalar(SizeInBits: ValRegTy.getSizeInBits());
1198 ValReg = MIRBuilder.buildPtrToInt(Dst: IntPtrTy, Src: ValReg).getReg(Idx: 0);
1199 }
1200
1201 switch (VA.getLocInfo()) {
1202 default: break;
1203 case CCValAssign::Full:
1204 case CCValAssign::BCvt:
1205 // FIXME: bitconverting between vector types may or may not be a
1206 // nop in big-endian situations.
1207 return ValReg;
1208 case CCValAssign::AExt: {
1209 auto MIB = MIRBuilder.buildAnyExt(Res: LocTy, Op: ValReg);
1210 return MIB.getReg(Idx: 0);
1211 }
1212 case CCValAssign::SExt: {
1213 Register NewReg = MRI.createGenericVirtualRegister(Ty: LocTy);
1214 MIRBuilder.buildSExt(Res: NewReg, Op: ValReg);
1215 return NewReg;
1216 }
1217 case CCValAssign::ZExt: {
1218 Register NewReg = MRI.createGenericVirtualRegister(Ty: LocTy);
1219 MIRBuilder.buildZExt(Res: NewReg, Op: ValReg);
1220 return NewReg;
1221 }
1222 }
1223 llvm_unreachable("unable to extend register");
1224}
1225
1226void CallLowering::ValueAssigner::anchor() {}
1227
1228Register CallLowering::IncomingValueHandler::buildExtensionHint(
1229 const CCValAssign &VA, Register SrcReg, LLT NarrowTy) {
1230 switch (VA.getLocInfo()) {
1231 case CCValAssign::LocInfo::ZExt: {
1232 return MIRBuilder
1233 .buildAssertZExt(Res: MRI.cloneVirtualRegister(VReg: SrcReg), Op: SrcReg,
1234 Size: NarrowTy.getScalarSizeInBits())
1235 .getReg(Idx: 0);
1236 }
1237 case CCValAssign::LocInfo::SExt: {
1238 return MIRBuilder
1239 .buildAssertSExt(Res: MRI.cloneVirtualRegister(VReg: SrcReg), Op: SrcReg,
1240 Size: NarrowTy.getScalarSizeInBits())
1241 .getReg(Idx: 0);
1242 break;
1243 }
1244 default:
1245 return SrcReg;
1246 }
1247}
1248
1249/// Check if we can use a basic COPY instruction between the two types.
1250///
1251/// We're currently building on top of the infrastructure using MVT, which loses
1252/// pointer information in the CCValAssign. We accept copies from physical
1253/// registers that have been reported as integers if it's to an equivalent sized
1254/// pointer LLT.
1255static bool isCopyCompatibleType(LLT SrcTy, LLT DstTy) {
1256 if (SrcTy == DstTy)
1257 return true;
1258
1259 if (SrcTy.getSizeInBits() != DstTy.getSizeInBits())
1260 return false;
1261
1262 SrcTy = SrcTy.getScalarType();
1263 DstTy = DstTy.getScalarType();
1264
1265 return (SrcTy.isPointer() && DstTy.isScalar()) ||
1266 (DstTy.isPointer() && SrcTy.isScalar());
1267}
1268
1269void CallLowering::IncomingValueHandler::assignValueToReg(
1270 Register ValVReg, Register PhysReg, const CCValAssign &VA) {
1271 const MVT LocVT = VA.getLocVT();
1272 const LLT LocTy(LocVT);
1273 const LLT RegTy = MRI.getType(Reg: ValVReg);
1274
1275 if (isCopyCompatibleType(SrcTy: RegTy, DstTy: LocTy)) {
1276 MIRBuilder.buildCopy(Res: ValVReg, Op: PhysReg);
1277 return;
1278 }
1279
1280 auto Copy = MIRBuilder.buildCopy(Res: LocTy, Op: PhysReg);
1281 auto Hint = buildExtensionHint(VA, SrcReg: Copy.getReg(Idx: 0), NarrowTy: RegTy);
1282 MIRBuilder.buildTrunc(Res: ValVReg, Op: Hint);
1283}
1284

source code of llvm/lib/CodeGen/GlobalISel/CallLowering.cpp