1//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements routines for translating from LLVM IR into SelectionDAG IR.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SelectionDAGBuilder.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/BitVector.h"
19#include "llvm/ADT/DenseMap.h"
20#include "llvm/ADT/None.h"
21#include "llvm/ADT/Optional.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/SmallPtrSet.h"
24#include "llvm/ADT/SmallSet.h"
25#include "llvm/ADT/SmallVector.h"
26#include "llvm/ADT/StringRef.h"
27#include "llvm/ADT/Triple.h"
28#include "llvm/ADT/Twine.h"
29#include "llvm/Analysis/AliasAnalysis.h"
30#include "llvm/Analysis/BranchProbabilityInfo.h"
31#include "llvm/Analysis/ConstantFolding.h"
32#include "llvm/Analysis/EHPersonalities.h"
33#include "llvm/Analysis/Loads.h"
34#include "llvm/Analysis/MemoryLocation.h"
35#include "llvm/Analysis/TargetLibraryInfo.h"
36#include "llvm/Analysis/ValueTracking.h"
37#include "llvm/Analysis/VectorUtils.h"
38#include "llvm/CodeGen/Analysis.h"
39#include "llvm/CodeGen/FunctionLoweringInfo.h"
40#include "llvm/CodeGen/GCMetadata.h"
41#include "llvm/CodeGen/ISDOpcodes.h"
42#include "llvm/CodeGen/MachineBasicBlock.h"
43#include "llvm/CodeGen/MachineFrameInfo.h"
44#include "llvm/CodeGen/MachineFunction.h"
45#include "llvm/CodeGen/MachineInstr.h"
46#include "llvm/CodeGen/MachineInstrBuilder.h"
47#include "llvm/CodeGen/MachineJumpTableInfo.h"
48#include "llvm/CodeGen/MachineMemOperand.h"
49#include "llvm/CodeGen/MachineModuleInfo.h"
50#include "llvm/CodeGen/MachineOperand.h"
51#include "llvm/CodeGen/MachineRegisterInfo.h"
52#include "llvm/CodeGen/RuntimeLibcalls.h"
53#include "llvm/CodeGen/SelectionDAG.h"
54#include "llvm/CodeGen/SelectionDAGNodes.h"
55#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
56#include "llvm/CodeGen/StackMaps.h"
57#include "llvm/CodeGen/TargetFrameLowering.h"
58#include "llvm/CodeGen/TargetInstrInfo.h"
59#include "llvm/CodeGen/TargetLowering.h"
60#include "llvm/CodeGen/TargetOpcodes.h"
61#include "llvm/CodeGen/TargetRegisterInfo.h"
62#include "llvm/CodeGen/TargetSubtargetInfo.h"
63#include "llvm/CodeGen/ValueTypes.h"
64#include "llvm/CodeGen/WinEHFuncInfo.h"
65#include "llvm/IR/Argument.h"
66#include "llvm/IR/Attributes.h"
67#include "llvm/IR/BasicBlock.h"
68#include "llvm/IR/CFG.h"
69#include "llvm/IR/CallSite.h"
70#include "llvm/IR/CallingConv.h"
71#include "llvm/IR/Constant.h"
72#include "llvm/IR/ConstantRange.h"
73#include "llvm/IR/Constants.h"
74#include "llvm/IR/DataLayout.h"
75#include "llvm/IR/DebugInfoMetadata.h"
76#include "llvm/IR/DebugLoc.h"
77#include "llvm/IR/DerivedTypes.h"
78#include "llvm/IR/Function.h"
79#include "llvm/IR/GetElementPtrTypeIterator.h"
80#include "llvm/IR/InlineAsm.h"
81#include "llvm/IR/InstrTypes.h"
82#include "llvm/IR/Instruction.h"
83#include "llvm/IR/Instructions.h"
84#include "llvm/IR/IntrinsicInst.h"
85#include "llvm/IR/Intrinsics.h"
86#include "llvm/IR/LLVMContext.h"
87#include "llvm/IR/Metadata.h"
88#include "llvm/IR/Module.h"
89#include "llvm/IR/Operator.h"
90#include "llvm/IR/PatternMatch.h"
91#include "llvm/IR/Statepoint.h"
92#include "llvm/IR/Type.h"
93#include "llvm/IR/User.h"
94#include "llvm/IR/Value.h"
95#include "llvm/MC/MCContext.h"
96#include "llvm/MC/MCSymbol.h"
97#include "llvm/Support/AtomicOrdering.h"
98#include "llvm/Support/BranchProbability.h"
99#include "llvm/Support/Casting.h"
100#include "llvm/Support/CodeGen.h"
101#include "llvm/Support/CommandLine.h"
102#include "llvm/Support/Compiler.h"
103#include "llvm/Support/Debug.h"
104#include "llvm/Support/ErrorHandling.h"
105#include "llvm/Support/MachineValueType.h"
106#include "llvm/Support/MathExtras.h"
107#include "llvm/Support/raw_ostream.h"
108#include "llvm/Target/TargetIntrinsicInfo.h"
109#include "llvm/Target/TargetMachine.h"
110#include "llvm/Target/TargetOptions.h"
111#include "llvm/Transforms/Utils/Local.h"
112#include <algorithm>
113#include <cassert>
114#include <cstddef>
115#include <cstdint>
116#include <cstring>
117#include <iterator>
118#include <limits>
119#include <numeric>
120#include <tuple>
121#include <utility>
122#include <vector>
123
124using namespace llvm;
125using namespace PatternMatch;
126
127#define DEBUG_TYPE "isel"
128
129/// LimitFloatPrecision - Generate low-precision inline sequences for
130/// some float libcalls (6, 8 or 12 bits).
131static unsigned LimitFloatPrecision;
132
133static cl::opt<unsigned, true>
134 LimitFPPrecision("limit-float-precision",
135 cl::desc("Generate low-precision inline sequences "
136 "for some float libcalls"),
137 cl::location(LimitFloatPrecision), cl::Hidden,
138 cl::init(0));
139
140static cl::opt<unsigned> SwitchPeelThreshold(
141 "switch-peel-threshold", cl::Hidden, cl::init(66),
142 cl::desc("Set the case probability threshold for peeling the case from a "
143 "switch statement. A value greater than 100 will void this "
144 "optimization"));
145
146// Limit the width of DAG chains. This is important in general to prevent
147// DAG-based analysis from blowing up. For example, alias analysis and
148// load clustering may not complete in reasonable time. It is difficult to
149// recognize and avoid this situation within each individual analysis, and
150// future analyses are likely to have the same behavior. Limiting DAG width is
151// the safe approach and will be especially important with global DAGs.
152//
153// MaxParallelChains default is arbitrarily high to avoid affecting
154// optimization, but could be lowered to improve compile time. Any ld-ld-st-st
155// sequence over this should have been converted to llvm.memcpy by the
156// frontend. It is easy to induce this behavior with .ll code such as:
157// %buffer = alloca [4096 x i8]
158// %data = load [4096 x i8]* %argPtr
159// store [4096 x i8] %data, [4096 x i8]* %buffer
160static const unsigned MaxParallelChains = 64;
161
162// Return the calling convention if the Value passed requires ABI mangling as it
163// is a parameter to a function or a return value from a function which is not
164// an intrinsic.
165static Optional<CallingConv::ID> getABIRegCopyCC(const Value *V) {
166 if (auto *R = dyn_cast<ReturnInst>(V))
167 return R->getParent()->getParent()->getCallingConv();
168
169 if (auto *CI = dyn_cast<CallInst>(V)) {
170 const bool IsInlineAsm = CI->isInlineAsm();
171 const bool IsIndirectFunctionCall =
172 !IsInlineAsm && !CI->getCalledFunction();
173
174 // It is possible that the call instruction is an inline asm statement or an
175 // indirect function call in which case the return value of
176 // getCalledFunction() would be nullptr.
177 const bool IsInstrinsicCall =
178 !IsInlineAsm && !IsIndirectFunctionCall &&
179 CI->getCalledFunction()->getIntrinsicID() != Intrinsic::not_intrinsic;
180
181 if (!IsInlineAsm && !IsInstrinsicCall)
182 return CI->getCallingConv();
183 }
184
185 return None;
186}
187
188static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
189 const SDValue *Parts, unsigned NumParts,
190 MVT PartVT, EVT ValueVT, const Value *V,
191 Optional<CallingConv::ID> CC);
192
193/// getCopyFromParts - Create a value that contains the specified legal parts
194/// combined into the value they represent. If the parts combine to a type
195/// larger than ValueVT then AssertOp can be used to specify whether the extra
196/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
197/// (ISD::AssertSext).
198static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL,
199 const SDValue *Parts, unsigned NumParts,
200 MVT PartVT, EVT ValueVT, const Value *V,
201 Optional<CallingConv::ID> CC = None,
202 Optional<ISD::NodeType> AssertOp = None) {
203 if (ValueVT.isVector())
204 return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
205 CC);
206
207 assert(NumParts > 0 && "No parts to assemble!");
208 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
209 SDValue Val = Parts[0];
210
211 if (NumParts > 1) {
212 // Assemble the value from multiple parts.
213 if (ValueVT.isInteger()) {
214 unsigned PartBits = PartVT.getSizeInBits();
215 unsigned ValueBits = ValueVT.getSizeInBits();
216
217 // Assemble the power of 2 part.
218 unsigned RoundParts = NumParts & (NumParts - 1) ?
219 1 << Log2_32(NumParts) : NumParts;
220 unsigned RoundBits = PartBits * RoundParts;
221 EVT RoundVT = RoundBits == ValueBits ?
222 ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
223 SDValue Lo, Hi;
224
225 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
226
227 if (RoundParts > 2) {
228 Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
229 PartVT, HalfVT, V);
230 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
231 RoundParts / 2, PartVT, HalfVT, V);
232 } else {
233 Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
234 Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
235 }
236
237 if (DAG.getDataLayout().isBigEndian())
238 std::swap(Lo, Hi);
239
240 Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
241
242 if (RoundParts < NumParts) {
243 // Assemble the trailing non-power-of-2 part.
244 unsigned OddParts = NumParts - RoundParts;
245 EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
246 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
247 OddVT, V, CC);
248
249 // Combine the round and odd parts.
250 Lo = Val;
251 if (DAG.getDataLayout().isBigEndian())
252 std::swap(Lo, Hi);
253 EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
254 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
255 Hi =
256 DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
257 DAG.getConstant(Lo.getValueSizeInBits(), DL,
258 TLI.getPointerTy(DAG.getDataLayout())));
259 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
260 Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
261 }
262 } else if (PartVT.isFloatingPoint()) {
263 // FP split into multiple FP parts (for ppcf128)
264 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
265 "Unexpected split");
266 SDValue Lo, Hi;
267 Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
268 Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
269 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
270 std::swap(Lo, Hi);
271 Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
272 } else {
273 // FP split into integer parts (soft fp)
274 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
275 !PartVT.isVector() && "Unexpected split");
276 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
277 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC);
278 }
279 }
280
281 // There is now one part, held in Val. Correct it to match ValueVT.
282 // PartEVT is the type of the register class that holds the value.
283 // ValueVT is the type of the inline asm operation.
284 EVT PartEVT = Val.getValueType();
285
286 if (PartEVT == ValueVT)
287 return Val;
288
289 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
290 ValueVT.bitsLT(PartEVT)) {
291 // For an FP value in an integer part, we need to truncate to the right
292 // width first.
293 PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
294 Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
295 }
296
297 // Handle types that have the same size.
298 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
299 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
300
301 // Handle types with different sizes.
302 if (PartEVT.isInteger() && ValueVT.isInteger()) {
303 if (ValueVT.bitsLT(PartEVT)) {
304 // For a truncate, see if we have any information to
305 // indicate whether the truncated bits will always be
306 // zero or sign-extension.
307 if (AssertOp.hasValue())
308 Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
309 DAG.getValueType(ValueVT));
310 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
311 }
312 return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
313 }
314
315 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
316 // FP_ROUND's are always exact here.
317 if (ValueVT.bitsLT(Val.getValueType()))
318 return DAG.getNode(
319 ISD::FP_ROUND, DL, ValueVT, Val,
320 DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
321
322 return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
323 }
324
325 llvm_unreachable("Unknown mismatch!");
326}
327
328static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
329 const Twine &ErrMsg) {
330 const Instruction *I = dyn_cast_or_null<Instruction>(V);
331 if (!V)
332 return Ctx.emitError(ErrMsg);
333
334 const char *AsmError = ", possible invalid constraint for vector type";
335 if (const CallInst *CI = dyn_cast<CallInst>(I))
336 if (isa<InlineAsm>(CI->getCalledValue()))
337 return Ctx.emitError(I, ErrMsg + AsmError);
338
339 return Ctx.emitError(I, ErrMsg);
340}
341
342/// getCopyFromPartsVector - Create a value that contains the specified legal
343/// parts combined into the value they represent. If the parts combine to a
344/// type larger than ValueVT then AssertOp can be used to specify whether the
345/// extra bits are known to be zero (ISD::AssertZext) or sign extended from
346/// ValueVT (ISD::AssertSext).
347static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
348 const SDValue *Parts, unsigned NumParts,
349 MVT PartVT, EVT ValueVT, const Value *V,
350 Optional<CallingConv::ID> CallConv) {
351 assert(ValueVT.isVector() && "Not a vector value");
352 assert(NumParts > 0 && "No parts to assemble!");
353 const bool IsABIRegCopy = CallConv.hasValue();
354
355 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
356 SDValue Val = Parts[0];
357
358 // Handle a multi-element vector.
359 if (NumParts > 1) {
360 EVT IntermediateVT;
361 MVT RegisterVT;
362 unsigned NumIntermediates;
363 unsigned NumRegs;
364
365 if (IsABIRegCopy) {
366 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
367 *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,
368 NumIntermediates, RegisterVT);
369 } else {
370 NumRegs =
371 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
372 NumIntermediates, RegisterVT);
373 }
374
375 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
376 NumParts = NumRegs; // Silence a compiler warning.
377 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
378 assert(RegisterVT.getSizeInBits() ==
379 Parts[0].getSimpleValueType().getSizeInBits() &&
380 "Part type sizes don't match!");
381
382 // Assemble the parts into intermediate operands.
383 SmallVector<SDValue, 8> Ops(NumIntermediates);
384 if (NumIntermediates == NumParts) {
385 // If the register was not expanded, truncate or copy the value,
386 // as appropriate.
387 for (unsigned i = 0; i != NumParts; ++i)
388 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
389 PartVT, IntermediateVT, V);
390 } else if (NumParts > 0) {
391 // If the intermediate type was expanded, build the intermediate
392 // operands from the parts.
393 assert(NumParts % NumIntermediates == 0 &&
394 "Must expand into a divisible number of parts!");
395 unsigned Factor = NumParts / NumIntermediates;
396 for (unsigned i = 0; i != NumIntermediates; ++i)
397 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
398 PartVT, IntermediateVT, V);
399 }
400
401 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
402 // intermediate operands.
403 EVT BuiltVectorTy =
404 EVT::getVectorVT(*DAG.getContext(), IntermediateVT.getScalarType(),
405 (IntermediateVT.isVector()
406 ? IntermediateVT.getVectorNumElements() * NumParts
407 : NumIntermediates));
408 Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
409 : ISD::BUILD_VECTOR,
410 DL, BuiltVectorTy, Ops);
411 }
412
413 // There is now one part, held in Val. Correct it to match ValueVT.
414 EVT PartEVT = Val.getValueType();
415
416 if (PartEVT == ValueVT)
417 return Val;
418
419 if (PartEVT.isVector()) {
420 // If the element type of the source/dest vectors are the same, but the
421 // parts vector has more elements than the value vector, then we have a
422 // vector widening case (e.g. <2 x float> -> <4 x float>). Extract the
423 // elements we want.
424 if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
425 assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&
426 "Cannot narrow, it would be a lossy transformation");
427 return DAG.getNode(
428 ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
429 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
430 }
431
432 // Vector/Vector bitcast.
433 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
434 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
435
436 assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&
437 "Cannot handle this kind of promotion");
438 // Promoted vector extract
439 return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
440
441 }
442
443 // Trivial bitcast if the types are the same size and the destination
444 // vector type is legal.
445 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
446 TLI.isTypeLegal(ValueVT))
447 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
448
449 if (ValueVT.getVectorNumElements() != 1) {
450 // Certain ABIs require that vectors are passed as integers. For vectors
451 // are the same size, this is an obvious bitcast.
452 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
453 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
454 } else if (ValueVT.getSizeInBits() < PartEVT.getSizeInBits()) {
455 // Bitcast Val back the original type and extract the corresponding
456 // vector we want.
457 unsigned Elts = PartEVT.getSizeInBits() / ValueVT.getScalarSizeInBits();
458 EVT WiderVecType = EVT::getVectorVT(*DAG.getContext(),
459 ValueVT.getVectorElementType(), Elts);
460 Val = DAG.getBitcast(WiderVecType, Val);
461 return DAG.getNode(
462 ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
463 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
464 }
465
466 diagnosePossiblyInvalidConstraint(
467 *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
468 return DAG.getUNDEF(ValueVT);
469 }
470
471 // Handle cases such as i8 -> <1 x i1>
472 EVT ValueSVT = ValueVT.getVectorElementType();
473 if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT)
474 Val = ValueVT.isFloatingPoint() ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
475 : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
476
477 return DAG.getBuildVector(ValueVT, DL, Val);
478}
479
480static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
481 SDValue Val, SDValue *Parts, unsigned NumParts,
482 MVT PartVT, const Value *V,
483 Optional<CallingConv::ID> CallConv);
484
485/// getCopyToParts - Create a series of nodes that contain the specified value
486/// split into legal parts. If the parts contain more bits than Val, then, for
487/// integers, ExtendKind can be used to specify how to generate the extra bits.
488static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
489 SDValue *Parts, unsigned NumParts, MVT PartVT,
490 const Value *V,
491 Optional<CallingConv::ID> CallConv = None,
492 ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
493 EVT ValueVT = Val.getValueType();
494
495 // Handle the vector case separately.
496 if (ValueVT.isVector())
497 return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
498 CallConv);
499
500 unsigned PartBits = PartVT.getSizeInBits();
501 unsigned OrigNumParts = NumParts;
502 assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
503 "Copying to an illegal type!");
504
505 if (NumParts == 0)
506 return;
507
508 assert(!ValueVT.isVector() && "Vector case handled elsewhere");
509 EVT PartEVT = PartVT;
510 if (PartEVT == ValueVT) {
511 assert(NumParts == 1 && "No-op copy with multiple parts!");
512 Parts[0] = Val;
513 return;
514 }
515
516 if (NumParts * PartBits > ValueVT.getSizeInBits()) {
517 // If the parts cover more bits than the value has, promote the value.
518 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
519 assert(NumParts == 1 && "Do not know what to promote to!");
520 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
521 } else {
522 if (ValueVT.isFloatingPoint()) {
523 // FP values need to be bitcast, then extended if they are being put
524 // into a larger container.
525 ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
526 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
527 }
528 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
529 ValueVT.isInteger() &&
530 "Unknown mismatch!");
531 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
532 Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
533 if (PartVT == MVT::x86mmx)
534 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
535 }
536 } else if (PartBits == ValueVT.getSizeInBits()) {
537 // Different types of the same size.
538 assert(NumParts == 1 && PartEVT != ValueVT);
539 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
540 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
541 // If the parts cover less bits than value has, truncate the value.
542 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
543 ValueVT.isInteger() &&
544 "Unknown mismatch!");
545 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
546 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
547 if (PartVT == MVT::x86mmx)
548 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
549 }
550
551 // The value may have changed - recompute ValueVT.
552 ValueVT = Val.getValueType();
553 assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
554 "Failed to tile the value with PartVT!");
555
556 if (NumParts == 1) {
557 if (PartEVT != ValueVT) {
558 diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
559 "scalar-to-vector conversion failed");
560 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
561 }
562
563 Parts[0] = Val;
564 return;
565 }
566
567 // Expand the value into multiple parts.
568 if (NumParts & (NumParts - 1)) {
569 // The number of parts is not a power of 2. Split off and copy the tail.
570 assert(PartVT.isInteger() && ValueVT.isInteger() &&
571 "Do not know what to expand to!");
572 unsigned RoundParts = 1 << Log2_32(NumParts);
573 unsigned RoundBits = RoundParts * PartBits;
574 unsigned OddParts = NumParts - RoundParts;
575 SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
576 DAG.getIntPtrConstant(RoundBits, DL));
577 getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
578 CallConv);
579
580 if (DAG.getDataLayout().isBigEndian())
581 // The odd parts were reversed by getCopyToParts - unreverse them.
582 std::reverse(Parts + RoundParts, Parts + NumParts);
583
584 NumParts = RoundParts;
585 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
586 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
587 }
588
589 // The number of parts is a power of 2. Repeatedly bisect the value using
590 // EXTRACT_ELEMENT.
591 Parts[0] = DAG.getNode(ISD::BITCAST, DL,
592 EVT::getIntegerVT(*DAG.getContext(),
593 ValueVT.getSizeInBits()),
594 Val);
595
596 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
597 for (unsigned i = 0; i < NumParts; i += StepSize) {
598 unsigned ThisBits = StepSize * PartBits / 2;
599 EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
600 SDValue &Part0 = Parts[i];
601 SDValue &Part1 = Parts[i+StepSize/2];
602
603 Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
604 ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
605 Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
606 ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
607
608 if (ThisBits == PartBits && ThisVT != PartVT) {
609 Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
610 Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
611 }
612 }
613 }
614
615 if (DAG.getDataLayout().isBigEndian())
616 std::reverse(Parts, Parts + OrigNumParts);
617}
618
619static SDValue widenVectorToPartType(SelectionDAG &DAG,
620 SDValue Val, const SDLoc &DL, EVT PartVT) {
621 if (!PartVT.isVector())
622 return SDValue();
623
624 EVT ValueVT = Val.getValueType();
625 unsigned PartNumElts = PartVT.getVectorNumElements();
626 unsigned ValueNumElts = ValueVT.getVectorNumElements();
627 if (PartNumElts > ValueNumElts &&
628 PartVT.getVectorElementType() == ValueVT.getVectorElementType()) {
629 EVT ElementVT = PartVT.getVectorElementType();
630 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
631 // undef elements.
632 SmallVector<SDValue, 16> Ops;
633 DAG.ExtractVectorElements(Val, Ops);
634 SDValue EltUndef = DAG.getUNDEF(ElementVT);
635 for (unsigned i = ValueNumElts, e = PartNumElts; i != e; ++i)
636 Ops.push_back(EltUndef);
637
638 // FIXME: Use CONCAT for 2x -> 4x.
639 return DAG.getBuildVector(PartVT, DL, Ops);
640 }
641
642 return SDValue();
643}
644
645/// getCopyToPartsVector - Create a series of nodes that contain the specified
646/// value split into legal parts.
647static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
648 SDValue Val, SDValue *Parts, unsigned NumParts,
649 MVT PartVT, const Value *V,
650 Optional<CallingConv::ID> CallConv) {
651 EVT ValueVT = Val.getValueType();
652 assert(ValueVT.isVector() && "Not a vector");
653 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
654 const bool IsABIRegCopy = CallConv.hasValue();
655
656 if (NumParts == 1) {
657 EVT PartEVT = PartVT;
658 if (PartEVT == ValueVT) {
659 // Nothing to do.
660 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
661 // Bitconvert vector->vector case.
662 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
663 } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
664 Val = Widened;
665 } else if (PartVT.isVector() &&
666 PartEVT.getVectorElementType().bitsGE(
667 ValueVT.getVectorElementType()) &&
668 PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
669
670 // Promoted vector extract
671 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
672 } else {
673 if (ValueVT.getVectorNumElements() == 1) {
674 Val = DAG.getNode(
675 ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
676 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
677 } else {
678 assert(PartVT.getSizeInBits() > ValueVT.getSizeInBits() &&
679 "lossy conversion of vector to scalar type");
680 EVT IntermediateType =
681 EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
682 Val = DAG.getBitcast(IntermediateType, Val);
683 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
684 }
685 }
686
687 assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
688 Parts[0] = Val;
689 return;
690 }
691
692 // Handle a multi-element vector.
693 EVT IntermediateVT;
694 MVT RegisterVT;
695 unsigned NumIntermediates;
696 unsigned NumRegs;
697 if (IsABIRegCopy) {
698 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
699 *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,
700 NumIntermediates, RegisterVT);
701 } else {
702 NumRegs =
703 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
704 NumIntermediates, RegisterVT);
705 }
706
707 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
708 NumParts = NumRegs; // Silence a compiler warning.
709 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
710
711 unsigned IntermediateNumElts = IntermediateVT.isVector() ?
712 IntermediateVT.getVectorNumElements() : 1;
713
714 // Convert the vector to the appropiate type if necessary.
715 unsigned DestVectorNoElts = NumIntermediates * IntermediateNumElts;
716
717 EVT BuiltVectorTy = EVT::getVectorVT(
718 *DAG.getContext(), IntermediateVT.getScalarType(), DestVectorNoElts);
719 MVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
720 if (ValueVT != BuiltVectorTy) {
721 if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy))
722 Val = Widened;
723
724 Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
725 }
726
727 // Split the vector into intermediate operands.
728 SmallVector<SDValue, 8> Ops(NumIntermediates);
729 for (unsigned i = 0; i != NumIntermediates; ++i) {
730 if (IntermediateVT.isVector()) {
731 Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
732 DAG.getConstant(i * IntermediateNumElts, DL, IdxVT));
733 } else {
734 Ops[i] = DAG.getNode(
735 ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
736 DAG.getConstant(i, DL, IdxVT));
737 }
738 }
739
740 // Split the intermediate operands into legal parts.
741 if (NumParts == NumIntermediates) {
742 // If the register was not expanded, promote or copy the value,
743 // as appropriate.
744 for (unsigned i = 0; i != NumParts; ++i)
745 getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
746 } else if (NumParts > 0) {
747 // If the intermediate type was expanded, split each the value into
748 // legal parts.
749 assert(NumIntermediates != 0 && "division by zero");
750 assert(NumParts % NumIntermediates == 0 &&
751 "Must expand into a divisible number of parts!");
752 unsigned Factor = NumParts / NumIntermediates;
753 for (unsigned i = 0; i != NumIntermediates; ++i)
754 getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
755 CallConv);
756 }
757}
758
759RegsForValue::RegsForValue(const SmallVector<unsigned, 4> &regs, MVT regvt,
760 EVT valuevt, Optional<CallingConv::ID> CC)
761 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
762 RegCount(1, regs.size()), CallConv(CC) {}
763
764RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,
765 const DataLayout &DL, unsigned Reg, Type *Ty,
766 Optional<CallingConv::ID> CC) {
767 ComputeValueVTs(TLI, DL, Ty, ValueVTs);
768
769 CallConv = CC;
770
771 for (EVT ValueVT : ValueVTs) {
772 unsigned NumRegs =
773 isABIMangled()
774 ? TLI.getNumRegistersForCallingConv(Context, CC.getValue(), ValueVT)
775 : TLI.getNumRegisters(Context, ValueVT);
776 MVT RegisterVT =
777 isABIMangled()
778 ? TLI.getRegisterTypeForCallingConv(Context, CC.getValue(), ValueVT)
779 : TLI.getRegisterType(Context, ValueVT);
780 for (unsigned i = 0; i != NumRegs; ++i)
781 Regs.push_back(Reg + i);
782 RegVTs.push_back(RegisterVT);
783 RegCount.push_back(NumRegs);
784 Reg += NumRegs;
785 }
786}
787
788SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
789 FunctionLoweringInfo &FuncInfo,
790 const SDLoc &dl, SDValue &Chain,
791 SDValue *Flag, const Value *V) const {
792 // A Value with type {} or [0 x %t] needs no registers.
793 if (ValueVTs.empty())
794 return SDValue();
795
796 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
797
798 // Assemble the legal parts into the final values.
799 SmallVector<SDValue, 4> Values(ValueVTs.size());
800 SmallVector<SDValue, 8> Parts;
801 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
802 // Copy the legal parts from the registers.
803 EVT ValueVT = ValueVTs[Value];
804 unsigned NumRegs = RegCount[Value];
805 MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(
806 *DAG.getContext(),
807 CallConv.getValue(), RegVTs[Value])
808 : RegVTs[Value];
809
810 Parts.resize(NumRegs);
811 for (unsigned i = 0; i != NumRegs; ++i) {
812 SDValue P;
813 if (!Flag) {
814 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
815 } else {
816 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
817 *Flag = P.getValue(2);
818 }
819
820 Chain = P.getValue(1);
821 Parts[i] = P;
822
823 // If the source register was virtual and if we know something about it,
824 // add an assert node.
825 if (!TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) ||
826 !RegisterVT.isInteger())
827 continue;
828
829 const FunctionLoweringInfo::LiveOutInfo *LOI =
830 FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
831 if (!LOI)
832 continue;
833
834 unsigned RegSize = RegisterVT.getScalarSizeInBits();
835 unsigned NumSignBits = LOI->NumSignBits;
836 unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
837
838 if (NumZeroBits == RegSize) {
839 // The current value is a zero.
840 // Explicitly express that as it would be easier for
841 // optimizations to kick in.
842 Parts[i] = DAG.getConstant(0, dl, RegisterVT);
843 continue;
844 }
845
846 // FIXME: We capture more information than the dag can represent. For
847 // now, just use the tightest assertzext/assertsext possible.
848 bool isSExt;
849 EVT FromVT(MVT::Other);
850 if (NumZeroBits) {
851 FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
852 isSExt = false;
853 } else if (NumSignBits > 1) {
854 FromVT =
855 EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
856 isSExt = true;
857 } else {
858 continue;
859 }
860 // Add an assertion node.
861 assert(FromVT != MVT::Other);
862 Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
863 RegisterVT, P, DAG.getValueType(FromVT));
864 }
865
866 Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
867 RegisterVT, ValueVT, V, CallConv);
868 Part += NumRegs;
869 Parts.clear();
870 }
871
872 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
873}
874
875void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,
876 const SDLoc &dl, SDValue &Chain, SDValue *Flag,
877 const Value *V,
878 ISD::NodeType PreferredExtendType) const {
879 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
880 ISD::NodeType ExtendKind = PreferredExtendType;
881
882 // Get the list of the values's legal parts.
883 unsigned NumRegs = Regs.size();
884 SmallVector<SDValue, 8> Parts(NumRegs);
885 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
886 unsigned NumParts = RegCount[Value];
887
888 MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(
889 *DAG.getContext(),
890 CallConv.getValue(), RegVTs[Value])
891 : RegVTs[Value];
892
893 if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
894 ExtendKind = ISD::ZERO_EXTEND;
895
896 getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
897 NumParts, RegisterVT, V, CallConv, ExtendKind);
898 Part += NumParts;
899 }
900
901 // Copy the parts into the registers.
902 SmallVector<SDValue, 8> Chains(NumRegs);
903 for (unsigned i = 0; i != NumRegs; ++i) {
904 SDValue Part;
905 if (!Flag) {
906 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
907 } else {
908 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
909 *Flag = Part.getValue(1);
910 }
911
912 Chains[i] = Part.getValue(0);
913 }
914
915 if (NumRegs == 1 || Flag)
916 // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
917 // flagged to it. That is the CopyToReg nodes and the user are considered
918 // a single scheduling unit. If we create a TokenFactor and return it as
919 // chain, then the TokenFactor is both a predecessor (operand) of the
920 // user as well as a successor (the TF operands are flagged to the user).
921 // c1, f1 = CopyToReg
922 // c2, f2 = CopyToReg
923 // c3 = TokenFactor c1, c2
924 // ...
925 // = op c3, ..., f2
926 Chain = Chains[NumRegs-1];
927 else
928 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
929}
930
931void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
932 unsigned MatchingIdx, const SDLoc &dl,
933 SelectionDAG &DAG,
934 std::vector<SDValue> &Ops) const {
935 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
936
937 unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
938 if (HasMatching)
939 Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
940 else if (!Regs.empty() &&
941 TargetRegisterInfo::isVirtualRegister(Regs.front())) {
942 // Put the register class of the virtual registers in the flag word. That
943 // way, later passes can recompute register class constraints for inline
944 // assembly as well as normal instructions.
945 // Don't do this for tied operands that can use the regclass information
946 // from the def.
947 const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
948 const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
949 Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
950 }
951
952 SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
953 Ops.push_back(Res);
954
955 if (Code == InlineAsm::Kind_Clobber) {
956 // Clobbers should always have a 1:1 mapping with registers, and may
957 // reference registers that have illegal (e.g. vector) types. Hence, we
958 // shouldn't try to apply any sort of splitting logic to them.
959 assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
960 "No 1:1 mapping from clobbers to regs?");
961 unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
962 (void)SP;
963 for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
964 Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
965 assert(
966 (Regs[I] != SP ||
967 DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) &&
968 "If we clobbered the stack pointer, MFI should know about it.");
969 }
970 return;
971 }
972
973 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
974 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
975 MVT RegisterVT = RegVTs[Value];
976 for (unsigned i = 0; i != NumRegs; ++i) {
977 assert(Reg < Regs.size() && "Mismatch in # registers expected");
978 unsigned TheReg = Regs[Reg++];
979 Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
980 }
981 }
982}
983
984SmallVector<std::pair<unsigned, unsigned>, 4>
985RegsForValue::getRegsAndSizes() const {
986 SmallVector<std::pair<unsigned, unsigned>, 4> OutVec;
987 unsigned I = 0;
988 for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
989 unsigned RegCount = std::get<0>(CountAndVT);
990 MVT RegisterVT = std::get<1>(CountAndVT);
991 unsigned RegisterSize = RegisterVT.getSizeInBits();
992 for (unsigned E = I + RegCount; I != E; ++I)
993 OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
994 }
995 return OutVec;
996}
997
998void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa,
999 const TargetLibraryInfo *li) {
1000 AA = aa;
1001 GFI = gfi;
1002 LibInfo = li;
1003 DL = &DAG.getDataLayout();
1004 Context = DAG.getContext();
1005 LPadToCallSiteMap.clear();
1006}
1007
1008void SelectionDAGBuilder::clear() {
1009 NodeMap.clear();
1010 UnusedArgNodeMap.clear();
1011 PendingLoads.clear();
1012 PendingExports.clear();
1013 CurInst = nullptr;
1014 HasTailCall = false;
1015 SDNodeOrder = LowestSDNodeOrder;
1016 StatepointLowering.clear();
1017}
1018
1019void SelectionDAGBuilder::clearDanglingDebugInfo() {
1020 DanglingDebugInfoMap.clear();
1021}
1022
1023SDValue SelectionDAGBuilder::getRoot() {
1024 if (PendingLoads.empty())
1025 return DAG.getRoot();
1026
1027 if (PendingLoads.size() == 1) {
1028 SDValue Root = PendingLoads[0];
1029 DAG.setRoot(Root);
1030 PendingLoads.clear();
1031 return Root;
1032 }
1033
1034 // Otherwise, we have to make a token factor node.
1035 SDValue Root = DAG.getTokenFactor(getCurSDLoc(), PendingLoads);
1036 PendingLoads.clear();
1037 DAG.setRoot(Root);
1038 return Root;
1039}
1040
1041SDValue SelectionDAGBuilder::getControlRoot() {
1042 SDValue Root = DAG.getRoot();
1043
1044 if (PendingExports.empty())
1045 return Root;
1046
1047 // Turn all of the CopyToReg chains into one factored node.
1048 if (Root.getOpcode() != ISD::EntryToken) {
1049 unsigned i = 0, e = PendingExports.size();
1050 for (; i != e; ++i) {
1051 assert(PendingExports[i].getNode()->getNumOperands() > 1);
1052 if (PendingExports[i].getNode()->getOperand(0) == Root)
1053 break; // Don't add the root if we already indirectly depend on it.
1054 }
1055
1056 if (i == e)
1057 PendingExports.push_back(Root);
1058 }
1059
1060 Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
1061 PendingExports);
1062 PendingExports.clear();
1063 DAG.setRoot(Root);
1064 return Root;
1065}
1066
1067void SelectionDAGBuilder::visit(const Instruction &I) {
1068 // Set up outgoing PHI node register values before emitting the terminator.
1069 if (I.isTerminator()) {
1070 HandlePHINodesInSuccessorBlocks(I.getParent());
1071 }
1072
1073 // Increase the SDNodeOrder if dealing with a non-debug instruction.
1074 if (!isa<DbgInfoIntrinsic>(I))
1075 ++SDNodeOrder;
1076
1077 CurInst = &I;
1078
1079 visit(I.getOpcode(), I);
1080
1081 if (auto *FPMO = dyn_cast<FPMathOperator>(&I)) {
1082 // Propagate the fast-math-flags of this IR instruction to the DAG node that
1083 // maps to this instruction.
1084 // TODO: We could handle all flags (nsw, etc) here.
1085 // TODO: If an IR instruction maps to >1 node, only the final node will have
1086 // flags set.
1087 if (SDNode *Node = getNodeForIRValue(&I)) {
1088 SDNodeFlags IncomingFlags;
1089 IncomingFlags.copyFMF(*FPMO);
1090 if (!Node->getFlags().isDefined())
1091 Node->setFlags(IncomingFlags);
1092 else
1093 Node->intersectFlagsWith(IncomingFlags);
1094 }
1095 }
1096
1097 if (!I.isTerminator() && !HasTailCall &&
1098 !isStatepoint(&I)) // statepoints handle their exports internally
1099 CopyToExportRegsIfNeeded(&I);
1100
1101 CurInst = nullptr;
1102}
1103
1104void SelectionDAGBuilder::visitPHI(const PHINode &) {
1105 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1106}
1107
1108void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1109 // Note: this doesn't use InstVisitor, because it has to work with
1110 // ConstantExpr's in addition to instructions.
1111 switch (Opcode) {
1112 default: llvm_unreachable("Unknown instruction type encountered!");
1113 // Build the switch statement using the Instruction.def file.
1114#define HANDLE_INST(NUM, OPCODE, CLASS) \
1115 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1116#include "llvm/IR/Instruction.def"
1117 }
1118}
1119
1120void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable,
1121 const DIExpression *Expr) {
1122 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1123 const DbgValueInst *DI = DDI.getDI();
1124 DIVariable *DanglingVariable = DI->getVariable();
1125 DIExpression *DanglingExpr = DI->getExpression();
1126 if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1127 LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << *DI << "\n");
1128 return true;
1129 }
1130 return false;
1131 };
1132
1133 for (auto &DDIMI : DanglingDebugInfoMap) {
1134 DanglingDebugInfoVector &DDIV = DDIMI.second;
1135
1136 // If debug info is to be dropped, run it through final checks to see
1137 // whether it can be salvaged.
1138 for (auto &DDI : DDIV)
1139 if (isMatchingDbgValue(DDI))
1140 salvageUnresolvedDbgValue(DDI);
1141
1142 DDIV.erase(remove_if(DDIV, isMatchingDbgValue), DDIV.end());
1143 }
1144}
1145
1146// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1147// generate the debug data structures now that we've seen its definition.
1148void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
1149 SDValue Val) {
1150 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1151 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1152 return;
1153
1154 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1155 for (auto &DDI : DDIV) {
1156 const DbgValueInst *DI = DDI.getDI();
1157 assert(DI && "Ill-formed DanglingDebugInfo");
1158 DebugLoc dl = DDI.getdl();
1159 unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1160 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1161 DILocalVariable *Variable = DI->getVariable();
1162 DIExpression *Expr = DI->getExpression();
1163 assert(Variable->isValidLocationForIntrinsic(dl) &&
1164 "Expected inlined-at fields to agree");
1165 SDDbgValue *SDV;
1166 if (Val.getNode()) {
1167 // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1168 // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1169 // we couldn't resolve it directly when examining the DbgValue intrinsic
1170 // in the first place we should not be more successful here). Unless we
1171 // have some test case that prove this to be correct we should avoid
1172 // calling EmitFuncArgumentDbgValue here.
1173 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, false, Val)) {
1174 LLVM_DEBUG(dbgs() << "Resolve dangling debug info [order="
1175 << DbgSDNodeOrder << "] for:\n " << *DI << "\n");
1176 LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump());
1177 // Increase the SDNodeOrder for the DbgValue here to make sure it is
1178 // inserted after the definition of Val when emitting the instructions
1179 // after ISel. An alternative could be to teach
1180 // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1181 LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1182 << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1183 << ValSDNodeOrder << "\n");
1184 SDV = getDbgValue(Val, Variable, Expr, dl,
1185 std::max(DbgSDNodeOrder, ValSDNodeOrder));
1186 DAG.AddDbgValue(SDV, Val.getNode(), false);
1187 } else
1188 LLVM_DEBUG(dbgs() << "Resolved dangling debug info for " << *DI
1189 << "in EmitFuncArgumentDbgValue\n");
1190 } else {
1191 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1192 auto Undef =
1193 UndefValue::get(DDI.getDI()->getVariableLocation()->getType());
1194 auto SDV =
1195 DAG.getConstantDbgValue(Variable, Expr, Undef, dl, DbgSDNodeOrder);
1196 DAG.AddDbgValue(SDV, nullptr, false);
1197 }
1198 }
1199 DDIV.clear();
1200}
1201
1202void SelectionDAGBuilder::salvageUnresolvedDbgValue(DanglingDebugInfo &DDI) {
1203 Value *V = DDI.getDI()->getValue();
1204 DILocalVariable *Var = DDI.getDI()->getVariable();
1205 DIExpression *Expr = DDI.getDI()->getExpression();
1206 DebugLoc DL = DDI.getdl();
1207 DebugLoc InstDL = DDI.getDI()->getDebugLoc();
1208 unsigned SDOrder = DDI.getSDNodeOrder();
1209
1210 // Currently we consider only dbg.value intrinsics -- we tell the salvager
1211 // that DW_OP_stack_value is desired.
1212 assert(isa<DbgValueInst>(DDI.getDI()));
1213 bool StackValue = true;
1214
1215 // Can this Value can be encoded without any further work?
1216 if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder))
1217 return;
1218
1219 // Attempt to salvage back through as many instructions as possible. Bail if
1220 // a non-instruction is seen, such as a constant expression or global
1221 // variable. FIXME: Further work could recover those too.
1222 while (isa<Instruction>(V)) {
1223 Instruction &VAsInst = *cast<Instruction>(V);
1224 DIExpression *NewExpr = salvageDebugInfoImpl(VAsInst, Expr, StackValue);
1225
1226 // If we cannot salvage any further, and haven't yet found a suitable debug
1227 // expression, bail out.
1228 if (!NewExpr)
1229 break;
1230
1231 // New value and expr now represent this debuginfo.
1232 V = VAsInst.getOperand(0);
1233 Expr = NewExpr;
1234
1235 // Some kind of simplification occurred: check whether the operand of the
1236 // salvaged debug expression can be encoded in this DAG.
1237 if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder)) {
1238 LLVM_DEBUG(dbgs() << "Salvaged debug location info for:\n "
1239 << DDI.getDI() << "\nBy stripping back to:\n " << V);
1240 return;
1241 }
1242 }
1243
1244 // This was the final opportunity to salvage this debug information, and it
1245 // couldn't be done. Place an undef DBG_VALUE at this location to terminate
1246 // any earlier variable location.
1247 auto Undef = UndefValue::get(DDI.getDI()->getVariableLocation()->getType());
1248 auto SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);
1249 DAG.AddDbgValue(SDV, nullptr, false);
1250
1251 LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n " << DDI.getDI()
1252 << "\n");
1253 LLVM_DEBUG(dbgs() << " Last seen at:\n " << *DDI.getDI()->getOperand(0)
1254 << "\n");
1255}
1256
1257bool SelectionDAGBuilder::handleDebugValue(const Value *V, DILocalVariable *Var,
1258 DIExpression *Expr, DebugLoc dl,
1259 DebugLoc InstDL, unsigned Order) {
1260 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1261 SDDbgValue *SDV;
1262 if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
1263 isa<ConstantPointerNull>(V)) {
1264 SDV = DAG.getConstantDbgValue(Var, Expr, V, dl, SDNodeOrder);
1265 DAG.AddDbgValue(SDV, nullptr, false);
1266 return true;
1267 }
1268
1269 // If the Value is a frame index, we can create a FrameIndex debug value
1270 // without relying on the DAG at all.
1271 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1272 auto SI = FuncInfo.StaticAllocaMap.find(AI);
1273 if (SI != FuncInfo.StaticAllocaMap.end()) {
1274 auto SDV =
1275 DAG.getFrameIndexDbgValue(Var, Expr, SI->second,
1276 /*IsIndirect*/ false, dl, SDNodeOrder);
1277 // Do not attach the SDNodeDbgValue to an SDNode: this variable location
1278 // is still available even if the SDNode gets optimized out.
1279 DAG.AddDbgValue(SDV, nullptr, false);
1280 return true;
1281 }
1282 }
1283
1284 // Do not use getValue() in here; we don't want to generate code at
1285 // this point if it hasn't been done yet.
1286 SDValue N = NodeMap[V];
1287 if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
1288 N = UnusedArgNodeMap[V];
1289 if (N.getNode()) {
1290 if (EmitFuncArgumentDbgValue(V, Var, Expr, dl, false, N))
1291 return true;
1292 SDV = getDbgValue(N, Var, Expr, dl, SDNodeOrder);
1293 DAG.AddDbgValue(SDV, N.getNode(), false);
1294 return true;
1295 }
1296
1297 // Special rules apply for the first dbg.values of parameter variables in a
1298 // function. Identify them by the fact they reference Argument Values, that
1299 // they're parameters, and they are parameters of the current function. We
1300 // need to let them dangle until they get an SDNode.
1301 bool IsParamOfFunc = isa<Argument>(V) && Var->isParameter() &&
1302 !InstDL.getInlinedAt();
1303 if (!IsParamOfFunc) {
1304 // The value is not used in this block yet (or it would have an SDNode).
1305 // We still want the value to appear for the user if possible -- if it has
1306 // an associated VReg, we can refer to that instead.
1307 auto VMI = FuncInfo.ValueMap.find(V);
1308 if (VMI != FuncInfo.ValueMap.end()) {
1309 unsigned Reg = VMI->second;
1310 // If this is a PHI node, it may be split up into several MI PHI nodes
1311 // (in FunctionLoweringInfo::set).
1312 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1313 V->getType(), None);
1314 if (RFV.occupiesMultipleRegs()) {
1315 unsigned Offset = 0;
1316 unsigned BitsToDescribe = 0;
1317 if (auto VarSize = Var->getSizeInBits())
1318 BitsToDescribe = *VarSize;
1319 if (auto Fragment = Expr->getFragmentInfo())
1320 BitsToDescribe = Fragment->SizeInBits;
1321 for (auto RegAndSize : RFV.getRegsAndSizes()) {
1322 unsigned RegisterSize = RegAndSize.second;
1323 // Bail out if all bits are described already.
1324 if (Offset >= BitsToDescribe)
1325 break;
1326 unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1327 ? BitsToDescribe - Offset
1328 : RegisterSize;
1329 auto FragmentExpr = DIExpression::createFragmentExpression(
1330 Expr, Offset, FragmentSize);
1331 if (!FragmentExpr)
1332 continue;
1333 SDV = DAG.getVRegDbgValue(Var, *FragmentExpr, RegAndSize.first,
1334 false, dl, SDNodeOrder);
1335 DAG.AddDbgValue(SDV, nullptr, false);
1336 Offset += RegisterSize;
1337 }
1338 } else {
1339 SDV = DAG.getVRegDbgValue(Var, Expr, Reg, false, dl, SDNodeOrder);
1340 DAG.AddDbgValue(SDV, nullptr, false);
1341 }
1342 return true;
1343 }
1344 }
1345
1346 return false;
1347}
1348
1349void SelectionDAGBuilder::resolveOrClearDbgInfo() {
1350 // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1351 for (auto &Pair : DanglingDebugInfoMap)
1352 for (auto &DDI : Pair.getSecond())
1353 salvageUnresolvedDbgValue(DDI);
1354 clearDanglingDebugInfo();
1355}
1356
1357/// getCopyFromRegs - If there was virtual register allocated for the value V
1358/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1359SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
1360 DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
1361 SDValue Result;
1362
1363 if (It != FuncInfo.ValueMap.end()) {
1364 unsigned InReg = It->second;
1365
1366 RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1367 DAG.getDataLayout(), InReg, Ty,
1368 None); // This is not an ABI copy.
1369 SDValue Chain = DAG.getEntryNode();
1370 Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1371 V);
1372 resolveDanglingDebugInfo(V, Result);
1373 }
1374
1375 return Result;
1376}
1377
1378/// getValue - Return an SDValue for the given Value.
1379SDValue SelectionDAGBuilder::getValue(const Value *V) {
1380 // If we already have an SDValue for this value, use it. It's important
1381 // to do this first, so that we don't create a CopyFromReg if we already
1382 // have a regular SDValue.
1383 SDValue &N = NodeMap[V];
1384 if (N.getNode()) return N;
1385
1386 // If there's a virtual register allocated and initialized for this
1387 // value, use it.
1388 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1389 return copyFromReg;
1390
1391 // Otherwise create a new SDValue and remember it.
1392 SDValue Val = getValueImpl(V);
1393 NodeMap[V] = Val;
1394 resolveDanglingDebugInfo(V, Val);
1395 return Val;
1396}
1397
1398// Return true if SDValue exists for the given Value
1399bool SelectionDAGBuilder::findValue(const Value *V) const {
1400 return (NodeMap.find(V) != NodeMap.end()) ||
1401 (FuncInfo.ValueMap.find(V) != FuncInfo.ValueMap.end());
1402}
1403
1404/// getNonRegisterValue - Return an SDValue for the given Value, but
1405/// don't look in FuncInfo.ValueMap for a virtual register.
1406SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
1407 // If we already have an SDValue for this value, use it.
1408 SDValue &N = NodeMap[V];
1409 if (N.getNode()) {
1410 if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {
1411 // Remove the debug location from the node as the node is about to be used
1412 // in a location which may differ from the original debug location. This
1413 // is relevant to Constant and ConstantFP nodes because they can appear
1414 // as constant expressions inside PHI nodes.
1415 N->setDebugLoc(DebugLoc());
1416 }
1417 return N;
1418 }
1419
1420 // Otherwise create a new SDValue and remember it.
1421 SDValue Val = getValueImpl(V);
1422 NodeMap[V] = Val;
1423 resolveDanglingDebugInfo(V, Val);
1424 return Val;
1425}
1426
1427/// getValueImpl - Helper function for getValue and getNonRegisterValue.
1428/// Create an SDValue for the given value.
1429SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
1430 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1431
1432 if (const Constant *C = dyn_cast<Constant>(V)) {
1433 EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1434
1435 if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1436 return DAG.getConstant(*CI, getCurSDLoc(), VT);
1437
1438 if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1439 return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1440
1441 if (isa<ConstantPointerNull>(C)) {
1442 unsigned AS = V->getType()->getPointerAddressSpace();
1443 return DAG.getConstant(0, getCurSDLoc(),
1444 TLI.getPointerTy(DAG.getDataLayout(), AS));
1445 }
1446
1447 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1448 return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1449
1450 if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1451 return DAG.getUNDEF(VT);
1452
1453 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1454 visit(CE->getOpcode(), *CE);
1455 SDValue N1 = NodeMap[V];
1456 assert(N1.getNode() && "visit didn't populate the NodeMap!");
1457 return N1;
1458 }
1459
1460 if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1461 SmallVector<SDValue, 4> Constants;
1462 for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
1463 OI != OE; ++OI) {
1464 SDNode *Val = getValue(*OI).getNode();
1465 // If the operand is an empty aggregate, there are no values.
1466 if (!Val) continue;
1467 // Add each leaf value from the operand to the Constants list
1468 // to form a flattened list of all the values.
1469 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1470 Constants.push_back(SDValue(Val, i));
1471 }
1472
1473 return DAG.getMergeValues(Constants, getCurSDLoc());
1474 }
1475
1476 if (const ConstantDataSequential *CDS =
1477 dyn_cast<ConstantDataSequential>(C)) {
1478 SmallVector<SDValue, 4> Ops;
1479 for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1480 SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1481 // Add each leaf value from the operand to the Constants list
1482 // to form a flattened list of all the values.
1483 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1484 Ops.push_back(SDValue(Val, i));
1485 }
1486
1487 if (isa<ArrayType>(CDS->getType()))
1488 return DAG.getMergeValues(Ops, getCurSDLoc());
1489 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1490 }
1491
1492 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1493 assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1494 "Unknown struct or array constant!");
1495
1496 SmallVector<EVT, 4> ValueVTs;
1497 ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1498 unsigned NumElts = ValueVTs.size();
1499 if (NumElts == 0)
1500 return SDValue(); // empty struct
1501 SmallVector<SDValue, 4> Constants(NumElts);
1502 for (unsigned i = 0; i != NumElts; ++i) {
1503 EVT EltVT = ValueVTs[i];
1504 if (isa<UndefValue>(C))
1505 Constants[i] = DAG.getUNDEF(EltVT);
1506 else if (EltVT.isFloatingPoint())
1507 Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1508 else
1509 Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1510 }
1511
1512 return DAG.getMergeValues(Constants, getCurSDLoc());
1513 }
1514
1515 if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1516 return DAG.getBlockAddress(BA, VT);
1517
1518 VectorType *VecTy = cast<VectorType>(V->getType());
1519 unsigned NumElements = VecTy->getNumElements();
1520
1521 // Now that we know the number and type of the elements, get that number of
1522 // elements into the Ops array based on what kind of constant it is.
1523 SmallVector<SDValue, 16> Ops;
1524 if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1525 for (unsigned i = 0; i != NumElements; ++i)
1526 Ops.push_back(getValue(CV->getOperand(i)));
1527 } else {
1528 assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
1529 EVT EltVT =
1530 TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
1531
1532 SDValue Op;
1533 if (EltVT.isFloatingPoint())
1534 Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1535 else
1536 Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1537 Ops.assign(NumElements, Op);
1538 }
1539
1540 // Create a BUILD_VECTOR node.
1541 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1542 }
1543
1544 // If this is a static alloca, generate it as the frameindex instead of
1545 // computation.
1546 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1547 DenseMap<const AllocaInst*, int>::iterator SI =
1548 FuncInfo.StaticAllocaMap.find(AI);
1549 if (SI != FuncInfo.StaticAllocaMap.end())
1550 return DAG.getFrameIndex(SI->second,
1551 TLI.getFrameIndexTy(DAG.getDataLayout()));
1552 }
1553
1554 // If this is an instruction which fast-isel has deferred, select it now.
1555 if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1556 unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
1557
1558 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1559 Inst->getType(), getABIRegCopyCC(V));
1560 SDValue Chain = DAG.getEntryNode();
1561 return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1562 }
1563
1564 llvm_unreachable("Can't get register for value!");
1565}
1566
1567void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1568 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1569 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1570 bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1571 bool IsSEH = isAsynchronousEHPersonality(Pers);
1572 bool IsWasmCXX = Pers == EHPersonality::Wasm_CXX;
1573 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1574 if (!IsSEH)
1575 CatchPadMBB->setIsEHScopeEntry();
1576 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1577 if (IsMSVCCXX || IsCoreCLR)
1578 CatchPadMBB->setIsEHFuncletEntry();
1579 // Wasm does not need catchpads anymore
1580 if (!IsWasmCXX)
1581 DAG.setRoot(DAG.getNode(ISD::CATCHPAD, getCurSDLoc(), MVT::Other,
1582 getControlRoot()));
1583}
1584
1585void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1586 // Update machine-CFG edge.
1587 MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1588 FuncInfo.MBB->addSuccessor(TargetMBB);
1589
1590 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1591 bool IsSEH = isAsynchronousEHPersonality(Pers);
1592 if (IsSEH) {
1593 // If this is not a fall-through branch or optimizations are switched off,
1594 // emit the branch.
1595 if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1596 TM.getOptLevel() == CodeGenOpt::None)
1597 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1598 getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1599 return;
1600 }
1601
1602 // Figure out the funclet membership for the catchret's successor.
1603 // This will be used by the FuncletLayout pass to determine how to order the
1604 // BB's.
1605 // A 'catchret' returns to the outer scope's color.
1606 Value *ParentPad = I.getCatchSwitchParentPad();
1607 const BasicBlock *SuccessorColor;
1608 if (isa<ConstantTokenNone>(ParentPad))
1609 SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1610 else
1611 SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1612 assert(SuccessorColor && "No parent funclet for catchret!");
1613 MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1614 assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
1615
1616 // Create the terminator node.
1617 SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
1618 getControlRoot(), DAG.getBasicBlock(TargetMBB),
1619 DAG.getBasicBlock(SuccessorColorMBB));
1620 DAG.setRoot(Ret);
1621}
1622
1623void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1624 // Don't emit any special code for the cleanuppad instruction. It just marks
1625 // the start of an EH scope/funclet.
1626 FuncInfo.MBB->setIsEHScopeEntry();
1627 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1628 if (Pers != EHPersonality::Wasm_CXX) {
1629 FuncInfo.MBB->setIsEHFuncletEntry();
1630 FuncInfo.MBB->setIsCleanupFuncletEntry();
1631 }
1632}
1633
1634// For wasm, there's alwyas a single catch pad attached to a catchswitch, and
1635// the control flow always stops at the single catch pad, as it does for a
1636// cleanup pad. In case the exception caught is not of the types the catch pad
1637// catches, it will be rethrown by a rethrow.
1638static void findWasmUnwindDestinations(
1639 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1640 BranchProbability Prob,
1641 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1642 &UnwindDests) {
1643 while (EHPadBB) {
1644 const Instruction *Pad = EHPadBB->getFirstNonPHI();
1645 if (isa<CleanupPadInst>(Pad)) {
1646 // Stop on cleanup pads.
1647 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1648 UnwindDests.back().first->setIsEHScopeEntry();
1649 break;
1650 } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1651 // Add the catchpad handlers to the possible destinations. We don't
1652 // continue to the unwind destination of the catchswitch for wasm.
1653 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1654 UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1655 UnwindDests.back().first->setIsEHScopeEntry();
1656 }
1657 break;
1658 } else {
1659 continue;
1660 }
1661 }
1662}
1663
1664/// When an invoke or a cleanupret unwinds to the next EH pad, there are
1665/// many places it could ultimately go. In the IR, we have a single unwind
1666/// destination, but in the machine CFG, we enumerate all the possible blocks.
1667/// This function skips over imaginary basic blocks that hold catchswitch
1668/// instructions, and finds all the "real" machine
1669/// basic block destinations. As those destinations may not be successors of
1670/// EHPadBB, here we also calculate the edge probability to those destinations.
1671/// The passed-in Prob is the edge probability to EHPadBB.
1672static void findUnwindDestinations(
1673 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1674 BranchProbability Prob,
1675 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1676 &UnwindDests) {
1677 EHPersonality Personality =
1678 classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1679 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
1680 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
1681 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
1682 bool IsSEH = isAsynchronousEHPersonality(Personality);
1683
1684 if (IsWasmCXX) {
1685 findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);
1686 return;
1687 }
1688
1689 while (EHPadBB) {
1690 const Instruction *Pad = EHPadBB->getFirstNonPHI();
1691 BasicBlock *NewEHPadBB = nullptr;
1692 if (isa<LandingPadInst>(Pad)) {
1693 // Stop on landingpads. They are not funclets.
1694 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1695 break;
1696 } else if (isa<CleanupPadInst>(Pad)) {
1697 // Stop on cleanup pads. Cleanups are always funclet entries for all known
1698 // personalities.
1699 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1700 UnwindDests.back().first->setIsEHScopeEntry();
1701 UnwindDests.back().first->setIsEHFuncletEntry();
1702 break;
1703 } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1704 // Add the catchpad handlers to the possible destinations.
1705 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1706 UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1707 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
1708 if (IsMSVCCXX || IsCoreCLR)
1709 UnwindDests.back().first->setIsEHFuncletEntry();
1710 if (!IsSEH)
1711 UnwindDests.back().first->setIsEHScopeEntry();
1712 }
1713 NewEHPadBB = CatchSwitch->getUnwindDest();
1714 } else {
1715 continue;
1716 }
1717
1718 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1719 if (BPI && NewEHPadBB)
1720 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
1721 EHPadBB = NewEHPadBB;
1722 }
1723}
1724
1725void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
1726 // Update successor info.
1727 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
1728 auto UnwindDest = I.getUnwindDest();
1729 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1730 BranchProbability UnwindDestProb =
1731 (BPI && UnwindDest)
1732 ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
1733 : BranchProbability::getZero();
1734 findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
1735 for (auto &UnwindDest : UnwindDests) {
1736 UnwindDest.first->setIsEHPad();
1737 addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
1738 }
1739 FuncInfo.MBB->normalizeSuccProbs();
1740
1741 // Create the terminator node.
1742 SDValue Ret =
1743 DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());
1744 DAG.setRoot(Ret);
1745}
1746
1747void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
1748 report_fatal_error("visitCatchSwitch not yet implemented!");
1749}
1750
1751void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
1752 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1753 auto &DL = DAG.getDataLayout();
1754 SDValue Chain = getControlRoot();
1755 SmallVector<ISD::OutputArg, 8> Outs;
1756 SmallVector<SDValue, 8> OutVals;
1757
1758 // Calls to @llvm.experimental.deoptimize don't generate a return value, so
1759 // lower
1760 //
1761 // %val = call <ty> @llvm.experimental.deoptimize()
1762 // ret <ty> %val
1763 //
1764 // differently.
1765 if (I.getParent()->getTerminatingDeoptimizeCall()) {
1766 LowerDeoptimizingReturn();
1767 return;
1768 }
1769
1770 if (!FuncInfo.CanLowerReturn) {
1771 unsigned DemoteReg = FuncInfo.DemoteRegister;
1772 const Function *F = I.getParent()->getParent();
1773
1774 // Emit a store of the return value through the virtual register.
1775 // Leave Outs empty so that LowerReturn won't try to load return
1776 // registers the usual way.
1777 SmallVector<EVT, 1> PtrValueVTs;
1778 ComputeValueVTs(TLI, DL,
1779 F->getReturnType()->getPointerTo(
1780 DAG.getDataLayout().getAllocaAddrSpace()),
1781 PtrValueVTs);
1782
1783 SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
1784 DemoteReg, PtrValueVTs[0]);
1785 SDValue RetOp = getValue(I.getOperand(0));
1786
1787 SmallVector<EVT, 4> ValueVTs;
1788 SmallVector<uint64_t, 4> Offsets;
1789 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &Offsets);
1790 unsigned NumValues = ValueVTs.size();
1791
1792 SmallVector<SDValue, 4> Chains(NumValues);
1793 for (unsigned i = 0; i != NumValues; ++i) {
1794 // An aggregate return value cannot wrap around the address space, so
1795 // offsets to its parts don't wrap either.
1796 SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr, Offsets[i]);
1797 Chains[i] = DAG.getStore(
1798 Chain, getCurSDLoc(), SDValue(RetOp.getNode(), RetOp.getResNo() + i),
1799 // FIXME: better loc info would be nice.
1800 Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
1801 }
1802
1803 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
1804 MVT::Other, Chains);
1805 } else if (I.getNumOperands() != 0) {
1806 SmallVector<EVT, 4> ValueVTs;
1807 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
1808 unsigned NumValues = ValueVTs.size();
1809 if (NumValues) {
1810 SDValue RetOp = getValue(I.getOperand(0));
1811
1812 const Function *F = I.getParent()->getParent();
1813
1814 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1815 if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1816 Attribute::SExt))
1817 ExtendKind = ISD::SIGN_EXTEND;
1818 else if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1819 Attribute::ZExt))
1820 ExtendKind = ISD::ZERO_EXTEND;
1821
1822 LLVMContext &Context = F->getContext();
1823 bool RetInReg = F->getAttributes().hasAttribute(
1824 AttributeList::ReturnIndex, Attribute::InReg);
1825
1826 for (unsigned j = 0; j != NumValues; ++j) {
1827 EVT VT = ValueVTs[j];
1828
1829 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
1830 VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
1831
1832 CallingConv::ID CC = F->getCallingConv();
1833
1834 unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
1835 MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
1836 SmallVector<SDValue, 4> Parts(NumParts);
1837 getCopyToParts(DAG, getCurSDLoc(),
1838 SDValue(RetOp.getNode(), RetOp.getResNo() + j),
1839 &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
1840
1841 // 'inreg' on function refers to return value
1842 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1843 if (RetInReg)
1844 Flags.setInReg();
1845
1846 // Propagate extension type if any
1847 if (ExtendKind == ISD::SIGN_EXTEND)
1848 Flags.setSExt();
1849 else if (ExtendKind == ISD::ZERO_EXTEND)
1850 Flags.setZExt();
1851
1852 for (unsigned i = 0; i < NumParts; ++i) {
1853 Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
1854 VT, /*isfixed=*/true, 0, 0));
1855 OutVals.push_back(Parts[i]);
1856 }
1857 }
1858 }
1859 }
1860
1861 // Push in swifterror virtual register as the last element of Outs. This makes
1862 // sure swifterror virtual register will be returned in the swifterror
1863 // physical register.
1864 const Function *F = I.getParent()->getParent();
1865 if (TLI.supportSwiftError() &&
1866 F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
1867 assert(FuncInfo.SwiftErrorArg && "Need a swift error argument");
1868 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1869 Flags.setSwiftError();
1870 Outs.push_back(ISD::OutputArg(Flags, EVT(TLI.getPointerTy(DL)) /*vt*/,
1871 EVT(TLI.getPointerTy(DL)) /*argvt*/,
1872 true /*isfixed*/, 1 /*origidx*/,
1873 0 /*partOffs*/));
1874 // Create SDNode for the swifterror virtual register.
1875 OutVals.push_back(
1876 DAG.getRegister(FuncInfo.getOrCreateSwiftErrorVRegUseAt(
1877 &I, FuncInfo.MBB, FuncInfo.SwiftErrorArg).first,
1878 EVT(TLI.getPointerTy(DL))));
1879 }
1880
1881 bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
1882 CallingConv::ID CallConv =
1883 DAG.getMachineFunction().getFunction().getCallingConv();
1884 Chain = DAG.getTargetLoweringInfo().LowerReturn(
1885 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
1886
1887 // Verify that the target's LowerReturn behaved as expected.
1888 assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
1889 "LowerReturn didn't return a valid chain!");
1890
1891 // Update the DAG with the new chain value resulting from return lowering.
1892 DAG.setRoot(Chain);
1893}
1894
1895/// CopyToExportRegsIfNeeded - If the given value has virtual registers
1896/// created for it, emit nodes to copy the value into the virtual
1897/// registers.
1898void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
1899 // Skip empty types
1900 if (V->getType()->isEmptyTy())
1901 return;
1902
1903 DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.