1//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::Legalize method.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/APFloat.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/FloatingPointMode.h"
17#include "llvm/ADT/SetVector.h"
18#include "llvm/ADT/SmallPtrSet.h"
19#include "llvm/ADT/SmallSet.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/Analysis/ConstantFolding.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/CodeGen/ISDOpcodes.h"
24#include "llvm/CodeGen/MachineFrameInfo.h"
25#include "llvm/CodeGen/MachineFunction.h"
26#include "llvm/CodeGen/MachineJumpTableInfo.h"
27#include "llvm/CodeGen/MachineMemOperand.h"
28#include "llvm/CodeGen/RuntimeLibcalls.h"
29#include "llvm/CodeGen/SelectionDAG.h"
30#include "llvm/CodeGen/SelectionDAGNodes.h"
31#include "llvm/CodeGen/TargetFrameLowering.h"
32#include "llvm/CodeGen/TargetLowering.h"
33#include "llvm/CodeGen/TargetSubtargetInfo.h"
34#include "llvm/CodeGen/ValueTypes.h"
35#include "llvm/CodeGenTypes/MachineValueType.h"
36#include "llvm/IR/CallingConv.h"
37#include "llvm/IR/Constants.h"
38#include "llvm/IR/DataLayout.h"
39#include "llvm/IR/DerivedTypes.h"
40#include "llvm/IR/Function.h"
41#include "llvm/IR/Metadata.h"
42#include "llvm/IR/Type.h"
43#include "llvm/Support/Casting.h"
44#include "llvm/Support/Compiler.h"
45#include "llvm/Support/Debug.h"
46#include "llvm/Support/ErrorHandling.h"
47#include "llvm/Support/MathExtras.h"
48#include "llvm/Support/raw_ostream.h"
49#include "llvm/Target/TargetMachine.h"
50#include "llvm/Target/TargetOptions.h"
51#include <cassert>
52#include <cstdint>
53#include <tuple>
54#include <utility>
55
56using namespace llvm;
57
58#define DEBUG_TYPE "legalizedag"
59
60namespace {
61
62/// Keeps track of state when getting the sign of a floating-point value as an
63/// integer.
64struct FloatSignAsInt {
65 EVT FloatVT;
66 SDValue Chain;
67 SDValue FloatPtr;
68 SDValue IntPtr;
69 MachinePointerInfo IntPointerInfo;
70 MachinePointerInfo FloatPointerInfo;
71 SDValue IntValue;
72 APInt SignMask;
73 uint8_t SignBit;
74};
75
76//===----------------------------------------------------------------------===//
77/// This takes an arbitrary SelectionDAG as input and
78/// hacks on it until the target machine can handle it. This involves
79/// eliminating value sizes the machine cannot handle (promoting small sizes to
80/// large sizes or splitting up large values into small values) as well as
81/// eliminating operations the machine cannot handle.
82///
83/// This code also does a small amount of optimization and recognition of idioms
84/// as part of its processing. For example, if a target does not support a
85/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
86/// will attempt merge setcc and brc instructions into brcc's.
87class SelectionDAGLegalize {
88 const TargetMachine &TM;
89 const TargetLowering &TLI;
90 SelectionDAG &DAG;
91
92 /// The set of nodes which have already been legalized. We hold a
93 /// reference to it in order to update as necessary on node deletion.
94 SmallPtrSetImpl<SDNode *> &LegalizedNodes;
95
96 /// A set of all the nodes updated during legalization.
97 SmallSetVector<SDNode *, 16> *UpdatedNodes;
98
99 EVT getSetCCResultType(EVT VT) const {
100 return TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
101 }
102
103 // Libcall insertion helpers.
104
105public:
106 SelectionDAGLegalize(SelectionDAG &DAG,
107 SmallPtrSetImpl<SDNode *> &LegalizedNodes,
108 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
109 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
110 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
111
112 /// Legalizes the given operation.
113 void LegalizeOp(SDNode *Node);
114
115private:
116 SDValue OptimizeFloatStore(StoreSDNode *ST);
117
118 void LegalizeLoadOps(SDNode *Node);
119 void LegalizeStoreOps(SDNode *Node);
120
121 /// Some targets cannot handle a variable
122 /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it
123 /// is necessary to spill the vector being inserted into to memory, perform
124 /// the insert there, and then read the result back.
125 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
126 const SDLoc &dl);
127 SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
128 const SDLoc &dl);
129
130 /// Return a vector shuffle operation which
131 /// performs the same shuffe in terms of order or result bytes, but on a type
132 /// whose vector element type is narrower than the original shuffle type.
133 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
134 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
135 SDValue N1, SDValue N2,
136 ArrayRef<int> Mask) const;
137
138 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
139 TargetLowering::ArgListTy &&Args, bool isSigned);
140 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
141
142 void ExpandFrexpLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
143 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
144 SmallVectorImpl<SDValue> &Results);
145 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
146 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
147 RTLIB::Libcall Call_F128,
148 RTLIB::Libcall Call_PPCF128,
149 SmallVectorImpl<SDValue> &Results);
150 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
151 RTLIB::Libcall Call_I8,
152 RTLIB::Libcall Call_I16,
153 RTLIB::Libcall Call_I32,
154 RTLIB::Libcall Call_I64,
155 RTLIB::Libcall Call_I128);
156 void ExpandArgFPLibCall(SDNode *Node,
157 RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
158 RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
159 RTLIB::Libcall Call_PPCF128,
160 SmallVectorImpl<SDValue> &Results);
161 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
162 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
163
164 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
165 const SDLoc &dl);
166 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
167 const SDLoc &dl, SDValue ChainIn);
168 SDValue ExpandBUILD_VECTOR(SDNode *Node);
169 SDValue ExpandSPLAT_VECTOR(SDNode *Node);
170 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
171 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
172 SmallVectorImpl<SDValue> &Results);
173 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
174 SDValue Value) const;
175 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
176 SDValue NewIntValue) const;
177 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
178 SDValue ExpandFABS(SDNode *Node) const;
179 SDValue ExpandFNEG(SDNode *Node) const;
180 SDValue expandLdexp(SDNode *Node) const;
181 SDValue expandFrexp(SDNode *Node) const;
182
183 SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
184 void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl,
185 SmallVectorImpl<SDValue> &Results);
186 void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
187 SmallVectorImpl<SDValue> &Results);
188 SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
189
190 SDValue ExpandPARITY(SDValue Op, const SDLoc &dl);
191
192 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
193 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
194 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
195
196 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
197 SDValue ExpandConstant(ConstantSDNode *CP);
198
199 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
200 bool ExpandNode(SDNode *Node);
201 void ConvertNodeToLibcall(SDNode *Node);
202 void PromoteNode(SDNode *Node);
203
204public:
205 // Node replacement helpers
206
207 void ReplacedNode(SDNode *N) {
208 LegalizedNodes.erase(Ptr: N);
209 if (UpdatedNodes)
210 UpdatedNodes->insert(X: N);
211 }
212
213 void ReplaceNode(SDNode *Old, SDNode *New) {
214 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
215 dbgs() << " with: "; New->dump(&DAG));
216
217 assert(Old->getNumValues() == New->getNumValues() &&
218 "Replacing one node with another that produces a different number "
219 "of values!");
220 DAG.ReplaceAllUsesWith(From: Old, To: New);
221 if (UpdatedNodes)
222 UpdatedNodes->insert(X: New);
223 ReplacedNode(N: Old);
224 }
225
226 void ReplaceNode(SDValue Old, SDValue New) {
227 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
228 dbgs() << " with: "; New->dump(&DAG));
229
230 DAG.ReplaceAllUsesWith(From: Old, To: New);
231 if (UpdatedNodes)
232 UpdatedNodes->insert(X: New.getNode());
233 ReplacedNode(N: Old.getNode());
234 }
235
236 void ReplaceNode(SDNode *Old, const SDValue *New) {
237 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
238
239 DAG.ReplaceAllUsesWith(From: Old, To: New);
240 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
241 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");
242 New[i]->dump(&DAG));
243 if (UpdatedNodes)
244 UpdatedNodes->insert(X: New[i].getNode());
245 }
246 ReplacedNode(N: Old);
247 }
248
249 void ReplaceNodeWithValue(SDValue Old, SDValue New) {
250 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
251 dbgs() << " with: "; New->dump(&DAG));
252
253 DAG.ReplaceAllUsesOfValueWith(From: Old, To: New);
254 if (UpdatedNodes)
255 UpdatedNodes->insert(X: New.getNode());
256 ReplacedNode(N: Old.getNode());
257 }
258};
259
260} // end anonymous namespace
261
262/// Return a vector shuffle operation which
263/// performs the same shuffle in terms of order or result bytes, but on a type
264/// whose vector element type is narrower than the original shuffle type.
265/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
266SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
267 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
268 ArrayRef<int> Mask) const {
269 unsigned NumMaskElts = VT.getVectorNumElements();
270 unsigned NumDestElts = NVT.getVectorNumElements();
271 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
272
273 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
274
275 if (NumEltsGrowth == 1)
276 return DAG.getVectorShuffle(VT: NVT, dl, N1, N2, Mask);
277
278 SmallVector<int, 8> NewMask;
279 for (unsigned i = 0; i != NumMaskElts; ++i) {
280 int Idx = Mask[i];
281 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
282 if (Idx < 0)
283 NewMask.push_back(Elt: -1);
284 else
285 NewMask.push_back(Elt: Idx * NumEltsGrowth + j);
286 }
287 }
288 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
289 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
290 return DAG.getVectorShuffle(VT: NVT, dl, N1, N2, Mask: NewMask);
291}
292
293/// Expands the ConstantFP node to an integer constant or
294/// a load from the constant pool.
295SDValue
296SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
297 bool Extend = false;
298 SDLoc dl(CFP);
299
300 // If a FP immediate is precise when represented as a float and if the
301 // target can do an extending load from float to double, we put it into
302 // the constant pool as a float, even if it's is statically typed as a
303 // double. This shrinks FP constants and canonicalizes them for targets where
304 // an FP extending load is the same cost as a normal load (such as on the x87
305 // fp stack or PPC FP unit).
306 EVT VT = CFP->getValueType(ResNo: 0);
307 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
308 if (!UseCP) {
309 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
310 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
311 (VT == MVT::f64) ? MVT::i64 : MVT::i32);
312 }
313
314 APFloat APF = CFP->getValueAPF();
315 EVT OrigVT = VT;
316 EVT SVT = VT;
317
318 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
319 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
320 if (!APF.isSignaling()) {
321 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
322 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
323 if (ConstantFPSDNode::isValueValidForType(VT: SVT, Val: APF) &&
324 // Only do this if the target has a native EXTLOAD instruction from
325 // smaller type.
326 TLI.isLoadExtLegal(ExtType: ISD::EXTLOAD, ValVT: OrigVT, MemVT: SVT) &&
327 TLI.ShouldShrinkFPConstant(OrigVT)) {
328 Type *SType = SVT.getTypeForEVT(Context&: *DAG.getContext());
329 LLVMC = cast<ConstantFP>(Val: ConstantFoldCastOperand(
330 Opcode: Instruction::FPTrunc, C: LLVMC, DestTy: SType, DL: DAG.getDataLayout()));
331 VT = SVT;
332 Extend = true;
333 }
334 }
335 }
336
337 SDValue CPIdx =
338 DAG.getConstantPool(C: LLVMC, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
339 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
340 if (Extend) {
341 SDValue Result = DAG.getExtLoad(
342 ExtType: ISD::EXTLOAD, dl, VT: OrigVT, Chain: DAG.getEntryNode(), Ptr: CPIdx,
343 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), MemVT: VT,
344 Alignment);
345 return Result;
346 }
347 SDValue Result = DAG.getLoad(
348 VT: OrigVT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
349 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), Alignment);
350 return Result;
351}
352
353/// Expands the Constant node to a load from the constant pool.
354SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
355 SDLoc dl(CP);
356 EVT VT = CP->getValueType(ResNo: 0);
357 SDValue CPIdx = DAG.getConstantPool(C: CP->getConstantIntValue(),
358 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
359 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
360 SDValue Result = DAG.getLoad(
361 VT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
362 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), Alignment);
363 return Result;
364}
365
366/// Some target cannot handle a variable insertion index for the
367/// INSERT_VECTOR_ELT instruction. In this case, it
368/// is necessary to spill the vector being inserted into to memory, perform
369/// the insert there, and then read the result back.
370SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
371 SDValue Val,
372 SDValue Idx,
373 const SDLoc &dl) {
374 SDValue Tmp1 = Vec;
375 SDValue Tmp2 = Val;
376 SDValue Tmp3 = Idx;
377
378 // If the target doesn't support this, we have to spill the input vector
379 // to a temporary stack slot, update the element, then reload it. This is
380 // badness. We could also load the value into a vector register (either
381 // with a "move to register" or "extload into register" instruction, then
382 // permute it into place, if the idx is a constant and if the idx is
383 // supported by the target.
384 EVT VT = Tmp1.getValueType();
385 EVT EltVT = VT.getVectorElementType();
386 SDValue StackPtr = DAG.CreateStackTemporary(VT);
387
388 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
389
390 // Store the vector.
391 SDValue Ch = DAG.getStore(
392 Chain: DAG.getEntryNode(), dl, Val: Tmp1, Ptr: StackPtr,
393 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
394
395 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT: VT, Index: Tmp3);
396
397 // Store the scalar value.
398 Ch = DAG.getTruncStore(
399 Chain: Ch, dl, Val: Tmp2, Ptr: StackPtr2,
400 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()), SVT: EltVT);
401 // Load the updated vector.
402 return DAG.getLoad(VT, dl, Chain: Ch, Ptr: StackPtr, PtrInfo: MachinePointerInfo::getFixedStack(
403 MF&: DAG.getMachineFunction(), FI: SPFI));
404}
405
406SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
407 SDValue Idx,
408 const SDLoc &dl) {
409 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Val&: Idx)) {
410 // SCALAR_TO_VECTOR requires that the type of the value being inserted
411 // match the element type of the vector being created, except for
412 // integers in which case the inserted value can be over width.
413 EVT EltVT = Vec.getValueType().getVectorElementType();
414 if (Val.getValueType() == EltVT ||
415 (EltVT.isInteger() && Val.getValueType().bitsGE(VT: EltVT))) {
416 SDValue ScVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl,
417 VT: Vec.getValueType(), Operand: Val);
418
419 unsigned NumElts = Vec.getValueType().getVectorNumElements();
420 // We generate a shuffle of InVec and ScVec, so the shuffle mask
421 // should be 0,1,2,3,4,5... with the appropriate element replaced with
422 // elt 0 of the RHS.
423 SmallVector<int, 8> ShufOps;
424 for (unsigned i = 0; i != NumElts; ++i)
425 ShufOps.push_back(Elt: i != InsertPos->getZExtValue() ? i : NumElts);
426
427 return DAG.getVectorShuffle(VT: Vec.getValueType(), dl, N1: Vec, N2: ScVec, Mask: ShufOps);
428 }
429 }
430 return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
431}
432
433SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
434 if (!ISD::isNormalStore(N: ST))
435 return SDValue();
436
437 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
438 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
439 // FIXME: move this to the DAG Combiner! Note that we can't regress due
440 // to phase ordering between legalized code and the dag combiner. This
441 // probably means that we need to integrate dag combiner and legalizer
442 // together.
443 // We generally can't do this one for long doubles.
444 SDValue Chain = ST->getChain();
445 SDValue Ptr = ST->getBasePtr();
446 SDValue Value = ST->getValue();
447 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
448 AAMDNodes AAInfo = ST->getAAInfo();
449 SDLoc dl(ST);
450
451 // Don't optimise TargetConstantFP
452 if (Value.getOpcode() == ISD::TargetConstantFP)
453 return SDValue();
454
455 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val&: Value)) {
456 if (CFP->getValueType(ResNo: 0) == MVT::f32 &&
457 TLI.isTypeLegal(MVT::VT: i32)) {
458 SDValue Con = DAG.getConstant(CFP->getValueAPF().
459 bitcastToAPInt().zextOrTrunc(width: 32),
460 SDLoc(CFP), MVT::i32);
461 return DAG.getStore(Chain, dl, Val: Con, Ptr, PtrInfo: ST->getPointerInfo(),
462 Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
463 }
464
465 if (CFP->getValueType(ResNo: 0) == MVT::f64 &&
466 !TLI.isFPImmLegal(CFP->getValueAPF(), MVT::f64)) {
467 // If this target supports 64-bit registers, do a single 64-bit store.
468 if (TLI.isTypeLegal(MVT::VT: i64)) {
469 SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
470 zextOrTrunc(width: 64), SDLoc(CFP), MVT::i64);
471 return DAG.getStore(Chain, dl, Val: Con, Ptr, PtrInfo: ST->getPointerInfo(),
472 Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
473 }
474
475 if (TLI.isTypeLegal(MVT::VT: i32) && !ST->isVolatile()) {
476 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
477 // stores. If the target supports neither 32- nor 64-bits, this
478 // xform is certainly not worth it.
479 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
480 SDValue Lo = DAG.getConstant(IntVal.trunc(width: 32), dl, MVT::i32);
481 SDValue Hi = DAG.getConstant(IntVal.lshr(shiftAmt: 32).trunc(width: 32), dl, MVT::i32);
482 if (DAG.getDataLayout().isBigEndian())
483 std::swap(a&: Lo, b&: Hi);
484
485 Lo = DAG.getStore(Chain, dl, Val: Lo, Ptr, PtrInfo: ST->getPointerInfo(),
486 Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
487 Ptr = DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: 4), DL: dl);
488 Hi = DAG.getStore(Chain, dl, Val: Hi, Ptr,
489 PtrInfo: ST->getPointerInfo().getWithOffset(O: 4),
490 Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
491
492 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
493 }
494 }
495 }
496 return SDValue();
497}
498
499void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
500 StoreSDNode *ST = cast<StoreSDNode>(Val: Node);
501 SDValue Chain = ST->getChain();
502 SDValue Ptr = ST->getBasePtr();
503 SDLoc dl(Node);
504
505 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
506 AAMDNodes AAInfo = ST->getAAInfo();
507
508 if (!ST->isTruncatingStore()) {
509 LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
510 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
511 ReplaceNode(Old: ST, New: OptStore);
512 return;
513 }
514
515 SDValue Value = ST->getValue();
516 MVT VT = Value.getSimpleValueType();
517 switch (TLI.getOperationAction(Op: ISD::STORE, VT)) {
518 default: llvm_unreachable("This action is not supported yet!");
519 case TargetLowering::Legal: {
520 // If this is an unaligned store and the target doesn't support it,
521 // expand it.
522 EVT MemVT = ST->getMemoryVT();
523 const DataLayout &DL = DAG.getDataLayout();
524 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
525 MMO: *ST->getMemOperand())) {
526 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
527 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
528 ReplaceNode(Old: SDValue(ST, 0), New: Result);
529 } else
530 LLVM_DEBUG(dbgs() << "Legal store\n");
531 break;
532 }
533 case TargetLowering::Custom: {
534 LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
535 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
536 if (Res && Res != SDValue(Node, 0))
537 ReplaceNode(Old: SDValue(Node, 0), New: Res);
538 return;
539 }
540 case TargetLowering::Promote: {
541 MVT NVT = TLI.getTypeToPromoteTo(Op: ISD::STORE, VT);
542 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
543 "Can only promote stores to same size type");
544 Value = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Value);
545 SDValue Result = DAG.getStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
546 Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
547 ReplaceNode(Old: SDValue(Node, 0), New: Result);
548 break;
549 }
550 }
551 return;
552 }
553
554 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
555 SDValue Value = ST->getValue();
556 EVT StVT = ST->getMemoryVT();
557 TypeSize StWidth = StVT.getSizeInBits();
558 TypeSize StSize = StVT.getStoreSizeInBits();
559 auto &DL = DAG.getDataLayout();
560
561 if (StWidth != StSize) {
562 // Promote to a byte-sized store with upper bits zero if not
563 // storing an integral number of bytes. For example, promote
564 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
565 EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StSize.getFixedValue());
566 Value = DAG.getZeroExtendInReg(Op: Value, DL: dl, VT: StVT);
567 SDValue Result =
568 DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(), SVT: NVT,
569 Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
570 ReplaceNode(Old: SDValue(Node, 0), New: Result);
571 } else if (!StVT.isVector() && !isPowerOf2_64(Value: StWidth.getFixedValue())) {
572 // If not storing a power-of-2 number of bits, expand as two stores.
573 assert(!StVT.isVector() && "Unsupported truncstore!");
574 unsigned StWidthBits = StWidth.getFixedValue();
575 unsigned LogStWidth = Log2_32(Value: StWidthBits);
576 assert(LogStWidth < 32);
577 unsigned RoundWidth = 1 << LogStWidth;
578 assert(RoundWidth < StWidthBits);
579 unsigned ExtraWidth = StWidthBits - RoundWidth;
580 assert(ExtraWidth < RoundWidth);
581 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
582 "Store size not an integral number of bytes!");
583 EVT RoundVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundWidth);
584 EVT ExtraVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExtraWidth);
585 SDValue Lo, Hi;
586 unsigned IncrementSize;
587
588 if (DL.isLittleEndian()) {
589 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
590 // Store the bottom RoundWidth bits.
591 Lo = DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
592 SVT: RoundVT, Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
593
594 // Store the remaining ExtraWidth bits.
595 IncrementSize = RoundWidth / 8;
596 Ptr =
597 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
598 Hi = DAG.getNode(
599 Opcode: ISD::SRL, DL: dl, VT: Value.getValueType(), N1: Value,
600 N2: DAG.getConstant(Val: RoundWidth, DL: dl,
601 VT: TLI.getShiftAmountTy(LHSTy: Value.getValueType(), DL)));
602 Hi = DAG.getTruncStore(Chain, dl, Val: Hi, Ptr,
603 PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize),
604 SVT: ExtraVT, Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
605 } else {
606 // Big endian - avoid unaligned stores.
607 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
608 // Store the top RoundWidth bits.
609 Hi = DAG.getNode(
610 Opcode: ISD::SRL, DL: dl, VT: Value.getValueType(), N1: Value,
611 N2: DAG.getConstant(Val: ExtraWidth, DL: dl,
612 VT: TLI.getShiftAmountTy(LHSTy: Value.getValueType(), DL)));
613 Hi = DAG.getTruncStore(Chain, dl, Val: Hi, Ptr, PtrInfo: ST->getPointerInfo(), SVT: RoundVT,
614 Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
615
616 // Store the remaining ExtraWidth bits.
617 IncrementSize = RoundWidth / 8;
618 Ptr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Ptr.getValueType(), N1: Ptr,
619 N2: DAG.getConstant(Val: IncrementSize, DL: dl,
620 VT: Ptr.getValueType()));
621 Lo = DAG.getTruncStore(Chain, dl, Val: Value, Ptr,
622 PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize),
623 SVT: ExtraVT, Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
624 }
625
626 // The order of the stores doesn't matter.
627 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
628 ReplaceNode(Old: SDValue(Node, 0), New: Result);
629 } else {
630 switch (TLI.getTruncStoreAction(ValVT: ST->getValue().getValueType(), MemVT: StVT)) {
631 default: llvm_unreachable("This action is not supported yet!");
632 case TargetLowering::Legal: {
633 EVT MemVT = ST->getMemoryVT();
634 // If this is an unaligned store and the target doesn't support it,
635 // expand it.
636 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
637 MMO: *ST->getMemOperand())) {
638 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
639 ReplaceNode(Old: SDValue(ST, 0), New: Result);
640 }
641 break;
642 }
643 case TargetLowering::Custom: {
644 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
645 if (Res && Res != SDValue(Node, 0))
646 ReplaceNode(Old: SDValue(Node, 0), New: Res);
647 return;
648 }
649 case TargetLowering::Expand:
650 assert(!StVT.isVector() &&
651 "Vector Stores are handled in LegalizeVectorOps");
652
653 SDValue Result;
654
655 // TRUNCSTORE:i16 i32 -> STORE i16
656 if (TLI.isTypeLegal(VT: StVT)) {
657 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: StVT, Operand: Value);
658 Result = DAG.getStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
659 Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
660 } else {
661 // The in-memory type isn't legal. Truncate to the type it would promote
662 // to, and then do a truncstore.
663 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl,
664 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StVT),
665 Operand: Value);
666 Result =
667 DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(), SVT: StVT,
668 Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo);
669 }
670
671 ReplaceNode(Old: SDValue(Node, 0), New: Result);
672 break;
673 }
674 }
675}
676
677void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
678 LoadSDNode *LD = cast<LoadSDNode>(Val: Node);
679 SDValue Chain = LD->getChain(); // The chain.
680 SDValue Ptr = LD->getBasePtr(); // The base pointer.
681 SDValue Value; // The value returned by the load op.
682 SDLoc dl(Node);
683
684 ISD::LoadExtType ExtType = LD->getExtensionType();
685 if (ExtType == ISD::NON_EXTLOAD) {
686 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
687 MVT VT = Node->getSimpleValueType(ResNo: 0);
688 SDValue RVal = SDValue(Node, 0);
689 SDValue RChain = SDValue(Node, 1);
690
691 switch (TLI.getOperationAction(Op: Node->getOpcode(), VT)) {
692 default: llvm_unreachable("This action is not supported yet!");
693 case TargetLowering::Legal: {
694 EVT MemVT = LD->getMemoryVT();
695 const DataLayout &DL = DAG.getDataLayout();
696 // If this is an unaligned load and the target doesn't support it,
697 // expand it.
698 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
699 MMO: *LD->getMemOperand())) {
700 std::tie(args&: RVal, args&: RChain) = TLI.expandUnalignedLoad(LD, DAG);
701 }
702 break;
703 }
704 case TargetLowering::Custom:
705 if (SDValue Res = TLI.LowerOperation(Op: RVal, DAG)) {
706 RVal = Res;
707 RChain = Res.getValue(R: 1);
708 }
709 break;
710
711 case TargetLowering::Promote: {
712 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
713 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
714 "Can only promote loads to same size type");
715
716 SDValue Res = DAG.getLoad(VT: NVT, dl, Chain, Ptr, MMO: LD->getMemOperand());
717 RVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Res);
718 RChain = Res.getValue(R: 1);
719 break;
720 }
721 }
722 if (RChain.getNode() != Node) {
723 assert(RVal.getNode() != Node && "Load must be completely replaced");
724 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: RVal);
725 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: RChain);
726 if (UpdatedNodes) {
727 UpdatedNodes->insert(X: RVal.getNode());
728 UpdatedNodes->insert(X: RChain.getNode());
729 }
730 ReplacedNode(N: Node);
731 }
732 return;
733 }
734
735 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
736 EVT SrcVT = LD->getMemoryVT();
737 TypeSize SrcWidth = SrcVT.getSizeInBits();
738 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
739 AAMDNodes AAInfo = LD->getAAInfo();
740
741 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
742 // Some targets pretend to have an i1 loading operation, and actually
743 // load an i8. This trick is correct for ZEXTLOAD because the top 7
744 // bits are guaranteed to be zero; it helps the optimizers understand
745 // that these bits are zero. It is also useful for EXTLOAD, since it
746 // tells the optimizers that those bits are undefined. It would be
747 // nice to have an effective generic way of getting these benefits...
748 // Until such a way is found, don't insist on promoting i1 here.
749 (SrcVT != MVT::i1 ||
750 TLI.getLoadExtAction(ExtType, ValVT: Node->getValueType(ResNo: 0), MVT::MemVT: i1) ==
751 TargetLowering::Promote)) {
752 // Promote to a byte-sized load if not loading an integral number of
753 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
754 unsigned NewWidth = SrcVT.getStoreSizeInBits();
755 EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NewWidth);
756 SDValue Ch;
757
758 // The extra bits are guaranteed to be zero, since we stored them that
759 // way. A zext load from NVT thus automatically gives zext from SrcVT.
760
761 ISD::LoadExtType NewExtType =
762 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
763
764 SDValue Result = DAG.getExtLoad(ExtType: NewExtType, dl, VT: Node->getValueType(ResNo: 0),
765 Chain, Ptr, PtrInfo: LD->getPointerInfo(), MemVT: NVT,
766 Alignment: LD->getOriginalAlign(), MMOFlags, AAInfo);
767
768 Ch = Result.getValue(R: 1); // The chain.
769
770 if (ExtType == ISD::SEXTLOAD)
771 // Having the top bits zero doesn't help when sign extending.
772 Result = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl,
773 VT: Result.getValueType(),
774 N1: Result, N2: DAG.getValueType(SrcVT));
775 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
776 // All the top bits are guaranteed to be zero - inform the optimizers.
777 Result = DAG.getNode(Opcode: ISD::AssertZext, DL: dl,
778 VT: Result.getValueType(), N1: Result,
779 N2: DAG.getValueType(SrcVT));
780
781 Value = Result;
782 Chain = Ch;
783 } else if (!isPowerOf2_64(Value: SrcWidth.getKnownMinValue())) {
784 // If not loading a power-of-2 number of bits, expand as two loads.
785 assert(!SrcVT.isVector() && "Unsupported extload!");
786 unsigned SrcWidthBits = SrcWidth.getFixedValue();
787 unsigned LogSrcWidth = Log2_32(Value: SrcWidthBits);
788 assert(LogSrcWidth < 32);
789 unsigned RoundWidth = 1 << LogSrcWidth;
790 assert(RoundWidth < SrcWidthBits);
791 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
792 assert(ExtraWidth < RoundWidth);
793 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
794 "Load size not an integral number of bytes!");
795 EVT RoundVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundWidth);
796 EVT ExtraVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExtraWidth);
797 SDValue Lo, Hi, Ch;
798 unsigned IncrementSize;
799 auto &DL = DAG.getDataLayout();
800
801 if (DL.isLittleEndian()) {
802 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
803 // Load the bottom RoundWidth bits.
804 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
805 PtrInfo: LD->getPointerInfo(), MemVT: RoundVT, Alignment: LD->getOriginalAlign(),
806 MMOFlags, AAInfo);
807
808 // Load the remaining ExtraWidth bits.
809 IncrementSize = RoundWidth / 8;
810 Ptr =
811 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
812 Hi = DAG.getExtLoad(ExtType, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
813 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
814 MemVT: ExtraVT, Alignment: LD->getOriginalAlign(), MMOFlags, AAInfo);
815
816 // Build a factor node to remember that this load is independent of
817 // the other one.
818 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(R: 1),
819 Hi.getValue(R: 1));
820
821 // Move the top bits to the right place.
822 Hi = DAG.getNode(
823 Opcode: ISD::SHL, DL: dl, VT: Hi.getValueType(), N1: Hi,
824 N2: DAG.getConstant(Val: RoundWidth, DL: dl,
825 VT: TLI.getShiftAmountTy(LHSTy: Hi.getValueType(), DL)));
826
827 // Join the hi and lo parts.
828 Value = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Lo, N2: Hi);
829 } else {
830 // Big endian - avoid unaligned loads.
831 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
832 // Load the top RoundWidth bits.
833 Hi = DAG.getExtLoad(ExtType, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
834 PtrInfo: LD->getPointerInfo(), MemVT: RoundVT, Alignment: LD->getOriginalAlign(),
835 MMOFlags, AAInfo);
836
837 // Load the remaining ExtraWidth bits.
838 IncrementSize = RoundWidth / 8;
839 Ptr =
840 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
841 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
842 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
843 MemVT: ExtraVT, Alignment: LD->getOriginalAlign(), MMOFlags, AAInfo);
844
845 // Build a factor node to remember that this load is independent of
846 // the other one.
847 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(R: 1),
848 Hi.getValue(R: 1));
849
850 // Move the top bits to the right place.
851 Hi = DAG.getNode(
852 Opcode: ISD::SHL, DL: dl, VT: Hi.getValueType(), N1: Hi,
853 N2: DAG.getConstant(Val: ExtraWidth, DL: dl,
854 VT: TLI.getShiftAmountTy(LHSTy: Hi.getValueType(), DL)));
855
856 // Join the hi and lo parts.
857 Value = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Lo, N2: Hi);
858 }
859
860 Chain = Ch;
861 } else {
862 bool isCustom = false;
863 switch (TLI.getLoadExtAction(ExtType, ValVT: Node->getValueType(ResNo: 0),
864 MemVT: SrcVT.getSimpleVT())) {
865 default: llvm_unreachable("This action is not supported yet!");
866 case TargetLowering::Custom:
867 isCustom = true;
868 [[fallthrough]];
869 case TargetLowering::Legal:
870 Value = SDValue(Node, 0);
871 Chain = SDValue(Node, 1);
872
873 if (isCustom) {
874 if (SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG)) {
875 Value = Res;
876 Chain = Res.getValue(R: 1);
877 }
878 } else {
879 // If this is an unaligned load and the target doesn't support it,
880 // expand it.
881 EVT MemVT = LD->getMemoryVT();
882 const DataLayout &DL = DAG.getDataLayout();
883 if (!TLI.allowsMemoryAccess(Context&: *DAG.getContext(), DL, VT: MemVT,
884 MMO: *LD->getMemOperand())) {
885 std::tie(args&: Value, args&: Chain) = TLI.expandUnalignedLoad(LD, DAG);
886 }
887 }
888 break;
889
890 case TargetLowering::Expand: {
891 EVT DestVT = Node->getValueType(ResNo: 0);
892 if (!TLI.isLoadExtLegal(ExtType: ISD::EXTLOAD, ValVT: DestVT, MemVT: SrcVT)) {
893 // If the source type is not legal, see if there is a legal extload to
894 // an intermediate type that we can then extend further.
895 EVT LoadVT = TLI.getRegisterType(VT: SrcVT.getSimpleVT());
896 if ((LoadVT.isFloatingPoint() == SrcVT.isFloatingPoint()) &&
897 (TLI.isTypeLegal(VT: SrcVT) || // Same as SrcVT == LoadVT?
898 TLI.isLoadExtLegal(ExtType, ValVT: LoadVT, MemVT: SrcVT))) {
899 // If we are loading a legal type, this is a non-extload followed by a
900 // full extend.
901 ISD::LoadExtType MidExtType =
902 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
903
904 SDValue Load = DAG.getExtLoad(ExtType: MidExtType, dl, VT: LoadVT, Chain, Ptr,
905 MemVT: SrcVT, MMO: LD->getMemOperand());
906 unsigned ExtendOp =
907 ISD::getExtForLoadExtType(IsFP: SrcVT.isFloatingPoint(), ExtType);
908 Value = DAG.getNode(Opcode: ExtendOp, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Load);
909 Chain = Load.getValue(R: 1);
910 break;
911 }
912
913 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
914 // normal undefined upper bits behavior to allow using an in-reg extend
915 // with the illegal FP type, so load as an integer and do the
916 // from-integer conversion.
917 EVT SVT = SrcVT.getScalarType();
918 if (SVT == MVT::f16 || SVT == MVT::bf16) {
919 EVT ISrcVT = SrcVT.changeTypeToInteger();
920 EVT IDestVT = DestVT.changeTypeToInteger();
921 EVT ILoadVT = TLI.getRegisterType(VT: IDestVT.getSimpleVT());
922
923 SDValue Result = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: ILoadVT, Chain,
924 Ptr, MemVT: ISrcVT, MMO: LD->getMemOperand());
925 Value =
926 DAG.getNode(SVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP,
927 dl, DestVT, Result);
928 Chain = Result.getValue(R: 1);
929 break;
930 }
931 }
932
933 assert(!SrcVT.isVector() &&
934 "Vector Loads are handled in LegalizeVectorOps");
935
936 // FIXME: This does not work for vectors on most targets. Sign-
937 // and zero-extend operations are currently folded into extending
938 // loads, whether they are legal or not, and then we end up here
939 // without any support for legalizing them.
940 assert(ExtType != ISD::EXTLOAD &&
941 "EXTLOAD should always be supported!");
942 // Turn the unsupported load into an EXTLOAD followed by an
943 // explicit zero/sign extend inreg.
944 SDValue Result = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl,
945 VT: Node->getValueType(ResNo: 0),
946 Chain, Ptr, MemVT: SrcVT,
947 MMO: LD->getMemOperand());
948 SDValue ValRes;
949 if (ExtType == ISD::SEXTLOAD)
950 ValRes = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl,
951 VT: Result.getValueType(),
952 N1: Result, N2: DAG.getValueType(SrcVT));
953 else
954 ValRes = DAG.getZeroExtendInReg(Op: Result, DL: dl, VT: SrcVT);
955 Value = ValRes;
956 Chain = Result.getValue(R: 1);
957 break;
958 }
959 }
960 }
961
962 // Since loads produce two values, make sure to remember that we legalized
963 // both of them.
964 if (Chain.getNode() != Node) {
965 assert(Value.getNode() != Node && "Load must be completely replaced");
966 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: Value);
967 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: Chain);
968 if (UpdatedNodes) {
969 UpdatedNodes->insert(X: Value.getNode());
970 UpdatedNodes->insert(X: Chain.getNode());
971 }
972 ReplacedNode(N: Node);
973 }
974}
975
976/// Return a legal replacement for the given operation, with all legal operands.
977void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
978 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
979
980 // Allow illegal target nodes and illegal registers.
981 if (Node->getOpcode() == ISD::TargetConstant ||
982 Node->getOpcode() == ISD::Register)
983 return;
984
985#ifndef NDEBUG
986 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
987 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
988 TargetLowering::TypeLegal &&
989 "Unexpected illegal type!");
990
991 for (const SDValue &Op : Node->op_values())
992 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
993 TargetLowering::TypeLegal ||
994 Op.getOpcode() == ISD::TargetConstant ||
995 Op.getOpcode() == ISD::Register) &&
996 "Unexpected illegal type!");
997#endif
998
999 // Figure out the correct action; the way to query this varies by opcode
1000 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
1001 bool SimpleFinishLegalizing = true;
1002 switch (Node->getOpcode()) {
1003 case ISD::INTRINSIC_W_CHAIN:
1004 case ISD::INTRINSIC_WO_CHAIN:
1005 case ISD::INTRINSIC_VOID:
1006 case ISD::STACKSAVE:
1007 Action = TLI.getOperationAction(Op: Node->getOpcode(), MVT::VT: Other);
1008 break;
1009 case ISD::GET_DYNAMIC_AREA_OFFSET:
1010 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1011 VT: Node->getValueType(ResNo: 0));
1012 break;
1013 case ISD::VAARG:
1014 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1015 VT: Node->getValueType(ResNo: 0));
1016 if (Action != TargetLowering::Promote)
1017 Action = TLI.getOperationAction(Op: Node->getOpcode(), MVT::VT: Other);
1018 break;
1019 case ISD::SET_FPENV:
1020 case ISD::SET_FPMODE:
1021 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1022 VT: Node->getOperand(Num: 1).getValueType());
1023 break;
1024 case ISD::FP_TO_FP16:
1025 case ISD::FP_TO_BF16:
1026 case ISD::SINT_TO_FP:
1027 case ISD::UINT_TO_FP:
1028 case ISD::EXTRACT_VECTOR_ELT:
1029 case ISD::LROUND:
1030 case ISD::LLROUND:
1031 case ISD::LRINT:
1032 case ISD::LLRINT:
1033 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1034 VT: Node->getOperand(Num: 0).getValueType());
1035 break;
1036 case ISD::STRICT_FP_TO_FP16:
1037 case ISD::STRICT_SINT_TO_FP:
1038 case ISD::STRICT_UINT_TO_FP:
1039 case ISD::STRICT_LRINT:
1040 case ISD::STRICT_LLRINT:
1041 case ISD::STRICT_LROUND:
1042 case ISD::STRICT_LLROUND:
1043 // These pseudo-ops are the same as the other STRICT_ ops except
1044 // they are registered with setOperationAction() using the input type
1045 // instead of the output type.
1046 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1047 VT: Node->getOperand(Num: 1).getValueType());
1048 break;
1049 case ISD::SIGN_EXTEND_INREG: {
1050 EVT InnerType = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
1051 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: InnerType);
1052 break;
1053 }
1054 case ISD::ATOMIC_STORE:
1055 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1056 VT: Node->getOperand(Num: 1).getValueType());
1057 break;
1058 case ISD::SELECT_CC:
1059 case ISD::STRICT_FSETCC:
1060 case ISD::STRICT_FSETCCS:
1061 case ISD::SETCC:
1062 case ISD::SETCCCARRY:
1063 case ISD::VP_SETCC:
1064 case ISD::BR_CC: {
1065 unsigned Opc = Node->getOpcode();
1066 unsigned CCOperand = Opc == ISD::SELECT_CC ? 4
1067 : Opc == ISD::STRICT_FSETCC ? 3
1068 : Opc == ISD::STRICT_FSETCCS ? 3
1069 : Opc == ISD::SETCCCARRY ? 3
1070 : (Opc == ISD::SETCC || Opc == ISD::VP_SETCC) ? 2
1071 : 1;
1072 unsigned CompareOperand = Opc == ISD::BR_CC ? 2
1073 : Opc == ISD::STRICT_FSETCC ? 1
1074 : Opc == ISD::STRICT_FSETCCS ? 1
1075 : 0;
1076 MVT OpVT = Node->getOperand(Num: CompareOperand).getSimpleValueType();
1077 ISD::CondCode CCCode =
1078 cast<CondCodeSDNode>(Val: Node->getOperand(Num: CCOperand))->get();
1079 Action = TLI.getCondCodeAction(CC: CCCode, VT: OpVT);
1080 if (Action == TargetLowering::Legal) {
1081 if (Node->getOpcode() == ISD::SELECT_CC)
1082 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1083 VT: Node->getValueType(ResNo: 0));
1084 else
1085 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: OpVT);
1086 }
1087 break;
1088 }
1089 case ISD::LOAD:
1090 case ISD::STORE:
1091 // FIXME: Model these properly. LOAD and STORE are complicated, and
1092 // STORE expects the unlegalized operand in some cases.
1093 SimpleFinishLegalizing = false;
1094 break;
1095 case ISD::CALLSEQ_START:
1096 case ISD::CALLSEQ_END:
1097 // FIXME: This shouldn't be necessary. These nodes have special properties
1098 // dealing with the recursive nature of legalization. Removing this
1099 // special case should be done as part of making LegalizeDAG non-recursive.
1100 SimpleFinishLegalizing = false;
1101 break;
1102 case ISD::EXTRACT_ELEMENT:
1103 case ISD::GET_ROUNDING:
1104 case ISD::MERGE_VALUES:
1105 case ISD::EH_RETURN:
1106 case ISD::FRAME_TO_ARGS_OFFSET:
1107 case ISD::EH_DWARF_CFA:
1108 case ISD::EH_SJLJ_SETJMP:
1109 case ISD::EH_SJLJ_LONGJMP:
1110 case ISD::EH_SJLJ_SETUP_DISPATCH:
1111 // These operations lie about being legal: when they claim to be legal,
1112 // they should actually be expanded.
1113 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1114 if (Action == TargetLowering::Legal)
1115 Action = TargetLowering::Expand;
1116 break;
1117 case ISD::INIT_TRAMPOLINE:
1118 case ISD::ADJUST_TRAMPOLINE:
1119 case ISD::FRAMEADDR:
1120 case ISD::RETURNADDR:
1121 case ISD::ADDROFRETURNADDR:
1122 case ISD::SPONENTRY:
1123 // These operations lie about being legal: when they claim to be legal,
1124 // they should actually be custom-lowered.
1125 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1126 if (Action == TargetLowering::Legal)
1127 Action = TargetLowering::Custom;
1128 break;
1129 case ISD::READCYCLECOUNTER:
1130 case ISD::READSTEADYCOUNTER:
1131 // READCYCLECOUNTER and READSTEADYCOUNTER return a i64, even if type
1132 // legalization might have expanded that to several smaller types.
1133 Action = TLI.getOperationAction(Op: Node->getOpcode(), MVT::VT: i64);
1134 break;
1135 case ISD::READ_REGISTER:
1136 case ISD::WRITE_REGISTER:
1137 // Named register is legal in the DAG, but blocked by register name
1138 // selection if not implemented by target (to chose the correct register)
1139 // They'll be converted to Copy(To/From)Reg.
1140 Action = TargetLowering::Legal;
1141 break;
1142 case ISD::UBSANTRAP:
1143 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1144 if (Action == TargetLowering::Expand) {
1145 // replace ISD::UBSANTRAP with ISD::TRAP
1146 SDValue NewVal;
1147 NewVal = DAG.getNode(Opcode: ISD::TRAP, DL: SDLoc(Node), VTList: Node->getVTList(),
1148 N: Node->getOperand(Num: 0));
1149 ReplaceNode(Old: Node, New: NewVal.getNode());
1150 LegalizeOp(Node: NewVal.getNode());
1151 return;
1152 }
1153 break;
1154 case ISD::DEBUGTRAP:
1155 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1156 if (Action == TargetLowering::Expand) {
1157 // replace ISD::DEBUGTRAP with ISD::TRAP
1158 SDValue NewVal;
1159 NewVal = DAG.getNode(Opcode: ISD::TRAP, DL: SDLoc(Node), VTList: Node->getVTList(),
1160 N: Node->getOperand(Num: 0));
1161 ReplaceNode(Old: Node, New: NewVal.getNode());
1162 LegalizeOp(Node: NewVal.getNode());
1163 return;
1164 }
1165 break;
1166 case ISD::SADDSAT:
1167 case ISD::UADDSAT:
1168 case ISD::SSUBSAT:
1169 case ISD::USUBSAT:
1170 case ISD::SSHLSAT:
1171 case ISD::USHLSAT:
1172 case ISD::FP_TO_SINT_SAT:
1173 case ISD::FP_TO_UINT_SAT:
1174 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1175 break;
1176 case ISD::SMULFIX:
1177 case ISD::SMULFIXSAT:
1178 case ISD::UMULFIX:
1179 case ISD::UMULFIXSAT:
1180 case ISD::SDIVFIX:
1181 case ISD::SDIVFIXSAT:
1182 case ISD::UDIVFIX:
1183 case ISD::UDIVFIXSAT: {
1184 unsigned Scale = Node->getConstantOperandVal(Num: 2);
1185 Action = TLI.getFixedPointOperationAction(Op: Node->getOpcode(),
1186 VT: Node->getValueType(ResNo: 0), Scale);
1187 break;
1188 }
1189 case ISD::MSCATTER:
1190 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1191 VT: cast<MaskedScatterSDNode>(Val: Node)->getValue().getValueType());
1192 break;
1193 case ISD::MSTORE:
1194 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1195 VT: cast<MaskedStoreSDNode>(Val: Node)->getValue().getValueType());
1196 break;
1197 case ISD::VP_SCATTER:
1198 Action = TLI.getOperationAction(
1199 Op: Node->getOpcode(),
1200 VT: cast<VPScatterSDNode>(Val: Node)->getValue().getValueType());
1201 break;
1202 case ISD::VP_STORE:
1203 Action = TLI.getOperationAction(
1204 Op: Node->getOpcode(),
1205 VT: cast<VPStoreSDNode>(Val: Node)->getValue().getValueType());
1206 break;
1207 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1208 Action = TLI.getOperationAction(
1209 Op: Node->getOpcode(),
1210 VT: cast<VPStridedStoreSDNode>(Val: Node)->getValue().getValueType());
1211 break;
1212 case ISD::VECREDUCE_FADD:
1213 case ISD::VECREDUCE_FMUL:
1214 case ISD::VECREDUCE_ADD:
1215 case ISD::VECREDUCE_MUL:
1216 case ISD::VECREDUCE_AND:
1217 case ISD::VECREDUCE_OR:
1218 case ISD::VECREDUCE_XOR:
1219 case ISD::VECREDUCE_SMAX:
1220 case ISD::VECREDUCE_SMIN:
1221 case ISD::VECREDUCE_UMAX:
1222 case ISD::VECREDUCE_UMIN:
1223 case ISD::VECREDUCE_FMAX:
1224 case ISD::VECREDUCE_FMIN:
1225 case ISD::VECREDUCE_FMAXIMUM:
1226 case ISD::VECREDUCE_FMINIMUM:
1227 case ISD::IS_FPCLASS:
1228 Action = TLI.getOperationAction(
1229 Op: Node->getOpcode(), VT: Node->getOperand(Num: 0).getValueType());
1230 break;
1231 case ISD::VECREDUCE_SEQ_FADD:
1232 case ISD::VECREDUCE_SEQ_FMUL:
1233 case ISD::VP_REDUCE_FADD:
1234 case ISD::VP_REDUCE_FMUL:
1235 case ISD::VP_REDUCE_ADD:
1236 case ISD::VP_REDUCE_MUL:
1237 case ISD::VP_REDUCE_AND:
1238 case ISD::VP_REDUCE_OR:
1239 case ISD::VP_REDUCE_XOR:
1240 case ISD::VP_REDUCE_SMAX:
1241 case ISD::VP_REDUCE_SMIN:
1242 case ISD::VP_REDUCE_UMAX:
1243 case ISD::VP_REDUCE_UMIN:
1244 case ISD::VP_REDUCE_FMAX:
1245 case ISD::VP_REDUCE_FMIN:
1246 case ISD::VP_REDUCE_SEQ_FADD:
1247 case ISD::VP_REDUCE_SEQ_FMUL:
1248 Action = TLI.getOperationAction(
1249 Op: Node->getOpcode(), VT: Node->getOperand(Num: 1).getValueType());
1250 break;
1251 default:
1252 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1253 Action = TLI.getCustomOperationAction(Op&: *Node);
1254 } else {
1255 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1256 }
1257 break;
1258 }
1259
1260 if (SimpleFinishLegalizing) {
1261 SDNode *NewNode = Node;
1262 switch (Node->getOpcode()) {
1263 default: break;
1264 case ISD::SHL:
1265 case ISD::SRL:
1266 case ISD::SRA:
1267 case ISD::ROTL:
1268 case ISD::ROTR: {
1269 // Legalizing shifts/rotates requires adjusting the shift amount
1270 // to the appropriate width.
1271 SDValue Op0 = Node->getOperand(Num: 0);
1272 SDValue Op1 = Node->getOperand(Num: 1);
1273 if (!Op1.getValueType().isVector()) {
1274 SDValue SAO = DAG.getShiftAmountOperand(LHSTy: Op0.getValueType(), Op: Op1);
1275 // The getShiftAmountOperand() may create a new operand node or
1276 // return the existing one. If new operand is created we need
1277 // to update the parent node.
1278 // Do not try to legalize SAO here! It will be automatically legalized
1279 // in the next round.
1280 if (SAO != Op1)
1281 NewNode = DAG.UpdateNodeOperands(N: Node, Op1: Op0, Op2: SAO);
1282 }
1283 }
1284 break;
1285 case ISD::FSHL:
1286 case ISD::FSHR:
1287 case ISD::SRL_PARTS:
1288 case ISD::SRA_PARTS:
1289 case ISD::SHL_PARTS: {
1290 // Legalizing shifts/rotates requires adjusting the shift amount
1291 // to the appropriate width.
1292 SDValue Op0 = Node->getOperand(Num: 0);
1293 SDValue Op1 = Node->getOperand(Num: 1);
1294 SDValue Op2 = Node->getOperand(Num: 2);
1295 if (!Op2.getValueType().isVector()) {
1296 SDValue SAO = DAG.getShiftAmountOperand(LHSTy: Op0.getValueType(), Op: Op2);
1297 // The getShiftAmountOperand() may create a new operand node or
1298 // return the existing one. If new operand is created we need
1299 // to update the parent node.
1300 if (SAO != Op2)
1301 NewNode = DAG.UpdateNodeOperands(N: Node, Op1: Op0, Op2: Op1, Op3: SAO);
1302 }
1303 break;
1304 }
1305 }
1306
1307 if (NewNode != Node) {
1308 ReplaceNode(Old: Node, New: NewNode);
1309 Node = NewNode;
1310 }
1311 switch (Action) {
1312 case TargetLowering::Legal:
1313 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1314 return;
1315 case TargetLowering::Custom:
1316 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1317 // FIXME: The handling for custom lowering with multiple results is
1318 // a complete mess.
1319 if (SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG)) {
1320 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1321 return;
1322
1323 if (Node->getNumValues() == 1) {
1324 // Verify the new types match the original. Glue is waived because
1325 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1326 assert((Res.getValueType() == Node->getValueType(0) ||
1327 Node->getValueType(0) == MVT::Glue) &&
1328 "Type mismatch for custom legalized operation");
1329 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1330 // We can just directly replace this node with the lowered value.
1331 ReplaceNode(Old: SDValue(Node, 0), New: Res);
1332 return;
1333 }
1334
1335 SmallVector<SDValue, 8> ResultVals;
1336 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
1337 // Verify the new types match the original. Glue is waived because
1338 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1339 assert((Res->getValueType(i) == Node->getValueType(i) ||
1340 Node->getValueType(i) == MVT::Glue) &&
1341 "Type mismatch for custom legalized operation");
1342 ResultVals.push_back(Elt: Res.getValue(R: i));
1343 }
1344 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1345 ReplaceNode(Old: Node, New: ResultVals.data());
1346 return;
1347 }
1348 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1349 [[fallthrough]];
1350 case TargetLowering::Expand:
1351 if (ExpandNode(Node))
1352 return;
1353 [[fallthrough]];
1354 case TargetLowering::LibCall:
1355 ConvertNodeToLibcall(Node);
1356 return;
1357 case TargetLowering::Promote:
1358 PromoteNode(Node);
1359 return;
1360 }
1361 }
1362
1363 switch (Node->getOpcode()) {
1364 default:
1365#ifndef NDEBUG
1366 dbgs() << "NODE: ";
1367 Node->dump( G: &DAG);
1368 dbgs() << "\n";
1369#endif
1370 llvm_unreachable("Do not know how to legalize this operator!");
1371
1372 case ISD::CALLSEQ_START:
1373 case ISD::CALLSEQ_END:
1374 break;
1375 case ISD::LOAD:
1376 return LegalizeLoadOps(Node);
1377 case ISD::STORE:
1378 return LegalizeStoreOps(Node);
1379 }
1380}
1381
1382// Helper function that generates an MMO that considers the alignment of the
1383// stack, and the size of the stack object
1384static MachineMemOperand *getStackAlignedMMO(SDValue StackPtr,
1385 MachineFunction &MF,
1386 bool isObjectScalable) {
1387 auto &MFI = MF.getFrameInfo();
1388 int FI = cast<FrameIndexSDNode>(Val&: StackPtr)->getIndex();
1389 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
1390 uint64_t ObjectSize = isObjectScalable ? ~UINT64_C(0) : MFI.getObjectSize(ObjectIdx: FI);
1391 return MF.getMachineMemOperand(PtrInfo, f: MachineMemOperand::MOStore,
1392 s: ObjectSize, base_alignment: MFI.getObjectAlign(ObjectIdx: FI));
1393}
1394
1395SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1396 SDValue Vec = Op.getOperand(i: 0);
1397 SDValue Idx = Op.getOperand(i: 1);
1398 SDLoc dl(Op);
1399
1400 // Before we generate a new store to a temporary stack slot, see if there is
1401 // already one that we can use. There often is because when we scalarize
1402 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1403 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1404 // the vector. If all are expanded here, we don't want one store per vector
1405 // element.
1406
1407 // Caches for hasPredecessorHelper
1408 SmallPtrSet<const SDNode *, 32> Visited;
1409 SmallVector<const SDNode *, 16> Worklist;
1410 Visited.insert(Ptr: Op.getNode());
1411 Worklist.push_back(Elt: Idx.getNode());
1412 SDValue StackPtr, Ch;
1413 for (SDNode *User : Vec.getNode()->uses()) {
1414 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: User)) {
1415 if (ST->isIndexed() || ST->isTruncatingStore() ||
1416 ST->getValue() != Vec)
1417 continue;
1418
1419 // Make sure that nothing else could have stored into the destination of
1420 // this store.
1421 if (!ST->getChain().reachesChainWithoutSideEffects(Dest: DAG.getEntryNode()))
1422 continue;
1423
1424 // If the index is dependent on the store we will introduce a cycle when
1425 // creating the load (the load uses the index, and by replacing the chain
1426 // we will make the index dependent on the load). Also, the store might be
1427 // dependent on the extractelement and introduce a cycle when creating
1428 // the load.
1429 if (SDNode::hasPredecessorHelper(N: ST, Visited, Worklist) ||
1430 ST->hasPredecessor(N: Op.getNode()))
1431 continue;
1432
1433 StackPtr = ST->getBasePtr();
1434 Ch = SDValue(ST, 0);
1435 break;
1436 }
1437 }
1438
1439 EVT VecVT = Vec.getValueType();
1440
1441 if (!Ch.getNode()) {
1442 // Store the value to a temporary stack slot, then LOAD the returned part.
1443 StackPtr = DAG.CreateStackTemporary(VT: VecVT);
1444 MachineMemOperand *StoreMMO = getStackAlignedMMO(
1445 StackPtr, MF&: DAG.getMachineFunction(), isObjectScalable: VecVT.isScalableVector());
1446 Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, MMO: StoreMMO);
1447 }
1448
1449 SDValue NewLoad;
1450 Align ElementAlignment =
1451 std::min(a: cast<StoreSDNode>(Val&: Ch)->getAlign(),
1452 b: DAG.getDataLayout().getPrefTypeAlign(
1453 Ty: Op.getValueType().getTypeForEVT(Context&: *DAG.getContext())));
1454
1455 if (Op.getValueType().isVector()) {
1456 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT,
1457 SubVecVT: Op.getValueType(), Index: Idx);
1458 NewLoad = DAG.getLoad(VT: Op.getValueType(), dl, Chain: Ch, Ptr: StackPtr,
1459 PtrInfo: MachinePointerInfo(), Alignment: ElementAlignment);
1460 } else {
1461 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
1462 NewLoad = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: Op.getValueType(), Chain: Ch, Ptr: StackPtr,
1463 PtrInfo: MachinePointerInfo(), MemVT: VecVT.getVectorElementType(),
1464 Alignment: ElementAlignment);
1465 }
1466
1467 // Replace the chain going out of the store, by the one out of the load.
1468 DAG.ReplaceAllUsesOfValueWith(From: Ch, To: SDValue(NewLoad.getNode(), 1));
1469
1470 // We introduced a cycle though, so update the loads operands, making sure
1471 // to use the original store's chain as an incoming chain.
1472 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1473 NewLoad->op_end());
1474 NewLoadOperands[0] = Ch;
1475 NewLoad =
1476 SDValue(DAG.UpdateNodeOperands(N: NewLoad.getNode(), Ops: NewLoadOperands), 0);
1477 return NewLoad;
1478}
1479
1480SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1481 assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1482
1483 SDValue Vec = Op.getOperand(i: 0);
1484 SDValue Part = Op.getOperand(i: 1);
1485 SDValue Idx = Op.getOperand(i: 2);
1486 SDLoc dl(Op);
1487
1488 // Store the value to a temporary stack slot, then LOAD the returned part.
1489 EVT VecVT = Vec.getValueType();
1490 EVT SubVecVT = Part.getValueType();
1491 SDValue StackPtr = DAG.CreateStackTemporary(VT: VecVT);
1492 int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1493 MachinePointerInfo PtrInfo =
1494 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
1495
1496 // First store the whole vector.
1497 SDValue Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo);
1498
1499 // Then store the inserted part.
1500 SDValue SubStackPtr =
1501 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT, Index: Idx);
1502
1503 // Store the subvector.
1504 Ch = DAG.getStore(
1505 Chain: Ch, dl, Val: Part, Ptr: SubStackPtr,
1506 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
1507
1508 // Finally, load the updated vector.
1509 return DAG.getLoad(VT: Op.getValueType(), dl, Chain: Ch, Ptr: StackPtr, PtrInfo);
1510}
1511
1512SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1513 assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
1514 Node->getOpcode() == ISD::CONCAT_VECTORS) &&
1515 "Unexpected opcode!");
1516
1517 // We can't handle this case efficiently. Allocate a sufficiently
1518 // aligned object on the stack, store each operand into it, then load
1519 // the result as a vector.
1520 // Create the stack frame object.
1521 EVT VT = Node->getValueType(ResNo: 0);
1522 EVT MemVT = isa<BuildVectorSDNode>(Val: Node) ? VT.getVectorElementType()
1523 : Node->getOperand(Num: 0).getValueType();
1524 SDLoc dl(Node);
1525 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1526 int FI = cast<FrameIndexSDNode>(Val: FIPtr.getNode())->getIndex();
1527 MachinePointerInfo PtrInfo =
1528 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
1529
1530 // Emit a store of each element to the stack slot.
1531 SmallVector<SDValue, 8> Stores;
1532 unsigned TypeByteSize = MemVT.getSizeInBits() / 8;
1533 assert(TypeByteSize > 0 && "Vector element type too small for stack store!");
1534
1535 // If the destination vector element type of a BUILD_VECTOR is narrower than
1536 // the source element type, only store the bits necessary.
1537 bool Truncate = isa<BuildVectorSDNode>(Val: Node) &&
1538 MemVT.bitsLT(VT: Node->getOperand(Num: 0).getValueType());
1539
1540 // Store (in the right endianness) the elements to memory.
1541 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1542 // Ignore undef elements.
1543 if (Node->getOperand(Num: i).isUndef()) continue;
1544
1545 unsigned Offset = TypeByteSize*i;
1546
1547 SDValue Idx =
1548 DAG.getMemBasePlusOffset(Base: FIPtr, Offset: TypeSize::getFixed(ExactSize: Offset), DL: dl);
1549
1550 if (Truncate)
1551 Stores.push_back(Elt: DAG.getTruncStore(Chain: DAG.getEntryNode(), dl,
1552 Val: Node->getOperand(Num: i), Ptr: Idx,
1553 PtrInfo: PtrInfo.getWithOffset(O: Offset), SVT: MemVT));
1554 else
1555 Stores.push_back(Elt: DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Node->getOperand(Num: i),
1556 Ptr: Idx, PtrInfo: PtrInfo.getWithOffset(O: Offset)));
1557 }
1558
1559 SDValue StoreChain;
1560 if (!Stores.empty()) // Not all undef elements?
1561 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1562 else
1563 StoreChain = DAG.getEntryNode();
1564
1565 // Result is a load from the stack slot.
1566 return DAG.getLoad(VT, dl, Chain: StoreChain, Ptr: FIPtr, PtrInfo);
1567}
1568
1569/// Bitcast a floating-point value to an integer value. Only bitcast the part
1570/// containing the sign bit if the target has no integer value capable of
1571/// holding all bits of the floating-point value.
1572void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1573 const SDLoc &DL,
1574 SDValue Value) const {
1575 EVT FloatVT = Value.getValueType();
1576 unsigned NumBits = FloatVT.getScalarSizeInBits();
1577 State.FloatVT = FloatVT;
1578 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumBits);
1579 // Convert to an integer of the same size.
1580 if (TLI.isTypeLegal(VT: IVT)) {
1581 State.IntValue = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IVT, Operand: Value);
1582 State.SignMask = APInt::getSignMask(BitWidth: NumBits);
1583 State.SignBit = NumBits - 1;
1584 return;
1585 }
1586
1587 auto &DataLayout = DAG.getDataLayout();
1588 // Store the float to memory, then load the sign part out as an integer.
1589 MVT LoadTy = TLI.getRegisterType(MVT::i8);
1590 // First create a temporary that is aligned for both the load and store.
1591 SDValue StackPtr = DAG.CreateStackTemporary(VT1: FloatVT, VT2: LoadTy);
1592 int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1593 // Then store the float to it.
1594 State.FloatPtr = StackPtr;
1595 MachineFunction &MF = DAG.getMachineFunction();
1596 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1597 State.Chain = DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: Value, Ptr: State.FloatPtr,
1598 PtrInfo: State.FloatPointerInfo);
1599
1600 SDValue IntPtr;
1601 if (DataLayout.isBigEndian()) {
1602 assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1603 // Load out a legal integer with the same sign bit as the float.
1604 IntPtr = StackPtr;
1605 State.IntPointerInfo = State.FloatPointerInfo;
1606 } else {
1607 // Advance the pointer so that the loaded byte will contain the sign bit.
1608 unsigned ByteOffset = (NumBits / 8) - 1;
1609 IntPtr =
1610 DAG.getMemBasePlusOffset(Base: StackPtr, Offset: TypeSize::getFixed(ExactSize: ByteOffset), DL);
1611 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1612 Offset: ByteOffset);
1613 }
1614
1615 State.IntPtr = IntPtr;
1616 State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1617 State.IntPointerInfo, MVT::i8);
1618 State.SignMask = APInt::getOneBitSet(numBits: LoadTy.getScalarSizeInBits(), BitNo: 7);
1619 State.SignBit = 7;
1620}
1621
1622/// Replace the integer value produced by getSignAsIntValue() with a new value
1623/// and cast the result back to a floating-point type.
1624SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1625 const SDLoc &DL,
1626 SDValue NewIntValue) const {
1627 if (!State.Chain)
1628 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: State.FloatVT, Operand: NewIntValue);
1629
1630 // Override the part containing the sign bit in the value stored on the stack.
1631 SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1632 State.IntPointerInfo, MVT::i8);
1633 return DAG.getLoad(VT: State.FloatVT, dl: DL, Chain, Ptr: State.FloatPtr,
1634 PtrInfo: State.FloatPointerInfo);
1635}
1636
1637SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1638 SDLoc DL(Node);
1639 SDValue Mag = Node->getOperand(Num: 0);
1640 SDValue Sign = Node->getOperand(Num: 1);
1641
1642 // Get sign bit into an integer value.
1643 FloatSignAsInt SignAsInt;
1644 getSignAsIntValue(State&: SignAsInt, DL, Value: Sign);
1645
1646 EVT IntVT = SignAsInt.IntValue.getValueType();
1647 SDValue SignMask = DAG.getConstant(Val: SignAsInt.SignMask, DL, VT: IntVT);
1648 SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: SignAsInt.IntValue,
1649 N2: SignMask);
1650
1651 // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1652 EVT FloatVT = Mag.getValueType();
1653 if (TLI.isOperationLegalOrCustom(Op: ISD::FABS, VT: FloatVT) &&
1654 TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT: FloatVT)) {
1655 SDValue AbsValue = DAG.getNode(Opcode: ISD::FABS, DL, VT: FloatVT, Operand: Mag);
1656 SDValue NegValue = DAG.getNode(Opcode: ISD::FNEG, DL, VT: FloatVT, Operand: AbsValue);
1657 SDValue Cond = DAG.getSetCC(DL, VT: getSetCCResultType(VT: IntVT), LHS: SignBit,
1658 RHS: DAG.getConstant(Val: 0, DL, VT: IntVT), Cond: ISD::SETNE);
1659 return DAG.getSelect(DL, VT: FloatVT, Cond, LHS: NegValue, RHS: AbsValue);
1660 }
1661
1662 // Transform Mag value to integer, and clear the sign bit.
1663 FloatSignAsInt MagAsInt;
1664 getSignAsIntValue(State&: MagAsInt, DL, Value: Mag);
1665 EVT MagVT = MagAsInt.IntValue.getValueType();
1666 SDValue ClearSignMask = DAG.getConstant(Val: ~MagAsInt.SignMask, DL, VT: MagVT);
1667 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: MagVT, N1: MagAsInt.IntValue,
1668 N2: ClearSignMask);
1669
1670 // Get the signbit at the right position for MagAsInt.
1671 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1672 EVT ShiftVT = IntVT;
1673 if (SignBit.getScalarValueSizeInBits() <
1674 ClearedSign.getScalarValueSizeInBits()) {
1675 SignBit = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MagVT, Operand: SignBit);
1676 ShiftVT = MagVT;
1677 }
1678 if (ShiftAmount > 0) {
1679 SDValue ShiftCnst = DAG.getConstant(Val: ShiftAmount, DL, VT: ShiftVT);
1680 SignBit = DAG.getNode(Opcode: ISD::SRL, DL, VT: ShiftVT, N1: SignBit, N2: ShiftCnst);
1681 } else if (ShiftAmount < 0) {
1682 SDValue ShiftCnst = DAG.getConstant(Val: -ShiftAmount, DL, VT: ShiftVT);
1683 SignBit = DAG.getNode(Opcode: ISD::SHL, DL, VT: ShiftVT, N1: SignBit, N2: ShiftCnst);
1684 }
1685 if (SignBit.getScalarValueSizeInBits() >
1686 ClearedSign.getScalarValueSizeInBits()) {
1687 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MagVT, Operand: SignBit);
1688 }
1689
1690 // Store the part with the modified sign and convert back to float.
1691 SDValue CopiedSign = DAG.getNode(Opcode: ISD::OR, DL, VT: MagVT, N1: ClearedSign, N2: SignBit);
1692 return modifySignAsInt(State: MagAsInt, DL, NewIntValue: CopiedSign);
1693}
1694
1695SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {
1696 // Get the sign bit as an integer.
1697 SDLoc DL(Node);
1698 FloatSignAsInt SignAsInt;
1699 getSignAsIntValue(State&: SignAsInt, DL, Value: Node->getOperand(Num: 0));
1700 EVT IntVT = SignAsInt.IntValue.getValueType();
1701
1702 // Flip the sign.
1703 SDValue SignMask = DAG.getConstant(Val: SignAsInt.SignMask, DL, VT: IntVT);
1704 SDValue SignFlip =
1705 DAG.getNode(Opcode: ISD::XOR, DL, VT: IntVT, N1: SignAsInt.IntValue, N2: SignMask);
1706
1707 // Convert back to float.
1708 return modifySignAsInt(State: SignAsInt, DL, NewIntValue: SignFlip);
1709}
1710
1711SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1712 SDLoc DL(Node);
1713 SDValue Value = Node->getOperand(Num: 0);
1714
1715 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1716 EVT FloatVT = Value.getValueType();
1717 if (TLI.isOperationLegalOrCustom(Op: ISD::FCOPYSIGN, VT: FloatVT)) {
1718 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL, VT: FloatVT);
1719 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT: FloatVT, N1: Value, N2: Zero);
1720 }
1721
1722 // Transform value to integer, clear the sign bit and transform back.
1723 FloatSignAsInt ValueAsInt;
1724 getSignAsIntValue(State&: ValueAsInt, DL, Value);
1725 EVT IntVT = ValueAsInt.IntValue.getValueType();
1726 SDValue ClearSignMask = DAG.getConstant(Val: ~ValueAsInt.SignMask, DL, VT: IntVT);
1727 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: ValueAsInt.IntValue,
1728 N2: ClearSignMask);
1729 return modifySignAsInt(State: ValueAsInt, DL, NewIntValue: ClearedSign);
1730}
1731
1732void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1733 SmallVectorImpl<SDValue> &Results) {
1734 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1735 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1736 " not tell us which reg is the stack pointer!");
1737 SDLoc dl(Node);
1738 EVT VT = Node->getValueType(ResNo: 0);
1739 SDValue Tmp1 = SDValue(Node, 0);
1740 SDValue Tmp2 = SDValue(Node, 1);
1741 SDValue Tmp3 = Node->getOperand(Num: 2);
1742 SDValue Chain = Tmp1.getOperand(i: 0);
1743
1744 // Chain the dynamic stack allocation so that it doesn't modify the stack
1745 // pointer when other instructions are using the stack.
1746 Chain = DAG.getCALLSEQ_START(Chain, InSize: 0, OutSize: 0, DL: dl);
1747
1748 SDValue Size = Tmp2.getOperand(i: 1);
1749 SDValue SP = DAG.getCopyFromReg(Chain, dl, Reg: SPReg, VT);
1750 Chain = SP.getValue(R: 1);
1751 Align Alignment = cast<ConstantSDNode>(Val&: Tmp3)->getAlignValue();
1752 const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering();
1753 unsigned Opc =
1754 TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp ?
1755 ISD::ADD : ISD::SUB;
1756
1757 Align StackAlign = TFL->getStackAlign();
1758 Tmp1 = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: SP, N2: Size); // Value
1759 if (Alignment > StackAlign)
1760 Tmp1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp1,
1761 N2: DAG.getConstant(Val: -Alignment.value(), DL: dl, VT));
1762 Chain = DAG.getCopyToReg(Chain, dl, Reg: SPReg, N: Tmp1); // Output chain
1763
1764 Tmp2 = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: SDValue(), DL: dl);
1765
1766 Results.push_back(Elt: Tmp1);
1767 Results.push_back(Elt: Tmp2);
1768}
1769
1770/// Emit a store/load combination to the stack. This stores
1771/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1772/// a load from the stack slot to DestVT, extending it if needed.
1773/// The resultant code need not be legal.
1774SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1775 EVT DestVT, const SDLoc &dl) {
1776 return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, ChainIn: DAG.getEntryNode());
1777}
1778
1779SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1780 EVT DestVT, const SDLoc &dl,
1781 SDValue Chain) {
1782 EVT SrcVT = SrcOp.getValueType();
1783 Type *DestType = DestVT.getTypeForEVT(Context&: *DAG.getContext());
1784 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(Ty: DestType);
1785
1786 // Don't convert with stack if the load/store is expensive.
1787 if ((SrcVT.bitsGT(VT: SlotVT) &&
1788 !TLI.isTruncStoreLegalOrCustom(ValVT: SrcOp.getValueType(), MemVT: SlotVT)) ||
1789 (SlotVT.bitsLT(VT: DestVT) &&
1790 !TLI.isLoadExtLegalOrCustom(ExtType: ISD::EXTLOAD, ValVT: DestVT, MemVT: SlotVT)))
1791 return SDValue();
1792
1793 // Create the stack frame object.
1794 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1795 Ty: SrcOp.getValueType().getTypeForEVT(Context&: *DAG.getContext()));
1796 SDValue FIPtr = DAG.CreateStackTemporary(Bytes: SlotVT.getStoreSize(), Alignment: SrcAlign);
1797
1798 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(Val&: FIPtr);
1799 int SPFI = StackPtrFI->getIndex();
1800 MachinePointerInfo PtrInfo =
1801 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI);
1802
1803 // Emit a store to the stack slot. Use a truncstore if the input value is
1804 // later than DestVT.
1805 SDValue Store;
1806
1807 if (SrcVT.bitsGT(VT: SlotVT))
1808 Store = DAG.getTruncStore(Chain, dl, Val: SrcOp, Ptr: FIPtr, PtrInfo,
1809 SVT: SlotVT, Alignment: SrcAlign);
1810 else {
1811 assert(SrcVT.bitsEq(SlotVT) && "Invalid store");
1812 Store = DAG.getStore(Chain, dl, Val: SrcOp, Ptr: FIPtr, PtrInfo, Alignment: SrcAlign);
1813 }
1814
1815 // Result is a load from the stack slot.
1816 if (SlotVT.bitsEq(VT: DestVT))
1817 return DAG.getLoad(VT: DestVT, dl, Chain: Store, Ptr: FIPtr, PtrInfo, Alignment: DestAlign);
1818
1819 assert(SlotVT.bitsLT(DestVT) && "Unknown extension!");
1820 return DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: DestVT, Chain: Store, Ptr: FIPtr, PtrInfo, MemVT: SlotVT,
1821 Alignment: DestAlign);
1822}
1823
1824SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1825 SDLoc dl(Node);
1826 // Create a vector sized/aligned stack slot, store the value to element #0,
1827 // then load the whole vector back out.
1828 SDValue StackPtr = DAG.CreateStackTemporary(VT: Node->getValueType(ResNo: 0));
1829
1830 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(Val&: StackPtr);
1831 int SPFI = StackPtrFI->getIndex();
1832
1833 SDValue Ch = DAG.getTruncStore(
1834 Chain: DAG.getEntryNode(), dl, Val: Node->getOperand(Num: 0), Ptr: StackPtr,
1835 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI),
1836 SVT: Node->getValueType(ResNo: 0).getVectorElementType());
1837 return DAG.getLoad(
1838 VT: Node->getValueType(ResNo: 0), dl, Chain: Ch, Ptr: StackPtr,
1839 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
1840}
1841
1842static bool
1843ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1844 const TargetLowering &TLI, SDValue &Res) {
1845 unsigned NumElems = Node->getNumOperands();
1846 SDLoc dl(Node);
1847 EVT VT = Node->getValueType(ResNo: 0);
1848
1849 // Try to group the scalars into pairs, shuffle the pairs together, then
1850 // shuffle the pairs of pairs together, etc. until the vector has
1851 // been built. This will work only if all of the necessary shuffle masks
1852 // are legal.
1853
1854 // We do this in two phases; first to check the legality of the shuffles,
1855 // and next, assuming that all shuffles are legal, to create the new nodes.
1856 for (int Phase = 0; Phase < 2; ++Phase) {
1857 SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
1858 NewIntermedVals;
1859 for (unsigned i = 0; i < NumElems; ++i) {
1860 SDValue V = Node->getOperand(Num: i);
1861 if (V.isUndef())
1862 continue;
1863
1864 SDValue Vec;
1865 if (Phase)
1866 Vec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: V);
1867 IntermedVals.push_back(Elt: std::make_pair(x&: Vec, y: SmallVector<int, 16>(1, i)));
1868 }
1869
1870 while (IntermedVals.size() > 2) {
1871 NewIntermedVals.clear();
1872 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1873 // This vector and the next vector are shuffled together (simply to
1874 // append the one to the other).
1875 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1876
1877 SmallVector<int, 16> FinalIndices;
1878 FinalIndices.reserve(N: IntermedVals[i].second.size() +
1879 IntermedVals[i+1].second.size());
1880
1881 int k = 0;
1882 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1883 ++j, ++k) {
1884 ShuffleVec[k] = j;
1885 FinalIndices.push_back(Elt: IntermedVals[i].second[j]);
1886 }
1887 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1888 ++j, ++k) {
1889 ShuffleVec[k] = NumElems + j;
1890 FinalIndices.push_back(Elt: IntermedVals[i+1].second[j]);
1891 }
1892
1893 SDValue Shuffle;
1894 if (Phase)
1895 Shuffle = DAG.getVectorShuffle(VT, dl, N1: IntermedVals[i].first,
1896 N2: IntermedVals[i+1].first,
1897 Mask: ShuffleVec);
1898 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1899 return false;
1900 NewIntermedVals.push_back(
1901 Elt: std::make_pair(x&: Shuffle, y: std::move(FinalIndices)));
1902 }
1903
1904 // If we had an odd number of defined values, then append the last
1905 // element to the array of new vectors.
1906 if ((IntermedVals.size() & 1) != 0)
1907 NewIntermedVals.push_back(Elt: IntermedVals.back());
1908
1909 IntermedVals.swap(RHS&: NewIntermedVals);
1910 }
1911
1912 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1913 "Invalid number of intermediate vectors");
1914 SDValue Vec1 = IntermedVals[0].first;
1915 SDValue Vec2;
1916 if (IntermedVals.size() > 1)
1917 Vec2 = IntermedVals[1].first;
1918 else if (Phase)
1919 Vec2 = DAG.getUNDEF(VT);
1920
1921 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1922 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1923 ShuffleVec[IntermedVals[0].second[i]] = i;
1924 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1925 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1926
1927 if (Phase)
1928 Res = DAG.getVectorShuffle(VT, dl, N1: Vec1, N2: Vec2, Mask: ShuffleVec);
1929 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1930 return false;
1931 }
1932
1933 return true;
1934}
1935
1936/// Expand a BUILD_VECTOR node on targets that don't
1937/// support the operation, but do support the resultant vector type.
1938SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1939 unsigned NumElems = Node->getNumOperands();
1940 SDValue Value1, Value2;
1941 SDLoc dl(Node);
1942 EVT VT = Node->getValueType(ResNo: 0);
1943 EVT OpVT = Node->getOperand(Num: 0).getValueType();
1944 EVT EltVT = VT.getVectorElementType();
1945
1946 // If the only non-undef value is the low element, turn this into a
1947 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1948 bool isOnlyLowElement = true;
1949 bool MoreThanTwoValues = false;
1950 bool isConstant = true;
1951 for (unsigned i = 0; i < NumElems; ++i) {
1952 SDValue V = Node->getOperand(Num: i);
1953 if (V.isUndef())
1954 continue;
1955 if (i > 0)
1956 isOnlyLowElement = false;
1957 if (!isa<ConstantFPSDNode>(Val: V) && !isa<ConstantSDNode>(Val: V))
1958 isConstant = false;
1959
1960 if (!Value1.getNode()) {
1961 Value1 = V;
1962 } else if (!Value2.getNode()) {
1963 if (V != Value1)
1964 Value2 = V;
1965 } else if (V != Value1 && V != Value2) {
1966 MoreThanTwoValues = true;
1967 }
1968 }
1969
1970 if (!Value1.getNode())
1971 return DAG.getUNDEF(VT);
1972
1973 if (isOnlyLowElement)
1974 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Node->getOperand(Num: 0));
1975
1976 // If all elements are constants, create a load from the constant pool.
1977 if (isConstant) {
1978 SmallVector<Constant*, 16> CV;
1979 for (unsigned i = 0, e = NumElems; i != e; ++i) {
1980 if (ConstantFPSDNode *V =
1981 dyn_cast<ConstantFPSDNode>(Val: Node->getOperand(Num: i))) {
1982 CV.push_back(Elt: const_cast<ConstantFP *>(V->getConstantFPValue()));
1983 } else if (ConstantSDNode *V =
1984 dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: i))) {
1985 if (OpVT==EltVT)
1986 CV.push_back(Elt: const_cast<ConstantInt *>(V->getConstantIntValue()));
1987 else {
1988 // If OpVT and EltVT don't match, EltVT is not legal and the
1989 // element values have been promoted/truncated earlier. Undo this;
1990 // we don't want a v16i8 to become a v16i32 for example.
1991 const ConstantInt *CI = V->getConstantIntValue();
1992 CV.push_back(Elt: ConstantInt::get(Ty: EltVT.getTypeForEVT(Context&: *DAG.getContext()),
1993 V: CI->getZExtValue()));
1994 }
1995 } else {
1996 assert(Node->getOperand(i).isUndef());
1997 Type *OpNTy = EltVT.getTypeForEVT(Context&: *DAG.getContext());
1998 CV.push_back(Elt: UndefValue::get(T: OpNTy));
1999 }
2000 }
2001 Constant *CP = ConstantVector::get(V: CV);
2002 SDValue CPIdx =
2003 DAG.getConstantPool(C: CP, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2004 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
2005 return DAG.getLoad(
2006 VT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
2007 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()),
2008 Alignment);
2009 }
2010
2011 SmallSet<SDValue, 16> DefinedValues;
2012 for (unsigned i = 0; i < NumElems; ++i) {
2013 if (Node->getOperand(Num: i).isUndef())
2014 continue;
2015 DefinedValues.insert(V: Node->getOperand(Num: i));
2016 }
2017
2018 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues: DefinedValues.size())) {
2019 if (!MoreThanTwoValues) {
2020 SmallVector<int, 8> ShuffleVec(NumElems, -1);
2021 for (unsigned i = 0; i < NumElems; ++i) {
2022 SDValue V = Node->getOperand(Num: i);
2023 if (V.isUndef())
2024 continue;
2025 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
2026 }
2027 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(ResNo: 0))) {
2028 // Get the splatted value into the low element of a vector register.
2029 SDValue Vec1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value1);
2030 SDValue Vec2;
2031 if (Value2.getNode())
2032 Vec2 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value2);
2033 else
2034 Vec2 = DAG.getUNDEF(VT);
2035
2036 // Return shuffle(LowValVec, undef, <0,0,0,0>)
2037 return DAG.getVectorShuffle(VT, dl, N1: Vec1, N2: Vec2, Mask: ShuffleVec);
2038 }
2039 } else {
2040 SDValue Res;
2041 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
2042 return Res;
2043 }
2044 }
2045
2046 // Otherwise, we can't handle this case efficiently.
2047 return ExpandVectorBuildThroughStack(Node);
2048}
2049
2050SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
2051 SDLoc DL(Node);
2052 EVT VT = Node->getValueType(ResNo: 0);
2053 SDValue SplatVal = Node->getOperand(Num: 0);
2054
2055 return DAG.getSplatBuildVector(VT, DL, Op: SplatVal);
2056}
2057
2058// Expand a node into a call to a libcall, returning the value as the first
2059// result and the chain as the second. If the result value does not fit into a
2060// register, return the lo part and set the hi part to the by-reg argument in
2061// the first. If it does fit into a single register, return the result and
2062// leave the Hi part unset.
2063std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2064 TargetLowering::ArgListTy &&Args,
2065 bool isSigned) {
2066 SDValue Callee = DAG.getExternalSymbol(Sym: TLI.getLibcallName(Call: LC),
2067 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2068
2069 EVT RetVT = Node->getValueType(ResNo: 0);
2070 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2071
2072 // By default, the input chain to this libcall is the entry node of the
2073 // function. If the libcall is going to be emitted as a tail call then
2074 // TLI.isUsedByReturnOnly will change it to the right chain if the return
2075 // node which is being folded has a non-entry input chain.
2076 SDValue InChain = DAG.getEntryNode();
2077
2078 // isTailCall may be true since the callee does not reference caller stack
2079 // frame. Check if it's in the right position and that the return types match.
2080 SDValue TCChain = InChain;
2081 const Function &F = DAG.getMachineFunction().getFunction();
2082 bool isTailCall =
2083 TLI.isInTailCallPosition(DAG, Node, Chain&: TCChain) &&
2084 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
2085 if (isTailCall)
2086 InChain = TCChain;
2087
2088 TargetLowering::CallLoweringInfo CLI(DAG);
2089 bool signExtend = TLI.shouldSignExtendTypeInLibCall(Type: RetVT, IsSigned: isSigned);
2090 CLI.setDebugLoc(SDLoc(Node))
2091 .setChain(InChain)
2092 .setLibCallee(CC: TLI.getLibcallCallingConv(Call: LC), ResultType: RetTy, Target: Callee,
2093 ArgsList: std::move(Args))
2094 .setTailCall(isTailCall)
2095 .setSExtResult(signExtend)
2096 .setZExtResult(!signExtend)
2097 .setIsPostTypeLegalization(true);
2098
2099 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2100
2101 if (!CallInfo.second.getNode()) {
2102 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG));
2103 // It's a tailcall, return the chain (which is the DAG root).
2104 return {DAG.getRoot(), DAG.getRoot()};
2105 }
2106
2107 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG));
2108 return CallInfo;
2109}
2110
2111std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2112 bool isSigned) {
2113 TargetLowering::ArgListTy Args;
2114 TargetLowering::ArgListEntry Entry;
2115 for (const SDValue &Op : Node->op_values()) {
2116 EVT ArgVT = Op.getValueType();
2117 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2118 Entry.Node = Op;
2119 Entry.Ty = ArgTy;
2120 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(Type: ArgVT, IsSigned: isSigned);
2121 Entry.IsZExt = !Entry.IsSExt;
2122 Args.push_back(x: Entry);
2123 }
2124
2125 return ExpandLibCall(LC, Node, Args: std::move(Args), isSigned);
2126}
2127
2128void SelectionDAGLegalize::ExpandFrexpLibCall(
2129 SDNode *Node, SmallVectorImpl<SDValue> &Results) {
2130 SDLoc dl(Node);
2131 EVT VT = Node->getValueType(ResNo: 0);
2132 EVT ExpVT = Node->getValueType(ResNo: 1);
2133
2134 SDValue FPOp = Node->getOperand(Num: 0);
2135
2136 EVT ArgVT = FPOp.getValueType();
2137 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2138
2139 TargetLowering::ArgListEntry FPArgEntry;
2140 FPArgEntry.Node = FPOp;
2141 FPArgEntry.Ty = ArgTy;
2142
2143 SDValue StackSlot = DAG.CreateStackTemporary(VT: ExpVT);
2144 TargetLowering::ArgListEntry PtrArgEntry;
2145 PtrArgEntry.Node = StackSlot;
2146 PtrArgEntry.Ty = PointerType::get(C&: *DAG.getContext(),
2147 AddressSpace: DAG.getDataLayout().getAllocaAddrSpace());
2148
2149 TargetLowering::ArgListTy Args = {FPArgEntry, PtrArgEntry};
2150
2151 RTLIB::Libcall LC = RTLIB::getFREXP(RetVT: VT);
2152 auto [Call, Chain] = ExpandLibCall(LC, Node, Args: std::move(Args), isSigned: false);
2153
2154 // FIXME: Get type of int for libcall declaration and cast
2155
2156 int FrameIdx = cast<FrameIndexSDNode>(Val&: StackSlot)->getIndex();
2157 auto PtrInfo =
2158 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
2159
2160 SDValue LoadExp = DAG.getLoad(VT: ExpVT, dl, Chain, Ptr: StackSlot, PtrInfo);
2161 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
2162 LoadExp.getValue(R: 1), DAG.getRoot());
2163 DAG.setRoot(OutputChain);
2164
2165 Results.push_back(Elt: Call);
2166 Results.push_back(Elt: LoadExp);
2167}
2168
2169void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2170 RTLIB::Libcall LC,
2171 SmallVectorImpl<SDValue> &Results) {
2172 if (LC == RTLIB::UNKNOWN_LIBCALL)
2173 llvm_unreachable("Can't create an unknown libcall!");
2174
2175 if (Node->isStrictFPOpcode()) {
2176 EVT RetVT = Node->getValueType(ResNo: 0);
2177 SmallVector<SDValue, 4> Ops(drop_begin(RangeOrContainer: Node->ops()));
2178 TargetLowering::MakeLibCallOptions CallOptions;
2179 // FIXME: This doesn't support tail calls.
2180 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2181 Ops, CallOptions,
2182 dl: SDLoc(Node),
2183 Chain: Node->getOperand(Num: 0));
2184 Results.push_back(Elt: Tmp.first);
2185 Results.push_back(Elt: Tmp.second);
2186 } else {
2187 SDValue Tmp = ExpandLibCall(LC, Node, isSigned: false).first;
2188 Results.push_back(Elt: Tmp);
2189 }
2190}
2191
2192/// Expand the node to a libcall based on the result type.
2193void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2194 RTLIB::Libcall Call_F32,
2195 RTLIB::Libcall Call_F64,
2196 RTLIB::Libcall Call_F80,
2197 RTLIB::Libcall Call_F128,
2198 RTLIB::Libcall Call_PPCF128,
2199 SmallVectorImpl<SDValue> &Results) {
2200 RTLIB::Libcall LC = RTLIB::getFPLibCall(VT: Node->getSimpleValueType(ResNo: 0),
2201 Call_F32, Call_F64, Call_F80,
2202 Call_F128, Call_PPCF128);
2203 ExpandFPLibCall(Node, LC, Results);
2204}
2205
2206SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2207 RTLIB::Libcall Call_I8,
2208 RTLIB::Libcall Call_I16,
2209 RTLIB::Libcall Call_I32,
2210 RTLIB::Libcall Call_I64,
2211 RTLIB::Libcall Call_I128) {
2212 RTLIB::Libcall LC;
2213 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2214 default: llvm_unreachable("Unexpected request for libcall!");
2215 case MVT::i8: LC = Call_I8; break;
2216 case MVT::i16: LC = Call_I16; break;
2217 case MVT::i32: LC = Call_I32; break;
2218 case MVT::i64: LC = Call_I64; break;
2219 case MVT::i128: LC = Call_I128; break;
2220 }
2221 return ExpandLibCall(LC, Node, isSigned).first;
2222}
2223
2224/// Expand the node to a libcall based on first argument type (for instance
2225/// lround and its variant).
2226void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2227 RTLIB::Libcall Call_F32,
2228 RTLIB::Libcall Call_F64,
2229 RTLIB::Libcall Call_F80,
2230 RTLIB::Libcall Call_F128,
2231 RTLIB::Libcall Call_PPCF128,
2232 SmallVectorImpl<SDValue> &Results) {
2233 EVT InVT = Node->getOperand(Num: Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2234 RTLIB::Libcall LC = RTLIB::getFPLibCall(VT: InVT.getSimpleVT(),
2235 Call_F32, Call_F64, Call_F80,
2236 Call_F128, Call_PPCF128);
2237 ExpandFPLibCall(Node, LC, Results);
2238}
2239
2240/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2241void
2242SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2243 SmallVectorImpl<SDValue> &Results) {
2244 unsigned Opcode = Node->getOpcode();
2245 bool isSigned = Opcode == ISD::SDIVREM;
2246
2247 RTLIB::Libcall LC;
2248 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2249 default: llvm_unreachable("Unexpected request for libcall!");
2250 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2251 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2252 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2253 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2254 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2255 }
2256
2257 // The input chain to this libcall is the entry node of the function.
2258 // Legalizing the call will automatically add the previous call to the
2259 // dependence.
2260 SDValue InChain = DAG.getEntryNode();
2261
2262 EVT RetVT = Node->getValueType(ResNo: 0);
2263 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2264
2265 TargetLowering::ArgListTy Args;
2266 TargetLowering::ArgListEntry Entry;
2267 for (const SDValue &Op : Node->op_values()) {
2268 EVT ArgVT = Op.getValueType();
2269 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2270 Entry.Node = Op;
2271 Entry.Ty = ArgTy;
2272 Entry.IsSExt = isSigned;
2273 Entry.IsZExt = !isSigned;
2274 Args.push_back(x: Entry);
2275 }
2276
2277 // Also pass the return address of the remainder.
2278 SDValue FIPtr = DAG.CreateStackTemporary(VT: RetVT);
2279 Entry.Node = FIPtr;
2280 Entry.Ty = PointerType::getUnqual(C&: RetTy->getContext());
2281 Entry.IsSExt = isSigned;
2282 Entry.IsZExt = !isSigned;
2283 Args.push_back(x: Entry);
2284
2285 SDValue Callee = DAG.getExternalSymbol(Sym: TLI.getLibcallName(Call: LC),
2286 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2287
2288 SDLoc dl(Node);
2289 TargetLowering::CallLoweringInfo CLI(DAG);
2290 CLI.setDebugLoc(dl)
2291 .setChain(InChain)
2292 .setLibCallee(CC: TLI.getLibcallCallingConv(Call: LC), ResultType: RetTy, Target: Callee,
2293 ArgsList: std::move(Args))
2294 .setSExtResult(isSigned)
2295 .setZExtResult(!isSigned);
2296
2297 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2298
2299 // Remainder is loaded back from the stack frame.
2300 SDValue Rem =
2301 DAG.getLoad(VT: RetVT, dl, Chain: CallInfo.second, Ptr: FIPtr, PtrInfo: MachinePointerInfo());
2302 Results.push_back(Elt: CallInfo.first);
2303 Results.push_back(Elt: Rem);
2304}
2305
2306/// Return true if sincos libcall is available.
2307static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
2308 RTLIB::Libcall LC;
2309 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2310 default: llvm_unreachable("Unexpected request for libcall!");
2311 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2312 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2313 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2314 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2315 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2316 }
2317 return TLI.getLibcallName(Call: LC) != nullptr;
2318}
2319
2320/// Only issue sincos libcall if both sin and cos are needed.
2321static bool useSinCos(SDNode *Node) {
2322 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2323 ? ISD::FCOS : ISD::FSIN;
2324
2325 SDValue Op0 = Node->getOperand(Num: 0);
2326 for (const SDNode *User : Op0.getNode()->uses()) {
2327 if (User == Node)
2328 continue;
2329 // The other user might have been turned into sincos already.
2330 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2331 return true;
2332 }
2333 return false;
2334}
2335
2336/// Issue libcalls to sincos to compute sin / cos pairs.
2337void
2338SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2339 SmallVectorImpl<SDValue> &Results) {
2340 RTLIB::Libcall LC;
2341 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2342 default: llvm_unreachable("Unexpected request for libcall!");
2343 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2344 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2345 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2346 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2347 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2348 }
2349
2350 // The input chain to this libcall is the entry node of the function.
2351 // Legalizing the call will automatically add the previous call to the
2352 // dependence.
2353 SDValue InChain = DAG.getEntryNode();
2354
2355 EVT RetVT = Node->getValueType(ResNo: 0);
2356 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2357
2358 TargetLowering::ArgListTy Args;
2359 TargetLowering::ArgListEntry Entry;
2360
2361 // Pass the argument.
2362 Entry.Node = Node->getOperand(Num: 0);
2363 Entry.Ty = RetTy;
2364 Entry.IsSExt = false;
2365 Entry.IsZExt = false;
2366 Args.push_back(x: Entry);
2367
2368 // Pass the return address of sin.
2369 SDValue SinPtr = DAG.CreateStackTemporary(VT: RetVT);
2370 Entry.Node = SinPtr;
2371 Entry.Ty = PointerType::getUnqual(C&: RetTy->getContext());
2372 Entry.IsSExt = false;
2373 Entry.IsZExt = false;
2374 Args.push_back(x: Entry);
2375
2376 // Also pass the return address of the cos.
2377 SDValue CosPtr = DAG.CreateStackTemporary(VT: RetVT);
2378 Entry.Node = CosPtr;
2379 Entry.Ty = PointerType::getUnqual(C&: RetTy->getContext());
2380 Entry.IsSExt = false;
2381 Entry.IsZExt = false;
2382 Args.push_back(x: Entry);
2383
2384 SDValue Callee = DAG.getExternalSymbol(Sym: TLI.getLibcallName(Call: LC),
2385 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2386
2387 SDLoc dl(Node);
2388 TargetLowering::CallLoweringInfo CLI(DAG);
2389 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2390 CC: TLI.getLibcallCallingConv(Call: LC), ResultType: Type::getVoidTy(C&: *DAG.getContext()), Target: Callee,
2391 ArgsList: std::move(Args));
2392
2393 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2394
2395 Results.push_back(
2396 Elt: DAG.getLoad(VT: RetVT, dl, Chain: CallInfo.second, Ptr: SinPtr, PtrInfo: MachinePointerInfo()));
2397 Results.push_back(
2398 Elt: DAG.getLoad(VT: RetVT, dl, Chain: CallInfo.second, Ptr: CosPtr, PtrInfo: MachinePointerInfo()));
2399}
2400
2401SDValue SelectionDAGLegalize::expandLdexp(SDNode *Node) const {
2402 SDLoc dl(Node);
2403 EVT VT = Node->getValueType(ResNo: 0);
2404 SDValue X = Node->getOperand(Num: 0);
2405 SDValue N = Node->getOperand(Num: 1);
2406 EVT ExpVT = N.getValueType();
2407 EVT AsIntVT = VT.changeTypeToInteger();
2408 if (AsIntVT == EVT()) // TODO: How to handle f80?
2409 return SDValue();
2410
2411 if (Node->getOpcode() == ISD::STRICT_FLDEXP) // TODO
2412 return SDValue();
2413
2414 SDNodeFlags NSW;
2415 NSW.setNoSignedWrap(true);
2416 SDNodeFlags NUW_NSW;
2417 NUW_NSW.setNoUnsignedWrap(true);
2418 NUW_NSW.setNoSignedWrap(true);
2419
2420 EVT SetCCVT =
2421 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: ExpVT);
2422 const fltSemantics &FltSem = SelectionDAG::EVTToAPFloatSemantics(VT);
2423
2424 const APFloat::ExponentType MaxExpVal = APFloat::semanticsMaxExponent(FltSem);
2425 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2426 const int Precision = APFloat::semanticsPrecision(FltSem);
2427
2428 const SDValue MaxExp = DAG.getConstant(Val: MaxExpVal, DL: dl, VT: ExpVT);
2429 const SDValue MinExp = DAG.getConstant(Val: MinExpVal, DL: dl, VT: ExpVT);
2430
2431 const SDValue DoubleMaxExp = DAG.getConstant(Val: 2 * MaxExpVal, DL: dl, VT: ExpVT);
2432
2433 const APFloat One(FltSem, "1.0");
2434 APFloat ScaleUpK = scalbn(X: One, Exp: MaxExpVal, RM: APFloat::rmNearestTiesToEven);
2435
2436 // Offset by precision to avoid denormal range.
2437 APFloat ScaleDownK =
2438 scalbn(X: One, Exp: MinExpVal + Precision, RM: APFloat::rmNearestTiesToEven);
2439
2440 // TODO: Should really introduce control flow and use a block for the >
2441 // MaxExp, < MinExp cases
2442
2443 // First, handle exponents Exp > MaxExp and scale down.
2444 SDValue NGtMaxExp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: MaxExp, Cond: ISD::SETGT);
2445
2446 SDValue DecN0 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ExpVT, N1: N, N2: MaxExp, Flags: NSW);
2447 SDValue ClampMaxVal = DAG.getConstant(Val: 3 * MaxExpVal, DL: dl, VT: ExpVT);
2448 SDValue ClampN_Big = DAG.getNode(Opcode: ISD::SMIN, DL: dl, VT: ExpVT, N1: N, N2: ClampMaxVal);
2449 SDValue DecN1 =
2450 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ExpVT, N1: ClampN_Big, N2: DoubleMaxExp, Flags: NSW);
2451
2452 SDValue ScaleUpTwice =
2453 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: DoubleMaxExp, Cond: ISD::SETUGT);
2454
2455 const SDValue ScaleUpVal = DAG.getConstantFP(Val: ScaleUpK, DL: dl, VT);
2456 SDValue ScaleUp0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: X, N2: ScaleUpVal);
2457 SDValue ScaleUp1 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: ScaleUp0, N2: ScaleUpVal);
2458
2459 SDValue SelectN_Big =
2460 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: ScaleUpTwice, N2: DecN1, N3: DecN0);
2461 SDValue SelectX_Big =
2462 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: ScaleUpTwice, N2: ScaleUp1, N3: ScaleUp0);
2463
2464 // Now handle exponents Exp < MinExp
2465 SDValue NLtMinExp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: MinExp, Cond: ISD::SETLT);
2466
2467 SDValue Increment0 = DAG.getConstant(Val: -(MinExpVal + Precision), DL: dl, VT: ExpVT);
2468 SDValue Increment1 = DAG.getConstant(Val: -2 * (MinExpVal + Precision), DL: dl, VT: ExpVT);
2469
2470 SDValue IncN0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: N, N2: Increment0, Flags: NUW_NSW);
2471
2472 SDValue ClampMinVal =
2473 DAG.getConstant(Val: 3 * MinExpVal + 2 * Precision, DL: dl, VT: ExpVT);
2474 SDValue ClampN_Small = DAG.getNode(Opcode: ISD::SMAX, DL: dl, VT: ExpVT, N1: N, N2: ClampMinVal);
2475 SDValue IncN1 =
2476 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: ClampN_Small, N2: Increment1, Flags: NSW);
2477
2478 const SDValue ScaleDownVal = DAG.getConstantFP(Val: ScaleDownK, DL: dl, VT);
2479 SDValue ScaleDown0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: X, N2: ScaleDownVal);
2480 SDValue ScaleDown1 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: ScaleDown0, N2: ScaleDownVal);
2481
2482 SDValue ScaleDownTwice = DAG.getSetCC(
2483 DL: dl, VT: SetCCVT, LHS: N, RHS: DAG.getConstant(Val: 2 * MinExpVal + Precision, DL: dl, VT: ExpVT),
2484 Cond: ISD::SETULT);
2485
2486 SDValue SelectN_Small =
2487 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: ScaleDownTwice, N2: IncN1, N3: IncN0);
2488 SDValue SelectX_Small =
2489 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: ScaleDownTwice, N2: ScaleDown1, N3: ScaleDown0);
2490
2491 // Now combine the two out of range exponent handling cases with the base
2492 // case.
2493 SDValue NewX = DAG.getNode(
2494 Opcode: ISD::SELECT, DL: dl, VT, N1: NGtMaxExp, N2: SelectX_Big,
2495 N3: DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: NLtMinExp, N2: SelectX_Small, N3: X));
2496
2497 SDValue NewN = DAG.getNode(
2498 Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: NGtMaxExp, N2: SelectN_Big,
2499 N3: DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: NLtMinExp, N2: SelectN_Small, N3: N));
2500
2501 SDValue BiasedN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: NewN, N2: MaxExp, Flags: NSW);
2502
2503 SDValue ExponentShiftAmt =
2504 DAG.getShiftAmountConstant(Val: Precision - 1, VT: ExpVT, DL: dl);
2505 SDValue CastExpToValTy = DAG.getZExtOrTrunc(Op: BiasedN, DL: dl, VT: AsIntVT);
2506
2507 SDValue AsInt = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: AsIntVT, N1: CastExpToValTy,
2508 N2: ExponentShiftAmt, Flags: NUW_NSW);
2509 SDValue AsFP = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: AsInt);
2510 return DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: NewX, N2: AsFP);
2511}
2512
2513SDValue SelectionDAGLegalize::expandFrexp(SDNode *Node) const {
2514 SDLoc dl(Node);
2515 SDValue Val = Node->getOperand(Num: 0);
2516 EVT VT = Val.getValueType();
2517 EVT ExpVT = Node->getValueType(ResNo: 1);
2518 EVT AsIntVT = VT.changeTypeToInteger();
2519 if (AsIntVT == EVT()) // TODO: How to handle f80?
2520 return SDValue();
2521
2522 const fltSemantics &FltSem = SelectionDAG::EVTToAPFloatSemantics(VT);
2523 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2524 const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2525 const unsigned BitSize = VT.getScalarSizeInBits();
2526
2527 // TODO: Could introduce control flow and skip over the denormal handling.
2528
2529 // scale_up = fmul value, scalbn(1.0, precision + 1)
2530 // extracted_exp = (bitcast value to uint) >> precision - 1
2531 // biased_exp = extracted_exp + min_exp
2532 // extracted_fract = (bitcast value to uint) & (fract_mask | sign_mask)
2533 //
2534 // is_denormal = val < smallest_normalized
2535 // computed_fract = is_denormal ? scale_up : extracted_fract
2536 // computed_exp = is_denormal ? biased_exp + (-precision - 1) : biased_exp
2537 //
2538 // result_0 = (!isfinite(val) || iszero(val)) ? val : computed_fract
2539 // result_1 = (!isfinite(val) || iszero(val)) ? 0 : computed_exp
2540
2541 SDValue NegSmallestNormalizedInt = DAG.getConstant(
2542 Val: APFloat::getSmallestNormalized(Sem: FltSem, Negative: true).bitcastToAPInt(), DL: dl,
2543 VT: AsIntVT);
2544
2545 SDValue SmallestNormalizedInt = DAG.getConstant(
2546 Val: APFloat::getSmallestNormalized(Sem: FltSem, Negative: false).bitcastToAPInt(), DL: dl,
2547 VT: AsIntVT);
2548
2549 // Masks out the exponent bits.
2550 SDValue ExpMask =
2551 DAG.getConstant(Val: APFloat::getInf(Sem: FltSem).bitcastToAPInt(), DL: dl, VT: AsIntVT);
2552
2553 // Mask out the exponent part of the value.
2554 //
2555 // e.g, for f32 FractSignMaskVal = 0x807fffff
2556 APInt FractSignMaskVal = APInt::getBitsSet(numBits: BitSize, loBit: 0, hiBit: Precision - 1);
2557 FractSignMaskVal.setBit(BitSize - 1); // Set the sign bit
2558
2559 APInt SignMaskVal = APInt::getSignedMaxValue(numBits: BitSize);
2560 SDValue SignMask = DAG.getConstant(Val: SignMaskVal, DL: dl, VT: AsIntVT);
2561
2562 SDValue FractSignMask = DAG.getConstant(Val: FractSignMaskVal, DL: dl, VT: AsIntVT);
2563
2564 const APFloat One(FltSem, "1.0");
2565 // Scale a possible denormal input.
2566 // e.g., for f64, 0x1p+54
2567 APFloat ScaleUpKVal =
2568 scalbn(X: One, Exp: Precision + 1, RM: APFloat::rmNearestTiesToEven);
2569
2570 SDValue ScaleUpK = DAG.getConstantFP(Val: ScaleUpKVal, DL: dl, VT);
2571 SDValue ScaleUp = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: Val, N2: ScaleUpK);
2572
2573 EVT SetCCVT =
2574 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2575
2576 SDValue AsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AsIntVT, Operand: Val);
2577
2578 SDValue Abs = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: AsInt, N2: SignMask);
2579
2580 SDValue AddNegSmallestNormal =
2581 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AsIntVT, N1: Abs, N2: NegSmallestNormalizedInt);
2582 SDValue DenormOrZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AddNegSmallestNormal,
2583 RHS: NegSmallestNormalizedInt, Cond: ISD::SETULE);
2584
2585 SDValue IsDenormal =
2586 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Abs, RHS: SmallestNormalizedInt, Cond: ISD::SETULT);
2587
2588 SDValue MinExp = DAG.getConstant(Val: MinExpVal, DL: dl, VT: ExpVT);
2589 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: ExpVT);
2590
2591 SDValue ScaledAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AsIntVT, Operand: ScaleUp);
2592 SDValue ScaledSelect =
2593 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: AsIntVT, N1: IsDenormal, N2: ScaledAsInt, N3: AsInt);
2594
2595 SDValue ExpMaskScaled =
2596 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: ScaledAsInt, N2: ExpMask);
2597
2598 SDValue ScaledValue =
2599 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: AsIntVT, N1: IsDenormal, N2: ExpMaskScaled, N3: Abs);
2600
2601 // Extract the exponent bits.
2602 SDValue ExponentShiftAmt =
2603 DAG.getShiftAmountConstant(Val: Precision - 1, VT: AsIntVT, DL: dl);
2604 SDValue ShiftedExp =
2605 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: AsIntVT, N1: ScaledValue, N2: ExponentShiftAmt);
2606 SDValue Exp = DAG.getSExtOrTrunc(Op: ShiftedExp, DL: dl, VT: ExpVT);
2607
2608 SDValue NormalBiasedExp = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: Exp, N2: MinExp);
2609 SDValue DenormalOffset = DAG.getConstant(Val: -Precision - 1, DL: dl, VT: ExpVT);
2610 SDValue DenormalExpBias =
2611 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: IsDenormal, N2: DenormalOffset, N3: Zero);
2612
2613 SDValue MaskedFractAsInt =
2614 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: ScaledSelect, N2: FractSignMask);
2615 const APFloat Half(FltSem, "0.5");
2616 SDValue FPHalf = DAG.getConstant(Val: Half.bitcastToAPInt(), DL: dl, VT: AsIntVT);
2617 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: AsIntVT, N1: MaskedFractAsInt, N2: FPHalf);
2618 SDValue MaskedFract = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Or);
2619
2620 SDValue ComputedExp =
2621 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: NormalBiasedExp, N2: DenormalExpBias);
2622
2623 SDValue Result0 =
2624 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: DenormOrZero, N2: Val, N3: MaskedFract);
2625
2626 SDValue Result1 =
2627 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: DenormOrZero, N2: Zero, N3: ComputedExp);
2628
2629 return DAG.getMergeValues(Ops: {Result0, Result1}, dl);
2630}
2631
2632/// This function is responsible for legalizing a
2633/// INT_TO_FP operation of the specified operand when the target requests that
2634/// we expand it. At this point, we know that the result and operand types are
2635/// legal for the target.
2636SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node,
2637 SDValue &Chain) {
2638 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
2639 Node->getOpcode() == ISD::SINT_TO_FP);
2640 EVT DestVT = Node->getValueType(ResNo: 0);
2641 SDLoc dl(Node);
2642 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
2643 SDValue Op0 = Node->getOperand(Num: OpNo);
2644 EVT SrcVT = Op0.getValueType();
2645
2646 // TODO: Should any fast-math-flags be set for the created nodes?
2647 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2648 if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) &&
2649 (DestVT.bitsLE(MVT::f64) ||
2650 TLI.isOperationLegal(Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND
2651 : ISD::FP_EXTEND,
2652 DestVT))) {
2653 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2654 "expansion\n");
2655
2656 // Get the stack frame index of a 8 byte buffer.
2657 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2658
2659 SDValue Lo = Op0;
2660 // if signed map to unsigned space
2661 if (isSigned) {
2662 // Invert sign bit (signed to unsigned mapping).
2663 Lo = DAG.getNode(ISD::XOR, dl, MVT::i32, Lo,
2664 DAG.getConstant(0x80000000u, dl, MVT::i32));
2665 }
2666 // Initial hi portion of constructed double.
2667 SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2668
2669 // If this a big endian target, swap the lo and high data.
2670 if (DAG.getDataLayout().isBigEndian())
2671 std::swap(a&: Lo, b&: Hi);
2672
2673 SDValue MemChain = DAG.getEntryNode();
2674
2675 // Store the lo of the constructed double.
2676 SDValue Store1 = DAG.getStore(Chain: MemChain, dl, Val: Lo, Ptr: StackSlot,
2677 PtrInfo: MachinePointerInfo());
2678 // Store the hi of the constructed double.
2679 SDValue HiPtr =
2680 DAG.getMemBasePlusOffset(Base: StackSlot, Offset: TypeSize::getFixed(ExactSize: 4), DL: dl);
2681 SDValue Store2 =
2682 DAG.getStore(Chain: MemChain, dl, Val: Hi, Ptr: HiPtr, PtrInfo: MachinePointerInfo());
2683 MemChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
2684
2685 // load the constructed double
2686 SDValue Load =
2687 DAG.getLoad(MVT::f64, dl, MemChain, StackSlot, MachinePointerInfo());
2688 // FP constant to bias correct the final result
2689 SDValue Bias = DAG.getConstantFP(
2690 isSigned ? llvm::bit_cast<double>(0x4330000080000000ULL)
2691 : llvm::bit_cast<double>(0x4330000000000000ULL),
2692 dl, MVT::f64);
2693 // Subtract the bias and get the final result.
2694 SDValue Sub;
2695 SDValue Result;
2696 if (Node->isStrictFPOpcode()) {
2697 Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other},
2698 {Node->getOperand(0), Load, Bias});
2699 Chain = Sub.getValue(R: 1);
2700 if (DestVT != Sub.getValueType()) {
2701 std::pair<SDValue, SDValue> ResultPair;
2702 ResultPair =
2703 DAG.getStrictFPExtendOrRound(Op: Sub, Chain, DL: dl, VT: DestVT);
2704 Result = ResultPair.first;
2705 Chain = ResultPair.second;
2706 }
2707 else
2708 Result = Sub;
2709 } else {
2710 Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2711 Result = DAG.getFPExtendOrRound(Op: Sub, DL: dl, VT: DestVT);
2712 }
2713 return Result;
2714 }
2715
2716 if (isSigned)
2717 return SDValue();
2718
2719 // TODO: Generalize this for use with other types.
2720 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2721 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2722 LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n");
2723 // For unsigned conversions, convert them to signed conversions using the
2724 // algorithm from the x86_64 __floatundisf in compiler_rt. That method
2725 // should be valid for i32->f32 as well.
2726
2727 // More generally this transform should be valid if there are 3 more bits
2728 // in the integer type than the significand. Rounding uses the first bit
2729 // after the width of the significand and the OR of all bits after that. So
2730 // we need to be able to OR the shifted out bit into one of the bits that
2731 // participate in the OR.
2732
2733 // TODO: This really should be implemented using a branch rather than a
2734 // select. We happen to get lucky and machinesink does the right
2735 // thing most of the time. This would be a good candidate for a
2736 // pseudo-op, or, even better, for whole-function isel.
2737 EVT SetCCVT = getSetCCResultType(VT: SrcVT);
2738
2739 SDValue SignBitTest = DAG.getSetCC(
2740 DL: dl, VT: SetCCVT, LHS: Op0, RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT), Cond: ISD::SETLT);
2741
2742 EVT ShiftVT = TLI.getShiftAmountTy(LHSTy: SrcVT, DL: DAG.getDataLayout());
2743 SDValue ShiftConst = DAG.getConstant(Val: 1, DL: dl, VT: ShiftVT);
2744 SDValue Shr = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: SrcVT, N1: Op0, N2: ShiftConst);
2745 SDValue AndConst = DAG.getConstant(Val: 1, DL: dl, VT: SrcVT);
2746 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SrcVT, N1: Op0, N2: AndConst);
2747 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SrcVT, N1: And, N2: Shr);
2748
2749 SDValue Slow, Fast;
2750 if (Node->isStrictFPOpcode()) {
2751 // In strict mode, we must avoid spurious exceptions, and therefore
2752 // must make sure to only emit a single STRICT_SINT_TO_FP.
2753 SDValue InCvt = DAG.getSelect(DL: dl, VT: SrcVT, Cond: SignBitTest, LHS: Or, RHS: Op0);
2754 Fast = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2755 { Node->getOperand(0), InCvt });
2756 Slow = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2757 { Fast.getValue(1), Fast, Fast });
2758 Chain = Slow.getValue(R: 1);
2759 // The STRICT_SINT_TO_FP inherits the exception mode from the
2760 // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can
2761 // never raise any exception.
2762 SDNodeFlags Flags;
2763 Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept());
2764 Fast->setFlags(Flags);
2765 Flags.setNoFPExcept(true);
2766 Slow->setFlags(Flags);
2767 } else {
2768 SDValue SignCvt = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Or);
2769 Slow = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DestVT, N1: SignCvt, N2: SignCvt);
2770 Fast = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Op0);
2771 }
2772
2773 return DAG.getSelect(DL: dl, VT: DestVT, Cond: SignBitTest, LHS: Slow, RHS: Fast);
2774 }
2775
2776 // Don't expand it if there isn't cheap fadd.
2777 if (!TLI.isOperationLegalOrCustom(
2778 Op: Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, VT: DestVT))
2779 return SDValue();
2780
2781 // The following optimization is valid only if every value in SrcVT (when
2782 // treated as signed) is representable in DestVT. Check that the mantissa
2783 // size of DestVT is >= than the number of bits in SrcVT -1.
2784 assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >=
2785 SrcVT.getSizeInBits() - 1 &&
2786 "Cannot perform lossless SINT_TO_FP!");
2787
2788 SDValue Tmp1;
2789 if (Node->isStrictFPOpcode()) {
2790 Tmp1 = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2791 { Node->getOperand(0), Op0 });
2792 } else
2793 Tmp1 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Op0);
2794
2795 SDValue SignSet = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: SrcVT), LHS: Op0,
2796 RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT), Cond: ISD::SETLT);
2797 SDValue Zero = DAG.getIntPtrConstant(Val: 0, DL: dl),
2798 Four = DAG.getIntPtrConstant(Val: 4, DL: dl);
2799 SDValue CstOffset = DAG.getSelect(DL: dl, VT: Zero.getValueType(),
2800 Cond: SignSet, LHS: Four, RHS: Zero);
2801
2802 // If the sign bit of the integer is set, the large number will be treated
2803 // as a negative number. To counteract this, the dynamic code adds an
2804 // offset depending on the data type.
2805 uint64_t FF;
2806 switch (SrcVT.getSimpleVT().SimpleTy) {
2807 default:
2808 return SDValue();
2809 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2810 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2811 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2812 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2813 }
2814 if (DAG.getDataLayout().isLittleEndian())
2815 FF <<= 32;
2816 Constant *FudgeFactor = ConstantInt::get(
2817 Ty: Type::getInt64Ty(C&: *DAG.getContext()), V: FF);
2818
2819 SDValue CPIdx =
2820 DAG.getConstantPool(C: FudgeFactor, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2821 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
2822 CPIdx = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: CPIdx.getValueType(), N1: CPIdx, N2: CstOffset);
2823 Alignment = commonAlignment(A: Alignment, Offset: 4);
2824 SDValue FudgeInReg;
2825 if (DestVT == MVT::f32)
2826 FudgeInReg = DAG.getLoad(
2827 MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2828 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2829 Alignment);
2830 else {
2831 SDValue Load = DAG.getExtLoad(
2832 ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2833 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2834 Alignment);
2835 HandleSDNode Handle(Load);
2836 LegalizeOp(Node: Load.getNode());
2837 FudgeInReg = Handle.getValue();
2838 }
2839
2840 if (Node->isStrictFPOpcode()) {
2841 SDValue Result = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2842 { Tmp1.getValue(1), Tmp1, FudgeInReg });
2843 Chain = Result.getValue(R: 1);
2844 return Result;
2845 }
2846
2847 return DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DestVT, N1: Tmp1, N2: FudgeInReg);
2848}
2849
2850/// This function is responsible for legalizing a
2851/// *INT_TO_FP operation of the specified operand when the target requests that
2852/// we promote it. At this point, we know that the result and operand types are
2853/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2854/// operation that takes a larger input.
2855void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2856 SDNode *N, const SDLoc &dl, SmallVectorImpl<SDValue> &Results) {
2857 bool IsStrict = N->isStrictFPOpcode();
2858 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
2859 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
2860 EVT DestVT = N->getValueType(ResNo: 0);
2861 SDValue LegalOp = N->getOperand(Num: IsStrict ? 1 : 0);
2862 unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP;
2863 unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP;
2864
2865 // First step, figure out the appropriate *INT_TO_FP operation to use.
2866 EVT NewInTy = LegalOp.getValueType();
2867
2868 unsigned OpToUse = 0;
2869
2870 // Scan for the appropriate larger type to use.
2871 while (true) {
2872 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2873 assert(NewInTy.isInteger() && "Ran out of possibilities!");
2874
2875 // If the target supports SINT_TO_FP of this type, use it.
2876 if (TLI.isOperationLegalOrCustom(Op: SIntOp, VT: NewInTy)) {
2877 OpToUse = SIntOp;
2878 break;
2879 }
2880 if (IsSigned)
2881 continue;
2882
2883 // If the target supports UINT_TO_FP of this type, use it.
2884 if (TLI.isOperationLegalOrCustom(Op: UIntOp, VT: NewInTy)) {
2885 OpToUse = UIntOp;
2886 break;
2887 }
2888
2889 // Otherwise, try a larger type.
2890 }
2891
2892 // Okay, we found the operation and type to use. Zero extend our input to the
2893 // desired type then run the operation on it.
2894 if (IsStrict) {
2895 SDValue Res =
2896 DAG.getNode(OpToUse, dl, {DestVT, MVT::Other},
2897 {N->getOperand(0),
2898 DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2899 dl, NewInTy, LegalOp)});
2900 Results.push_back(Elt: Res);
2901 Results.push_back(Elt: Res.getValue(R: 1));
2902 return;
2903 }
2904
2905 Results.push_back(
2906 Elt: DAG.getNode(Opcode: OpToUse, DL: dl, VT: DestVT,
2907 Operand: DAG.getNode(Opcode: IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2908 DL: dl, VT: NewInTy, Operand: LegalOp)));
2909}
2910
2911/// This function is responsible for legalizing a
2912/// FP_TO_*INT operation of the specified operand when the target requests that
2913/// we promote it. At this point, we know that the result and operand types are
2914/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2915/// operation that returns a larger result.
2916void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
2917 SmallVectorImpl<SDValue> &Results) {
2918 bool IsStrict = N->isStrictFPOpcode();
2919 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
2920 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2921 EVT DestVT = N->getValueType(ResNo: 0);
2922 SDValue LegalOp = N->getOperand(Num: IsStrict ? 1 : 0);
2923 // First step, figure out the appropriate FP_TO*INT operation to use.
2924 EVT NewOutTy = DestVT;
2925
2926 unsigned OpToUse = 0;
2927
2928 // Scan for the appropriate larger type to use.
2929 while (true) {
2930 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2931 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2932
2933 // A larger signed type can hold all unsigned values of the requested type,
2934 // so using FP_TO_SINT is valid
2935 OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT;
2936 if (TLI.isOperationLegalOrCustom(Op: OpToUse, VT: NewOutTy))
2937 break;
2938
2939 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2940 OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT;
2941 if (!IsSigned && TLI.isOperationLegalOrCustom(Op: OpToUse, VT: NewOutTy))
2942 break;
2943
2944 // Otherwise, try a larger type.
2945 }
2946
2947 // Okay, we found the operation and type to use.
2948 SDValue Operation;
2949 if (IsStrict) {
2950 SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other);
2951 Operation = DAG.getNode(Opcode: OpToUse, DL: dl, VTList: VTs, N1: N->getOperand(Num: 0), N2: LegalOp);
2952 } else
2953 Operation = DAG.getNode(Opcode: OpToUse, DL: dl, VT: NewOutTy, Operand: LegalOp);
2954
2955 // Truncate the result of the extended FP_TO_*INT operation to the desired
2956 // size.
2957 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: DestVT, Operand: Operation);
2958 Results.push_back(Elt: Trunc);
2959 if (IsStrict)
2960 Results.push_back(Elt: Operation.getValue(R: 1));
2961}
2962
2963/// Promote FP_TO_*INT_SAT operation to a larger result type. At this point
2964/// the result and operand types are legal and there must be a legal
2965/// FP_TO_*INT_SAT operation for a larger result type.
2966SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node,
2967 const SDLoc &dl) {
2968 unsigned Opcode = Node->getOpcode();
2969
2970 // Scan for the appropriate larger type to use.
2971 EVT NewOutTy = Node->getValueType(ResNo: 0);
2972 while (true) {
2973 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1);
2974 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2975
2976 if (TLI.isOperationLegalOrCustom(Op: Opcode, VT: NewOutTy))
2977 break;
2978 }
2979
2980 // Saturation width is determined by second operand, so we don't have to
2981 // perform any fixup and can directly truncate the result.
2982 SDValue Result = DAG.getNode(Opcode, DL: dl, VT: NewOutTy, N1: Node->getOperand(Num: 0),
2983 N2: Node->getOperand(Num: 1));
2984 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Result);
2985}
2986
2987/// Open code the operations for PARITY of the specified operation.
2988SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) {
2989 EVT VT = Op.getValueType();
2990 EVT ShVT = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
2991 unsigned Sz = VT.getScalarSizeInBits();
2992
2993 // If CTPOP is legal, use it. Otherwise use shifts and xor.
2994 SDValue Result;
2995 if (TLI.isOperationLegalOrPromote(Op: ISD::CTPOP, VT)) {
2996 Result = DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Op);
2997 } else {
2998 Result = Op;
2999 for (unsigned i = Log2_32_Ceil(Value: Sz); i != 0;) {
3000 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Result,
3001 N2: DAG.getConstant(Val: 1ULL << (--i), DL: dl, VT: ShVT));
3002 Result = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Result, N2: Shift);
3003 }
3004 }
3005
3006 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Result, N2: DAG.getConstant(Val: 1, DL: dl, VT));
3007}
3008
3009bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
3010 LLVM_DEBUG(dbgs() << "Trying to expand node\n");
3011 SmallVector<SDValue, 8> Results;
3012 SDLoc dl(Node);
3013 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
3014 bool NeedInvert;
3015 switch (Node->getOpcode()) {
3016 case ISD::ABS:
3017 if ((Tmp1 = TLI.expandABS(N: Node, DAG)))
3018 Results.push_back(Elt: Tmp1);
3019 break;
3020 case ISD::ABDS:
3021 case ISD::ABDU:
3022 if ((Tmp1 = TLI.expandABD(N: Node, DAG)))
3023 Results.push_back(Elt: Tmp1);
3024 break;
3025 case ISD::CTPOP:
3026 if ((Tmp1 = TLI.expandCTPOP(N: Node, DAG)))
3027 Results.push_back(Elt: Tmp1);
3028 break;
3029 case ISD::CTLZ:
3030 case ISD::CTLZ_ZERO_UNDEF:
3031 if ((Tmp1 = TLI.expandCTLZ(N: Node, DAG)))
3032 Results.push_back(Elt: Tmp1);
3033 break;
3034 case ISD::CTTZ:
3035 case ISD::CTTZ_ZERO_UNDEF:
3036 if ((Tmp1 = TLI.expandCTTZ(N: Node, DAG)))
3037 Results.push_back(Elt: Tmp1);
3038 break;
3039 case ISD::BITREVERSE:
3040 if ((Tmp1 = TLI.expandBITREVERSE(N: Node, DAG)))
3041 Results.push_back(Elt: Tmp1);
3042 break;
3043 case ISD::BSWAP:
3044 if ((Tmp1 = TLI.expandBSWAP(N: Node, DAG)))
3045 Results.push_back(Elt: Tmp1);
3046 break;
3047 case ISD::PARITY:
3048 Results.push_back(Elt: ExpandPARITY(Op: Node->getOperand(Num: 0), dl));
3049 break;
3050 case ISD::FRAMEADDR:
3051 case ISD::RETURNADDR:
3052 case ISD::FRAME_TO_ARGS_OFFSET:
3053 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3054 break;
3055 case ISD::EH_DWARF_CFA: {
3056 SDValue CfaArg = DAG.getSExtOrTrunc(Op: Node->getOperand(Num: 0), DL: dl,
3057 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
3058 SDValue Offset = DAG.getNode(Opcode: ISD::ADD, DL: dl,
3059 VT: CfaArg.getValueType(),
3060 N1: DAG.getNode(Opcode: ISD::FRAME_TO_ARGS_OFFSET, DL: dl,
3061 VT: CfaArg.getValueType()),
3062 N2: CfaArg);
3063 SDValue FA = DAG.getNode(
3064 Opcode: ISD::FRAMEADDR, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()),
3065 Operand: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
3066 Results.push_back(Elt: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: FA.getValueType(),
3067 N1: FA, N2: Offset));
3068 break;
3069 }
3070 case ISD::GET_ROUNDING:
3071 Results.push_back(Elt: DAG.getConstant(Val: 1, DL: dl, VT: Node->getValueType(ResNo: 0)));
3072 Results.push_back(Elt: Node->getOperand(Num: 0));
3073 break;
3074 case ISD::EH_RETURN:
3075 case ISD::EH_LABEL:
3076 case ISD::PREFETCH:
3077 case ISD::VAEND:
3078 case ISD::EH_SJLJ_LONGJMP:
3079 // If the target didn't expand these, there's nothing to do, so just
3080 // preserve the chain and be done.
3081 Results.push_back(Elt: Node->getOperand(Num: 0));
3082 break;
3083 case ISD::READCYCLECOUNTER:
3084 case ISD::READSTEADYCOUNTER:
3085 // If the target didn't expand this, just return 'zero' and preserve the
3086 // chain.
3087 Results.append(NumInputs: Node->getNumValues() - 1,
3088 Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3089 Results.push_back(Elt: Node->getOperand(Num: 0));
3090 break;
3091 case ISD::EH_SJLJ_SETJMP:
3092 // If the target didn't expand this, just return 'zero' and preserve the
3093 // chain.
3094 Results.push_back(DAG.getConstant(0, dl, MVT::i32));
3095 Results.push_back(Elt: Node->getOperand(Num: 0));
3096 break;
3097 case ISD::ATOMIC_LOAD: {
3098 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
3099 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0));
3100 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
3101 SDValue Swap = DAG.getAtomicCmpSwap(
3102 Opcode: ISD::ATOMIC_CMP_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(), VTs,
3103 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1), Cmp: Zero, Swp: Zero,
3104 MMO: cast<AtomicSDNode>(Val: Node)->getMemOperand());
3105 Results.push_back(Elt: Swap.getValue(R: 0));
3106 Results.push_back(Elt: Swap.getValue(R: 1));
3107 break;
3108 }
3109 case ISD::ATOMIC_STORE: {
3110 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
3111 SDValue Swap = DAG.getAtomic(
3112 Opcode: ISD::ATOMIC_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(),
3113 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 2), Val: Node->getOperand(Num: 1),
3114 MMO: cast<AtomicSDNode>(Val: Node)->getMemOperand());
3115 Results.push_back(Elt: Swap.getValue(R: 1));
3116 break;
3117 }
3118 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
3119 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
3120 // splits out the success value as a comparison. Expanding the resulting
3121 // ATOMIC_CMP_SWAP will produce a libcall.
3122 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
3123 SDValue Res = DAG.getAtomicCmpSwap(
3124 Opcode: ISD::ATOMIC_CMP_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(), VTs,
3125 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1), Cmp: Node->getOperand(Num: 2),
3126 Swp: Node->getOperand(Num: 3), MMO: cast<MemSDNode>(Val: Node)->getMemOperand());
3127
3128 SDValue ExtRes = Res;
3129 SDValue LHS = Res;
3130 SDValue RHS = Node->getOperand(Num: 1);
3131
3132 EVT AtomicType = cast<AtomicSDNode>(Val: Node)->getMemoryVT();
3133 EVT OuterType = Node->getValueType(ResNo: 0);
3134 switch (TLI.getExtendForAtomicOps()) {
3135 case ISD::SIGN_EXTEND:
3136 LHS = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: OuterType, N1: Res,
3137 N2: DAG.getValueType(AtomicType));
3138 RHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: OuterType,
3139 N1: Node->getOperand(Num: 2), N2: DAG.getValueType(AtomicType));
3140 ExtRes = LHS;
3141 break;
3142 case ISD::ZERO_EXTEND:
3143 LHS = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: OuterType, N1: Res,
3144 N2: DAG.getValueType(AtomicType));
3145 RHS = DAG.getZeroExtendInReg(Op: Node->getOperand(Num: 2), DL: dl, VT: AtomicType);
3146 ExtRes = LHS;
3147 break;
3148 case ISD::ANY_EXTEND:
3149 LHS = DAG.getZeroExtendInReg(Op: Res, DL: dl, VT: AtomicType);
3150 RHS = DAG.getZeroExtendInReg(Op: Node->getOperand(Num: 2), DL: dl, VT: AtomicType);
3151 break;
3152 default:
3153 llvm_unreachable("Invalid atomic op extension");
3154 }
3155
3156 SDValue Success =
3157 DAG.getSetCC(DL: dl, VT: Node->getValueType(ResNo: 1), LHS, RHS, Cond: ISD::SETEQ);
3158
3159 Results.push_back(Elt: ExtRes.getValue(R: 0));
3160 Results.push_back(Elt: Success);
3161 Results.push_back(Elt: Res.getValue(R: 1));
3162 break;
3163 }
3164 case ISD::ATOMIC_LOAD_SUB: {
3165 SDLoc DL(Node);
3166 EVT VT = Node->getValueType(ResNo: 0);
3167 SDValue RHS = Node->getOperand(Num: 2);
3168 AtomicSDNode *AN = cast<AtomicSDNode>(Val: Node);
3169 if (RHS->getOpcode() == ISD::SIGN_EXTEND_INREG &&
3170 cast<VTSDNode>(Val: RHS->getOperand(Num: 1))->getVT() == AN->getMemoryVT())
3171 RHS = RHS->getOperand(Num: 0);
3172 SDValue NewRHS =
3173 DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: RHS);
3174 SDValue Res = DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD_ADD, dl: DL, MemVT: AN->getMemoryVT(),
3175 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1),
3176 Val: NewRHS, MMO: AN->getMemOperand());
3177 Results.push_back(Elt: Res);
3178 Results.push_back(Elt: Res.getValue(R: 1));
3179 break;
3180 }
3181 case ISD::DYNAMIC_STACKALLOC:
3182 ExpandDYNAMIC_STACKALLOC(Node, Results);
3183 break;
3184 case ISD::MERGE_VALUES:
3185 for (unsigned i = 0; i < Node->getNumValues(); i++)
3186 Results.push_back(Elt: Node->getOperand(Num: i));
3187 break;
3188 case ISD::UNDEF: {
3189 EVT VT = Node->getValueType(ResNo: 0);
3190 if (VT.isInteger())
3191 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT));
3192 else {
3193 assert(VT.isFloatingPoint() && "Unknown value type!");
3194 Results.push_back(Elt: DAG.getConstantFP(Val: 0, DL: dl, VT));
3195 }
3196 break;
3197 }
3198 case ISD::STRICT_FP_ROUND:
3199 // When strict mode is enforced we can't do expansion because it
3200 // does not honor the "strict" properties. Only libcall is allowed.
3201 if (TLI.isStrictFPEnabled())
3202 break;
3203 // We might as well mutate to FP_ROUND when FP_ROUND operation is legal
3204 // since this operation is more efficient than stack operation.
3205 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
3206 VT: Node->getValueType(ResNo: 0))
3207 == TargetLowering::Legal)
3208 break;
3209 // We fall back to use stack operation when the FP_ROUND operation
3210 // isn't available.
3211 if ((Tmp1 = EmitStackConvert(SrcOp: Node->getOperand(Num: 1), SlotVT: Node->getValueType(ResNo: 0),
3212 DestVT: Node->getValueType(ResNo: 0), dl,
3213 Chain: Node->getOperand(Num: 0)))) {
3214 ReplaceNode(Old: Node, New: Tmp1.getNode());
3215 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");
3216 return true;
3217 }
3218 break;
3219 case ISD::FP_ROUND: {
3220 EVT VT = Node->getValueType(ResNo: 0);
3221 if (VT.getScalarType() == MVT::bf16) {
3222 Results.push_back(
3223 Elt: DAG.getNode(Opcode: ISD::FP_TO_BF16, DL: SDLoc(Node), VT, Operand: Node->getOperand(Num: 0)));
3224 break;
3225 }
3226
3227 LLVM_FALLTHROUGH;
3228 }
3229 case ISD::BITCAST:
3230 if ((Tmp1 = EmitStackConvert(SrcOp: Node->getOperand(Num: 0), SlotVT: Node->getValueType(ResNo: 0),
3231 DestVT: Node->getValueType(ResNo: 0), dl)))
3232 Results.push_back(Elt: Tmp1);
3233 break;
3234 case ISD::STRICT_FP_EXTEND:
3235 // When strict mode is enforced we can't do expansion because it
3236 // does not honor the "strict" properties. Only libcall is allowed.
3237 if (TLI.isStrictFPEnabled())
3238 break;
3239 // We might as well mutate to FP_EXTEND when FP_EXTEND operation is legal
3240 // since this operation is more efficient than stack operation.
3241 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
3242 VT: Node->getValueType(ResNo: 0))
3243 == TargetLowering::Legal)
3244 break;
3245 // We fall back to use stack operation when the FP_EXTEND operation
3246 // isn't available.
3247 if ((Tmp1 = EmitStackConvert(
3248 SrcOp: Node->getOperand(Num: 1), SlotVT: Node->getOperand(Num: 1).getValueType(),
3249 DestVT: Node->getValueType(ResNo: 0), dl, Chain: Node->getOperand(Num: 0)))) {
3250 ReplaceNode(Old: Node, New: Tmp1.getNode());
3251 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");
3252 return true;
3253 }
3254 break;
3255 case ISD::FP_EXTEND: {
3256 SDValue Op = Node->getOperand(Num: 0);
3257 EVT SrcVT = Op.getValueType();
3258 EVT DstVT = Node->getValueType(ResNo: 0);
3259 if (SrcVT.getScalarType() == MVT::bf16) {
3260 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BF16_TO_FP, DL: SDLoc(Node), VT: DstVT, Operand: Op));
3261 break;
3262 }
3263
3264 if ((Tmp1 = EmitStackConvert(SrcOp: Op, SlotVT: SrcVT, DestVT: DstVT, dl)))
3265 Results.push_back(Elt: Tmp1);
3266 break;
3267 }
3268 case ISD::BF16_TO_FP: {
3269 // Always expand bf16 to f32 casts, they lower to ext + shift.
3270 //
3271 // Note that the operand of this code can be bf16 or an integer type in case
3272 // bf16 is not supported on the target and was softened.
3273 SDValue Op = Node->getOperand(Num: 0);
3274 if (Op.getValueType() == MVT::bf16) {
3275 Op = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32,
3276 DAG.getNode(ISD::BITCAST, dl, MVT::i16, Op));
3277 } else {
3278 Op = DAG.getAnyExtOrTrunc(Op, dl, MVT::i32);
3279 }
3280 Op = DAG.getNode(
3281 ISD::SHL, dl, MVT::i32, Op,
3282 DAG.getConstant(16, dl,
3283 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3284 Op = DAG.getNode(ISD::BITCAST, dl, MVT::f32, Op);
3285 // Add fp_extend in case the output is bigger than f32.
3286 if (Node->getValueType(0) != MVT::f32)
3287 Op = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Op);
3288 Results.push_back(Elt: Op);
3289 break;
3290 }
3291 case ISD::FP_TO_BF16: {
3292 SDValue Op = Node->getOperand(Num: 0);
3293 if (Op.getValueType() != MVT::f32)
3294 Op = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3295 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
3296 Op = DAG.getNode(
3297 ISD::SRL, dl, MVT::i32, DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op),
3298 DAG.getConstant(16, dl,
3299 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3300 // The result of this node can be bf16 or an integer type in case bf16 is
3301 // not supported on the target and was softened to i16 for storage.
3302 if (Node->getValueType(0) == MVT::bf16) {
3303 Op = DAG.getNode(ISD::BITCAST, dl, MVT::bf16,
3304 DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Op));
3305 } else {
3306 Op = DAG.getAnyExtOrTrunc(Op, DL: dl, VT: Node->getValueType(ResNo: 0));
3307 }
3308 Results.push_back(Elt: Op);
3309 break;
3310 }
3311 case ISD::SIGN_EXTEND_INREG: {
3312 EVT ExtraVT = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
3313 EVT VT = Node->getValueType(ResNo: 0);
3314
3315 // An in-register sign-extend of a boolean is a negation:
3316 // 'true' (1) sign-extended is -1.
3317 // 'false' (0) sign-extended is 0.
3318 // However, we must mask the high bits of the source operand because the
3319 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
3320
3321 // TODO: Do this for vectors too?
3322 if (ExtraVT.isScalarInteger() && ExtraVT.getSizeInBits() == 1) {
3323 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
3324 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: One);
3325 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
3326 SDValue Neg = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: And);
3327 Results.push_back(Elt: Neg);
3328 break;
3329 }
3330
3331 // NOTE: we could fall back on load/store here too for targets without
3332 // SRA. However, it is doubtful that any exist.
3333 EVT ShiftAmountTy = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
3334 unsigned BitsDiff = VT.getScalarSizeInBits() -
3335 ExtraVT.getScalarSizeInBits();
3336 SDValue ShiftCst = DAG.getConstant(Val: BitsDiff, DL: dl, VT: ShiftAmountTy);
3337 Tmp1 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: Node->getValueType(ResNo: 0),
3338 N1: Node->getOperand(Num: 0), N2: ShiftCst);
3339 Tmp1 = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1, N2: ShiftCst);
3340 Results.push_back(Elt: Tmp1);
3341 break;
3342 }
3343 case ISD::UINT_TO_FP:
3344 case ISD::STRICT_UINT_TO_FP:
3345 if (TLI.expandUINT_TO_FP(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG)) {
3346 Results.push_back(Elt: Tmp1);
3347 if (Node->isStrictFPOpcode())
3348 Results.push_back(Elt: Tmp2);
3349 break;
3350 }
3351 [[fallthrough]];
3352 case ISD::SINT_TO_FP:
3353 case ISD::STRICT_SINT_TO_FP:
3354 if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Chain&: Tmp2))) {
3355 Results.push_back(Elt: Tmp1);
3356 if (Node->isStrictFPOpcode())
3357 Results.push_back(Elt: Tmp2);
3358 }
3359 break;
3360 case ISD::FP_TO_SINT:
3361 if (TLI.expandFP_TO_SINT(N: Node, Result&: Tmp1, DAG))
3362 Results.push_back(Elt: Tmp1);
3363 break;
3364 case ISD::STRICT_FP_TO_SINT:
3365 if (TLI.expandFP_TO_SINT(N: Node, Result&: Tmp1, DAG)) {
3366 ReplaceNode(Old: Node, New: Tmp1.getNode());
3367 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n");
3368 return true;
3369 }
3370 break;
3371 case ISD::FP_TO_UINT:
3372 if (TLI.expandFP_TO_UINT(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG))
3373 Results.push_back(Elt: Tmp1);
3374 break;
3375 case ISD::STRICT_FP_TO_UINT:
3376 if (TLI.expandFP_TO_UINT(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG)) {
3377 // Relink the chain.
3378 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node,1), To: Tmp2);
3379 // Replace the new UINT result.
3380 ReplaceNodeWithValue(Old: SDValue(Node, 0), New: Tmp1);
3381 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n");
3382 return true;
3383 }
3384 break;
3385 case ISD::FP_TO_SINT_SAT:
3386 case ISD::FP_TO_UINT_SAT:
3387 Results.push_back(Elt: TLI.expandFP_TO_INT_SAT(N: Node, DAG));
3388 break;
3389 case ISD::VAARG:
3390 Results.push_back(Elt: DAG.expandVAArg(Node));
3391 Results.push_back(Elt: Results[0].getValue(R: 1));
3392 break;
3393 case ISD::VACOPY:
3394 Results.push_back(Elt: DAG.expandVACopy(Node));
3395 break;
3396 case ISD::EXTRACT_VECTOR_ELT:
3397 if (Node->getOperand(Num: 0).getValueType().getVectorElementCount().isScalar())
3398 // This must be an access of the only element. Return it.
3399 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Node->getValueType(ResNo: 0),
3400 Operand: Node->getOperand(Num: 0));
3401 else
3402 Tmp1 = ExpandExtractFromVectorThroughStack(Op: SDValue(Node, 0));
3403 Results.push_back(Elt: Tmp1);
3404 break;
3405 case ISD::EXTRACT_SUBVECTOR:
3406 Results.push_back(Elt: ExpandExtractFromVectorThroughStack(Op: SDValue(Node, 0)));
3407 break;
3408 case ISD::INSERT_SUBVECTOR:
3409 Results.push_back(Elt: ExpandInsertToVectorThroughStack(Op: SDValue(Node, 0)));
3410 break;
3411 case ISD::CONCAT_VECTORS:
3412 Results.push_back(Elt: ExpandVectorBuildThroughStack(Node));
3413 break;
3414 case ISD::SCALAR_TO_VECTOR:
3415 Results.push_back(Elt: ExpandSCALAR_TO_VECTOR(Node));
3416 break;
3417 case ISD::INSERT_VECTOR_ELT:
3418 Results.push_back(Elt: ExpandINSERT_VECTOR_ELT(Vec: Node->getOperand(Num: 0),
3419 Val: Node->getOperand(Num: 1),
3420 Idx: Node->getOperand(Num: 2), dl));
3421 break;
3422 case ISD::VECTOR_SHUFFLE: {
3423 SmallVector<int, 32> NewMask;
3424 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Node)->getMask();
3425
3426 EVT VT = Node->getValueType(ResNo: 0);
3427 EVT EltVT = VT.getVectorElementType();
3428 SDValue Op0 = Node->getOperand(Num: 0);
3429 SDValue Op1 = Node->getOperand(Num: 1);
3430 if (!TLI.isTypeLegal(VT: EltVT)) {
3431 EVT NewEltVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: EltVT);
3432
3433 // BUILD_VECTOR operands are allowed to be wider than the element type.
3434 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3435 // it.
3436 if (NewEltVT.bitsLT(VT: EltVT)) {
3437 // Convert shuffle node.
3438 // If original node was v4i64 and the new EltVT is i32,
3439 // cast operands to v8i32 and re-build the mask.
3440
3441 // Calculate new VT, the size of the new VT should be equal to original.
3442 EVT NewVT =
3443 EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewEltVT,
3444 NumElements: VT.getSizeInBits() / NewEltVT.getSizeInBits());
3445 assert(NewVT.bitsEq(VT));
3446
3447 // cast operands to new VT
3448 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: Op0);
3449 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: Op1);
3450
3451 // Convert the shuffle mask
3452 unsigned int factor =
3453 NewVT.getVectorNumElements()/VT.getVectorNumElements();
3454
3455 // EltVT gets smaller
3456 assert(factor > 0);
3457
3458 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3459 if (Mask[i] < 0) {
3460 for (unsigned fi = 0; fi < factor; ++fi)
3461 NewMask.push_back(Elt: Mask[i]);
3462 }
3463 else {
3464 for (unsigned fi = 0; fi < factor; ++fi)
3465 NewMask.push_back(Elt: Mask[i]*factor+fi);
3466 }
3467 }
3468 Mask = NewMask;
3469 VT = NewVT;
3470 }
3471 EltVT = NewEltVT;
3472 }
3473 unsigned NumElems = VT.getVectorNumElements();
3474 SmallVector<SDValue, 16> Ops;
3475 for (unsigned i = 0; i != NumElems; ++i) {
3476 if (Mask[i] < 0) {
3477 Ops.push_back(Elt: DAG.getUNDEF(VT: EltVT));
3478 continue;
3479 }
3480 unsigned Idx = Mask[i];
3481 if (Idx < NumElems)
3482 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
3483 N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl)));
3484 else
3485 Ops.push_back(
3486 Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op1,
3487 N2: DAG.getVectorIdxConstant(Val: Idx - NumElems, DL: dl)));
3488 }
3489
3490 Tmp1 = DAG.getBuildVector(VT, DL: dl, Ops);
3491 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3492 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
3493 Results.push_back(Elt: Tmp1);
3494 break;
3495 }
3496 case ISD::VECTOR_SPLICE: {
3497 Results.push_back(Elt: TLI.expandVectorSplice(Node, DAG));
3498 break;
3499 }
3500 case ISD::EXTRACT_ELEMENT: {
3501 EVT OpTy = Node->getOperand(Num: 0).getValueType();
3502 if (Node->getConstantOperandVal(Num: 1)) {
3503 // 1 -> Hi
3504 Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: OpTy, N1: Node->getOperand(Num: 0),
3505 N2: DAG.getConstant(Val: OpTy.getSizeInBits() / 2, DL: dl,
3506 VT: TLI.getShiftAmountTy(
3507 LHSTy: Node->getOperand(Num: 0).getValueType(),
3508 DL: DAG.getDataLayout())));
3509 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
3510 } else {
3511 // 0 -> Lo
3512 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0),
3513 Operand: Node->getOperand(Num: 0));
3514 }
3515 Results.push_back(Elt: Tmp1);
3516 break;
3517 }
3518 case ISD::STACKSAVE:
3519 // Expand to CopyFromReg if the target set
3520 // StackPointerRegisterToSaveRestore.
3521 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3522 Results.push_back(Elt: DAG.getCopyFromReg(Chain: Node->getOperand(Num: 0), dl, Reg: SP,
3523 VT: Node->getValueType(ResNo: 0)));
3524 Results.push_back(Elt: Results[0].getValue(R: 1));
3525 } else {
3526 Results.push_back(Elt: DAG.getUNDEF(VT: Node->getValueType(ResNo: 0)));
3527 Results.push_back(Elt: Node->getOperand(Num: 0));
3528 }
3529 break;
3530 case ISD::STACKRESTORE:
3531 // Expand to CopyToReg if the target set
3532 // StackPointerRegisterToSaveRestore.
3533 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3534 Results.push_back(Elt: DAG.getCopyToReg(Chain: Node->getOperand(Num: 0), dl, Reg: SP,
3535 N: Node->getOperand(Num: 1)));
3536 } else {
3537 Results.push_back(Elt: Node->getOperand(Num: 0));
3538 }
3539 break;
3540 case ISD::GET_DYNAMIC_AREA_OFFSET:
3541 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3542 Results.push_back(Elt: Results[0].getValue(R: 0));
3543 break;
3544 case ISD::FCOPYSIGN:
3545 Results.push_back(Elt: ExpandFCOPYSIGN(Node));
3546 break;
3547 case ISD::FNEG:
3548 Results.push_back(Elt: ExpandFNEG(Node));
3549 break;
3550 case ISD::FABS:
3551 Results.push_back(Elt: ExpandFABS(Node));
3552 break;
3553 case ISD::IS_FPCLASS: {
3554 auto Test = static_cast<FPClassTest>(Node->getConstantOperandVal(Num: 1));
3555 if (SDValue Expanded =
3556 TLI.expandIS_FPCLASS(ResultVT: Node->getValueType(ResNo: 0), Op: Node->getOperand(Num: 0),
3557 Test, Flags: Node->getFlags(), DL: SDLoc(Node), DAG))
3558 Results.push_back(Elt: Expanded);
3559 break;
3560 }
3561 case ISD::SMIN:
3562 case ISD::SMAX:
3563 case ISD::UMIN:
3564 case ISD::UMAX: {
3565 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3566 ISD::CondCode Pred;
3567 switch (Node->getOpcode()) {
3568 default: llvm_unreachable("How did we get here?");
3569 case ISD::SMAX: Pred = ISD::SETGT; break;
3570 case ISD::SMIN: Pred = ISD::SETLT; break;
3571 case ISD::UMAX: Pred = ISD::SETUGT; break;
3572 case ISD::UMIN: Pred = ISD::SETULT; break;
3573 }
3574 Tmp1 = Node->getOperand(Num: 0);
3575 Tmp2 = Node->getOperand(Num: 1);
3576 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: Tmp2, True: Tmp1, False: Tmp2, Cond: Pred);
3577 Results.push_back(Elt: Tmp1);
3578 break;
3579 }
3580 case ISD::FMINNUM:
3581 case ISD::FMAXNUM: {
3582 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(N: Node, DAG))
3583 Results.push_back(Elt: Expanded);
3584 break;
3585 }
3586 case ISD::FSIN:
3587 case ISD::FCOS: {
3588 EVT VT = Node->getValueType(ResNo: 0);
3589 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3590 // fcos which share the same operand and both are used.
3591 if ((TLI.isOperationLegalOrCustom(Op: ISD::FSINCOS, VT) ||
3592 isSinCosLibcallAvailable(Node, TLI))
3593 && useSinCos(Node)) {
3594 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
3595 Tmp1 = DAG.getNode(Opcode: ISD::FSINCOS, DL: dl, VTList: VTs, N: Node->getOperand(Num: 0));
3596 if (Node->getOpcode() == ISD::FCOS)
3597 Tmp1 = Tmp1.getValue(R: 1);
3598 Results.push_back(Elt: Tmp1);
3599 }
3600 break;
3601 }
3602 case ISD::FLDEXP:
3603 case ISD::STRICT_FLDEXP: {
3604 EVT VT = Node->getValueType(ResNo: 0);
3605 RTLIB::Libcall LC = RTLIB::getLDEXP(RetVT: VT);
3606 // Use the LibCall instead, it is very likely faster
3607 // FIXME: Use separate LibCall action.
3608 if (TLI.getLibcallName(Call: LC))
3609 break;
3610
3611 if (SDValue Expanded = expandLdexp(Node)) {
3612 Results.push_back(Elt: Expanded);
3613 if (Node->getOpcode() == ISD::STRICT_FLDEXP)
3614 Results.push_back(Elt: Expanded.getValue(R: 1));
3615 }
3616
3617 break;
3618 }
3619 case ISD::FFREXP: {
3620 RTLIB::Libcall LC = RTLIB::getFREXP(RetVT: Node->getValueType(ResNo: 0));
3621 // Use the LibCall instead, it is very likely faster
3622 // FIXME: Use separate LibCall action.
3623 if (TLI.getLibcallName(Call: LC))
3624 break;
3625
3626 if (SDValue Expanded = expandFrexp(Node)) {
3627 Results.push_back(Elt: Expanded);
3628 Results.push_back(Elt: Expanded.getValue(R: 1));
3629 }
3630 break;
3631 }
3632 case ISD::FMAD:
3633 llvm_unreachable("Illegal fmad should never be formed");
3634
3635 case ISD::FP16_TO_FP:
3636 if (Node->getValueType(0) != MVT::f32) {
3637 // We can extend to types bigger than f32 in two steps without changing
3638 // the result. Since "f16 -> f32" is much more commonly available, give
3639 // CodeGen the option of emitting that before resorting to a libcall.
3640 SDValue Res =
3641 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3642 Results.push_back(
3643 Elt: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Res));
3644 }
3645 break;
3646 case ISD::STRICT_FP16_TO_FP:
3647 if (Node->getValueType(0) != MVT::f32) {
3648 // We can extend to types bigger than f32 in two steps without changing
3649 // the result. Since "f16 -> f32" is much more commonly available, give
3650 // CodeGen the option of emitting that before resorting to a libcall.
3651 SDValue Res =
3652 DAG.getNode(ISD::STRICT_FP16_TO_FP, dl, {MVT::f32, MVT::Other},
3653 {Node->getOperand(0), Node->getOperand(1)});
3654 Res = DAG.getNode(ISD::STRICT_FP_EXTEND, dl,
3655 {Node->getValueType(0), MVT::Other},
3656 {Res.getValue(1), Res});
3657 Results.push_back(Elt: Res);
3658 Results.push_back(Elt: Res.getValue(R: 1));
3659 }
3660 break;
3661 case ISD::FP_TO_FP16:
3662 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
3663 if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3664 SDValue Op = Node->getOperand(Num: 0);
3665 MVT SVT = Op.getSimpleValueType();
3666 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3667 TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3668 // Under fastmath, we can expand this node into a fround followed by
3669 // a float-half conversion.
3670 SDValue FloatVal =
3671 DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3672 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
3673 Results.push_back(
3674 Elt: DAG.getNode(Opcode: ISD::FP_TO_FP16, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: FloatVal));
3675 }
3676 }
3677 break;
3678 case ISD::ConstantFP: {
3679 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Val: Node);
3680 // Check to see if this FP immediate is already legal.
3681 // If this is a legal constant, turn it into a TargetConstantFP node.
3682 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(ResNo: 0),
3683 ForCodeSize: DAG.shouldOptForSize()))
3684 Results.push_back(Elt: ExpandConstantFP(CFP, UseCP: true));
3685 break;
3686 }
3687 case ISD::Constant: {
3688 ConstantSDNode *CP = cast<ConstantSDNode>(Val: Node);
3689 Results.push_back(Elt: ExpandConstant(CP));
3690 break;
3691 }
3692 case ISD::FSUB: {
3693 EVT VT = Node->getValueType(ResNo: 0);
3694 if (TLI.isOperationLegalOrCustom(Op: ISD::FADD, VT) &&
3695 TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT)) {
3696 const SDNodeFlags Flags = Node->getFlags();
3697 Tmp1 = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Node->getOperand(Num: 1));
3698 Tmp1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: Tmp1, Flags);
3699 Results.push_back(Elt: Tmp1);
3700 }
3701 break;
3702 }
3703 case ISD::SUB: {
3704 EVT VT = Node->getValueType(ResNo: 0);
3705 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
3706 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
3707 "Don't know how to expand this subtraction!");
3708 Tmp1 = DAG.getNOT(DL: dl, Val: Node->getOperand(Num: 1), VT);
3709 Tmp1 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Tmp1, N2: DAG.getConstant(Val: 1, DL: dl, VT));
3710 Results.push_back(Elt: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: Tmp1));
3711 break;
3712 }
3713 case ISD::UREM:
3714 case ISD::SREM:
3715 if (TLI.expandREM(Node, Result&: Tmp1, DAG))
3716 Results.push_back(Elt: Tmp1);
3717 break;
3718 case ISD::UDIV:
3719 case ISD::SDIV: {
3720 bool isSigned = Node->getOpcode() == ISD::SDIV;
3721 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3722 EVT VT = Node->getValueType(ResNo: 0);
3723 if (TLI.isOperationLegalOrCustom(Op: DivRemOpc, VT)) {
3724 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
3725 Tmp1 = DAG.getNode(Opcode: DivRemOpc, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
3726 N2: Node->getOperand(Num: 1));
3727 Results.push_back(Elt: Tmp1);
3728 }
3729 break;
3730 }
3731 case ISD::MULHU:
3732 case ISD::MULHS: {
3733 unsigned ExpandOpcode =
3734 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
3735 EVT VT = Node->getValueType(ResNo: 0);
3736 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
3737
3738 Tmp1 = DAG.getNode(Opcode: ExpandOpcode, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
3739 N2: Node->getOperand(Num: 1));
3740 Results.push_back(Elt: Tmp1.getValue(R: 1));
3741 break;
3742 }
3743 case ISD::UMUL_LOHI:
3744 case ISD::SMUL_LOHI: {
3745 SDValue LHS = Node->getOperand(Num: 0);
3746 SDValue RHS = Node->getOperand(Num: 1);
3747 MVT VT = LHS.getSimpleValueType();
3748 unsigned MULHOpcode =
3749 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
3750
3751 if (TLI.isOperationLegalOrCustom(Op: MULHOpcode, VT)) {
3752 Results.push_back(Elt: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS));
3753 Results.push_back(Elt: DAG.getNode(Opcode: MULHOpcode, DL: dl, VT, N1: LHS, N2: RHS));
3754 break;
3755 }
3756
3757 SmallVector<SDValue, 4> Halves;
3758 EVT HalfType = EVT(VT).getHalfSizedIntegerVT(Context&: *DAG.getContext());
3759 assert(TLI.isTypeLegal(HalfType));
3760 if (TLI.expandMUL_LOHI(Opcode: Node->getOpcode(), VT, dl, LHS, RHS, Result&: Halves,
3761 HiLoVT: HalfType, DAG,
3762 Kind: TargetLowering::MulExpansionKind::Always)) {
3763 for (unsigned i = 0; i < 2; ++i) {
3764 SDValue Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Halves[2 * i]);
3765 SDValue Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Halves[2 * i + 1]);
3766 SDValue Shift = DAG.getConstant(
3767 Val: HalfType.getScalarSizeInBits(), DL: dl,
3768 VT: TLI.getShiftAmountTy(LHSTy: HalfType, DL: DAG.getDataLayout()));
3769 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
3770 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi));
3771 }
3772 break;
3773 }
3774 break;
3775 }
3776 case ISD::MUL: {
3777 EVT VT = Node->getValueType(ResNo: 0);
3778 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
3779 // See if multiply or divide can be lowered using two-result operations.
3780 // We just need the low half of the multiply; try both the signed
3781 // and unsigned forms. If the target supports both SMUL_LOHI and
3782 // UMUL_LOHI, form a preference by checking which forms of plain
3783 // MULH it supports.
3784 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(Op: ISD::SMUL_LOHI, VT);
3785 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT);
3786 bool HasMULHS = TLI.isOperationLegalOrCustom(Op: ISD::MULHS, VT);
3787 bool HasMULHU = TLI.isOperationLegalOrCustom(Op: ISD::MULHU, VT);
3788 unsigned OpToUse = 0;
3789 if (HasSMUL_LOHI && !HasMULHS) {
3790 OpToUse = ISD::SMUL_LOHI;
3791 } else if (HasUMUL_LOHI && !HasMULHU) {
3792 OpToUse = ISD::UMUL_LOHI;
3793 } else if (HasSMUL_LOHI) {
3794 OpToUse = ISD::SMUL_LOHI;
3795 } else if (HasUMUL_LOHI) {
3796 OpToUse = ISD::UMUL_LOHI;
3797 }
3798 if (OpToUse) {
3799 Results.push_back(Elt: DAG.getNode(Opcode: OpToUse, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
3800 N2: Node->getOperand(Num: 1)));
3801 break;
3802 }
3803
3804 SDValue Lo, Hi;
3805 EVT HalfType = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
3806 if (TLI.isOperationLegalOrCustom(Op: ISD::ZERO_EXTEND, VT) &&
3807 TLI.isOperationLegalOrCustom(Op: ISD::ANY_EXTEND, VT) &&
3808 TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT) &&
3809 TLI.isOperationLegalOrCustom(Op: ISD::OR, VT) &&
3810 TLI.expandMUL(N: Node, Lo, Hi, HiLoVT: HalfType, DAG,
3811 Kind: TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3812 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Lo);
3813 Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Hi);
3814 SDValue Shift =
3815 DAG.getConstant(Val: HalfType.getSizeInBits(), DL: dl,
3816 VT: TLI.getShiftAmountTy(LHSTy: HalfType, DL: DAG.getDataLayout()));
3817 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
3818 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi));
3819 }
3820 break;
3821 }
3822 case ISD::FSHL:
3823 case ISD::FSHR:
3824 if (SDValue Expanded = TLI.expandFunnelShift(N: Node, DAG))
3825 Results.push_back(Elt: Expanded);
3826 break;
3827 case ISD::ROTL:
3828 case ISD::ROTR:
3829 if (SDValue Expanded = TLI.expandROT(N: Node, AllowVectorOps: true /*AllowVectorOps*/, DAG))
3830 Results.push_back(Elt: Expanded);
3831 break;
3832 case ISD::SADDSAT:
3833 case ISD::UADDSAT:
3834 case ISD::SSUBSAT:
3835 case ISD::USUBSAT:
3836 Results.push_back(Elt: TLI.expandAddSubSat(Node, DAG));
3837 break;
3838 case ISD::SSHLSAT:
3839 case ISD::USHLSAT:
3840 Results.push_back(Elt: TLI.expandShlSat(Node, DAG));
3841 break;
3842 case ISD::SMULFIX:
3843 case ISD::SMULFIXSAT:
3844 case ISD::UMULFIX:
3845 case ISD::UMULFIXSAT:
3846 Results.push_back(Elt: TLI.expandFixedPointMul(Node, DAG));
3847 break;
3848 case ISD::SDIVFIX:
3849 case ISD::SDIVFIXSAT:
3850 case ISD::UDIVFIX:
3851 case ISD::UDIVFIXSAT:
3852 if (SDValue V = TLI.expandFixedPointDiv(Opcode: Node->getOpcode(), dl: SDLoc(Node),
3853 LHS: Node->getOperand(Num: 0),
3854 RHS: Node->getOperand(Num: 1),
3855 Scale: Node->getConstantOperandVal(Num: 2),
3856 DAG)) {
3857 Results.push_back(Elt: V);
3858 break;
3859 }
3860 // FIXME: We might want to retry here with a wider type if we fail, if that
3861 // type is legal.
3862 // FIXME: Technically, so long as we only have sdivfixes where BW+Scale is
3863 // <= 128 (which is the case for all of the default Embedded-C types),
3864 // we will only get here with types and scales that we could always expand
3865 // if we were allowed to generate libcalls to division functions of illegal
3866 // type. But we cannot do that.
3867 llvm_unreachable("Cannot expand DIVFIX!");
3868 case ISD::UADDO_CARRY:
3869 case ISD::USUBO_CARRY: {
3870 SDValue LHS = Node->getOperand(Num: 0);
3871 SDValue RHS = Node->getOperand(Num: 1);
3872 SDValue Carry = Node->getOperand(Num: 2);
3873
3874 bool IsAdd = Node->getOpcode() == ISD::UADDO_CARRY;
3875
3876 // Initial add of the 2 operands.
3877 unsigned Op = IsAdd ? ISD::ADD : ISD::SUB;
3878 EVT VT = LHS.getValueType();
3879 SDValue Sum = DAG.getNode(Opcode: Op, DL: dl, VT, N1: LHS, N2: RHS);
3880
3881 // Initial check for overflow.
3882 EVT CarryType = Node->getValueType(ResNo: 1);
3883 EVT SetCCType = getSetCCResultType(VT: Node->getValueType(ResNo: 0));
3884 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
3885 SDValue Overflow = DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: LHS, Cond: CC);
3886
3887 // Add of the sum and the carry.
3888 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
3889 SDValue CarryExt =
3890 DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT), N2: One);
3891 SDValue Sum2 = DAG.getNode(Opcode: Op, DL: dl, VT, N1: Sum, N2: CarryExt);
3892
3893 // Second check for overflow. If we are adding, we can only overflow if the
3894 // initial sum is all 1s ang the carry is set, resulting in a new sum of 0.
3895 // If we are subtracting, we can only overflow if the initial sum is 0 and
3896 // the carry is set, resulting in a new sum of all 1s.
3897 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
3898 SDValue Overflow2 =
3899 IsAdd ? DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum2, RHS: Zero, Cond: ISD::SETEQ)
3900 : DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: Zero, Cond: ISD::SETEQ);
3901 Overflow2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCType, N1: Overflow2,
3902 N2: DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT: SetCCType));
3903
3904 SDValue ResultCarry =
3905 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SetCCType, N1: Overflow, N2: Overflow2);
3906
3907 Results.push_back(Elt: Sum2);
3908 Results.push_back(Elt: DAG.getBoolExtOrTrunc(Op: ResultCarry, SL: dl, VT: CarryType, OpVT: VT));
3909 break;
3910 }
3911 case ISD::SADDO:
3912 case ISD::SSUBO: {
3913 SDValue Result, Overflow;
3914 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
3915 Results.push_back(Elt: Result);
3916 Results.push_back(Elt: Overflow);
3917 break;
3918 }
3919 case ISD::UADDO:
3920 case ISD::USUBO: {
3921 SDValue Result, Overflow;
3922 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
3923 Results.push_back(Elt: Result);
3924 Results.push_back(Elt: Overflow);
3925 break;
3926 }
3927 case ISD::UMULO:
3928 case ISD::SMULO: {
3929 SDValue Result, Overflow;
3930 if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
3931 Results.push_back(Elt: Result);
3932 Results.push_back(Elt: Overflow);
3933 }
3934 break;
3935 }
3936 case ISD::BUILD_PAIR: {
3937 EVT PairTy = Node->getValueType(ResNo: 0);
3938 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: PairTy, Operand: Node->getOperand(Num: 0));
3939 Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: PairTy, Operand: Node->getOperand(Num: 1));
3940 Tmp2 = DAG.getNode(
3941 Opcode: ISD::SHL, DL: dl, VT: PairTy, N1: Tmp2,
3942 N2: DAG.getConstant(Val: PairTy.getSizeInBits() / 2, DL: dl,
3943 VT: TLI.getShiftAmountTy(LHSTy: PairTy, DL: DAG.getDataLayout())));
3944 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: PairTy, N1: Tmp1, N2: Tmp2));
3945 break;
3946 }
3947 case ISD::SELECT:
3948 Tmp1 = Node->getOperand(Num: 0);
3949 Tmp2 = Node->getOperand(Num: 1);
3950 Tmp3 = Node->getOperand(Num: 2);
3951 if (Tmp1.getOpcode() == ISD::SETCC) {
3952 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1.getOperand(i: 0), RHS: Tmp1.getOperand(i: 1),
3953 True: Tmp2, False: Tmp3,
3954 Cond: cast<CondCodeSDNode>(Val: Tmp1.getOperand(i: 2))->get());
3955 } else {
3956 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1,
3957 RHS: DAG.getConstant(Val: 0, DL: dl, VT: Tmp1.getValueType()),
3958 True: Tmp2, False: Tmp3, Cond: ISD::SETNE);
3959 }
3960 Tmp1->setFlags(Node->getFlags());
3961 Results.push_back(Elt: Tmp1);
3962 break;
3963 case ISD::BR_JT: {
3964 SDValue Chain = Node->getOperand(Num: 0);
3965 SDValue Table = Node->getOperand(Num: 1);
3966 SDValue Index = Node->getOperand(Num: 2);
3967 int JTI = cast<JumpTableSDNode>(Val: Table.getNode())->getIndex();
3968
3969 const DataLayout &TD = DAG.getDataLayout();
3970 EVT PTy = TLI.getPointerTy(DL: TD);
3971
3972 unsigned EntrySize =
3973 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3974
3975 // For power-of-two jumptable entry sizes convert multiplication to a shift.
3976 // This transformation needs to be done here since otherwise the MIPS
3977 // backend will end up emitting a three instruction multiply sequence
3978 // instead of a single shift and MSP430 will call a runtime function.
3979 if (llvm::isPowerOf2_32(Value: EntrySize))
3980 Index = DAG.getNode(
3981 Opcode: ISD::SHL, DL: dl, VT: Index.getValueType(), N1: Index,
3982 N2: DAG.getConstant(Val: llvm::Log2_32(Value: EntrySize), DL: dl, VT: Index.getValueType()));
3983 else
3984 Index = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: Index.getValueType(), N1: Index,
3985 N2: DAG.getConstant(Val: EntrySize, DL: dl, VT: Index.getValueType()));
3986 SDValue Addr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Index.getValueType(),
3987 N1: Index, N2: Table);
3988
3989 EVT MemVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: EntrySize * 8);
3990 SDValue LD = DAG.getExtLoad(
3991 ExtType: ISD::SEXTLOAD, dl, VT: PTy, Chain, Ptr: Addr,
3992 PtrInfo: MachinePointerInfo::getJumpTable(MF&: DAG.getMachineFunction()), MemVT);
3993 Addr = LD;
3994 if (TLI.isJumpTableRelative()) {
3995 // For PIC, the sequence is:
3996 // BRIND(load(Jumptable + index) + RelocBase)
3997 // RelocBase can be JumpTable, GOT or some sort of global base.
3998 Addr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PTy, N1: Addr,
3999 N2: TLI.getPICJumpTableRelocBase(Table, DAG));
4000 }
4001
4002 Tmp1 = TLI.expandIndirectJTBranch(dl, Value: LD.getValue(R: 1), Addr, JTI, DAG);
4003 Results.push_back(Elt: Tmp1);
4004 break;
4005 }
4006 case ISD::BRCOND:
4007 // Expand brcond's setcc into its constituent parts and create a BR_CC
4008 // Node.
4009 Tmp1 = Node->getOperand(Num: 0);
4010 Tmp2 = Node->getOperand(Num: 1);
4011 if (Tmp2.getOpcode() == ISD::SETCC &&
4012 TLI.isOperationLegalOrCustom(Op: ISD::BR_CC,
4013 VT: Tmp2.getOperand(i: 0).getValueType())) {
4014 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, Tmp2.getOperand(2),
4015 Tmp2.getOperand(0), Tmp2.getOperand(1),
4016 Node->getOperand(2));
4017 } else {
4018 // We test only the i1 bit. Skip the AND if UNDEF or another AND.
4019 if (Tmp2.isUndef() ||
4020 (Tmp2.getOpcode() == ISD::AND && isOneConstant(V: Tmp2.getOperand(i: 1))))
4021 Tmp3 = Tmp2;
4022 else
4023 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp2.getValueType(), N1: Tmp2,
4024 N2: DAG.getConstant(Val: 1, DL: dl, VT: Tmp2.getValueType()));
4025 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
4026 DAG.getCondCode(ISD::SETNE), Tmp3,
4027 DAG.getConstant(0, dl, Tmp3.getValueType()),
4028 Node->getOperand(2));
4029 }
4030 Results.push_back(Elt: Tmp1);
4031 break;
4032 case ISD::SETCC:
4033 case ISD::VP_SETCC:
4034 case ISD::STRICT_FSETCC:
4035 case ISD::STRICT_FSETCCS: {
4036 bool IsVP = Node->getOpcode() == ISD::VP_SETCC;
4037 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
4038 Node->getOpcode() == ISD::STRICT_FSETCCS;
4039 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
4040 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
4041 unsigned Offset = IsStrict ? 1 : 0;
4042 Tmp1 = Node->getOperand(Num: 0 + Offset);
4043 Tmp2 = Node->getOperand(Num: 1 + Offset);
4044 Tmp3 = Node->getOperand(Num: 2 + Offset);
4045 SDValue Mask, EVL;
4046 if (IsVP) {
4047 Mask = Node->getOperand(Num: 3 + Offset);
4048 EVL = Node->getOperand(Num: 4 + Offset);
4049 }
4050 bool Legalized = TLI.LegalizeSetCCCondCode(
4051 DAG, VT: Node->getValueType(ResNo: 0), LHS&: Tmp1, RHS&: Tmp2, CC&: Tmp3, Mask, EVL, NeedInvert, dl,
4052 Chain, IsSignaling);
4053
4054 if (Legalized) {
4055 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4056 // condition code, create a new SETCC node.
4057 if (Tmp3.getNode()) {
4058 if (IsStrict) {
4059 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: Node->getVTList(),
4060 Ops: {Chain, Tmp1, Tmp2, Tmp3}, Flags: Node->getFlags());
4061 Chain = Tmp1.getValue(R: 1);
4062 } else if (IsVP) {
4063 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0),
4064 Ops: {Tmp1, Tmp2, Tmp3, Mask, EVL}, Flags: Node->getFlags());
4065 } else {
4066 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4067 N2: Tmp2, N3: Tmp3, Flags: Node->getFlags());
4068 }
4069 }
4070
4071 // If we expanded the SETCC by inverting the condition code, then wrap
4072 // the existing SETCC in a NOT to restore the intended condition.
4073 if (NeedInvert) {
4074 if (!IsVP)
4075 Tmp1 = DAG.getLogicalNOT(DL: dl, Val: Tmp1, VT: Tmp1->getValueType(ResNo: 0));
4076 else
4077 Tmp1 =
4078 DAG.getVPLogicalNOT(DL: dl, Val: Tmp1, Mask, EVL, VT: Tmp1->getValueType(ResNo: 0));
4079 }
4080
4081 Results.push_back(Elt: Tmp1);
4082 if (IsStrict)
4083 Results.push_back(Elt: Chain);
4084
4085 break;
4086 }
4087
4088 // FIXME: It seems Legalized is false iff CCCode is Legal. I don't
4089 // understand if this code is useful for strict nodes.
4090 assert(!IsStrict && "Don't know how to expand for strict nodes.");
4091
4092 // Otherwise, SETCC for the given comparison type must be completely
4093 // illegal; expand it into a SELECT_CC.
4094 // FIXME: This drops the mask/evl for VP_SETCC.
4095 EVT VT = Node->getValueType(ResNo: 0);
4096 EVT Tmp1VT = Tmp1.getValueType();
4097 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT, N1: Tmp1, N2: Tmp2,
4098 N3: DAG.getBoolConstant(V: true, DL: dl, VT, OpVT: Tmp1VT),
4099 N4: DAG.getBoolConstant(V: false, DL: dl, VT, OpVT: Tmp1VT), N5: Tmp3);
4100 Tmp1->setFlags(Node->getFlags());
4101 Results.push_back(Elt: Tmp1);
4102 break;
4103 }
4104 case ISD::SELECT_CC: {
4105 // TODO: need to add STRICT_SELECT_CC and STRICT_SELECT_CCS
4106 Tmp1 = Node->getOperand(Num: 0); // LHS
4107 Tmp2 = Node->getOperand(Num: 1); // RHS
4108 Tmp3 = Node->getOperand(Num: 2); // True
4109 Tmp4 = Node->getOperand(Num: 3); // False
4110 EVT VT = Node->getValueType(ResNo: 0);
4111 SDValue Chain;
4112 SDValue CC = Node->getOperand(Num: 4);
4113 ISD::CondCode CCOp = cast<CondCodeSDNode>(Val&: CC)->get();
4114
4115 if (TLI.isCondCodeLegalOrCustom(CC: CCOp, VT: Tmp1.getSimpleValueType())) {
4116 // If the condition code is legal, then we need to expand this
4117 // node using SETCC and SELECT.
4118 EVT CmpVT = Tmp1.getValueType();
4119 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
4120 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4121 "expanded.");
4122 EVT CCVT = getSetCCResultType(VT: CmpVT);
4123 SDValue Cond = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: CCVT, N1: Tmp1, N2: Tmp2, N3: CC, Flags: Node->getFlags());
4124 Results.push_back(Elt: DAG.getSelect(DL: dl, VT, Cond, LHS: Tmp3, RHS: Tmp4));
4125 break;
4126 }
4127
4128 // SELECT_CC is legal, so the condition code must not be.
4129 bool Legalized = false;
4130 // Try to legalize by inverting the condition. This is for targets that
4131 // might support an ordered version of a condition, but not the unordered
4132 // version (or vice versa).
4133 ISD::CondCode InvCC = ISD::getSetCCInverse(Operation: CCOp, Type: Tmp1.getValueType());
4134 if (TLI.isCondCodeLegalOrCustom(CC: InvCC, VT: Tmp1.getSimpleValueType())) {
4135 // Use the new condition code and swap true and false
4136 Legalized = true;
4137 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: Tmp2, True: Tmp4, False: Tmp3, Cond: InvCC);
4138 Tmp1->setFlags(Node->getFlags());
4139 } else {
4140 // If The inverse is not legal, then try to swap the arguments using
4141 // the inverse condition code.
4142 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(Operation: InvCC);
4143 if (TLI.isCondCodeLegalOrCustom(CC: SwapInvCC, VT: Tmp1.getSimpleValueType())) {
4144 // The swapped inverse condition is legal, so swap true and false,
4145 // lhs and rhs.
4146 Legalized = true;
4147 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp2, RHS: Tmp1, True: Tmp4, False: Tmp3, Cond: SwapInvCC);
4148 Tmp1->setFlags(Node->getFlags());
4149 }
4150 }
4151
4152 if (!Legalized) {
4153 Legalized = TLI.LegalizeSetCCCondCode(
4154 DAG, VT: getSetCCResultType(VT: Tmp1.getValueType()), LHS&: Tmp1, RHS&: Tmp2, CC,
4155 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4156
4157 assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
4158
4159 // If we expanded the SETCC by inverting the condition code, then swap
4160 // the True/False operands to match.
4161 if (NeedInvert)
4162 std::swap(a&: Tmp3, b&: Tmp4);
4163
4164 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4165 // condition code, create a new SELECT_CC node.
4166 if (CC.getNode()) {
4167 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: Node->getValueType(ResNo: 0),
4168 N1: Tmp1, N2: Tmp2, N3: Tmp3, N4: Tmp4, N5: CC);
4169 } else {
4170 Tmp2 = DAG.getConstant(Val: 0, DL: dl, VT: Tmp1.getValueType());
4171 CC = DAG.getCondCode(Cond: ISD::SETNE);
4172 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4173 N2: Tmp2, N3: Tmp3, N4: Tmp4, N5: CC);
4174 }
4175 Tmp1->setFlags(Node->getFlags());
4176 }
4177 Results.push_back(Elt: Tmp1);
4178 break;
4179 }
4180 case ISD::BR_CC: {
4181 // TODO: need to add STRICT_BR_CC and STRICT_BR_CCS
4182 SDValue Chain;
4183 Tmp1 = Node->getOperand(Num: 0); // Chain
4184 Tmp2 = Node->getOperand(Num: 2); // LHS
4185 Tmp3 = Node->getOperand(Num: 3); // RHS
4186 Tmp4 = Node->getOperand(Num: 1); // CC
4187
4188 bool Legalized = TLI.LegalizeSetCCCondCode(
4189 DAG, VT: getSetCCResultType(VT: Tmp2.getValueType()), LHS&: Tmp2, RHS&: Tmp3, CC&: Tmp4,
4190 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4191 (void)Legalized;
4192 assert(Legalized && "Can't legalize BR_CC with legal condition!");
4193
4194 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
4195 // node.
4196 if (Tmp4.getNode()) {
4197 assert(!NeedInvert && "Don't know how to invert BR_CC!");
4198
4199 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4200 N2: Tmp4, N3: Tmp2, N4: Tmp3, N5: Node->getOperand(Num: 4));
4201 } else {
4202 Tmp3 = DAG.getConstant(Val: 0, DL: dl, VT: Tmp2.getValueType());
4203 Tmp4 = DAG.getCondCode(Cond: NeedInvert ? ISD::SETEQ : ISD::SETNE);
4204 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1, N2: Tmp4,
4205 N3: Tmp2, N4: Tmp3, N5: Node->getOperand(Num: 4));
4206 }
4207 Results.push_back(Elt: Tmp1);
4208 break;
4209 }
4210 case ISD::BUILD_VECTOR:
4211 Results.push_back(Elt: ExpandBUILD_VECTOR(Node));
4212 break;
4213 case ISD::SPLAT_VECTOR:
4214 Results.push_back(Elt: ExpandSPLAT_VECTOR(Node));
4215 break;
4216 case ISD::SRA:
4217 case ISD::SRL:
4218 case ISD::SHL: {
4219 // Scalarize vector SRA/SRL/SHL.
4220 EVT VT = Node->getValueType(ResNo: 0);
4221 assert(VT.isVector() && "Unable to legalize non-vector shift");
4222 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
4223 unsigned NumElem = VT.getVectorNumElements();
4224
4225 SmallVector<SDValue, 8> Scalars;
4226 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
4227 SDValue Ex =
4228 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: VT.getScalarType(),
4229 N1: Node->getOperand(Num: 0), N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4230 SDValue Sh =
4231 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: VT.getScalarType(),
4232 N1: Node->getOperand(Num: 1), N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4233 Scalars.push_back(Elt: DAG.getNode(Opcode: Node->getOpcode(), DL: dl,
4234 VT: VT.getScalarType(), N1: Ex, N2: Sh));
4235 }
4236
4237 SDValue Result = DAG.getBuildVector(VT: Node->getValueType(ResNo: 0), DL: dl, Ops: Scalars);
4238 Results.push_back(Elt: Result);
4239 break;
4240 }
4241 case ISD::VECREDUCE_FADD:
4242 case ISD::VECREDUCE_FMUL:
4243 case ISD::VECREDUCE_ADD:
4244 case ISD::VECREDUCE_MUL:
4245 case ISD::VECREDUCE_AND:
4246 case ISD::VECREDUCE_OR:
4247 case ISD::VECREDUCE_XOR:
4248 case ISD::VECREDUCE_SMAX:
4249 case ISD::VECREDUCE_SMIN:
4250 case ISD::VECREDUCE_UMAX:
4251 case ISD::VECREDUCE_UMIN:
4252 case ISD::VECREDUCE_FMAX:
4253 case ISD::VECREDUCE_FMIN:
4254 case ISD::VECREDUCE_FMAXIMUM:
4255 case ISD::VECREDUCE_FMINIMUM:
4256 Results.push_back(Elt: TLI.expandVecReduce(Node, DAG));
4257 break;
4258 case ISD::GLOBAL_OFFSET_TABLE:
4259 case ISD::GlobalAddress:
4260 case ISD::GlobalTLSAddress:
4261 case ISD::ExternalSymbol:
4262 case ISD::ConstantPool:
4263 case ISD::JumpTable:
4264 case ISD::INTRINSIC_W_CHAIN:
4265 case ISD::INTRINSIC_WO_CHAIN:
4266 case ISD::INTRINSIC_VOID:
4267 // FIXME: Custom lowering for these operations shouldn't return null!
4268 // Return true so that we don't call ConvertNodeToLibcall which also won't
4269 // do anything.
4270 return true;
4271 }
4272
4273 if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) {
4274 // FIXME: We were asked to expand a strict floating-point operation,
4275 // but there is currently no expansion implemented that would preserve
4276 // the "strict" properties. For now, we just fall back to the non-strict
4277 // version if that is legal on the target. The actual mutation of the
4278 // operation will happen in SelectionDAGISel::DoInstructionSelection.
4279 switch (Node->getOpcode()) {
4280 default:
4281 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
4282 VT: Node->getValueType(ResNo: 0))
4283 == TargetLowering::Legal)
4284 return true;
4285 break;
4286 case ISD::STRICT_FSUB: {
4287 if (TLI.getStrictFPOperationAction(
4288 Op: ISD::STRICT_FSUB, VT: Node->getValueType(ResNo: 0)) == TargetLowering::Legal)
4289 return true;
4290 if (TLI.getStrictFPOperationAction(
4291 Op: ISD::STRICT_FADD, VT: Node->getValueType(ResNo: 0)) != TargetLowering::Legal)
4292 break;
4293
4294 EVT VT = Node->getValueType(ResNo: 0);
4295 const SDNodeFlags Flags = Node->getFlags();
4296 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Node->getOperand(Num: 2), Flags);
4297 SDValue Fadd = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, VTList: Node->getVTList(),
4298 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1), Neg},
4299 Flags);
4300
4301 Results.push_back(Elt: Fadd);
4302 Results.push_back(Elt: Fadd.getValue(R: 1));
4303 break;
4304 }
4305 case ISD::STRICT_SINT_TO_FP:
4306 case ISD::STRICT_UINT_TO_FP:
4307 case ISD::STRICT_LRINT:
4308 case ISD::STRICT_LLRINT:
4309 case ISD::STRICT_LROUND:
4310 case ISD::STRICT_LLROUND:
4311 // These are registered by the operand type instead of the value
4312 // type. Reflect that here.
4313 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
4314 VT: Node->getOperand(Num: 1).getValueType())
4315 == TargetLowering::Legal)
4316 return true;
4317 break;
4318 }
4319 }
4320
4321 // Replace the original node with the legalized result.
4322 if (Results.empty()) {
4323 LLVM_DEBUG(dbgs() << "Cannot expand node\n");
4324 return false;
4325 }
4326
4327 LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
4328 ReplaceNode(Old: Node, New: Results.data());
4329 return true;
4330}
4331
4332void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
4333 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
4334 SmallVector<SDValue, 8> Results;
4335 SDLoc dl(Node);
4336 // FIXME: Check flags on the node to see if we can use a finite call.
4337 unsigned Opc = Node->getOpcode();
4338 switch (Opc) {
4339 case ISD::ATOMIC_FENCE: {
4340 // If the target didn't lower this, lower it to '__sync_synchronize()' call
4341 // FIXME: handle "fence singlethread" more efficiently.
4342 TargetLowering::ArgListTy Args;
4343
4344 TargetLowering::CallLoweringInfo CLI(DAG);
4345 CLI.setDebugLoc(dl)
4346 .setChain(Node->getOperand(Num: 0))
4347 .setLibCallee(
4348 CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
4349 Target: DAG.getExternalSymbol(Sym: "__sync_synchronize",
4350 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
4351 ArgsList: std::move(Args));
4352
4353 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4354
4355 Results.push_back(Elt: CallResult.second);
4356 break;
4357 }
4358 // By default, atomic intrinsics are marked Legal and lowered. Targets
4359 // which don't support them directly, however, may want libcalls, in which
4360 // case they mark them Expand, and we get here.
4361 case ISD::ATOMIC_SWAP:
4362 case ISD::ATOMIC_LOAD_ADD:
4363 case ISD::ATOMIC_LOAD_SUB:
4364 case ISD::ATOMIC_LOAD_AND:
4365 case ISD::ATOMIC_LOAD_CLR:
4366 case ISD::ATOMIC_LOAD_OR:
4367 case ISD::ATOMIC_LOAD_XOR:
4368 case ISD::ATOMIC_LOAD_NAND:
4369 case ISD::ATOMIC_LOAD_MIN:
4370 case ISD::ATOMIC_LOAD_MAX:
4371 case ISD::ATOMIC_LOAD_UMIN:
4372 case ISD::ATOMIC_LOAD_UMAX:
4373 case ISD::ATOMIC_CMP_SWAP: {
4374 MVT VT = cast<AtomicSDNode>(Val: Node)->getMemoryVT().getSimpleVT();
4375 AtomicOrdering Order = cast<AtomicSDNode>(Val: Node)->getMergedOrdering();
4376 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT);
4377 EVT RetVT = Node->getValueType(ResNo: 0);
4378 TargetLowering::MakeLibCallOptions CallOptions;
4379 SmallVector<SDValue, 4> Ops;
4380 if (TLI.getLibcallName(Call: LC)) {
4381 // If outline atomic available, prepare its arguments and expand.
4382 Ops.append(in_start: Node->op_begin() + 2, in_end: Node->op_end());
4383 Ops.push_back(Elt: Node->getOperand(Num: 1));
4384
4385 } else {
4386 LC = RTLIB::getSYNC(Opc, VT);
4387 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4388 "Unexpected atomic op or value type!");
4389 // Arguments for expansion to sync libcall
4390 Ops.append(in_start: Node->op_begin() + 1, in_end: Node->op_end());
4391 }
4392 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4393 Ops, CallOptions,
4394 dl: SDLoc(Node),
4395 Chain: Node->getOperand(Num: 0));
4396 Results.push_back(Elt: Tmp.first);
4397 Results.push_back(Elt: Tmp.second);
4398 break;
4399 }
4400 case ISD::TRAP: {
4401 // If this operation is not supported, lower it to 'abort()' call
4402 TargetLowering::ArgListTy Args;
4403 TargetLowering::CallLoweringInfo CLI(DAG);
4404 CLI.setDebugLoc(dl)
4405 .setChain(Node->getOperand(Num: 0))
4406 .setLibCallee(CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
4407 Target: DAG.getExternalSymbol(
4408 Sym: "abort", VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
4409 ArgsList: std::move(Args));
4410 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4411
4412 Results.push_back(Elt: CallResult.second);
4413 break;
4414 }
4415 case ISD::FMINNUM:
4416 case ISD::STRICT_FMINNUM:
4417 ExpandFPLibCall(Node, Call_F32: RTLIB::FMIN_F32, Call_F64: RTLIB::FMIN_F64,
4418 Call_F80: RTLIB::FMIN_F80, Call_F128: RTLIB::FMIN_F128,
4419 Call_PPCF128: RTLIB::FMIN_PPCF128, Results);
4420 break;
4421 // FIXME: We do not have libcalls for FMAXIMUM and FMINIMUM. So, we cannot use
4422 // libcall legalization for these nodes, but there is no default expasion for
4423 // these nodes either (see PR63267 for example).
4424 case ISD::FMAXNUM:
4425 case ISD::STRICT_FMAXNUM:
4426 ExpandFPLibCall(Node, Call_F32: RTLIB::FMAX_F32, Call_F64: RTLIB::FMAX_F64,
4427 Call_F80: RTLIB::FMAX_F80, Call_F128: RTLIB::FMAX_F128,
4428 Call_PPCF128: RTLIB::FMAX_PPCF128, Results);
4429 break;
4430 case ISD::FSQRT:
4431 case ISD::STRICT_FSQRT:
4432 ExpandFPLibCall(Node, Call_F32: RTLIB::SQRT_F32, Call_F64: RTLIB::SQRT_F64,
4433 Call_F80: RTLIB::SQRT_F80, Call_F128: RTLIB::SQRT_F128,
4434 Call_PPCF128: RTLIB::SQRT_PPCF128, Results);
4435 break;
4436 case ISD::FCBRT:
4437 ExpandFPLibCall(Node, Call_F32: RTLIB::CBRT_F32, Call_F64: RTLIB::CBRT_F64,
4438 Call_F80: RTLIB::CBRT_F80, Call_F128: RTLIB::CBRT_F128,
4439 Call_PPCF128: RTLIB::CBRT_PPCF128, Results);
4440 break;
4441 case ISD::FSIN:
4442 case ISD::STRICT_FSIN:
4443 ExpandFPLibCall(Node, Call_F32: RTLIB::SIN_F32, Call_F64: RTLIB::SIN_F64,
4444 Call_F80: RTLIB::SIN_F80, Call_F128: RTLIB::SIN_F128,
4445 Call_PPCF128: RTLIB::SIN_PPCF128, Results);
4446 break;
4447 case ISD::FCOS:
4448 case ISD::STRICT_FCOS:
4449 ExpandFPLibCall(Node, Call_F32: RTLIB::COS_F32, Call_F64: RTLIB::COS_F64,
4450 Call_F80: RTLIB::COS_F80, Call_F128: RTLIB::COS_F128,
4451 Call_PPCF128: RTLIB::COS_PPCF128, Results);
4452 break;
4453 case ISD::FSINCOS:
4454 // Expand into sincos libcall.
4455 ExpandSinCosLibCall(Node, Results);
4456 break;
4457 case ISD::FLOG:
4458 case ISD::STRICT_FLOG:
4459 ExpandFPLibCall(Node, Call_F32: RTLIB::LOG_F32, Call_F64: RTLIB::LOG_F64, Call_F80: RTLIB::LOG_F80,
4460 Call_F128: RTLIB::LOG_F128, Call_PPCF128: RTLIB::LOG_PPCF128, Results);
4461 break;
4462 case ISD::FLOG2:
4463 case ISD::STRICT_FLOG2:
4464 ExpandFPLibCall(Node, Call_F32: RTLIB::LOG2_F32, Call_F64: RTLIB::LOG2_F64, Call_F80: RTLIB::LOG2_F80,
4465 Call_F128: RTLIB::LOG2_F128, Call_PPCF128: RTLIB::LOG2_PPCF128, Results);
4466 break;
4467 case ISD::FLOG10:
4468 case ISD::STRICT_FLOG10:
4469 ExpandFPLibCall(Node, Call_F32: RTLIB::LOG10_F32, Call_F64: RTLIB::LOG10_F64, Call_F80: RTLIB::LOG10_F80,
4470 Call_F128: RTLIB::LOG10_F128, Call_PPCF128: RTLIB::LOG10_PPCF128, Results);
4471 break;
4472 case ISD::FEXP:
4473 case ISD::STRICT_FEXP:
4474 ExpandFPLibCall(Node, Call_F32: RTLIB::EXP_F32, Call_F64: RTLIB::EXP_F64, Call_F80: RTLIB::EXP_F80,
4475 Call_F128: RTLIB::EXP_F128, Call_PPCF128: RTLIB::EXP_PPCF128, Results);
4476 break;
4477 case ISD::FEXP2:
4478 case ISD::STRICT_FEXP2:
4479 ExpandFPLibCall(Node, Call_F32: RTLIB::EXP2_F32, Call_F64: RTLIB::EXP2_F64, Call_F80: RTLIB::EXP2_F80,
4480 Call_F128: RTLIB::EXP2_F128, Call_PPCF128: RTLIB::EXP2_PPCF128, Results);
4481 break;
4482 case ISD::FEXP10:
4483 ExpandFPLibCall(Node, Call_F32: RTLIB::EXP10_F32, Call_F64: RTLIB::EXP10_F64, Call_F80: RTLIB::EXP10_F80,
4484 Call_F128: RTLIB::EXP10_F128, Call_PPCF128: RTLIB::EXP10_PPCF128, Results);
4485 break;
4486 case ISD::FTRUNC:
4487 case ISD::STRICT_FTRUNC:
4488 ExpandFPLibCall(Node, Call_F32: RTLIB::TRUNC_F32, Call_F64: RTLIB::TRUNC_F64,
4489 Call_F80: RTLIB::TRUNC_F80, Call_F128: RTLIB::TRUNC_F128,
4490 Call_PPCF128: RTLIB::TRUNC_PPCF128, Results);
4491 break;
4492 case ISD::FFLOOR:
4493 case ISD::STRICT_FFLOOR:
4494 ExpandFPLibCall(Node, Call_F32: RTLIB::FLOOR_F32, Call_F64: RTLIB::FLOOR_F64,
4495 Call_F80: RTLIB::FLOOR_F80, Call_F128: RTLIB::FLOOR_F128,
4496 Call_PPCF128: RTLIB::FLOOR_PPCF128, Results);
4497 break;
4498 case ISD::FCEIL:
4499 case ISD::STRICT_FCEIL:
4500 ExpandFPLibCall(Node, Call_F32: RTLIB::CEIL_F32, Call_F64: RTLIB::CEIL_F64,
4501 Call_F80: RTLIB::CEIL_F80, Call_F128: RTLIB::CEIL_F128,
4502 Call_PPCF128: RTLIB::CEIL_PPCF128, Results);
4503 break;
4504 case ISD::FRINT:
4505 case ISD::STRICT_FRINT:
4506 ExpandFPLibCall(Node, Call_F32: RTLIB::RINT_F32, Call_F64: RTLIB::RINT_F64,
4507 Call_F80: RTLIB::RINT_F80, Call_F128: RTLIB::RINT_F128,
4508 Call_PPCF128: RTLIB::RINT_PPCF128, Results);
4509 break;
4510 case ISD::FNEARBYINT:
4511 case ISD::STRICT_FNEARBYINT:
4512 ExpandFPLibCall(Node, Call_F32: RTLIB::NEARBYINT_F32,
4513 Call_F64: RTLIB::NEARBYINT_F64,
4514 Call_F80: RTLIB::NEARBYINT_F80,
4515 Call_F128: RTLIB::NEARBYINT_F128,
4516 Call_PPCF128: RTLIB::NEARBYINT_PPCF128, Results);
4517 break;
4518 case ISD::FROUND:
4519 case ISD::STRICT_FROUND:
4520 ExpandFPLibCall(Node, Call_F32: RTLIB::ROUND_F32,
4521 Call_F64: RTLIB::ROUND_F64,
4522 Call_F80: RTLIB::ROUND_F80,
4523 Call_F128: RTLIB::ROUND_F128,
4524 Call_PPCF128: RTLIB::ROUND_PPCF128, Results);
4525 break;
4526 case ISD::FROUNDEVEN:
4527 case ISD::STRICT_FROUNDEVEN:
4528 ExpandFPLibCall(Node, Call_F32: RTLIB::ROUNDEVEN_F32,
4529 Call_F64: RTLIB::ROUNDEVEN_F64,
4530 Call_F80: RTLIB::ROUNDEVEN_F80,
4531 Call_F128: RTLIB::ROUNDEVEN_F128,
4532 Call_PPCF128: RTLIB::ROUNDEVEN_PPCF128, Results);
4533 break;
4534 case ISD::FLDEXP:
4535 case ISD::STRICT_FLDEXP:
4536 ExpandFPLibCall(Node, Call_F32: RTLIB::LDEXP_F32, Call_F64: RTLIB::LDEXP_F64, Call_F80: RTLIB::LDEXP_F80,
4537 Call_F128: RTLIB::LDEXP_F128, Call_PPCF128: RTLIB::LDEXP_PPCF128, Results);
4538 break;
4539 case ISD::FFREXP: {
4540 ExpandFrexpLibCall(Node, Results);
4541 break;
4542 }
4543 case ISD::FPOWI:
4544 case ISD::STRICT_FPOWI: {
4545 RTLIB::Libcall LC = RTLIB::getPOWI(RetVT: Node->getSimpleValueType(ResNo: 0));
4546 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
4547 if (!TLI.getLibcallName(Call: LC)) {
4548 // Some targets don't have a powi libcall; use pow instead.
4549 if (Node->isStrictFPOpcode()) {
4550 SDValue Exponent =
4551 DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: SDLoc(Node),
4552 ResultTys: {Node->getValueType(ResNo: 0), Node->getValueType(ResNo: 1)},
4553 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
4554 SDValue FPOW =
4555 DAG.getNode(Opcode: ISD::STRICT_FPOW, DL: SDLoc(Node),
4556 ResultTys: {Node->getValueType(ResNo: 0), Node->getValueType(ResNo: 1)},
4557 Ops: {Exponent.getValue(R: 1), Node->getOperand(Num: 1), Exponent});
4558 Results.push_back(Elt: FPOW);
4559 Results.push_back(Elt: FPOW.getValue(R: 1));
4560 } else {
4561 SDValue Exponent =
4562 DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0),
4563 Operand: Node->getOperand(Num: 1));
4564 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FPOW, DL: SDLoc(Node),
4565 VT: Node->getValueType(ResNo: 0),
4566 N1: Node->getOperand(Num: 0), N2: Exponent));
4567 }
4568 break;
4569 }
4570 unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0;
4571 bool ExponentHasSizeOfInt =
4572 DAG.getLibInfo().getIntSize() ==
4573 Node->getOperand(Num: 1 + Offset).getValueType().getSizeInBits();
4574 if (!ExponentHasSizeOfInt) {
4575 // If the exponent does not match with sizeof(int) a libcall to
4576 // RTLIB::POWI would use the wrong type for the argument.
4577 DAG.getContext()->emitError(ErrorStr: "POWI exponent does not match sizeof(int)");
4578 Results.push_back(Elt: DAG.getUNDEF(VT: Node->getValueType(ResNo: 0)));
4579 break;
4580 }
4581 ExpandFPLibCall(Node, LC, Results);
4582 break;
4583 }
4584 case ISD::FPOW:
4585 case ISD::STRICT_FPOW:
4586 ExpandFPLibCall(Node, Call_F32: RTLIB::POW_F32, Call_F64: RTLIB::POW_F64, Call_F80: RTLIB::POW_F80,
4587 Call_F128: RTLIB::POW_F128, Call_PPCF128: RTLIB::POW_PPCF128, Results);
4588 break;
4589 case ISD::LROUND:
4590 case ISD::STRICT_LROUND:
4591 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LROUND_F32,
4592 Call_F64: RTLIB::LROUND_F64, Call_F80: RTLIB::LROUND_F80,
4593 Call_F128: RTLIB::LROUND_F128,
4594 Call_PPCF128: RTLIB::LROUND_PPCF128, Results);
4595 break;
4596 case ISD::LLROUND:
4597 case ISD::STRICT_LLROUND:
4598 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LLROUND_F32,
4599 Call_F64: RTLIB::LLROUND_F64, Call_F80: RTLIB::LLROUND_F80,
4600 Call_F128: RTLIB::LLROUND_F128,
4601 Call_PPCF128: RTLIB::LLROUND_PPCF128, Results);
4602 break;
4603 case ISD::LRINT:
4604 case ISD::STRICT_LRINT:
4605 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LRINT_F32,
4606 Call_F64: RTLIB::LRINT_F64, Call_F80: RTLIB::LRINT_F80,
4607 Call_F128: RTLIB::LRINT_F128,
4608 Call_PPCF128: RTLIB::LRINT_PPCF128, Results);
4609 break;
4610 case ISD::LLRINT:
4611 case ISD::STRICT_LLRINT:
4612 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LLRINT_F32,
4613 Call_F64: RTLIB::LLRINT_F64, Call_F80: RTLIB::LLRINT_F80,
4614 Call_F128: RTLIB::LLRINT_F128,
4615 Call_PPCF128: RTLIB::LLRINT_PPCF128, Results);
4616 break;
4617 case ISD::FDIV:
4618 case ISD::STRICT_FDIV:
4619 ExpandFPLibCall(Node, Call_F32: RTLIB::DIV_F32, Call_F64: RTLIB::DIV_F64,
4620 Call_F80: RTLIB::DIV_F80, Call_F128: RTLIB::DIV_F128,
4621 Call_PPCF128: RTLIB::DIV_PPCF128, Results);
4622 break;
4623 case ISD::FREM:
4624 case ISD::STRICT_FREM:
4625 ExpandFPLibCall(Node, Call_F32: RTLIB::REM_F32, Call_F64: RTLIB::REM_F64,
4626 Call_F80: RTLIB::REM_F80, Call_F128: RTLIB::REM_F128,
4627 Call_PPCF128: RTLIB::REM_PPCF128, Results);
4628 break;
4629 case ISD::FMA:
4630 case ISD::STRICT_FMA:
4631 ExpandFPLibCall(Node, Call_F32: RTLIB::FMA_F32, Call_F64: RTLIB::FMA_F64,
4632 Call_F80: RTLIB::FMA_F80, Call_F128: RTLIB::FMA_F128,
4633 Call_PPCF128: RTLIB::FMA_PPCF128, Results);
4634 break;
4635 case ISD::FADD:
4636 case ISD::STRICT_FADD:
4637 ExpandFPLibCall(Node, Call_F32: RTLIB::ADD_F32, Call_F64: RTLIB::ADD_F64,
4638 Call_F80: RTLIB::ADD_F80, Call_F128: RTLIB::ADD_F128,
4639 Call_PPCF128: RTLIB::ADD_PPCF128, Results);
4640 break;
4641 case ISD::FMUL:
4642 case ISD::STRICT_FMUL:
4643 ExpandFPLibCall(Node, Call_F32: RTLIB::MUL_F32, Call_F64: RTLIB::MUL_F64,
4644 Call_F80: RTLIB::MUL_F80, Call_F128: RTLIB::MUL_F128,
4645 Call_PPCF128: RTLIB::MUL_PPCF128, Results);
4646 break;
4647 case ISD::FP16_TO_FP:
4648 if (Node->getValueType(0) == MVT::f32) {
4649 Results.push_back(Elt: ExpandLibCall(LC: RTLIB::FPEXT_F16_F32, Node, isSigned: false).first);
4650 }
4651 break;
4652 case ISD::STRICT_FP16_TO_FP: {
4653 if (Node->getValueType(0) == MVT::f32) {
4654 TargetLowering::MakeLibCallOptions CallOptions;
4655 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4656 DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Node->getOperand(1), CallOptions,
4657 SDLoc(Node), Node->getOperand(0));
4658 Results.push_back(Elt: Tmp.first);
4659 Results.push_back(Elt: Tmp.second);
4660 }
4661 break;
4662 }
4663 case ISD::FP_TO_FP16: {
4664 RTLIB::Libcall LC =
4665 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
4666 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
4667 Results.push_back(Elt: ExpandLibCall(LC, Node, isSigned: false).first);
4668 break;
4669 }
4670 case ISD::FP_TO_BF16: {
4671 RTLIB::Libcall LC =
4672 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::bf16);
4673 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_bf16");
4674 Results.push_back(Elt: ExpandLibCall(LC, Node, isSigned: false).first);
4675 break;
4676 }
4677 case ISD::STRICT_SINT_TO_FP:
4678 case ISD::STRICT_UINT_TO_FP:
4679 case ISD::SINT_TO_FP:
4680 case ISD::UINT_TO_FP: {
4681 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP
4682 bool IsStrict = Node->isStrictFPOpcode();
4683 bool Signed = Node->getOpcode() == ISD::SINT_TO_FP ||
4684 Node->getOpcode() == ISD::STRICT_SINT_TO_FP;
4685 EVT SVT = Node->getOperand(Num: IsStrict ? 1 : 0).getValueType();
4686 EVT RVT = Node->getValueType(ResNo: 0);
4687 EVT NVT = EVT();
4688 SDLoc dl(Node);
4689
4690 // Even if the input is legal, no libcall may exactly match, eg. we don't
4691 // have i1 -> fp conversions. So, it needs to be promoted to a larger type,
4692 // eg: i13 -> fp. Then, look for an appropriate libcall.
4693 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4694 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
4695 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4696 ++t) {
4697 NVT = (MVT::SimpleValueType)t;
4698 // The source needs to big enough to hold the operand.
4699 if (NVT.bitsGE(SVT))
4700 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT)
4701 : RTLIB::getUINTTOFP(NVT, RVT);
4702 }
4703 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4704
4705 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
4706 // Sign/zero extend the argument if the libcall takes a larger type.
4707 SDValue Op = DAG.getNode(Opcode: Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
4708 VT: NVT, Operand: Node->getOperand(Num: IsStrict ? 1 : 0));
4709 TargetLowering::MakeLibCallOptions CallOptions;
4710 CallOptions.setSExt(Signed);
4711 std::pair<SDValue, SDValue> Tmp =
4712 TLI.makeLibCall(DAG, LC, RetVT: RVT, Ops: Op, CallOptions, dl, Chain);
4713 Results.push_back(Elt: Tmp.first);
4714 if (IsStrict)
4715 Results.push_back(Elt: Tmp.second);
4716 break;
4717 }
4718 case ISD::FP_TO_SINT:
4719 case ISD::FP_TO_UINT:
4720 case ISD::STRICT_FP_TO_SINT:
4721 case ISD::STRICT_FP_TO_UINT: {
4722 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT.
4723 bool IsStrict = Node->isStrictFPOpcode();
4724 bool Signed = Node->getOpcode() == ISD::FP_TO_SINT ||
4725 Node->getOpcode() == ISD::STRICT_FP_TO_SINT;
4726
4727 SDValue Op = Node->getOperand(Num: IsStrict ? 1 : 0);
4728 EVT SVT = Op.getValueType();
4729 EVT RVT = Node->getValueType(ResNo: 0);
4730 EVT NVT = EVT();
4731 SDLoc dl(Node);
4732
4733 // Even if the result is legal, no libcall may exactly match, eg. we don't
4734 // have fp -> i1 conversions. So, it needs to be promoted to a larger type,
4735 // eg: fp -> i32. Then, look for an appropriate libcall.
4736 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4737 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
4738 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4739 ++IntVT) {
4740 NVT = (MVT::SimpleValueType)IntVT;
4741 // The type needs to big enough to hold the result.
4742 if (NVT.bitsGE(RVT))
4743 LC = Signed ? RTLIB::getFPTOSINT(SVT, NVT)
4744 : RTLIB::getFPTOUINT(SVT, NVT);
4745 }
4746 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4747
4748 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
4749 TargetLowering::MakeLibCallOptions CallOptions;
4750 std::pair<SDValue, SDValue> Tmp =
4751 TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op, CallOptions, dl, Chain);
4752
4753 // Truncate the result if the libcall returns a larger type.
4754 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RVT, Operand: Tmp.first));
4755 if (IsStrict)
4756 Results.push_back(Elt: Tmp.second);
4757 break;
4758 }
4759
4760 case ISD::FP_ROUND:
4761 case ISD::STRICT_FP_ROUND: {
4762 // X = FP_ROUND(Y, TRUNC)
4763 // TRUNC is a flag, which is always an integer that is zero or one.
4764 // If TRUNC is 0, this is a normal rounding, if it is 1, this FP_ROUND
4765 // is known to not change the value of Y.
4766 // We can only expand it into libcall if the TRUNC is 0.
4767 bool IsStrict = Node->isStrictFPOpcode();
4768 SDValue Op = Node->getOperand(Num: IsStrict ? 1 : 0);
4769 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
4770 EVT VT = Node->getValueType(ResNo: 0);
4771 assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
4772 "Unable to expand as libcall if it is not normal rounding");
4773
4774 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op.getValueType(), RetVT: VT);
4775 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4776
4777 TargetLowering::MakeLibCallOptions CallOptions;
4778 std::pair<SDValue, SDValue> Tmp =
4779 TLI.makeLibCall(DAG, LC, RetVT: VT, Ops: Op, CallOptions, dl: SDLoc(Node), Chain);
4780 Results.push_back(Elt: Tmp.first);
4781 if (IsStrict)
4782 Results.push_back(Elt: Tmp.second);
4783 break;
4784 }
4785 case ISD::FP_EXTEND: {
4786 Results.push_back(
4787 Elt: ExpandLibCall(LC: RTLIB::getFPEXT(OpVT: Node->getOperand(Num: 0).getValueType(),
4788 RetVT: Node->getValueType(ResNo: 0)),
4789 Node, isSigned: false).first);
4790 break;
4791 }
4792 case ISD::STRICT_FP_EXTEND:
4793 case ISD::STRICT_FP_TO_FP16: {
4794 RTLIB::Libcall LC =
4795 Node->getOpcode() == ISD::STRICT_FP_TO_FP16
4796 ? RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::f16)
4797 : RTLIB::getFPEXT(Node->getOperand(1).getValueType(),
4798 Node->getValueType(0));
4799 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4800
4801 TargetLowering::MakeLibCallOptions CallOptions;
4802 std::pair<SDValue, SDValue> Tmp =
4803 TLI.makeLibCall(DAG, LC, RetVT: Node->getValueType(ResNo: 0), Ops: Node->getOperand(Num: 1),
4804 CallOptions, dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
4805 Results.push_back(Elt: Tmp.first);
4806 Results.push_back(Elt: Tmp.second);
4807 break;
4808 }
4809 case ISD::FSUB:
4810 case ISD::STRICT_FSUB:
4811 ExpandFPLibCall(Node, Call_F32: RTLIB::SUB_F32, Call_F64: RTLIB::SUB_F64,
4812 Call_F80: RTLIB::SUB_F80, Call_F128: RTLIB::SUB_F128,
4813 Call_PPCF128: RTLIB::SUB_PPCF128, Results);
4814 break;
4815 case ISD::SREM:
4816 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: true,
4817 Call_I8: RTLIB::SREM_I8,
4818 Call_I16: RTLIB::SREM_I16, Call_I32: RTLIB::SREM_I32,
4819 Call_I64: RTLIB::SREM_I64, Call_I128: RTLIB::SREM_I128));
4820 break;
4821 case ISD::UREM:
4822 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
4823 Call_I8: RTLIB::UREM_I8,
4824 Call_I16: RTLIB::UREM_I16, Call_I32: RTLIB::UREM_I32,
4825 Call_I64: RTLIB::UREM_I64, Call_I128: RTLIB::UREM_I128));
4826 break;
4827 case ISD::SDIV:
4828 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: true,
4829 Call_I8: RTLIB::SDIV_I8,
4830 Call_I16: RTLIB::SDIV_I16, Call_I32: RTLIB::SDIV_I32,
4831 Call_I64: RTLIB::SDIV_I64, Call_I128: RTLIB::SDIV_I128));
4832 break;
4833 case ISD::UDIV:
4834 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
4835 Call_I8: RTLIB::UDIV_I8,
4836 Call_I16: RTLIB::UDIV_I16, Call_I32: RTLIB::UDIV_I32,
4837 Call_I64: RTLIB::UDIV_I64, Call_I128: RTLIB::UDIV_I128));
4838 break;
4839 case ISD::SDIVREM:
4840 case ISD::UDIVREM:
4841 // Expand into divrem libcall
4842 ExpandDivRemLibCall(Node, Results);
4843 break;
4844 case ISD::MUL:
4845 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
4846 Call_I8: RTLIB::MUL_I8,
4847 Call_I16: RTLIB::MUL_I16, Call_I32: RTLIB::MUL_I32,
4848 Call_I64: RTLIB::MUL_I64, Call_I128: RTLIB::MUL_I128));
4849 break;
4850 case ISD::CTLZ_ZERO_UNDEF:
4851 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
4852 default:
4853 llvm_unreachable("LibCall explicitly requested, but not available");
4854 case MVT::i32:
4855 Results.push_back(Elt: ExpandLibCall(LC: RTLIB::CTLZ_I32, Node, isSigned: false).first);
4856 break;
4857 case MVT::i64:
4858 Results.push_back(Elt: ExpandLibCall(LC: RTLIB::CTLZ_I64, Node, isSigned: false).first);
4859 break;
4860 case MVT::i128:
4861 Results.push_back(Elt: ExpandLibCall(LC: RTLIB::CTLZ_I128, Node, isSigned: false).first);
4862 break;
4863 }
4864 break;
4865 case ISD::RESET_FPENV: {
4866 // It is legalized to call 'fesetenv(FE_DFL_ENV)'. On most targets
4867 // FE_DFL_ENV is defined as '((const fenv_t *) -1)' in glibc.
4868 SDValue Ptr = DAG.getIntPtrConstant(Val: -1LL, DL: dl);
4869 SDValue Chain = Node->getOperand(Num: 0);
4870 Results.push_back(
4871 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETENV, Ptr, InChain: Chain, DLoc: dl));
4872 break;
4873 }
4874 case ISD::GET_FPENV_MEM: {
4875 SDValue Chain = Node->getOperand(Num: 0);
4876 SDValue EnvPtr = Node->getOperand(Num: 1);
4877 Results.push_back(
4878 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FEGETENV, Ptr: EnvPtr, InChain: Chain, DLoc: dl));
4879 break;
4880 }
4881 case ISD::SET_FPENV_MEM: {
4882 SDValue Chain = Node->getOperand(Num: 0);
4883 SDValue EnvPtr = Node->getOperand(Num: 1);
4884 Results.push_back(
4885 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETENV, Ptr: EnvPtr, InChain: Chain, DLoc: dl));
4886 break;
4887 }
4888 case ISD::GET_FPMODE: {
4889 // Call fegetmode, which saves control modes into a stack slot. Then load
4890 // the value to return from the stack.
4891 EVT ModeVT = Node->getValueType(ResNo: 0);
4892 SDValue StackPtr = DAG.CreateStackTemporary(VT: ModeVT);
4893 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
4894 SDValue Chain = DAG.makeStateFunctionCall(LibFunc: RTLIB::FEGETMODE, Ptr: StackPtr,
4895 InChain: Node->getOperand(Num: 0), DLoc: dl);
4896 SDValue LdInst = DAG.getLoad(
4897 VT: ModeVT, dl, Chain, Ptr: StackPtr,
4898 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
4899 Results.push_back(Elt: LdInst);
4900 Results.push_back(Elt: LdInst.getValue(R: 1));
4901 break;
4902 }
4903 case ISD::SET_FPMODE: {
4904 // Move control modes to stack slot and then call fesetmode with the pointer
4905 // to the slot as argument.
4906 SDValue Mode = Node->getOperand(Num: 1);
4907 EVT ModeVT = Mode.getValueType();
4908 SDValue StackPtr = DAG.CreateStackTemporary(VT: ModeVT);
4909 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
4910 SDValue StInst = DAG.getStore(
4911 Chain: Node->getOperand(Num: 0), dl, Val: Mode, Ptr: StackPtr,
4912 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
4913 Results.push_back(
4914 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETMODE, Ptr: StackPtr, InChain: StInst, DLoc: dl));
4915 break;
4916 }
4917 case ISD::RESET_FPMODE: {
4918 // It is legalized to a call 'fesetmode(FE_DFL_MODE)'. On most targets
4919 // FE_DFL_MODE is defined as '((const femode_t *) -1)' in glibc. If not, the
4920 // target must provide custom lowering.
4921 const DataLayout &DL = DAG.getDataLayout();
4922 EVT PtrTy = TLI.getPointerTy(DL);
4923 SDValue Mode = DAG.getConstant(Val: -1LL, DL: dl, VT: PtrTy);
4924 Results.push_back(Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETMODE, Ptr: Mode,
4925 InChain: Node->getOperand(Num: 0), DLoc: dl));
4926 break;
4927 }
4928 }
4929
4930 // Replace the original node with the legalized result.
4931 if (!Results.empty()) {
4932 LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n");
4933 ReplaceNode(Old: Node, New: Results.data());
4934 } else
4935 LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n");
4936}
4937
4938// Determine the vector type to use in place of an original scalar element when
4939// promoting equally sized vectors.
4940static MVT getPromotedVectorElementType(const TargetLowering &TLI,
4941 MVT EltVT, MVT NewEltVT) {
4942 unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
4943 MVT MidVT = OldEltsPerNewElt == 1
4944 ? NewEltVT
4945 : MVT::getVectorVT(VT: NewEltVT, NumElements: OldEltsPerNewElt);
4946 assert(TLI.isTypeLegal(MidVT) && "unexpected");
4947 return MidVT;
4948}
4949
4950void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
4951 LLVM_DEBUG(dbgs() << "Trying to promote node\n");
4952 SmallVector<SDValue, 8> Results;
4953 MVT OVT = Node->getSimpleValueType(ResNo: 0);
4954 if (Node->getOpcode() == ISD::UINT_TO_FP ||
4955 Node->getOpcode() == ISD::SINT_TO_FP ||
4956 Node->getOpcode() == ISD::SETCC ||
4957 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
4958 Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
4959 OVT = Node->getOperand(Num: 0).getSimpleValueType();
4960 }
4961 if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||
4962 Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
4963 Node->getOpcode() == ISD::STRICT_FSETCC ||
4964 Node->getOpcode() == ISD::STRICT_FSETCCS)
4965 OVT = Node->getOperand(Num: 1).getSimpleValueType();
4966 if (Node->getOpcode() == ISD::BR_CC ||
4967 Node->getOpcode() == ISD::SELECT_CC)
4968 OVT = Node->getOperand(Num: 2).getSimpleValueType();
4969 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: OVT);
4970 SDLoc dl(Node);
4971 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
4972 switch (Node->getOpcode()) {
4973 case ISD::CTTZ:
4974 case ISD::CTTZ_ZERO_UNDEF:
4975 case ISD::CTLZ:
4976 case ISD::CTLZ_ZERO_UNDEF:
4977 case ISD::CTPOP:
4978 // Zero extend the argument unless its cttz, then use any_extend.
4979 if (Node->getOpcode() == ISD::CTTZ ||
4980 Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF)
4981 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
4982 else
4983 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
4984
4985 if (Node->getOpcode() == ISD::CTTZ) {
4986 // The count is the same in the promoted type except if the original
4987 // value was zero. This can be handled by setting the bit just off
4988 // the top of the original type.
4989 auto TopBit = APInt::getOneBitSet(numBits: NVT.getSizeInBits(),
4990 BitNo: OVT.getSizeInBits());
4991 Tmp1 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, N1: Tmp1,
4992 N2: DAG.getConstant(Val: TopBit, DL: dl, VT: NVT));
4993 }
4994 // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
4995 // already the correct result.
4996 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
4997 if (Node->getOpcode() == ISD::CTLZ ||
4998 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
4999 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
5000 Tmp1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, N1: Tmp1,
5001 N2: DAG.getConstant(Val: NVT.getSizeInBits() -
5002 OVT.getSizeInBits(), DL: dl, VT: NVT));
5003 }
5004 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5005 break;
5006 case ISD::BITREVERSE:
5007 case ISD::BSWAP: {
5008 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
5009 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5010 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
5011 Tmp1 = DAG.getNode(
5012 Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Tmp1,
5013 N2: DAG.getConstant(Val: DiffBits, DL: dl,
5014 VT: TLI.getShiftAmountTy(LHSTy: NVT, DL: DAG.getDataLayout())));
5015
5016 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5017 break;
5018 }
5019 case ISD::FP_TO_UINT:
5020 case ISD::STRICT_FP_TO_UINT:
5021 case ISD::FP_TO_SINT:
5022 case ISD::STRICT_FP_TO_SINT:
5023 PromoteLegalFP_TO_INT(N: Node, dl, Results);
5024 break;
5025 case ISD::FP_TO_UINT_SAT:
5026 case ISD::FP_TO_SINT_SAT:
5027 Results.push_back(Elt: PromoteLegalFP_TO_INT_SAT(Node, dl));
5028 break;
5029 case ISD::UINT_TO_FP:
5030 case ISD::STRICT_UINT_TO_FP:
5031 case ISD::SINT_TO_FP:
5032 case ISD::STRICT_SINT_TO_FP:
5033 PromoteLegalINT_TO_FP(N: Node, dl, Results);
5034 break;
5035 case ISD::VAARG: {
5036 SDValue Chain = Node->getOperand(Num: 0); // Get the chain.
5037 SDValue Ptr = Node->getOperand(Num: 1); // Get the pointer.
5038
5039 unsigned TruncOp;
5040 if (OVT.isVector()) {
5041 TruncOp = ISD::BITCAST;
5042 } else {
5043 assert(OVT.isInteger()
5044 && "VAARG promotion is supported only for vectors or integer types");
5045 TruncOp = ISD::TRUNCATE;
5046 }
5047
5048 // Perform the larger operation, then convert back
5049 Tmp1 = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV: Node->getOperand(Num: 2),
5050 Align: Node->getConstantOperandVal(Num: 3));
5051 Chain = Tmp1.getValue(R: 1);
5052
5053 Tmp2 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: OVT, Operand: Tmp1);
5054
5055 // Modified the chain result - switch anything that used the old chain to
5056 // use the new one.
5057 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: Tmp2);
5058 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: Chain);
5059 if (UpdatedNodes) {
5060 UpdatedNodes->insert(X: Tmp2.getNode());
5061 UpdatedNodes->insert(X: Chain.getNode());
5062 }
5063 ReplacedNode(N: Node);
5064 break;
5065 }
5066 case ISD::MUL:
5067 case ISD::SDIV:
5068 case ISD::SREM:
5069 case ISD::UDIV:
5070 case ISD::UREM:
5071 case ISD::SMIN:
5072 case ISD::SMAX:
5073 case ISD::UMIN:
5074 case ISD::UMAX:
5075 case ISD::AND:
5076 case ISD::OR:
5077 case ISD::XOR: {
5078 unsigned ExtOp, TruncOp;
5079 if (OVT.isVector()) {
5080 ExtOp = ISD::BITCAST;
5081 TruncOp = ISD::BITCAST;
5082 } else {
5083 assert(OVT.isInteger() && "Cannot promote logic operation");
5084
5085 switch (Node->getOpcode()) {
5086 default:
5087 ExtOp = ISD::ANY_EXTEND;
5088 break;
5089 case ISD::SDIV:
5090 case ISD::SREM:
5091 case ISD::SMIN:
5092 case ISD::SMAX:
5093 ExtOp = ISD::SIGN_EXTEND;
5094 break;
5095 case ISD::UDIV:
5096 case ISD::UREM:
5097 ExtOp = ISD::ZERO_EXTEND;
5098 break;
5099 case ISD::UMIN:
5100 case ISD::UMAX:
5101 if (TLI.isSExtCheaperThanZExt(FromTy: OVT, ToTy: NVT))
5102 ExtOp = ISD::SIGN_EXTEND;
5103 else
5104 ExtOp = ISD::ZERO_EXTEND;
5105 break;
5106 }
5107 TruncOp = ISD::TRUNCATE;
5108 }
5109 // Promote each of the values to the new type.
5110 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5111 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5112 // Perform the larger operation, then convert back
5113 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5114 Results.push_back(Elt: DAG.getNode(Opcode: TruncOp, DL: dl, VT: OVT, Operand: Tmp1));
5115 break;
5116 }
5117 case ISD::UMUL_LOHI:
5118 case ISD::SMUL_LOHI: {
5119 // Promote to a multiply in a wider integer type.
5120 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
5121 : ISD::SIGN_EXTEND;
5122 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5123 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5124 Tmp1 = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5125
5126 auto &DL = DAG.getDataLayout();
5127 unsigned OriginalSize = OVT.getScalarSizeInBits();
5128 Tmp2 = DAG.getNode(
5129 Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Tmp1,
5130 N2: DAG.getConstant(Val: OriginalSize, DL: dl, VT: TLI.getScalarShiftAmountTy(DL, NVT)));
5131 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5132 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp2));
5133 break;
5134 }
5135 case ISD::SELECT: {
5136 unsigned ExtOp, TruncOp;
5137 if (Node->getValueType(ResNo: 0).isVector() ||
5138 Node->getValueType(ResNo: 0).getSizeInBits() == NVT.getSizeInBits()) {
5139 ExtOp = ISD::BITCAST;
5140 TruncOp = ISD::BITCAST;
5141 } else if (Node->getValueType(ResNo: 0).isInteger()) {
5142 ExtOp = ISD::ANY_EXTEND;
5143 TruncOp = ISD::TRUNCATE;
5144 } else {
5145 ExtOp = ISD::FP_EXTEND;
5146 TruncOp = ISD::FP_ROUND;
5147 }
5148 Tmp1 = Node->getOperand(Num: 0);
5149 // Promote each of the values to the new type.
5150 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5151 Tmp3 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5152 // Perform the larger operation, then round down.
5153 Tmp1 = DAG.getSelect(DL: dl, VT: NVT, Cond: Tmp1, LHS: Tmp2, RHS: Tmp3);
5154 Tmp1->setFlags(Node->getFlags());
5155 if (TruncOp != ISD::FP_ROUND)
5156 Tmp1 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
5157 else
5158 Tmp1 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
5159 N2: DAG.getIntPtrConstant(Val: 0, DL: dl));
5160 Results.push_back(Elt: Tmp1);
5161 break;
5162 }
5163 case ISD::VECTOR_SHUFFLE: {
5164 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Node)->getMask();
5165
5166 // Cast the two input vectors.
5167 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5168 Tmp2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5169
5170 // Convert the shuffle mask to the right # elements.
5171 Tmp1 = ShuffleWithNarrowerEltType(NVT, VT: OVT, dl, N1: Tmp1, N2: Tmp2, Mask);
5172 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OVT, Operand: Tmp1);
5173 Results.push_back(Elt: Tmp1);
5174 break;
5175 }
5176 case ISD::VECTOR_SPLICE: {
5177 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5178 Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5179 Tmp3 = DAG.getNode(Opcode: ISD::VECTOR_SPLICE, DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2,
5180 N3: Node->getOperand(Num: 2));
5181 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp3));
5182 break;
5183 }
5184 case ISD::SELECT_CC: {
5185 SDValue Cond = Node->getOperand(Num: 4);
5186 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val&: Cond)->get();
5187 // Type of the comparison operands.
5188 MVT CVT = Node->getSimpleValueType(ResNo: 0);
5189 assert(CVT == OVT && "not handled");
5190
5191 unsigned ExtOp = ISD::FP_EXTEND;
5192 if (NVT.isInteger()) {
5193 ExtOp = isSignedIntSetCC(Code: CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
5194 }
5195
5196 // Promote the comparison operands, if needed.
5197 if (TLI.isCondCodeLegal(CC: CCCode, VT: CVT)) {
5198 Tmp1 = Node->getOperand(Num: 0);
5199 Tmp2 = Node->getOperand(Num: 1);
5200 } else {
5201 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5202 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5203 }
5204 // Cast the true/false operands.
5205 Tmp3 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5206 Tmp4 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 3));
5207
5208 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: NVT, Ops: {Tmp1, Tmp2, Tmp3, Tmp4, Cond},
5209 Flags: Node->getFlags());
5210
5211 // Cast the result back to the original type.
5212 if (ExtOp != ISD::FP_EXTEND)
5213 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1);
5214 else
5215 Tmp1 = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp1,
5216 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5217
5218 Results.push_back(Elt: Tmp1);
5219 break;
5220 }
5221 case ISD::SETCC:
5222 case ISD::STRICT_FSETCC:
5223 case ISD::STRICT_FSETCCS: {
5224 unsigned ExtOp = ISD::FP_EXTEND;
5225 if (NVT.isInteger()) {
5226 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Node->getOperand(Num: 2))->get();
5227 if (isSignedIntSetCC(Code: CCCode) ||
5228 TLI.isSExtCheaperThanZExt(FromTy: Node->getOperand(Num: 0).getValueType(), ToTy: NVT))
5229 ExtOp = ISD::SIGN_EXTEND;
5230 else
5231 ExtOp = ISD::ZERO_EXTEND;
5232 }
5233 if (Node->isStrictFPOpcode()) {
5234 SDValue InChain = Node->getOperand(Num: 0);
5235 std::tie(args&: Tmp1, args: std::ignore) =
5236 DAG.getStrictFPExtendOrRound(Op: Node->getOperand(Num: 1), Chain: InChain, DL: dl, VT: NVT);
5237 std::tie(args&: Tmp2, args: std::ignore) =
5238 DAG.getStrictFPExtendOrRound(Op: Node->getOperand(Num: 2), Chain: InChain, DL: dl, VT: NVT);
5239 SmallVector<SDValue, 2> TmpChains = {Tmp1.getValue(R: 1), Tmp2.getValue(R: 1)};
5240 SDValue OutChain = DAG.getTokenFactor(DL: dl, Vals&: TmpChains);
5241 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
5242 Results.push_back(Elt: DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: VTs,
5243 Ops: {OutChain, Tmp1, Tmp2, Node->getOperand(Num: 3)},
5244 Flags: Node->getFlags()));
5245 Results.push_back(Elt: Results.back().getValue(R: 1));
5246 break;
5247 }
5248 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5249 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5250 Results.push_back(Elt: DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
5251 N2: Tmp2, N3: Node->getOperand(Num: 2), Flags: Node->getFlags()));
5252 break;
5253 }
5254 case ISD::BR_CC: {
5255 unsigned ExtOp = ISD::FP_EXTEND;
5256 if (NVT.isInteger()) {
5257 ISD::CondCode CCCode =
5258 cast<CondCodeSDNode>(Val: Node->getOperand(Num: 1))->get();
5259 ExtOp = isSignedIntSetCC(Code: CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
5260 }
5261 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5262 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 3));
5263 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0),
5264 N1: Node->getOperand(Num: 0), N2: Node->getOperand(Num: 1),
5265 N3: Tmp1, N4: Tmp2, N5: Node->getOperand(Num: 4)));
5266 break;
5267 }
5268 case ISD::FADD:
5269 case ISD::FSUB:
5270 case ISD::FMUL:
5271 case ISD::FDIV:
5272 case ISD::FREM:
5273 case ISD::FMINNUM:
5274 case ISD::FMAXNUM:
5275 case ISD::FMINIMUM:
5276 case ISD::FMAXIMUM:
5277 case ISD::FPOW:
5278 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5279 Tmp2 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5280 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2,
5281 Flags: Node->getFlags());
5282 Results.push_back(
5283 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp3,
5284 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5285 break;
5286 case ISD::STRICT_FADD:
5287 case ISD::STRICT_FSUB:
5288 case ISD::STRICT_FMUL:
5289 case ISD::STRICT_FDIV:
5290 case ISD::STRICT_FMINNUM:
5291 case ISD::STRICT_FMAXNUM:
5292 case ISD::STRICT_FREM:
5293 case ISD::STRICT_FPOW:
5294 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5295 {Node->getOperand(0), Node->getOperand(1)});
5296 Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5297 {Node->getOperand(0), Node->getOperand(2)});
5298 Tmp3 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1),
5299 Tmp2.getValue(1));
5300 Tmp1 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5301 {Tmp3, Tmp1, Tmp2});
5302 Tmp1 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
5303 {Tmp1.getValue(1), Tmp1, DAG.getIntPtrConstant(0, dl)});
5304 Results.push_back(Elt: Tmp1);
5305 Results.push_back(Elt: Tmp1.getValue(R: 1));
5306 break;
5307 case ISD::FMA:
5308 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5309 Tmp2 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5310 Tmp3 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5311 Results.push_back(
5312 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT,
5313 N1: DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2, N3: Tmp3),
5314 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5315 break;
5316 case ISD::STRICT_FMA:
5317 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5318 {Node->getOperand(0), Node->getOperand(1)});
5319 Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5320 {Node->getOperand(0), Node->getOperand(2)});
5321 Tmp3 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5322 {Node->getOperand(0), Node->getOperand(3)});
5323 Tmp4 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1),
5324 Tmp2.getValue(1), Tmp3.getValue(1));
5325 Tmp4 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5326 {Tmp4, Tmp1, Tmp2, Tmp3});
5327 Tmp4 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
5328 {Tmp4.getValue(1), Tmp4, DAG.getIntPtrConstant(0, dl)});
5329 Results.push_back(Elt: Tmp4);
5330 Results.push_back(Elt: Tmp4.getValue(R: 1));
5331 break;
5332 case ISD::FCOPYSIGN:
5333 case ISD::FLDEXP:
5334 case ISD::FPOWI: {
5335 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5336 Tmp2 = Node->getOperand(Num: 1);
5337 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5338
5339 // fcopysign doesn't change anything but the sign bit, so
5340 // (fp_round (fcopysign (fpext a), b))
5341 // is as precise as
5342 // (fp_round (fpext a))
5343 // which is a no-op. Mark it as a TRUNCating FP_ROUND.
5344 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
5345 Results.push_back(
5346 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp3,
5347 N2: DAG.getIntPtrConstant(Val: isTrunc, DL: dl, /*isTarget=*/true)));
5348 break;
5349 }
5350 case ISD::STRICT_FPOWI:
5351 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5352 {Node->getOperand(0), Node->getOperand(1)});
5353 Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5354 {Tmp1.getValue(1), Tmp1, Node->getOperand(2)});
5355 Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
5356 {Tmp2.getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
5357 Results.push_back(Elt: Tmp3);
5358 Results.push_back(Elt: Tmp3.getValue(R: 1));
5359 break;
5360 case ISD::FFREXP: {
5361 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5362 Tmp2 = DAG.getNode(Opcode: ISD::FFREXP, DL: dl, ResultTys: {NVT, Node->getValueType(ResNo: 1)}, Ops: Tmp1);
5363
5364 Results.push_back(
5365 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
5366 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5367
5368 Results.push_back(Elt: Tmp2.getValue(R: 1));
5369 break;
5370 }
5371 case ISD::FFLOOR:
5372 case ISD::FCEIL:
5373 case ISD::FRINT:
5374 case ISD::FNEARBYINT:
5375 case ISD::FROUND:
5376 case ISD::FROUNDEVEN:
5377 case ISD::FTRUNC:
5378 case ISD::FNEG:
5379 case ISD::FSQRT:
5380 case ISD::FSIN:
5381 case ISD::FCOS:
5382 case ISD::FLOG:
5383 case ISD::FLOG2:
5384 case ISD::FLOG10:
5385 case ISD::FABS:
5386 case ISD::FEXP:
5387 case ISD::FEXP2:
5388 case ISD::FEXP10:
5389 case ISD::FCANONICALIZE:
5390 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5391 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
5392 Results.push_back(
5393 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
5394 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5395 break;
5396 case ISD::STRICT_FFLOOR:
5397 case ISD::STRICT_FCEIL:
5398 case ISD::STRICT_FRINT:
5399 case ISD::STRICT_FNEARBYINT:
5400 case ISD::STRICT_FROUND:
5401 case ISD::STRICT_FROUNDEVEN:
5402 case ISD::STRICT_FTRUNC:
5403 case ISD::STRICT_FSQRT:
5404 case ISD::STRICT_FSIN:
5405 case ISD::STRICT_FCOS:
5406 case ISD::STRICT_FLOG:
5407 case ISD::STRICT_FLOG2:
5408 case ISD::STRICT_FLOG10:
5409 case ISD::STRICT_FEXP:
5410 case ISD::STRICT_FEXP2:
5411 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5412 {Node->getOperand(0), Node->getOperand(1)});
5413 Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5414 {Tmp1.getValue(1), Tmp1});
5415 Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
5416 {Tmp2.getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
5417 Results.push_back(Elt: Tmp3);
5418 Results.push_back(Elt: Tmp3.getValue(R: 1));
5419 break;
5420 case ISD::BUILD_VECTOR: {
5421 MVT EltVT = OVT.getVectorElementType();
5422 MVT NewEltVT = NVT.getVectorElementType();
5423
5424 // Handle bitcasts to a different vector type with the same total bit size
5425 //
5426 // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
5427 // =>
5428 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
5429
5430 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
5431 "Invalid promote type for build_vector");
5432 assert(NewEltVT.bitsLE(EltVT) && "not handled");
5433
5434 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5435
5436 SmallVector<SDValue, 8> NewOps;
5437 for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) {
5438 SDValue Op = Node->getOperand(Num: I);
5439 NewOps.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(Op), VT: MidVT, Operand: Op));
5440 }
5441
5442 SDLoc SL(Node);
5443 SDValue Concat =
5444 DAG.getNode(Opcode: MidVT == NewEltVT ? ISD::BUILD_VECTOR : ISD::CONCAT_VECTORS,
5445 DL: SL, VT: NVT, Ops: NewOps);
5446 SDValue CvtVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: Concat);
5447 Results.push_back(Elt: CvtVec);
5448 break;
5449 }
5450 case ISD::EXTRACT_VECTOR_ELT: {
5451 MVT EltVT = OVT.getVectorElementType();
5452 MVT NewEltVT = NVT.getVectorElementType();
5453
5454 // Handle bitcasts to a different vector type with the same total bit size.
5455 //
5456 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
5457 // =>
5458 // v4i32:castx = bitcast x:v2i64
5459 //
5460 // i64 = bitcast
5461 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
5462 // (i32 (extract_vector_elt castx, (2 * y + 1)))
5463 //
5464
5465 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
5466 "Invalid promote type for extract_vector_elt");
5467 assert(NewEltVT.bitsLT(EltVT) && "not handled");
5468
5469 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5470 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
5471
5472 SDValue Idx = Node->getOperand(Num: 1);
5473 EVT IdxVT = Idx.getValueType();
5474 SDLoc SL(Node);
5475 SDValue Factor = DAG.getConstant(Val: NewEltsPerOldElt, DL: SL, VT: IdxVT);
5476 SDValue NewBaseIdx = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: IdxVT, N1: Idx, N2: Factor);
5477
5478 SDValue CastVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: Node->getOperand(Num: 0));
5479
5480 SmallVector<SDValue, 8> NewOps;
5481 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
5482 SDValue IdxOffset = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
5483 SDValue TmpIdx = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: IdxVT, N1: NewBaseIdx, N2: IdxOffset);
5484
5485 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: NewEltVT,
5486 N1: CastVec, N2: TmpIdx);
5487 NewOps.push_back(Elt);
5488 }
5489
5490 SDValue NewVec = DAG.getBuildVector(VT: MidVT, DL: SL, Ops: NewOps);
5491 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: EltVT, Operand: NewVec));
5492 break;
5493 }
5494 case ISD::INSERT_VECTOR_ELT: {
5495 MVT EltVT = OVT.getVectorElementType();
5496 MVT NewEltVT = NVT.getVectorElementType();
5497
5498 // Handle bitcasts to a different vector type with the same total bit size
5499 //
5500 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
5501 // =>
5502 // v4i32:castx = bitcast x:v2i64
5503 // v2i32:casty = bitcast y:i64
5504 //
5505 // v2i64 = bitcast
5506 // (v4i32 insert_vector_elt
5507 // (v4i32 insert_vector_elt v4i32:castx,
5508 // (extract_vector_elt casty, 0), 2 * z),
5509 // (extract_vector_elt casty, 1), (2 * z + 1))
5510
5511 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
5512 "Invalid promote type for insert_vector_elt");
5513 assert(NewEltVT.bitsLT(EltVT) && "not handled");
5514
5515 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5516 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
5517
5518 SDValue Val = Node->getOperand(Num: 1);
5519 SDValue Idx = Node->getOperand(Num: 2);
5520 EVT IdxVT = Idx.getValueType();
5521 SDLoc SL(Node);
5522
5523 SDValue Factor = DAG.getConstant(Val: NewEltsPerOldElt, DL: SDLoc(), VT: IdxVT);
5524 SDValue NewBaseIdx = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: IdxVT, N1: Idx, N2: Factor);
5525
5526 SDValue CastVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: Node->getOperand(Num: 0));
5527 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MidVT, Operand: Val);
5528
5529 SDValue NewVec = CastVec;
5530 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
5531 SDValue IdxOffset = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
5532 SDValue InEltIdx = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: IdxVT, N1: NewBaseIdx, N2: IdxOffset);
5533
5534 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: NewEltVT,
5535 N1: CastVal, N2: IdxOffset);
5536
5537 NewVec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: NVT,
5538 N1: NewVec, N2: Elt, N3: InEltIdx);
5539 }
5540
5541 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewVec));
5542 break;
5543 }
5544 case ISD::SCALAR_TO_VECTOR: {
5545 MVT EltVT = OVT.getVectorElementType();
5546 MVT NewEltVT = NVT.getVectorElementType();
5547
5548 // Handle bitcasts to different vector type with the same total bit size.
5549 //
5550 // e.g. v2i64 = scalar_to_vector x:i64
5551 // =>
5552 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
5553 //
5554
5555 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5556 SDValue Val = Node->getOperand(Num: 0);
5557 SDLoc SL(Node);
5558
5559 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MidVT, Operand: Val);
5560 SDValue Undef = DAG.getUNDEF(VT: MidVT);
5561
5562 SmallVector<SDValue, 8> NewElts;
5563 NewElts.push_back(Elt: CastVal);
5564 for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
5565 NewElts.push_back(Elt: Undef);
5566
5567 SDValue Concat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT: NVT, Ops: NewElts);
5568 SDValue CvtVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: Concat);
5569 Results.push_back(Elt: CvtVec);
5570 break;
5571 }
5572 case ISD::ATOMIC_SWAP: {
5573 AtomicSDNode *AM = cast<AtomicSDNode>(Val: Node);
5574 SDLoc SL(Node);
5575 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: AM->getVal());
5576 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
5577 "unexpected promotion type");
5578 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
5579 "unexpected atomic_swap with illegal type");
5580
5581 SDValue NewAtomic
5582 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT,
5583 DAG.getVTList(NVT, MVT::Other),
5584 { AM->getChain(), AM->getBasePtr(), CastVal },
5585 AM->getMemOperand());
5586 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewAtomic));
5587 Results.push_back(Elt: NewAtomic.getValue(R: 1));
5588 break;
5589 }
5590 case ISD::SPLAT_VECTOR: {
5591 SDValue Scalar = Node->getOperand(Num: 0);
5592 MVT ScalarType = Scalar.getSimpleValueType();
5593 MVT NewScalarType = NVT.getVectorElementType();
5594 if (ScalarType.isInteger()) {
5595 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NewScalarType, Operand: Scalar);
5596 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
5597 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp2));
5598 break;
5599 }
5600 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NewScalarType, Operand: Scalar);
5601 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
5602 Results.push_back(
5603 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
5604 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5605 break;
5606 }
5607 }
5608
5609 // Replace the original node with the legalized result.
5610 if (!Results.empty()) {
5611 LLVM_DEBUG(dbgs() << "Successfully promoted node\n");
5612 ReplaceNode(Old: Node, New: Results.data());
5613 } else
5614 LLVM_DEBUG(dbgs() << "Could not promote node\n");
5615}
5616
5617/// This is the entry point for the file.
5618void SelectionDAG::Legalize() {
5619 AssignTopologicalOrder();
5620
5621 SmallPtrSet<SDNode *, 16> LegalizedNodes;
5622 // Use a delete listener to remove nodes which were deleted during
5623 // legalization from LegalizeNodes. This is needed to handle the situation
5624 // where a new node is allocated by the object pool to the same address of a
5625 // previously deleted node.
5626 DAGNodeDeletedListener DeleteListener(
5627 *this,
5628 [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(Ptr: N); });
5629
5630 SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
5631
5632 // Visit all the nodes. We start in topological order, so that we see
5633 // nodes with their original operands intact. Legalization can produce
5634 // new nodes which may themselves need to be legalized. Iterate until all
5635 // nodes have been legalized.
5636 while (true) {
5637 bool AnyLegalized = false;
5638 for (auto NI = allnodes_end(); NI != allnodes_begin();) {
5639 --NI;
5640
5641 SDNode *N = &*NI;
5642 if (N->use_empty() && N != getRoot().getNode()) {
5643 ++NI;
5644 DeleteNode(N);
5645 continue;
5646 }
5647
5648 if (LegalizedNodes.insert(Ptr: N).second) {
5649 AnyLegalized = true;
5650 Legalizer.LegalizeOp(Node: N);
5651
5652 if (N->use_empty() && N != getRoot().getNode()) {
5653 ++NI;
5654 DeleteNode(N);
5655 }
5656 }
5657 }
5658 if (!AnyLegalized)
5659 break;
5660
5661 }
5662
5663 // Remove dead nodes now.
5664 RemoveDeadNodes();
5665}
5666
5667bool SelectionDAG::LegalizeOp(SDNode *N,
5668 SmallSetVector<SDNode *, 16> &UpdatedNodes) {
5669 SmallPtrSet<SDNode *, 16> LegalizedNodes;
5670 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
5671
5672 // Directly insert the node in question, and legalize it. This will recurse
5673 // as needed through operands.
5674 LegalizedNodes.insert(Ptr: N);
5675 Legalizer.LegalizeOp(Node: N);
5676
5677 return LegalizedNodes.count(Ptr: N);
5678}
5679

source code of llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp