1 | //===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file implements the SelectionDAG::Legalize method. |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | |
13 | #include "llvm/ADT/APFloat.h" |
14 | #include "llvm/ADT/APInt.h" |
15 | #include "llvm/ADT/ArrayRef.h" |
16 | #include "llvm/ADT/FloatingPointMode.h" |
17 | #include "llvm/ADT/SetVector.h" |
18 | #include "llvm/ADT/SmallPtrSet.h" |
19 | #include "llvm/ADT/SmallSet.h" |
20 | #include "llvm/ADT/SmallVector.h" |
21 | #include "llvm/Analysis/ConstantFolding.h" |
22 | #include "llvm/Analysis/TargetLibraryInfo.h" |
23 | #include "llvm/CodeGen/ISDOpcodes.h" |
24 | #include "llvm/CodeGen/MachineFrameInfo.h" |
25 | #include "llvm/CodeGen/MachineFunction.h" |
26 | #include "llvm/CodeGen/MachineJumpTableInfo.h" |
27 | #include "llvm/CodeGen/MachineMemOperand.h" |
28 | #include "llvm/CodeGen/RuntimeLibcalls.h" |
29 | #include "llvm/CodeGen/SelectionDAG.h" |
30 | #include "llvm/CodeGen/SelectionDAGNodes.h" |
31 | #include "llvm/CodeGen/TargetFrameLowering.h" |
32 | #include "llvm/CodeGen/TargetLowering.h" |
33 | #include "llvm/CodeGen/TargetSubtargetInfo.h" |
34 | #include "llvm/CodeGen/ValueTypes.h" |
35 | #include "llvm/CodeGenTypes/MachineValueType.h" |
36 | #include "llvm/IR/CallingConv.h" |
37 | #include "llvm/IR/Constants.h" |
38 | #include "llvm/IR/DataLayout.h" |
39 | #include "llvm/IR/DerivedTypes.h" |
40 | #include "llvm/IR/Function.h" |
41 | #include "llvm/IR/Metadata.h" |
42 | #include "llvm/IR/Type.h" |
43 | #include "llvm/Support/Casting.h" |
44 | #include "llvm/Support/Compiler.h" |
45 | #include "llvm/Support/Debug.h" |
46 | #include "llvm/Support/ErrorHandling.h" |
47 | #include "llvm/Support/MathExtras.h" |
48 | #include "llvm/Support/raw_ostream.h" |
49 | #include "llvm/Target/TargetMachine.h" |
50 | #include "llvm/Target/TargetOptions.h" |
51 | #include <cassert> |
52 | #include <cstdint> |
53 | #include <tuple> |
54 | #include <utility> |
55 | |
56 | using namespace llvm; |
57 | |
58 | #define DEBUG_TYPE "legalizedag" |
59 | |
60 | namespace { |
61 | |
62 | /// Keeps track of state when getting the sign of a floating-point value as an |
63 | /// integer. |
64 | struct FloatSignAsInt { |
65 | EVT FloatVT; |
66 | SDValue Chain; |
67 | SDValue FloatPtr; |
68 | SDValue IntPtr; |
69 | MachinePointerInfo IntPointerInfo; |
70 | MachinePointerInfo FloatPointerInfo; |
71 | SDValue IntValue; |
72 | APInt SignMask; |
73 | uint8_t SignBit; |
74 | }; |
75 | |
76 | //===----------------------------------------------------------------------===// |
77 | /// This takes an arbitrary SelectionDAG as input and |
78 | /// hacks on it until the target machine can handle it. This involves |
79 | /// eliminating value sizes the machine cannot handle (promoting small sizes to |
80 | /// large sizes or splitting up large values into small values) as well as |
81 | /// eliminating operations the machine cannot handle. |
82 | /// |
83 | /// This code also does a small amount of optimization and recognition of idioms |
84 | /// as part of its processing. For example, if a target does not support a |
85 | /// 'setcc' instruction efficiently, but does support 'brcc' instruction, this |
86 | /// will attempt merge setcc and brc instructions into brcc's. |
87 | class SelectionDAGLegalize { |
88 | const TargetMachine &TM; |
89 | const TargetLowering &TLI; |
90 | SelectionDAG &DAG; |
91 | |
92 | /// The set of nodes which have already been legalized. We hold a |
93 | /// reference to it in order to update as necessary on node deletion. |
94 | SmallPtrSetImpl<SDNode *> &LegalizedNodes; |
95 | |
96 | /// A set of all the nodes updated during legalization. |
97 | SmallSetVector<SDNode *, 16> *UpdatedNodes; |
98 | |
99 | EVT getSetCCResultType(EVT VT) const { |
100 | return TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT); |
101 | } |
102 | |
103 | // Libcall insertion helpers. |
104 | |
105 | public: |
106 | SelectionDAGLegalize(SelectionDAG &DAG, |
107 | SmallPtrSetImpl<SDNode *> &LegalizedNodes, |
108 | SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr) |
109 | : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG), |
110 | LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {} |
111 | |
112 | /// Legalizes the given operation. |
113 | void LegalizeOp(SDNode *Node); |
114 | |
115 | private: |
116 | SDValue OptimizeFloatStore(StoreSDNode *ST); |
117 | |
118 | void LegalizeLoadOps(SDNode *Node); |
119 | void LegalizeStoreOps(SDNode *Node); |
120 | |
121 | /// Some targets cannot handle a variable |
122 | /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it |
123 | /// is necessary to spill the vector being inserted into to memory, perform |
124 | /// the insert there, and then read the result back. |
125 | SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx, |
126 | const SDLoc &dl); |
127 | SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx, |
128 | const SDLoc &dl); |
129 | |
130 | /// Return a vector shuffle operation which |
131 | /// performs the same shuffe in terms of order or result bytes, but on a type |
132 | /// whose vector element type is narrower than the original shuffle type. |
133 | /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> |
134 | SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl, |
135 | SDValue N1, SDValue N2, |
136 | ArrayRef<int> Mask) const; |
137 | |
138 | std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, |
139 | TargetLowering::ArgListTy &&Args, bool isSigned); |
140 | std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned); |
141 | |
142 | void ExpandFrexpLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); |
143 | void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC, |
144 | SmallVectorImpl<SDValue> &Results); |
145 | void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32, |
146 | RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, |
147 | RTLIB::Libcall Call_F128, |
148 | RTLIB::Libcall Call_PPCF128, |
149 | SmallVectorImpl<SDValue> &Results); |
150 | SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, |
151 | RTLIB::Libcall Call_I8, |
152 | RTLIB::Libcall Call_I16, |
153 | RTLIB::Libcall Call_I32, |
154 | RTLIB::Libcall Call_I64, |
155 | RTLIB::Libcall Call_I128); |
156 | void ExpandArgFPLibCall(SDNode *Node, |
157 | RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, |
158 | RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128, |
159 | RTLIB::Libcall Call_PPCF128, |
160 | SmallVectorImpl<SDValue> &Results); |
161 | void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); |
162 | void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); |
163 | |
164 | SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, |
165 | const SDLoc &dl); |
166 | SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, |
167 | const SDLoc &dl, SDValue ChainIn); |
168 | SDValue ExpandBUILD_VECTOR(SDNode *Node); |
169 | SDValue ExpandSPLAT_VECTOR(SDNode *Node); |
170 | SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node); |
171 | void ExpandDYNAMIC_STACKALLOC(SDNode *Node, |
172 | SmallVectorImpl<SDValue> &Results); |
173 | void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL, |
174 | SDValue Value) const; |
175 | SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL, |
176 | SDValue NewIntValue) const; |
177 | SDValue ExpandFCOPYSIGN(SDNode *Node) const; |
178 | SDValue ExpandFABS(SDNode *Node) const; |
179 | SDValue ExpandFNEG(SDNode *Node) const; |
180 | SDValue expandLdexp(SDNode *Node) const; |
181 | SDValue expandFrexp(SDNode *Node) const; |
182 | |
183 | SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain); |
184 | void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl, |
185 | SmallVectorImpl<SDValue> &Results); |
186 | void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl, |
187 | SmallVectorImpl<SDValue> &Results); |
188 | SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl); |
189 | |
190 | SDValue ExpandPARITY(SDValue Op, const SDLoc &dl); |
191 | |
192 | SDValue ExpandExtractFromVectorThroughStack(SDValue Op); |
193 | SDValue ExpandInsertToVectorThroughStack(SDValue Op); |
194 | SDValue ExpandVectorBuildThroughStack(SDNode* Node); |
195 | |
196 | SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP); |
197 | SDValue ExpandConstant(ConstantSDNode *CP); |
198 | |
199 | // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall |
200 | bool ExpandNode(SDNode *Node); |
201 | void ConvertNodeToLibcall(SDNode *Node); |
202 | void PromoteNode(SDNode *Node); |
203 | |
204 | public: |
205 | // Node replacement helpers |
206 | |
207 | void ReplacedNode(SDNode *N) { |
208 | LegalizedNodes.erase(Ptr: N); |
209 | if (UpdatedNodes) |
210 | UpdatedNodes->insert(X: N); |
211 | } |
212 | |
213 | void ReplaceNode(SDNode *Old, SDNode *New) { |
214 | LLVM_DEBUG(dbgs() << " ... replacing: " ; Old->dump(&DAG); |
215 | dbgs() << " with: " ; New->dump(&DAG)); |
216 | |
217 | assert(Old->getNumValues() == New->getNumValues() && |
218 | "Replacing one node with another that produces a different number " |
219 | "of values!" ); |
220 | DAG.ReplaceAllUsesWith(From: Old, To: New); |
221 | if (UpdatedNodes) |
222 | UpdatedNodes->insert(X: New); |
223 | ReplacedNode(N: Old); |
224 | } |
225 | |
226 | void ReplaceNode(SDValue Old, SDValue New) { |
227 | LLVM_DEBUG(dbgs() << " ... replacing: " ; Old->dump(&DAG); |
228 | dbgs() << " with: " ; New->dump(&DAG)); |
229 | |
230 | DAG.ReplaceAllUsesWith(From: Old, To: New); |
231 | if (UpdatedNodes) |
232 | UpdatedNodes->insert(X: New.getNode()); |
233 | ReplacedNode(N: Old.getNode()); |
234 | } |
235 | |
236 | void ReplaceNode(SDNode *Old, const SDValue *New) { |
237 | LLVM_DEBUG(dbgs() << " ... replacing: " ; Old->dump(&DAG)); |
238 | |
239 | DAG.ReplaceAllUsesWith(From: Old, To: New); |
240 | for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) { |
241 | LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: " ); |
242 | New[i]->dump(&DAG)); |
243 | if (UpdatedNodes) |
244 | UpdatedNodes->insert(X: New[i].getNode()); |
245 | } |
246 | ReplacedNode(N: Old); |
247 | } |
248 | |
249 | void ReplaceNodeWithValue(SDValue Old, SDValue New) { |
250 | LLVM_DEBUG(dbgs() << " ... replacing: " ; Old->dump(&DAG); |
251 | dbgs() << " with: " ; New->dump(&DAG)); |
252 | |
253 | DAG.ReplaceAllUsesOfValueWith(From: Old, To: New); |
254 | if (UpdatedNodes) |
255 | UpdatedNodes->insert(X: New.getNode()); |
256 | ReplacedNode(N: Old.getNode()); |
257 | } |
258 | }; |
259 | |
260 | } // end anonymous namespace |
261 | |
262 | /// Return a vector shuffle operation which |
263 | /// performs the same shuffle in terms of order or result bytes, but on a type |
264 | /// whose vector element type is narrower than the original shuffle type. |
265 | /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> |
266 | SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType( |
267 | EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, |
268 | ArrayRef<int> Mask) const { |
269 | unsigned NumMaskElts = VT.getVectorNumElements(); |
270 | unsigned NumDestElts = NVT.getVectorNumElements(); |
271 | unsigned NumEltsGrowth = NumDestElts / NumMaskElts; |
272 | |
273 | assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!" ); |
274 | |
275 | if (NumEltsGrowth == 1) |
276 | return DAG.getVectorShuffle(VT: NVT, dl, N1, N2, Mask); |
277 | |
278 | SmallVector<int, 8> NewMask; |
279 | for (unsigned i = 0; i != NumMaskElts; ++i) { |
280 | int Idx = Mask[i]; |
281 | for (unsigned j = 0; j != NumEltsGrowth; ++j) { |
282 | if (Idx < 0) |
283 | NewMask.push_back(Elt: -1); |
284 | else |
285 | NewMask.push_back(Elt: Idx * NumEltsGrowth + j); |
286 | } |
287 | } |
288 | assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?" ); |
289 | assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?" ); |
290 | return DAG.getVectorShuffle(VT: NVT, dl, N1, N2, Mask: NewMask); |
291 | } |
292 | |
293 | /// Expands the ConstantFP node to an integer constant or |
294 | /// a load from the constant pool. |
295 | SDValue |
296 | SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) { |
297 | bool Extend = false; |
298 | SDLoc dl(CFP); |
299 | |
300 | // If a FP immediate is precise when represented as a float and if the |
301 | // target can do an extending load from float to double, we put it into |
302 | // the constant pool as a float, even if it's is statically typed as a |
303 | // double. This shrinks FP constants and canonicalizes them for targets where |
304 | // an FP extending load is the same cost as a normal load (such as on the x87 |
305 | // fp stack or PPC FP unit). |
306 | EVT VT = CFP->getValueType(ResNo: 0); |
307 | ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue()); |
308 | if (!UseCP) { |
309 | assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion" ); |
310 | return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl, |
311 | (VT == MVT::f64) ? MVT::i64 : MVT::i32); |
312 | } |
313 | |
314 | APFloat APF = CFP->getValueAPF(); |
315 | EVT OrigVT = VT; |
316 | EVT SVT = VT; |
317 | |
318 | // We don't want to shrink SNaNs. Converting the SNaN back to its real type |
319 | // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ). |
320 | if (!APF.isSignaling()) { |
321 | while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) { |
322 | SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1); |
323 | if (ConstantFPSDNode::isValueValidForType(VT: SVT, Val: APF) && |
324 | // Only do this if the target has a native EXTLOAD instruction from |
325 | // smaller type. |
326 | TLI.isLoadExtLegal(ExtType: ISD::EXTLOAD, ValVT: OrigVT, MemVT: SVT) && |
327 | TLI.ShouldShrinkFPConstant(OrigVT)) { |
328 | Type *SType = SVT.getTypeForEVT(Context&: *DAG.getContext()); |
329 | LLVMC = cast<ConstantFP>(Val: ConstantFoldCastOperand( |
330 | Opcode: Instruction::FPTrunc, C: LLVMC, DestTy: SType, DL: DAG.getDataLayout())); |
331 | VT = SVT; |
332 | Extend = true; |
333 | } |
334 | } |
335 | } |
336 | |
337 | SDValue CPIdx = |
338 | DAG.getConstantPool(C: LLVMC, VT: TLI.getPointerTy(DL: DAG.getDataLayout())); |
339 | Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign(); |
340 | if (Extend) { |
341 | SDValue Result = DAG.getExtLoad( |
342 | ExtType: ISD::EXTLOAD, dl, VT: OrigVT, Chain: DAG.getEntryNode(), Ptr: CPIdx, |
343 | PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), MemVT: VT, |
344 | Alignment); |
345 | return Result; |
346 | } |
347 | SDValue Result = DAG.getLoad( |
348 | VT: OrigVT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx, |
349 | PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), Alignment); |
350 | return Result; |
351 | } |
352 | |
353 | /// Expands the Constant node to a load from the constant pool. |
354 | SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) { |
355 | SDLoc dl(CP); |
356 | EVT VT = CP->getValueType(ResNo: 0); |
357 | SDValue CPIdx = DAG.getConstantPool(C: CP->getConstantIntValue(), |
358 | VT: TLI.getPointerTy(DL: DAG.getDataLayout())); |
359 | Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign(); |
360 | SDValue Result = DAG.getLoad( |
361 | VT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx, |
362 | PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), Alignment); |
363 | return Result; |
364 | } |
365 | |
366 | /// Some target cannot handle a variable insertion index for the |
367 | /// INSERT_VECTOR_ELT instruction. In this case, it |
368 | /// is necessary to spill the vector being inserted into to memory, perform |
369 | /// the insert there, and then read the result back. |
370 | SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec, |
371 | SDValue Val, |
372 | SDValue Idx, |
373 | const SDLoc &dl) { |
374 | SDValue Tmp1 = Vec; |
375 | SDValue Tmp2 = Val; |
376 | SDValue Tmp3 = Idx; |
377 | |
378 | // If the target doesn't support this, we have to spill the input vector |
379 | // to a temporary stack slot, update the element, then reload it. This is |
380 | // badness. We could also load the value into a vector register (either |
381 | // with a "move to register" or "extload into register" instruction, then |
382 | // permute it into place, if the idx is a constant and if the idx is |
383 | // supported by the target. |
384 | EVT VT = Tmp1.getValueType(); |
385 | EVT EltVT = VT.getVectorElementType(); |
386 | SDValue StackPtr = DAG.CreateStackTemporary(VT); |
387 | |
388 | int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex(); |
389 | |
390 | // Store the vector. |
391 | SDValue Ch = DAG.getStore( |
392 | Chain: DAG.getEntryNode(), dl, Val: Tmp1, Ptr: StackPtr, |
393 | PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI)); |
394 | |
395 | SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT: VT, Index: Tmp3); |
396 | |
397 | // Store the scalar value. |
398 | Ch = DAG.getTruncStore( |
399 | Chain: Ch, dl, Val: Tmp2, Ptr: StackPtr2, |
400 | PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()), SVT: EltVT); |
401 | // Load the updated vector. |
402 | return DAG.getLoad(VT, dl, Chain: Ch, Ptr: StackPtr, PtrInfo: MachinePointerInfo::getFixedStack( |
403 | MF&: DAG.getMachineFunction(), FI: SPFI)); |
404 | } |
405 | |
406 | SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, |
407 | SDValue Idx, |
408 | const SDLoc &dl) { |
409 | if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Val&: Idx)) { |
410 | // SCALAR_TO_VECTOR requires that the type of the value being inserted |
411 | // match the element type of the vector being created, except for |
412 | // integers in which case the inserted value can be over width. |
413 | EVT EltVT = Vec.getValueType().getVectorElementType(); |
414 | if (Val.getValueType() == EltVT || |
415 | (EltVT.isInteger() && Val.getValueType().bitsGE(VT: EltVT))) { |
416 | SDValue ScVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, |
417 | VT: Vec.getValueType(), Operand: Val); |
418 | |
419 | unsigned NumElts = Vec.getValueType().getVectorNumElements(); |
420 | // We generate a shuffle of InVec and ScVec, so the shuffle mask |
421 | // should be 0,1,2,3,4,5... with the appropriate element replaced with |
422 | // elt 0 of the RHS. |
423 | SmallVector<int, 8> ShufOps; |
424 | for (unsigned i = 0; i != NumElts; ++i) |
425 | ShufOps.push_back(Elt: i != InsertPos->getZExtValue() ? i : NumElts); |
426 | |
427 | return DAG.getVectorShuffle(VT: Vec.getValueType(), dl, N1: Vec, N2: ScVec, Mask: ShufOps); |
428 | } |
429 | } |
430 | return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl); |
431 | } |
432 | |
433 | SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) { |
434 | if (!ISD::isNormalStore(N: ST)) |
435 | return SDValue(); |
436 | |
437 | LLVM_DEBUG(dbgs() << "Optimizing float store operations\n" ); |
438 | // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr' |
439 | // FIXME: move this to the DAG Combiner! Note that we can't regress due |
440 | // to phase ordering between legalized code and the dag combiner. This |
441 | // probably means that we need to integrate dag combiner and legalizer |
442 | // together. |
443 | // We generally can't do this one for long doubles. |
444 | SDValue Chain = ST->getChain(); |
445 | SDValue Ptr = ST->getBasePtr(); |
446 | SDValue Value = ST->getValue(); |
447 | MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags(); |
448 | AAMDNodes AAInfo = ST->getAAInfo(); |
449 | SDLoc dl(ST); |
450 | |
451 | // Don't optimise TargetConstantFP |
452 | if (Value.getOpcode() == ISD::TargetConstantFP) |
453 | return SDValue(); |
454 | |
455 | if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val&: Value)) { |
456 | if (CFP->getValueType(ResNo: 0) == MVT::f32 && |
457 | TLI.isTypeLegal(MVT::VT: i32)) { |
458 | SDValue Con = DAG.getConstant(CFP->getValueAPF(). |
459 | bitcastToAPInt().zextOrTrunc(width: 32), |
460 | SDLoc(CFP), MVT::i32); |
461 | return DAG.getStore(Chain, dl, Val: Con, Ptr, PtrInfo: ST->getPointerInfo(), |
462 | Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo); |
463 | } |
464 | |
465 | if (CFP->getValueType(ResNo: 0) == MVT::f64 && |
466 | !TLI.isFPImmLegal(CFP->getValueAPF(), MVT::f64)) { |
467 | // If this target supports 64-bit registers, do a single 64-bit store. |
468 | if (TLI.isTypeLegal(MVT::VT: i64)) { |
469 | SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt(). |
470 | zextOrTrunc(width: 64), SDLoc(CFP), MVT::i64); |
471 | return DAG.getStore(Chain, dl, Val: Con, Ptr, PtrInfo: ST->getPointerInfo(), |
472 | Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo); |
473 | } |
474 | |
475 | if (TLI.isTypeLegal(MVT::VT: i32) && !ST->isVolatile()) { |
476 | // Otherwise, if the target supports 32-bit registers, use 2 32-bit |
477 | // stores. If the target supports neither 32- nor 64-bits, this |
478 | // xform is certainly not worth it. |
479 | const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt(); |
480 | SDValue Lo = DAG.getConstant(IntVal.trunc(width: 32), dl, MVT::i32); |
481 | SDValue Hi = DAG.getConstant(IntVal.lshr(shiftAmt: 32).trunc(width: 32), dl, MVT::i32); |
482 | if (DAG.getDataLayout().isBigEndian()) |
483 | std::swap(a&: Lo, b&: Hi); |
484 | |
485 | Lo = DAG.getStore(Chain, dl, Val: Lo, Ptr, PtrInfo: ST->getPointerInfo(), |
486 | Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo); |
487 | Ptr = DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: 4), DL: dl); |
488 | Hi = DAG.getStore(Chain, dl, Val: Hi, Ptr, |
489 | PtrInfo: ST->getPointerInfo().getWithOffset(O: 4), |
490 | Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo); |
491 | |
492 | return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); |
493 | } |
494 | } |
495 | } |
496 | return SDValue(); |
497 | } |
498 | |
499 | void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) { |
500 | StoreSDNode *ST = cast<StoreSDNode>(Val: Node); |
501 | SDValue Chain = ST->getChain(); |
502 | SDValue Ptr = ST->getBasePtr(); |
503 | SDLoc dl(Node); |
504 | |
505 | MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags(); |
506 | AAMDNodes AAInfo = ST->getAAInfo(); |
507 | |
508 | if (!ST->isTruncatingStore()) { |
509 | LLVM_DEBUG(dbgs() << "Legalizing store operation\n" ); |
510 | if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) { |
511 | ReplaceNode(Old: ST, New: OptStore); |
512 | return; |
513 | } |
514 | |
515 | SDValue Value = ST->getValue(); |
516 | MVT VT = Value.getSimpleValueType(); |
517 | switch (TLI.getOperationAction(Op: ISD::STORE, VT)) { |
518 | default: llvm_unreachable("This action is not supported yet!" ); |
519 | case TargetLowering::Legal: { |
520 | // If this is an unaligned store and the target doesn't support it, |
521 | // expand it. |
522 | EVT MemVT = ST->getMemoryVT(); |
523 | const DataLayout &DL = DAG.getDataLayout(); |
524 | if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT, |
525 | MMO: *ST->getMemOperand())) { |
526 | LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n" ); |
527 | SDValue Result = TLI.expandUnalignedStore(ST, DAG); |
528 | ReplaceNode(Old: SDValue(ST, 0), New: Result); |
529 | } else |
530 | LLVM_DEBUG(dbgs() << "Legal store\n" ); |
531 | break; |
532 | } |
533 | case TargetLowering::Custom: { |
534 | LLVM_DEBUG(dbgs() << "Trying custom lowering\n" ); |
535 | SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG); |
536 | if (Res && Res != SDValue(Node, 0)) |
537 | ReplaceNode(Old: SDValue(Node, 0), New: Res); |
538 | return; |
539 | } |
540 | case TargetLowering::Promote: { |
541 | MVT NVT = TLI.getTypeToPromoteTo(Op: ISD::STORE, VT); |
542 | assert(NVT.getSizeInBits() == VT.getSizeInBits() && |
543 | "Can only promote stores to same size type" ); |
544 | Value = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Value); |
545 | SDValue Result = DAG.getStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(), |
546 | Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo); |
547 | ReplaceNode(Old: SDValue(Node, 0), New: Result); |
548 | break; |
549 | } |
550 | } |
551 | return; |
552 | } |
553 | |
554 | LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n" ); |
555 | SDValue Value = ST->getValue(); |
556 | EVT StVT = ST->getMemoryVT(); |
557 | TypeSize StWidth = StVT.getSizeInBits(); |
558 | TypeSize StSize = StVT.getStoreSizeInBits(); |
559 | auto &DL = DAG.getDataLayout(); |
560 | |
561 | if (StWidth != StSize) { |
562 | // Promote to a byte-sized store with upper bits zero if not |
563 | // storing an integral number of bytes. For example, promote |
564 | // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1) |
565 | EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StSize.getFixedValue()); |
566 | Value = DAG.getZeroExtendInReg(Op: Value, DL: dl, VT: StVT); |
567 | SDValue Result = |
568 | DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(), SVT: NVT, |
569 | Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo); |
570 | ReplaceNode(Old: SDValue(Node, 0), New: Result); |
571 | } else if (!StVT.isVector() && !isPowerOf2_64(Value: StWidth.getFixedValue())) { |
572 | // If not storing a power-of-2 number of bits, expand as two stores. |
573 | assert(!StVT.isVector() && "Unsupported truncstore!" ); |
574 | unsigned StWidthBits = StWidth.getFixedValue(); |
575 | unsigned LogStWidth = Log2_32(Value: StWidthBits); |
576 | assert(LogStWidth < 32); |
577 | unsigned RoundWidth = 1 << LogStWidth; |
578 | assert(RoundWidth < StWidthBits); |
579 | unsigned = StWidthBits - RoundWidth; |
580 | assert(ExtraWidth < RoundWidth); |
581 | assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && |
582 | "Store size not an integral number of bytes!" ); |
583 | EVT RoundVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundWidth); |
584 | EVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExtraWidth); |
585 | SDValue Lo, Hi; |
586 | unsigned IncrementSize; |
587 | |
588 | if (DL.isLittleEndian()) { |
589 | // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16) |
590 | // Store the bottom RoundWidth bits. |
591 | Lo = DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(), |
592 | SVT: RoundVT, Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo); |
593 | |
594 | // Store the remaining ExtraWidth bits. |
595 | IncrementSize = RoundWidth / 8; |
596 | Ptr = |
597 | DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl); |
598 | Hi = DAG.getNode( |
599 | Opcode: ISD::SRL, DL: dl, VT: Value.getValueType(), N1: Value, |
600 | N2: DAG.getConstant(Val: RoundWidth, DL: dl, |
601 | VT: TLI.getShiftAmountTy(LHSTy: Value.getValueType(), DL))); |
602 | Hi = DAG.getTruncStore(Chain, dl, Val: Hi, Ptr, |
603 | PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize), |
604 | SVT: ExtraVT, Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo); |
605 | } else { |
606 | // Big endian - avoid unaligned stores. |
607 | // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X |
608 | // Store the top RoundWidth bits. |
609 | Hi = DAG.getNode( |
610 | Opcode: ISD::SRL, DL: dl, VT: Value.getValueType(), N1: Value, |
611 | N2: DAG.getConstant(Val: ExtraWidth, DL: dl, |
612 | VT: TLI.getShiftAmountTy(LHSTy: Value.getValueType(), DL))); |
613 | Hi = DAG.getTruncStore(Chain, dl, Val: Hi, Ptr, PtrInfo: ST->getPointerInfo(), SVT: RoundVT, |
614 | Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo); |
615 | |
616 | // Store the remaining ExtraWidth bits. |
617 | IncrementSize = RoundWidth / 8; |
618 | Ptr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Ptr.getValueType(), N1: Ptr, |
619 | N2: DAG.getConstant(Val: IncrementSize, DL: dl, |
620 | VT: Ptr.getValueType())); |
621 | Lo = DAG.getTruncStore(Chain, dl, Val: Value, Ptr, |
622 | PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize), |
623 | SVT: ExtraVT, Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo); |
624 | } |
625 | |
626 | // The order of the stores doesn't matter. |
627 | SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); |
628 | ReplaceNode(Old: SDValue(Node, 0), New: Result); |
629 | } else { |
630 | switch (TLI.getTruncStoreAction(ValVT: ST->getValue().getValueType(), MemVT: StVT)) { |
631 | default: llvm_unreachable("This action is not supported yet!" ); |
632 | case TargetLowering::Legal: { |
633 | EVT MemVT = ST->getMemoryVT(); |
634 | // If this is an unaligned store and the target doesn't support it, |
635 | // expand it. |
636 | if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT, |
637 | MMO: *ST->getMemOperand())) { |
638 | SDValue Result = TLI.expandUnalignedStore(ST, DAG); |
639 | ReplaceNode(Old: SDValue(ST, 0), New: Result); |
640 | } |
641 | break; |
642 | } |
643 | case TargetLowering::Custom: { |
644 | SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG); |
645 | if (Res && Res != SDValue(Node, 0)) |
646 | ReplaceNode(Old: SDValue(Node, 0), New: Res); |
647 | return; |
648 | } |
649 | case TargetLowering::Expand: |
650 | assert(!StVT.isVector() && |
651 | "Vector Stores are handled in LegalizeVectorOps" ); |
652 | |
653 | SDValue Result; |
654 | |
655 | // TRUNCSTORE:i16 i32 -> STORE i16 |
656 | if (TLI.isTypeLegal(VT: StVT)) { |
657 | Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: StVT, Operand: Value); |
658 | Result = DAG.getStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(), |
659 | Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo); |
660 | } else { |
661 | // The in-memory type isn't legal. Truncate to the type it would promote |
662 | // to, and then do a truncstore. |
663 | Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, |
664 | VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StVT), |
665 | Operand: Value); |
666 | Result = |
667 | DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(), SVT: StVT, |
668 | Alignment: ST->getOriginalAlign(), MMOFlags, AAInfo); |
669 | } |
670 | |
671 | ReplaceNode(Old: SDValue(Node, 0), New: Result); |
672 | break; |
673 | } |
674 | } |
675 | } |
676 | |
677 | void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) { |
678 | LoadSDNode *LD = cast<LoadSDNode>(Val: Node); |
679 | SDValue Chain = LD->getChain(); // The chain. |
680 | SDValue Ptr = LD->getBasePtr(); // The base pointer. |
681 | SDValue Value; // The value returned by the load op. |
682 | SDLoc dl(Node); |
683 | |
684 | ISD::LoadExtType ExtType = LD->getExtensionType(); |
685 | if (ExtType == ISD::NON_EXTLOAD) { |
686 | LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n" ); |
687 | MVT VT = Node->getSimpleValueType(ResNo: 0); |
688 | SDValue RVal = SDValue(Node, 0); |
689 | SDValue RChain = SDValue(Node, 1); |
690 | |
691 | switch (TLI.getOperationAction(Op: Node->getOpcode(), VT)) { |
692 | default: llvm_unreachable("This action is not supported yet!" ); |
693 | case TargetLowering::Legal: { |
694 | EVT MemVT = LD->getMemoryVT(); |
695 | const DataLayout &DL = DAG.getDataLayout(); |
696 | // If this is an unaligned load and the target doesn't support it, |
697 | // expand it. |
698 | if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT, |
699 | MMO: *LD->getMemOperand())) { |
700 | std::tie(args&: RVal, args&: RChain) = TLI.expandUnalignedLoad(LD, DAG); |
701 | } |
702 | break; |
703 | } |
704 | case TargetLowering::Custom: |
705 | if (SDValue Res = TLI.LowerOperation(Op: RVal, DAG)) { |
706 | RVal = Res; |
707 | RChain = Res.getValue(R: 1); |
708 | } |
709 | break; |
710 | |
711 | case TargetLowering::Promote: { |
712 | MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT); |
713 | assert(NVT.getSizeInBits() == VT.getSizeInBits() && |
714 | "Can only promote loads to same size type" ); |
715 | |
716 | SDValue Res = DAG.getLoad(VT: NVT, dl, Chain, Ptr, MMO: LD->getMemOperand()); |
717 | RVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Res); |
718 | RChain = Res.getValue(R: 1); |
719 | break; |
720 | } |
721 | } |
722 | if (RChain.getNode() != Node) { |
723 | assert(RVal.getNode() != Node && "Load must be completely replaced" ); |
724 | DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: RVal); |
725 | DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: RChain); |
726 | if (UpdatedNodes) { |
727 | UpdatedNodes->insert(X: RVal.getNode()); |
728 | UpdatedNodes->insert(X: RChain.getNode()); |
729 | } |
730 | ReplacedNode(N: Node); |
731 | } |
732 | return; |
733 | } |
734 | |
735 | LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n" ); |
736 | EVT SrcVT = LD->getMemoryVT(); |
737 | TypeSize SrcWidth = SrcVT.getSizeInBits(); |
738 | MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); |
739 | AAMDNodes AAInfo = LD->getAAInfo(); |
740 | |
741 | if (SrcWidth != SrcVT.getStoreSizeInBits() && |
742 | // Some targets pretend to have an i1 loading operation, and actually |
743 | // load an i8. This trick is correct for ZEXTLOAD because the top 7 |
744 | // bits are guaranteed to be zero; it helps the optimizers understand |
745 | // that these bits are zero. It is also useful for EXTLOAD, since it |
746 | // tells the optimizers that those bits are undefined. It would be |
747 | // nice to have an effective generic way of getting these benefits... |
748 | // Until such a way is found, don't insist on promoting i1 here. |
749 | (SrcVT != MVT::i1 || |
750 | TLI.getLoadExtAction(ExtType, ValVT: Node->getValueType(ResNo: 0), MVT::MemVT: i1) == |
751 | TargetLowering::Promote)) { |
752 | // Promote to a byte-sized load if not loading an integral number of |
753 | // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24. |
754 | unsigned NewWidth = SrcVT.getStoreSizeInBits(); |
755 | EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NewWidth); |
756 | SDValue Ch; |
757 | |
758 | // The extra bits are guaranteed to be zero, since we stored them that |
759 | // way. A zext load from NVT thus automatically gives zext from SrcVT. |
760 | |
761 | ISD::LoadExtType NewExtType = |
762 | ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD; |
763 | |
764 | SDValue Result = DAG.getExtLoad(ExtType: NewExtType, dl, VT: Node->getValueType(ResNo: 0), |
765 | Chain, Ptr, PtrInfo: LD->getPointerInfo(), MemVT: NVT, |
766 | Alignment: LD->getOriginalAlign(), MMOFlags, AAInfo); |
767 | |
768 | Ch = Result.getValue(R: 1); // The chain. |
769 | |
770 | if (ExtType == ISD::SEXTLOAD) |
771 | // Having the top bits zero doesn't help when sign extending. |
772 | Result = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, |
773 | VT: Result.getValueType(), |
774 | N1: Result, N2: DAG.getValueType(SrcVT)); |
775 | else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType()) |
776 | // All the top bits are guaranteed to be zero - inform the optimizers. |
777 | Result = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, |
778 | VT: Result.getValueType(), N1: Result, |
779 | N2: DAG.getValueType(SrcVT)); |
780 | |
781 | Value = Result; |
782 | Chain = Ch; |
783 | } else if (!isPowerOf2_64(Value: SrcWidth.getKnownMinValue())) { |
784 | // If not loading a power-of-2 number of bits, expand as two loads. |
785 | assert(!SrcVT.isVector() && "Unsupported extload!" ); |
786 | unsigned SrcWidthBits = SrcWidth.getFixedValue(); |
787 | unsigned LogSrcWidth = Log2_32(Value: SrcWidthBits); |
788 | assert(LogSrcWidth < 32); |
789 | unsigned RoundWidth = 1 << LogSrcWidth; |
790 | assert(RoundWidth < SrcWidthBits); |
791 | unsigned = SrcWidthBits - RoundWidth; |
792 | assert(ExtraWidth < RoundWidth); |
793 | assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && |
794 | "Load size not an integral number of bytes!" ); |
795 | EVT RoundVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundWidth); |
796 | EVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExtraWidth); |
797 | SDValue Lo, Hi, Ch; |
798 | unsigned IncrementSize; |
799 | auto &DL = DAG.getDataLayout(); |
800 | |
801 | if (DL.isLittleEndian()) { |
802 | // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16) |
803 | // Load the bottom RoundWidth bits. |
804 | Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr, |
805 | PtrInfo: LD->getPointerInfo(), MemVT: RoundVT, Alignment: LD->getOriginalAlign(), |
806 | MMOFlags, AAInfo); |
807 | |
808 | // Load the remaining ExtraWidth bits. |
809 | IncrementSize = RoundWidth / 8; |
810 | Ptr = |
811 | DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl); |
812 | Hi = DAG.getExtLoad(ExtType, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr, |
813 | PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize), |
814 | MemVT: ExtraVT, Alignment: LD->getOriginalAlign(), MMOFlags, AAInfo); |
815 | |
816 | // Build a factor node to remember that this load is independent of |
817 | // the other one. |
818 | Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(R: 1), |
819 | Hi.getValue(R: 1)); |
820 | |
821 | // Move the top bits to the right place. |
822 | Hi = DAG.getNode( |
823 | Opcode: ISD::SHL, DL: dl, VT: Hi.getValueType(), N1: Hi, |
824 | N2: DAG.getConstant(Val: RoundWidth, DL: dl, |
825 | VT: TLI.getShiftAmountTy(LHSTy: Hi.getValueType(), DL))); |
826 | |
827 | // Join the hi and lo parts. |
828 | Value = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Lo, N2: Hi); |
829 | } else { |
830 | // Big endian - avoid unaligned loads. |
831 | // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8 |
832 | // Load the top RoundWidth bits. |
833 | Hi = DAG.getExtLoad(ExtType, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr, |
834 | PtrInfo: LD->getPointerInfo(), MemVT: RoundVT, Alignment: LD->getOriginalAlign(), |
835 | MMOFlags, AAInfo); |
836 | |
837 | // Load the remaining ExtraWidth bits. |
838 | IncrementSize = RoundWidth / 8; |
839 | Ptr = |
840 | DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl); |
841 | Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr, |
842 | PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize), |
843 | MemVT: ExtraVT, Alignment: LD->getOriginalAlign(), MMOFlags, AAInfo); |
844 | |
845 | // Build a factor node to remember that this load is independent of |
846 | // the other one. |
847 | Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(R: 1), |
848 | Hi.getValue(R: 1)); |
849 | |
850 | // Move the top bits to the right place. |
851 | Hi = DAG.getNode( |
852 | Opcode: ISD::SHL, DL: dl, VT: Hi.getValueType(), N1: Hi, |
853 | N2: DAG.getConstant(Val: ExtraWidth, DL: dl, |
854 | VT: TLI.getShiftAmountTy(LHSTy: Hi.getValueType(), DL))); |
855 | |
856 | // Join the hi and lo parts. |
857 | Value = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Lo, N2: Hi); |
858 | } |
859 | |
860 | Chain = Ch; |
861 | } else { |
862 | bool isCustom = false; |
863 | switch (TLI.getLoadExtAction(ExtType, ValVT: Node->getValueType(ResNo: 0), |
864 | MemVT: SrcVT.getSimpleVT())) { |
865 | default: llvm_unreachable("This action is not supported yet!" ); |
866 | case TargetLowering::Custom: |
867 | isCustom = true; |
868 | [[fallthrough]]; |
869 | case TargetLowering::Legal: |
870 | Value = SDValue(Node, 0); |
871 | Chain = SDValue(Node, 1); |
872 | |
873 | if (isCustom) { |
874 | if (SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG)) { |
875 | Value = Res; |
876 | Chain = Res.getValue(R: 1); |
877 | } |
878 | } else { |
879 | // If this is an unaligned load and the target doesn't support it, |
880 | // expand it. |
881 | EVT MemVT = LD->getMemoryVT(); |
882 | const DataLayout &DL = DAG.getDataLayout(); |
883 | if (!TLI.allowsMemoryAccess(Context&: *DAG.getContext(), DL, VT: MemVT, |
884 | MMO: *LD->getMemOperand())) { |
885 | std::tie(args&: Value, args&: Chain) = TLI.expandUnalignedLoad(LD, DAG); |
886 | } |
887 | } |
888 | break; |
889 | |
890 | case TargetLowering::Expand: { |
891 | EVT DestVT = Node->getValueType(ResNo: 0); |
892 | if (!TLI.isLoadExtLegal(ExtType: ISD::EXTLOAD, ValVT: DestVT, MemVT: SrcVT)) { |
893 | // If the source type is not legal, see if there is a legal extload to |
894 | // an intermediate type that we can then extend further. |
895 | EVT LoadVT = TLI.getRegisterType(VT: SrcVT.getSimpleVT()); |
896 | if ((LoadVT.isFloatingPoint() == SrcVT.isFloatingPoint()) && |
897 | (TLI.isTypeLegal(VT: SrcVT) || // Same as SrcVT == LoadVT? |
898 | TLI.isLoadExtLegal(ExtType, ValVT: LoadVT, MemVT: SrcVT))) { |
899 | // If we are loading a legal type, this is a non-extload followed by a |
900 | // full extend. |
901 | ISD::LoadExtType MidExtType = |
902 | (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType; |
903 | |
904 | SDValue Load = DAG.getExtLoad(ExtType: MidExtType, dl, VT: LoadVT, Chain, Ptr, |
905 | MemVT: SrcVT, MMO: LD->getMemOperand()); |
906 | unsigned ExtendOp = |
907 | ISD::getExtForLoadExtType(IsFP: SrcVT.isFloatingPoint(), ExtType); |
908 | Value = DAG.getNode(Opcode: ExtendOp, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Load); |
909 | Chain = Load.getValue(R: 1); |
910 | break; |
911 | } |
912 | |
913 | // Handle the special case of fp16 extloads. EXTLOAD doesn't have the |
914 | // normal undefined upper bits behavior to allow using an in-reg extend |
915 | // with the illegal FP type, so load as an integer and do the |
916 | // from-integer conversion. |
917 | EVT SVT = SrcVT.getScalarType(); |
918 | if (SVT == MVT::f16 || SVT == MVT::bf16) { |
919 | EVT ISrcVT = SrcVT.changeTypeToInteger(); |
920 | EVT IDestVT = DestVT.changeTypeToInteger(); |
921 | EVT ILoadVT = TLI.getRegisterType(VT: IDestVT.getSimpleVT()); |
922 | |
923 | SDValue Result = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: ILoadVT, Chain, |
924 | Ptr, MemVT: ISrcVT, MMO: LD->getMemOperand()); |
925 | Value = |
926 | DAG.getNode(SVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP, |
927 | dl, DestVT, Result); |
928 | Chain = Result.getValue(R: 1); |
929 | break; |
930 | } |
931 | } |
932 | |
933 | assert(!SrcVT.isVector() && |
934 | "Vector Loads are handled in LegalizeVectorOps" ); |
935 | |
936 | // FIXME: This does not work for vectors on most targets. Sign- |
937 | // and zero-extend operations are currently folded into extending |
938 | // loads, whether they are legal or not, and then we end up here |
939 | // without any support for legalizing them. |
940 | assert(ExtType != ISD::EXTLOAD && |
941 | "EXTLOAD should always be supported!" ); |
942 | // Turn the unsupported load into an EXTLOAD followed by an |
943 | // explicit zero/sign extend inreg. |
944 | SDValue Result = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, |
945 | VT: Node->getValueType(ResNo: 0), |
946 | Chain, Ptr, MemVT: SrcVT, |
947 | MMO: LD->getMemOperand()); |
948 | SDValue ValRes; |
949 | if (ExtType == ISD::SEXTLOAD) |
950 | ValRes = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, |
951 | VT: Result.getValueType(), |
952 | N1: Result, N2: DAG.getValueType(SrcVT)); |
953 | else |
954 | ValRes = DAG.getZeroExtendInReg(Op: Result, DL: dl, VT: SrcVT); |
955 | Value = ValRes; |
956 | Chain = Result.getValue(R: 1); |
957 | break; |
958 | } |
959 | } |
960 | } |
961 | |
962 | // Since loads produce two values, make sure to remember that we legalized |
963 | // both of them. |
964 | if (Chain.getNode() != Node) { |
965 | assert(Value.getNode() != Node && "Load must be completely replaced" ); |
966 | DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: Value); |
967 | DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: Chain); |
968 | if (UpdatedNodes) { |
969 | UpdatedNodes->insert(X: Value.getNode()); |
970 | UpdatedNodes->insert(X: Chain.getNode()); |
971 | } |
972 | ReplacedNode(N: Node); |
973 | } |
974 | } |
975 | |
976 | /// Return a legal replacement for the given operation, with all legal operands. |
977 | void SelectionDAGLegalize::LegalizeOp(SDNode *Node) { |
978 | LLVM_DEBUG(dbgs() << "\nLegalizing: " ; Node->dump(&DAG)); |
979 | |
980 | // Allow illegal target nodes and illegal registers. |
981 | if (Node->getOpcode() == ISD::TargetConstant || |
982 | Node->getOpcode() == ISD::Register) |
983 | return; |
984 | |
985 | #ifndef NDEBUG |
986 | for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) |
987 | assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == |
988 | TargetLowering::TypeLegal && |
989 | "Unexpected illegal type!" ); |
990 | |
991 | for (const SDValue &Op : Node->op_values()) |
992 | assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == |
993 | TargetLowering::TypeLegal || |
994 | Op.getOpcode() == ISD::TargetConstant || |
995 | Op.getOpcode() == ISD::Register) && |
996 | "Unexpected illegal type!" ); |
997 | #endif |
998 | |
999 | // Figure out the correct action; the way to query this varies by opcode |
1000 | TargetLowering::LegalizeAction Action = TargetLowering::Legal; |
1001 | bool SimpleFinishLegalizing = true; |
1002 | switch (Node->getOpcode()) { |
1003 | case ISD::INTRINSIC_W_CHAIN: |
1004 | case ISD::INTRINSIC_WO_CHAIN: |
1005 | case ISD::INTRINSIC_VOID: |
1006 | case ISD::STACKSAVE: |
1007 | Action = TLI.getOperationAction(Op: Node->getOpcode(), MVT::VT: Other); |
1008 | break; |
1009 | case ISD::GET_DYNAMIC_AREA_OFFSET: |
1010 | Action = TLI.getOperationAction(Op: Node->getOpcode(), |
1011 | VT: Node->getValueType(ResNo: 0)); |
1012 | break; |
1013 | case ISD::VAARG: |
1014 | Action = TLI.getOperationAction(Op: Node->getOpcode(), |
1015 | VT: Node->getValueType(ResNo: 0)); |
1016 | if (Action != TargetLowering::Promote) |
1017 | Action = TLI.getOperationAction(Op: Node->getOpcode(), MVT::VT: Other); |
1018 | break; |
1019 | case ISD::SET_FPENV: |
1020 | case ISD::SET_FPMODE: |
1021 | Action = TLI.getOperationAction(Op: Node->getOpcode(), |
1022 | VT: Node->getOperand(Num: 1).getValueType()); |
1023 | break; |
1024 | case ISD::FP_TO_FP16: |
1025 | case ISD::FP_TO_BF16: |
1026 | case ISD::SINT_TO_FP: |
1027 | case ISD::UINT_TO_FP: |
1028 | case ISD::EXTRACT_VECTOR_ELT: |
1029 | case ISD::LROUND: |
1030 | case ISD::LLROUND: |
1031 | case ISD::LRINT: |
1032 | case ISD::LLRINT: |
1033 | Action = TLI.getOperationAction(Op: Node->getOpcode(), |
1034 | VT: Node->getOperand(Num: 0).getValueType()); |
1035 | break; |
1036 | case ISD::STRICT_FP_TO_FP16: |
1037 | case ISD::STRICT_SINT_TO_FP: |
1038 | case ISD::STRICT_UINT_TO_FP: |
1039 | case ISD::STRICT_LRINT: |
1040 | case ISD::STRICT_LLRINT: |
1041 | case ISD::STRICT_LROUND: |
1042 | case ISD::STRICT_LLROUND: |
1043 | // These pseudo-ops are the same as the other STRICT_ ops except |
1044 | // they are registered with setOperationAction() using the input type |
1045 | // instead of the output type. |
1046 | Action = TLI.getOperationAction(Op: Node->getOpcode(), |
1047 | VT: Node->getOperand(Num: 1).getValueType()); |
1048 | break; |
1049 | case ISD::SIGN_EXTEND_INREG: { |
1050 | EVT InnerType = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT(); |
1051 | Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: InnerType); |
1052 | break; |
1053 | } |
1054 | case ISD::ATOMIC_STORE: |
1055 | Action = TLI.getOperationAction(Op: Node->getOpcode(), |
1056 | VT: Node->getOperand(Num: 1).getValueType()); |
1057 | break; |
1058 | case ISD::SELECT_CC: |
1059 | case ISD::STRICT_FSETCC: |
1060 | case ISD::STRICT_FSETCCS: |
1061 | case ISD::SETCC: |
1062 | case ISD::SETCCCARRY: |
1063 | case ISD::VP_SETCC: |
1064 | case ISD::BR_CC: { |
1065 | unsigned Opc = Node->getOpcode(); |
1066 | unsigned CCOperand = Opc == ISD::SELECT_CC ? 4 |
1067 | : Opc == ISD::STRICT_FSETCC ? 3 |
1068 | : Opc == ISD::STRICT_FSETCCS ? 3 |
1069 | : Opc == ISD::SETCCCARRY ? 3 |
1070 | : (Opc == ISD::SETCC || Opc == ISD::VP_SETCC) ? 2 |
1071 | : 1; |
1072 | unsigned CompareOperand = Opc == ISD::BR_CC ? 2 |
1073 | : Opc == ISD::STRICT_FSETCC ? 1 |
1074 | : Opc == ISD::STRICT_FSETCCS ? 1 |
1075 | : 0; |
1076 | MVT OpVT = Node->getOperand(Num: CompareOperand).getSimpleValueType(); |
1077 | ISD::CondCode CCCode = |
1078 | cast<CondCodeSDNode>(Val: Node->getOperand(Num: CCOperand))->get(); |
1079 | Action = TLI.getCondCodeAction(CC: CCCode, VT: OpVT); |
1080 | if (Action == TargetLowering::Legal) { |
1081 | if (Node->getOpcode() == ISD::SELECT_CC) |
1082 | Action = TLI.getOperationAction(Op: Node->getOpcode(), |
1083 | VT: Node->getValueType(ResNo: 0)); |
1084 | else |
1085 | Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: OpVT); |
1086 | } |
1087 | break; |
1088 | } |
1089 | case ISD::LOAD: |
1090 | case ISD::STORE: |
1091 | // FIXME: Model these properly. LOAD and STORE are complicated, and |
1092 | // STORE expects the unlegalized operand in some cases. |
1093 | SimpleFinishLegalizing = false; |
1094 | break; |
1095 | case ISD::CALLSEQ_START: |
1096 | case ISD::CALLSEQ_END: |
1097 | // FIXME: This shouldn't be necessary. These nodes have special properties |
1098 | // dealing with the recursive nature of legalization. Removing this |
1099 | // special case should be done as part of making LegalizeDAG non-recursive. |
1100 | SimpleFinishLegalizing = false; |
1101 | break; |
1102 | case ISD::EXTRACT_ELEMENT: |
1103 | case ISD::GET_ROUNDING: |
1104 | case ISD::MERGE_VALUES: |
1105 | case ISD::EH_RETURN: |
1106 | case ISD::FRAME_TO_ARGS_OFFSET: |
1107 | case ISD::EH_DWARF_CFA: |
1108 | case ISD::EH_SJLJ_SETJMP: |
1109 | case ISD::EH_SJLJ_LONGJMP: |
1110 | case ISD::EH_SJLJ_SETUP_DISPATCH: |
1111 | // These operations lie about being legal: when they claim to be legal, |
1112 | // they should actually be expanded. |
1113 | Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0)); |
1114 | if (Action == TargetLowering::Legal) |
1115 | Action = TargetLowering::Expand; |
1116 | break; |
1117 | case ISD::INIT_TRAMPOLINE: |
1118 | case ISD::ADJUST_TRAMPOLINE: |
1119 | case ISD::FRAMEADDR: |
1120 | case ISD::RETURNADDR: |
1121 | case ISD::ADDROFRETURNADDR: |
1122 | case ISD::SPONENTRY: |
1123 | // These operations lie about being legal: when they claim to be legal, |
1124 | // they should actually be custom-lowered. |
1125 | Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0)); |
1126 | if (Action == TargetLowering::Legal) |
1127 | Action = TargetLowering::Custom; |
1128 | break; |
1129 | case ISD::READCYCLECOUNTER: |
1130 | case ISD::READSTEADYCOUNTER: |
1131 | // READCYCLECOUNTER and READSTEADYCOUNTER return a i64, even if type |
1132 | // legalization might have expanded that to several smaller types. |
1133 | Action = TLI.getOperationAction(Op: Node->getOpcode(), MVT::VT: i64); |
1134 | break; |
1135 | case ISD::READ_REGISTER: |
1136 | case ISD::WRITE_REGISTER: |
1137 | // Named register is legal in the DAG, but blocked by register name |
1138 | // selection if not implemented by target (to chose the correct register) |
1139 | // They'll be converted to Copy(To/From)Reg. |
1140 | Action = TargetLowering::Legal; |
1141 | break; |
1142 | case ISD::UBSANTRAP: |
1143 | Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0)); |
1144 | if (Action == TargetLowering::Expand) { |
1145 | // replace ISD::UBSANTRAP with ISD::TRAP |
1146 | SDValue NewVal; |
1147 | NewVal = DAG.getNode(Opcode: ISD::TRAP, DL: SDLoc(Node), VTList: Node->getVTList(), |
1148 | N: Node->getOperand(Num: 0)); |
1149 | ReplaceNode(Old: Node, New: NewVal.getNode()); |
1150 | LegalizeOp(Node: NewVal.getNode()); |
1151 | return; |
1152 | } |
1153 | break; |
1154 | case ISD::DEBUGTRAP: |
1155 | Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0)); |
1156 | if (Action == TargetLowering::Expand) { |
1157 | // replace ISD::DEBUGTRAP with ISD::TRAP |
1158 | SDValue NewVal; |
1159 | NewVal = DAG.getNode(Opcode: ISD::TRAP, DL: SDLoc(Node), VTList: Node->getVTList(), |
1160 | N: Node->getOperand(Num: 0)); |
1161 | ReplaceNode(Old: Node, New: NewVal.getNode()); |
1162 | LegalizeOp(Node: NewVal.getNode()); |
1163 | return; |
1164 | } |
1165 | break; |
1166 | case ISD::SADDSAT: |
1167 | case ISD::UADDSAT: |
1168 | case ISD::SSUBSAT: |
1169 | case ISD::USUBSAT: |
1170 | case ISD::SSHLSAT: |
1171 | case ISD::USHLSAT: |
1172 | case ISD::FP_TO_SINT_SAT: |
1173 | case ISD::FP_TO_UINT_SAT: |
1174 | Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0)); |
1175 | break; |
1176 | case ISD::SMULFIX: |
1177 | case ISD::SMULFIXSAT: |
1178 | case ISD::UMULFIX: |
1179 | case ISD::UMULFIXSAT: |
1180 | case ISD::SDIVFIX: |
1181 | case ISD::SDIVFIXSAT: |
1182 | case ISD::UDIVFIX: |
1183 | case ISD::UDIVFIXSAT: { |
1184 | unsigned Scale = Node->getConstantOperandVal(Num: 2); |
1185 | Action = TLI.getFixedPointOperationAction(Op: Node->getOpcode(), |
1186 | VT: Node->getValueType(ResNo: 0), Scale); |
1187 | break; |
1188 | } |
1189 | case ISD::MSCATTER: |
1190 | Action = TLI.getOperationAction(Op: Node->getOpcode(), |
1191 | VT: cast<MaskedScatterSDNode>(Val: Node)->getValue().getValueType()); |
1192 | break; |
1193 | case ISD::MSTORE: |
1194 | Action = TLI.getOperationAction(Op: Node->getOpcode(), |
1195 | VT: cast<MaskedStoreSDNode>(Val: Node)->getValue().getValueType()); |
1196 | break; |
1197 | case ISD::VP_SCATTER: |
1198 | Action = TLI.getOperationAction( |
1199 | Op: Node->getOpcode(), |
1200 | VT: cast<VPScatterSDNode>(Val: Node)->getValue().getValueType()); |
1201 | break; |
1202 | case ISD::VP_STORE: |
1203 | Action = TLI.getOperationAction( |
1204 | Op: Node->getOpcode(), |
1205 | VT: cast<VPStoreSDNode>(Val: Node)->getValue().getValueType()); |
1206 | break; |
1207 | case ISD::EXPERIMENTAL_VP_STRIDED_STORE: |
1208 | Action = TLI.getOperationAction( |
1209 | Op: Node->getOpcode(), |
1210 | VT: cast<VPStridedStoreSDNode>(Val: Node)->getValue().getValueType()); |
1211 | break; |
1212 | case ISD::VECREDUCE_FADD: |
1213 | case ISD::VECREDUCE_FMUL: |
1214 | case ISD::VECREDUCE_ADD: |
1215 | case ISD::VECREDUCE_MUL: |
1216 | case ISD::VECREDUCE_AND: |
1217 | case ISD::VECREDUCE_OR: |
1218 | case ISD::VECREDUCE_XOR: |
1219 | case ISD::VECREDUCE_SMAX: |
1220 | case ISD::VECREDUCE_SMIN: |
1221 | case ISD::VECREDUCE_UMAX: |
1222 | case ISD::VECREDUCE_UMIN: |
1223 | case ISD::VECREDUCE_FMAX: |
1224 | case ISD::VECREDUCE_FMIN: |
1225 | case ISD::VECREDUCE_FMAXIMUM: |
1226 | case ISD::VECREDUCE_FMINIMUM: |
1227 | case ISD::IS_FPCLASS: |
1228 | Action = TLI.getOperationAction( |
1229 | Op: Node->getOpcode(), VT: Node->getOperand(Num: 0).getValueType()); |
1230 | break; |
1231 | case ISD::VECREDUCE_SEQ_FADD: |
1232 | case ISD::VECREDUCE_SEQ_FMUL: |
1233 | case ISD::VP_REDUCE_FADD: |
1234 | case ISD::VP_REDUCE_FMUL: |
1235 | case ISD::VP_REDUCE_ADD: |
1236 | case ISD::VP_REDUCE_MUL: |
1237 | case ISD::VP_REDUCE_AND: |
1238 | case ISD::VP_REDUCE_OR: |
1239 | case ISD::VP_REDUCE_XOR: |
1240 | case ISD::VP_REDUCE_SMAX: |
1241 | case ISD::VP_REDUCE_SMIN: |
1242 | case ISD::VP_REDUCE_UMAX: |
1243 | case ISD::VP_REDUCE_UMIN: |
1244 | case ISD::VP_REDUCE_FMAX: |
1245 | case ISD::VP_REDUCE_FMIN: |
1246 | case ISD::VP_REDUCE_SEQ_FADD: |
1247 | case ISD::VP_REDUCE_SEQ_FMUL: |
1248 | Action = TLI.getOperationAction( |
1249 | Op: Node->getOpcode(), VT: Node->getOperand(Num: 1).getValueType()); |
1250 | break; |
1251 | default: |
1252 | if (Node->getOpcode() >= ISD::BUILTIN_OP_END) { |
1253 | Action = TLI.getCustomOperationAction(Op&: *Node); |
1254 | } else { |
1255 | Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0)); |
1256 | } |
1257 | break; |
1258 | } |
1259 | |
1260 | if (SimpleFinishLegalizing) { |
1261 | SDNode *NewNode = Node; |
1262 | switch (Node->getOpcode()) { |
1263 | default: break; |
1264 | case ISD::SHL: |
1265 | case ISD::SRL: |
1266 | case ISD::SRA: |
1267 | case ISD::ROTL: |
1268 | case ISD::ROTR: { |
1269 | // Legalizing shifts/rotates requires adjusting the shift amount |
1270 | // to the appropriate width. |
1271 | SDValue Op0 = Node->getOperand(Num: 0); |
1272 | SDValue Op1 = Node->getOperand(Num: 1); |
1273 | if (!Op1.getValueType().isVector()) { |
1274 | SDValue SAO = DAG.getShiftAmountOperand(LHSTy: Op0.getValueType(), Op: Op1); |
1275 | // The getShiftAmountOperand() may create a new operand node or |
1276 | // return the existing one. If new operand is created we need |
1277 | // to update the parent node. |
1278 | // Do not try to legalize SAO here! It will be automatically legalized |
1279 | // in the next round. |
1280 | if (SAO != Op1) |
1281 | NewNode = DAG.UpdateNodeOperands(N: Node, Op1: Op0, Op2: SAO); |
1282 | } |
1283 | } |
1284 | break; |
1285 | case ISD::FSHL: |
1286 | case ISD::FSHR: |
1287 | case ISD::SRL_PARTS: |
1288 | case ISD::SRA_PARTS: |
1289 | case ISD::SHL_PARTS: { |
1290 | // Legalizing shifts/rotates requires adjusting the shift amount |
1291 | // to the appropriate width. |
1292 | SDValue Op0 = Node->getOperand(Num: 0); |
1293 | SDValue Op1 = Node->getOperand(Num: 1); |
1294 | SDValue Op2 = Node->getOperand(Num: 2); |
1295 | if (!Op2.getValueType().isVector()) { |
1296 | SDValue SAO = DAG.getShiftAmountOperand(LHSTy: Op0.getValueType(), Op: Op2); |
1297 | // The getShiftAmountOperand() may create a new operand node or |
1298 | // return the existing one. If new operand is created we need |
1299 | // to update the parent node. |
1300 | if (SAO != Op2) |
1301 | NewNode = DAG.UpdateNodeOperands(N: Node, Op1: Op0, Op2: Op1, Op3: SAO); |
1302 | } |
1303 | break; |
1304 | } |
1305 | } |
1306 | |
1307 | if (NewNode != Node) { |
1308 | ReplaceNode(Old: Node, New: NewNode); |
1309 | Node = NewNode; |
1310 | } |
1311 | switch (Action) { |
1312 | case TargetLowering::Legal: |
1313 | LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n" ); |
1314 | return; |
1315 | case TargetLowering::Custom: |
1316 | LLVM_DEBUG(dbgs() << "Trying custom legalization\n" ); |
1317 | // FIXME: The handling for custom lowering with multiple results is |
1318 | // a complete mess. |
1319 | if (SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG)) { |
1320 | if (!(Res.getNode() != Node || Res.getResNo() != 0)) |
1321 | return; |
1322 | |
1323 | if (Node->getNumValues() == 1) { |
1324 | // Verify the new types match the original. Glue is waived because |
1325 | // ISD::ADDC can be legalized by replacing Glue with an integer type. |
1326 | assert((Res.getValueType() == Node->getValueType(0) || |
1327 | Node->getValueType(0) == MVT::Glue) && |
1328 | "Type mismatch for custom legalized operation" ); |
1329 | LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n" ); |
1330 | // We can just directly replace this node with the lowered value. |
1331 | ReplaceNode(Old: SDValue(Node, 0), New: Res); |
1332 | return; |
1333 | } |
1334 | |
1335 | SmallVector<SDValue, 8> ResultVals; |
1336 | for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) { |
1337 | // Verify the new types match the original. Glue is waived because |
1338 | // ISD::ADDC can be legalized by replacing Glue with an integer type. |
1339 | assert((Res->getValueType(i) == Node->getValueType(i) || |
1340 | Node->getValueType(i) == MVT::Glue) && |
1341 | "Type mismatch for custom legalized operation" ); |
1342 | ResultVals.push_back(Elt: Res.getValue(R: i)); |
1343 | } |
1344 | LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n" ); |
1345 | ReplaceNode(Old: Node, New: ResultVals.data()); |
1346 | return; |
1347 | } |
1348 | LLVM_DEBUG(dbgs() << "Could not custom legalize node\n" ); |
1349 | [[fallthrough]]; |
1350 | case TargetLowering::Expand: |
1351 | if (ExpandNode(Node)) |
1352 | return; |
1353 | [[fallthrough]]; |
1354 | case TargetLowering::LibCall: |
1355 | ConvertNodeToLibcall(Node); |
1356 | return; |
1357 | case TargetLowering::Promote: |
1358 | PromoteNode(Node); |
1359 | return; |
1360 | } |
1361 | } |
1362 | |
1363 | switch (Node->getOpcode()) { |
1364 | default: |
1365 | #ifndef NDEBUG |
1366 | dbgs() << "NODE: " ; |
1367 | Node->dump( G: &DAG); |
1368 | dbgs() << "\n" ; |
1369 | #endif |
1370 | llvm_unreachable("Do not know how to legalize this operator!" ); |
1371 | |
1372 | case ISD::CALLSEQ_START: |
1373 | case ISD::CALLSEQ_END: |
1374 | break; |
1375 | case ISD::LOAD: |
1376 | return LegalizeLoadOps(Node); |
1377 | case ISD::STORE: |
1378 | return LegalizeStoreOps(Node); |
1379 | } |
1380 | } |
1381 | |
1382 | // Helper function that generates an MMO that considers the alignment of the |
1383 | // stack, and the size of the stack object |
1384 | static MachineMemOperand *getStackAlignedMMO(SDValue StackPtr, |
1385 | MachineFunction &MF, |
1386 | bool isObjectScalable) { |
1387 | auto &MFI = MF.getFrameInfo(); |
1388 | int FI = cast<FrameIndexSDNode>(Val&: StackPtr)->getIndex(); |
1389 | MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI); |
1390 | uint64_t ObjectSize = isObjectScalable ? ~UINT64_C(0) : MFI.getObjectSize(ObjectIdx: FI); |
1391 | return MF.getMachineMemOperand(PtrInfo, f: MachineMemOperand::MOStore, |
1392 | s: ObjectSize, base_alignment: MFI.getObjectAlign(ObjectIdx: FI)); |
1393 | } |
1394 | |
1395 | SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) { |
1396 | SDValue Vec = Op.getOperand(i: 0); |
1397 | SDValue Idx = Op.getOperand(i: 1); |
1398 | SDLoc dl(Op); |
1399 | |
1400 | // Before we generate a new store to a temporary stack slot, see if there is |
1401 | // already one that we can use. There often is because when we scalarize |
1402 | // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole |
1403 | // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in |
1404 | // the vector. If all are expanded here, we don't want one store per vector |
1405 | // element. |
1406 | |
1407 | // Caches for hasPredecessorHelper |
1408 | SmallPtrSet<const SDNode *, 32> Visited; |
1409 | SmallVector<const SDNode *, 16> Worklist; |
1410 | Visited.insert(Ptr: Op.getNode()); |
1411 | Worklist.push_back(Elt: Idx.getNode()); |
1412 | SDValue StackPtr, Ch; |
1413 | for (SDNode *User : Vec.getNode()->uses()) { |
1414 | if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: User)) { |
1415 | if (ST->isIndexed() || ST->isTruncatingStore() || |
1416 | ST->getValue() != Vec) |
1417 | continue; |
1418 | |
1419 | // Make sure that nothing else could have stored into the destination of |
1420 | // this store. |
1421 | if (!ST->getChain().reachesChainWithoutSideEffects(Dest: DAG.getEntryNode())) |
1422 | continue; |
1423 | |
1424 | // If the index is dependent on the store we will introduce a cycle when |
1425 | // creating the load (the load uses the index, and by replacing the chain |
1426 | // we will make the index dependent on the load). Also, the store might be |
1427 | // dependent on the extractelement and introduce a cycle when creating |
1428 | // the load. |
1429 | if (SDNode::hasPredecessorHelper(N: ST, Visited, Worklist) || |
1430 | ST->hasPredecessor(N: Op.getNode())) |
1431 | continue; |
1432 | |
1433 | StackPtr = ST->getBasePtr(); |
1434 | Ch = SDValue(ST, 0); |
1435 | break; |
1436 | } |
1437 | } |
1438 | |
1439 | EVT VecVT = Vec.getValueType(); |
1440 | |
1441 | if (!Ch.getNode()) { |
1442 | // Store the value to a temporary stack slot, then LOAD the returned part. |
1443 | StackPtr = DAG.CreateStackTemporary(VT: VecVT); |
1444 | MachineMemOperand *StoreMMO = getStackAlignedMMO( |
1445 | StackPtr, MF&: DAG.getMachineFunction(), isObjectScalable: VecVT.isScalableVector()); |
1446 | Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, MMO: StoreMMO); |
1447 | } |
1448 | |
1449 | SDValue NewLoad; |
1450 | Align ElementAlignment = |
1451 | std::min(a: cast<StoreSDNode>(Val&: Ch)->getAlign(), |
1452 | b: DAG.getDataLayout().getPrefTypeAlign( |
1453 | Ty: Op.getValueType().getTypeForEVT(Context&: *DAG.getContext()))); |
1454 | |
1455 | if (Op.getValueType().isVector()) { |
1456 | StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, |
1457 | SubVecVT: Op.getValueType(), Index: Idx); |
1458 | NewLoad = DAG.getLoad(VT: Op.getValueType(), dl, Chain: Ch, Ptr: StackPtr, |
1459 | PtrInfo: MachinePointerInfo(), Alignment: ElementAlignment); |
1460 | } else { |
1461 | StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx); |
1462 | NewLoad = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: Op.getValueType(), Chain: Ch, Ptr: StackPtr, |
1463 | PtrInfo: MachinePointerInfo(), MemVT: VecVT.getVectorElementType(), |
1464 | Alignment: ElementAlignment); |
1465 | } |
1466 | |
1467 | // Replace the chain going out of the store, by the one out of the load. |
1468 | DAG.ReplaceAllUsesOfValueWith(From: Ch, To: SDValue(NewLoad.getNode(), 1)); |
1469 | |
1470 | // We introduced a cycle though, so update the loads operands, making sure |
1471 | // to use the original store's chain as an incoming chain. |
1472 | SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(), |
1473 | NewLoad->op_end()); |
1474 | NewLoadOperands[0] = Ch; |
1475 | NewLoad = |
1476 | SDValue(DAG.UpdateNodeOperands(N: NewLoad.getNode(), Ops: NewLoadOperands), 0); |
1477 | return NewLoad; |
1478 | } |
1479 | |
1480 | SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) { |
1481 | assert(Op.getValueType().isVector() && "Non-vector insert subvector!" ); |
1482 | |
1483 | SDValue Vec = Op.getOperand(i: 0); |
1484 | SDValue Part = Op.getOperand(i: 1); |
1485 | SDValue Idx = Op.getOperand(i: 2); |
1486 | SDLoc dl(Op); |
1487 | |
1488 | // Store the value to a temporary stack slot, then LOAD the returned part. |
1489 | EVT VecVT = Vec.getValueType(); |
1490 | EVT SubVecVT = Part.getValueType(); |
1491 | SDValue StackPtr = DAG.CreateStackTemporary(VT: VecVT); |
1492 | int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex(); |
1493 | MachinePointerInfo PtrInfo = |
1494 | MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI); |
1495 | |
1496 | // First store the whole vector. |
1497 | SDValue Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo); |
1498 | |
1499 | // Then store the inserted part. |
1500 | SDValue SubStackPtr = |
1501 | TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT, Index: Idx); |
1502 | |
1503 | // Store the subvector. |
1504 | Ch = DAG.getStore( |
1505 | Chain: Ch, dl, Val: Part, Ptr: SubStackPtr, |
1506 | PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction())); |
1507 | |
1508 | // Finally, load the updated vector. |
1509 | return DAG.getLoad(VT: Op.getValueType(), dl, Chain: Ch, Ptr: StackPtr, PtrInfo); |
1510 | } |
1511 | |
1512 | SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) { |
1513 | assert((Node->getOpcode() == ISD::BUILD_VECTOR || |
1514 | Node->getOpcode() == ISD::CONCAT_VECTORS) && |
1515 | "Unexpected opcode!" ); |
1516 | |
1517 | // We can't handle this case efficiently. Allocate a sufficiently |
1518 | // aligned object on the stack, store each operand into it, then load |
1519 | // the result as a vector. |
1520 | // Create the stack frame object. |
1521 | EVT VT = Node->getValueType(ResNo: 0); |
1522 | EVT MemVT = isa<BuildVectorSDNode>(Val: Node) ? VT.getVectorElementType() |
1523 | : Node->getOperand(Num: 0).getValueType(); |
1524 | SDLoc dl(Node); |
1525 | SDValue FIPtr = DAG.CreateStackTemporary(VT); |
1526 | int FI = cast<FrameIndexSDNode>(Val: FIPtr.getNode())->getIndex(); |
1527 | MachinePointerInfo PtrInfo = |
1528 | MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI); |
1529 | |
1530 | // Emit a store of each element to the stack slot. |
1531 | SmallVector<SDValue, 8> Stores; |
1532 | unsigned TypeByteSize = MemVT.getSizeInBits() / 8; |
1533 | assert(TypeByteSize > 0 && "Vector element type too small for stack store!" ); |
1534 | |
1535 | // If the destination vector element type of a BUILD_VECTOR is narrower than |
1536 | // the source element type, only store the bits necessary. |
1537 | bool Truncate = isa<BuildVectorSDNode>(Val: Node) && |
1538 | MemVT.bitsLT(VT: Node->getOperand(Num: 0).getValueType()); |
1539 | |
1540 | // Store (in the right endianness) the elements to memory. |
1541 | for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { |
1542 | // Ignore undef elements. |
1543 | if (Node->getOperand(Num: i).isUndef()) continue; |
1544 | |
1545 | unsigned Offset = TypeByteSize*i; |
1546 | |
1547 | SDValue Idx = |
1548 | DAG.getMemBasePlusOffset(Base: FIPtr, Offset: TypeSize::getFixed(ExactSize: Offset), DL: dl); |
1549 | |
1550 | if (Truncate) |
1551 | Stores.push_back(Elt: DAG.getTruncStore(Chain: DAG.getEntryNode(), dl, |
1552 | Val: Node->getOperand(Num: i), Ptr: Idx, |
1553 | PtrInfo: PtrInfo.getWithOffset(O: Offset), SVT: MemVT)); |
1554 | else |
1555 | Stores.push_back(Elt: DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Node->getOperand(Num: i), |
1556 | Ptr: Idx, PtrInfo: PtrInfo.getWithOffset(O: Offset))); |
1557 | } |
1558 | |
1559 | SDValue StoreChain; |
1560 | if (!Stores.empty()) // Not all undef elements? |
1561 | StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); |
1562 | else |
1563 | StoreChain = DAG.getEntryNode(); |
1564 | |
1565 | // Result is a load from the stack slot. |
1566 | return DAG.getLoad(VT, dl, Chain: StoreChain, Ptr: FIPtr, PtrInfo); |
1567 | } |
1568 | |
1569 | /// Bitcast a floating-point value to an integer value. Only bitcast the part |
1570 | /// containing the sign bit if the target has no integer value capable of |
1571 | /// holding all bits of the floating-point value. |
1572 | void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State, |
1573 | const SDLoc &DL, |
1574 | SDValue Value) const { |
1575 | EVT FloatVT = Value.getValueType(); |
1576 | unsigned NumBits = FloatVT.getScalarSizeInBits(); |
1577 | State.FloatVT = FloatVT; |
1578 | EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumBits); |
1579 | // Convert to an integer of the same size. |
1580 | if (TLI.isTypeLegal(VT: IVT)) { |
1581 | State.IntValue = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IVT, Operand: Value); |
1582 | State.SignMask = APInt::getSignMask(BitWidth: NumBits); |
1583 | State.SignBit = NumBits - 1; |
1584 | return; |
1585 | } |
1586 | |
1587 | auto &DataLayout = DAG.getDataLayout(); |
1588 | // Store the float to memory, then load the sign part out as an integer. |
1589 | MVT LoadTy = TLI.getRegisterType(MVT::i8); |
1590 | // First create a temporary that is aligned for both the load and store. |
1591 | SDValue StackPtr = DAG.CreateStackTemporary(VT1: FloatVT, VT2: LoadTy); |
1592 | int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex(); |
1593 | // Then store the float to it. |
1594 | State.FloatPtr = StackPtr; |
1595 | MachineFunction &MF = DAG.getMachineFunction(); |
1596 | State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI); |
1597 | State.Chain = DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: Value, Ptr: State.FloatPtr, |
1598 | PtrInfo: State.FloatPointerInfo); |
1599 | |
1600 | SDValue IntPtr; |
1601 | if (DataLayout.isBigEndian()) { |
1602 | assert(FloatVT.isByteSized() && "Unsupported floating point type!" ); |
1603 | // Load out a legal integer with the same sign bit as the float. |
1604 | IntPtr = StackPtr; |
1605 | State.IntPointerInfo = State.FloatPointerInfo; |
1606 | } else { |
1607 | // Advance the pointer so that the loaded byte will contain the sign bit. |
1608 | unsigned ByteOffset = (NumBits / 8) - 1; |
1609 | IntPtr = |
1610 | DAG.getMemBasePlusOffset(Base: StackPtr, Offset: TypeSize::getFixed(ExactSize: ByteOffset), DL); |
1611 | State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI, |
1612 | Offset: ByteOffset); |
1613 | } |
1614 | |
1615 | State.IntPtr = IntPtr; |
1616 | State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr, |
1617 | State.IntPointerInfo, MVT::i8); |
1618 | State.SignMask = APInt::getOneBitSet(numBits: LoadTy.getScalarSizeInBits(), BitNo: 7); |
1619 | State.SignBit = 7; |
1620 | } |
1621 | |
1622 | /// Replace the integer value produced by getSignAsIntValue() with a new value |
1623 | /// and cast the result back to a floating-point type. |
1624 | SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State, |
1625 | const SDLoc &DL, |
1626 | SDValue NewIntValue) const { |
1627 | if (!State.Chain) |
1628 | return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: State.FloatVT, Operand: NewIntValue); |
1629 | |
1630 | // Override the part containing the sign bit in the value stored on the stack. |
1631 | SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr, |
1632 | State.IntPointerInfo, MVT::i8); |
1633 | return DAG.getLoad(VT: State.FloatVT, dl: DL, Chain, Ptr: State.FloatPtr, |
1634 | PtrInfo: State.FloatPointerInfo); |
1635 | } |
1636 | |
1637 | SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const { |
1638 | SDLoc DL(Node); |
1639 | SDValue Mag = Node->getOperand(Num: 0); |
1640 | SDValue Sign = Node->getOperand(Num: 1); |
1641 | |
1642 | // Get sign bit into an integer value. |
1643 | FloatSignAsInt SignAsInt; |
1644 | getSignAsIntValue(State&: SignAsInt, DL, Value: Sign); |
1645 | |
1646 | EVT IntVT = SignAsInt.IntValue.getValueType(); |
1647 | SDValue SignMask = DAG.getConstant(Val: SignAsInt.SignMask, DL, VT: IntVT); |
1648 | SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: SignAsInt.IntValue, |
1649 | N2: SignMask); |
1650 | |
1651 | // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X) |
1652 | EVT FloatVT = Mag.getValueType(); |
1653 | if (TLI.isOperationLegalOrCustom(Op: ISD::FABS, VT: FloatVT) && |
1654 | TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT: FloatVT)) { |
1655 | SDValue AbsValue = DAG.getNode(Opcode: ISD::FABS, DL, VT: FloatVT, Operand: Mag); |
1656 | SDValue NegValue = DAG.getNode(Opcode: ISD::FNEG, DL, VT: FloatVT, Operand: AbsValue); |
1657 | SDValue Cond = DAG.getSetCC(DL, VT: getSetCCResultType(VT: IntVT), LHS: SignBit, |
1658 | RHS: DAG.getConstant(Val: 0, DL, VT: IntVT), Cond: ISD::SETNE); |
1659 | return DAG.getSelect(DL, VT: FloatVT, Cond, LHS: NegValue, RHS: AbsValue); |
1660 | } |
1661 | |
1662 | // Transform Mag value to integer, and clear the sign bit. |
1663 | FloatSignAsInt MagAsInt; |
1664 | getSignAsIntValue(State&: MagAsInt, DL, Value: Mag); |
1665 | EVT MagVT = MagAsInt.IntValue.getValueType(); |
1666 | SDValue ClearSignMask = DAG.getConstant(Val: ~MagAsInt.SignMask, DL, VT: MagVT); |
1667 | SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: MagVT, N1: MagAsInt.IntValue, |
1668 | N2: ClearSignMask); |
1669 | |
1670 | // Get the signbit at the right position for MagAsInt. |
1671 | int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit; |
1672 | EVT ShiftVT = IntVT; |
1673 | if (SignBit.getScalarValueSizeInBits() < |
1674 | ClearedSign.getScalarValueSizeInBits()) { |
1675 | SignBit = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MagVT, Operand: SignBit); |
1676 | ShiftVT = MagVT; |
1677 | } |
1678 | if (ShiftAmount > 0) { |
1679 | SDValue ShiftCnst = DAG.getConstant(Val: ShiftAmount, DL, VT: ShiftVT); |
1680 | SignBit = DAG.getNode(Opcode: ISD::SRL, DL, VT: ShiftVT, N1: SignBit, N2: ShiftCnst); |
1681 | } else if (ShiftAmount < 0) { |
1682 | SDValue ShiftCnst = DAG.getConstant(Val: -ShiftAmount, DL, VT: ShiftVT); |
1683 | SignBit = DAG.getNode(Opcode: ISD::SHL, DL, VT: ShiftVT, N1: SignBit, N2: ShiftCnst); |
1684 | } |
1685 | if (SignBit.getScalarValueSizeInBits() > |
1686 | ClearedSign.getScalarValueSizeInBits()) { |
1687 | SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MagVT, Operand: SignBit); |
1688 | } |
1689 | |
1690 | // Store the part with the modified sign and convert back to float. |
1691 | SDValue CopiedSign = DAG.getNode(Opcode: ISD::OR, DL, VT: MagVT, N1: ClearedSign, N2: SignBit); |
1692 | return modifySignAsInt(State: MagAsInt, DL, NewIntValue: CopiedSign); |
1693 | } |
1694 | |
1695 | SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const { |
1696 | // Get the sign bit as an integer. |
1697 | SDLoc DL(Node); |
1698 | FloatSignAsInt SignAsInt; |
1699 | getSignAsIntValue(State&: SignAsInt, DL, Value: Node->getOperand(Num: 0)); |
1700 | EVT IntVT = SignAsInt.IntValue.getValueType(); |
1701 | |
1702 | // Flip the sign. |
1703 | SDValue SignMask = DAG.getConstant(Val: SignAsInt.SignMask, DL, VT: IntVT); |
1704 | SDValue SignFlip = |
1705 | DAG.getNode(Opcode: ISD::XOR, DL, VT: IntVT, N1: SignAsInt.IntValue, N2: SignMask); |
1706 | |
1707 | // Convert back to float. |
1708 | return modifySignAsInt(State: SignAsInt, DL, NewIntValue: SignFlip); |
1709 | } |
1710 | |
1711 | SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const { |
1712 | SDLoc DL(Node); |
1713 | SDValue Value = Node->getOperand(Num: 0); |
1714 | |
1715 | // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal. |
1716 | EVT FloatVT = Value.getValueType(); |
1717 | if (TLI.isOperationLegalOrCustom(Op: ISD::FCOPYSIGN, VT: FloatVT)) { |
1718 | SDValue Zero = DAG.getConstantFP(Val: 0.0, DL, VT: FloatVT); |
1719 | return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT: FloatVT, N1: Value, N2: Zero); |
1720 | } |
1721 | |
1722 | // Transform value to integer, clear the sign bit and transform back. |
1723 | FloatSignAsInt ValueAsInt; |
1724 | getSignAsIntValue(State&: ValueAsInt, DL, Value); |
1725 | EVT IntVT = ValueAsInt.IntValue.getValueType(); |
1726 | SDValue ClearSignMask = DAG.getConstant(Val: ~ValueAsInt.SignMask, DL, VT: IntVT); |
1727 | SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: ValueAsInt.IntValue, |
1728 | N2: ClearSignMask); |
1729 | return modifySignAsInt(State: ValueAsInt, DL, NewIntValue: ClearedSign); |
1730 | } |
1731 | |
1732 | void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node, |
1733 | SmallVectorImpl<SDValue> &Results) { |
1734 | Register SPReg = TLI.getStackPointerRegisterToSaveRestore(); |
1735 | assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and" |
1736 | " not tell us which reg is the stack pointer!" ); |
1737 | SDLoc dl(Node); |
1738 | EVT VT = Node->getValueType(ResNo: 0); |
1739 | SDValue Tmp1 = SDValue(Node, 0); |
1740 | SDValue Tmp2 = SDValue(Node, 1); |
1741 | SDValue Tmp3 = Node->getOperand(Num: 2); |
1742 | SDValue Chain = Tmp1.getOperand(i: 0); |
1743 | |
1744 | // Chain the dynamic stack allocation so that it doesn't modify the stack |
1745 | // pointer when other instructions are using the stack. |
1746 | Chain = DAG.getCALLSEQ_START(Chain, InSize: 0, OutSize: 0, DL: dl); |
1747 | |
1748 | SDValue Size = Tmp2.getOperand(i: 1); |
1749 | SDValue SP = DAG.getCopyFromReg(Chain, dl, Reg: SPReg, VT); |
1750 | Chain = SP.getValue(R: 1); |
1751 | Align Alignment = cast<ConstantSDNode>(Val&: Tmp3)->getAlignValue(); |
1752 | const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering(); |
1753 | unsigned Opc = |
1754 | TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp ? |
1755 | ISD::ADD : ISD::SUB; |
1756 | |
1757 | Align StackAlign = TFL->getStackAlign(); |
1758 | Tmp1 = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: SP, N2: Size); // Value |
1759 | if (Alignment > StackAlign) |
1760 | Tmp1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp1, |
1761 | N2: DAG.getConstant(Val: -Alignment.value(), DL: dl, VT)); |
1762 | Chain = DAG.getCopyToReg(Chain, dl, Reg: SPReg, N: Tmp1); // Output chain |
1763 | |
1764 | Tmp2 = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: SDValue(), DL: dl); |
1765 | |
1766 | Results.push_back(Elt: Tmp1); |
1767 | Results.push_back(Elt: Tmp2); |
1768 | } |
1769 | |
1770 | /// Emit a store/load combination to the stack. This stores |
1771 | /// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does |
1772 | /// a load from the stack slot to DestVT, extending it if needed. |
1773 | /// The resultant code need not be legal. |
1774 | SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT, |
1775 | EVT DestVT, const SDLoc &dl) { |
1776 | return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, ChainIn: DAG.getEntryNode()); |
1777 | } |
1778 | |
1779 | SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT, |
1780 | EVT DestVT, const SDLoc &dl, |
1781 | SDValue Chain) { |
1782 | EVT SrcVT = SrcOp.getValueType(); |
1783 | Type *DestType = DestVT.getTypeForEVT(Context&: *DAG.getContext()); |
1784 | Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(Ty: DestType); |
1785 | |
1786 | // Don't convert with stack if the load/store is expensive. |
1787 | if ((SrcVT.bitsGT(VT: SlotVT) && |
1788 | !TLI.isTruncStoreLegalOrCustom(ValVT: SrcOp.getValueType(), MemVT: SlotVT)) || |
1789 | (SlotVT.bitsLT(VT: DestVT) && |
1790 | !TLI.isLoadExtLegalOrCustom(ExtType: ISD::EXTLOAD, ValVT: DestVT, MemVT: SlotVT))) |
1791 | return SDValue(); |
1792 | |
1793 | // Create the stack frame object. |
1794 | Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign( |
1795 | Ty: SrcOp.getValueType().getTypeForEVT(Context&: *DAG.getContext())); |
1796 | SDValue FIPtr = DAG.CreateStackTemporary(Bytes: SlotVT.getStoreSize(), Alignment: SrcAlign); |
1797 | |
1798 | FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(Val&: FIPtr); |
1799 | int SPFI = StackPtrFI->getIndex(); |
1800 | MachinePointerInfo PtrInfo = |
1801 | MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI); |
1802 | |
1803 | // Emit a store to the stack slot. Use a truncstore if the input value is |
1804 | // later than DestVT. |
1805 | SDValue Store; |
1806 | |
1807 | if (SrcVT.bitsGT(VT: SlotVT)) |
1808 | Store = DAG.getTruncStore(Chain, dl, Val: SrcOp, Ptr: FIPtr, PtrInfo, |
1809 | SVT: SlotVT, Alignment: SrcAlign); |
1810 | else { |
1811 | assert(SrcVT.bitsEq(SlotVT) && "Invalid store" ); |
1812 | Store = DAG.getStore(Chain, dl, Val: SrcOp, Ptr: FIPtr, PtrInfo, Alignment: SrcAlign); |
1813 | } |
1814 | |
1815 | // Result is a load from the stack slot. |
1816 | if (SlotVT.bitsEq(VT: DestVT)) |
1817 | return DAG.getLoad(VT: DestVT, dl, Chain: Store, Ptr: FIPtr, PtrInfo, Alignment: DestAlign); |
1818 | |
1819 | assert(SlotVT.bitsLT(DestVT) && "Unknown extension!" ); |
1820 | return DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: DestVT, Chain: Store, Ptr: FIPtr, PtrInfo, MemVT: SlotVT, |
1821 | Alignment: DestAlign); |
1822 | } |
1823 | |
1824 | SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) { |
1825 | SDLoc dl(Node); |
1826 | // Create a vector sized/aligned stack slot, store the value to element #0, |
1827 | // then load the whole vector back out. |
1828 | SDValue StackPtr = DAG.CreateStackTemporary(VT: Node->getValueType(ResNo: 0)); |
1829 | |
1830 | FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(Val&: StackPtr); |
1831 | int SPFI = StackPtrFI->getIndex(); |
1832 | |
1833 | SDValue Ch = DAG.getTruncStore( |
1834 | Chain: DAG.getEntryNode(), dl, Val: Node->getOperand(Num: 0), Ptr: StackPtr, |
1835 | PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI), |
1836 | SVT: Node->getValueType(ResNo: 0).getVectorElementType()); |
1837 | return DAG.getLoad( |
1838 | VT: Node->getValueType(ResNo: 0), dl, Chain: Ch, Ptr: StackPtr, |
1839 | PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI)); |
1840 | } |
1841 | |
1842 | static bool |
1843 | ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, |
1844 | const TargetLowering &TLI, SDValue &Res) { |
1845 | unsigned NumElems = Node->getNumOperands(); |
1846 | SDLoc dl(Node); |
1847 | EVT VT = Node->getValueType(ResNo: 0); |
1848 | |
1849 | // Try to group the scalars into pairs, shuffle the pairs together, then |
1850 | // shuffle the pairs of pairs together, etc. until the vector has |
1851 | // been built. This will work only if all of the necessary shuffle masks |
1852 | // are legal. |
1853 | |
1854 | // We do this in two phases; first to check the legality of the shuffles, |
1855 | // and next, assuming that all shuffles are legal, to create the new nodes. |
1856 | for (int Phase = 0; Phase < 2; ++Phase) { |
1857 | SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals, |
1858 | NewIntermedVals; |
1859 | for (unsigned i = 0; i < NumElems; ++i) { |
1860 | SDValue V = Node->getOperand(Num: i); |
1861 | if (V.isUndef()) |
1862 | continue; |
1863 | |
1864 | SDValue Vec; |
1865 | if (Phase) |
1866 | Vec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: V); |
1867 | IntermedVals.push_back(Elt: std::make_pair(x&: Vec, y: SmallVector<int, 16>(1, i))); |
1868 | } |
1869 | |
1870 | while (IntermedVals.size() > 2) { |
1871 | NewIntermedVals.clear(); |
1872 | for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) { |
1873 | // This vector and the next vector are shuffled together (simply to |
1874 | // append the one to the other). |
1875 | SmallVector<int, 16> ShuffleVec(NumElems, -1); |
1876 | |
1877 | SmallVector<int, 16> FinalIndices; |
1878 | FinalIndices.reserve(N: IntermedVals[i].second.size() + |
1879 | IntermedVals[i+1].second.size()); |
1880 | |
1881 | int k = 0; |
1882 | for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f; |
1883 | ++j, ++k) { |
1884 | ShuffleVec[k] = j; |
1885 | FinalIndices.push_back(Elt: IntermedVals[i].second[j]); |
1886 | } |
1887 | for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f; |
1888 | ++j, ++k) { |
1889 | ShuffleVec[k] = NumElems + j; |
1890 | FinalIndices.push_back(Elt: IntermedVals[i+1].second[j]); |
1891 | } |
1892 | |
1893 | SDValue Shuffle; |
1894 | if (Phase) |
1895 | Shuffle = DAG.getVectorShuffle(VT, dl, N1: IntermedVals[i].first, |
1896 | N2: IntermedVals[i+1].first, |
1897 | Mask: ShuffleVec); |
1898 | else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT)) |
1899 | return false; |
1900 | NewIntermedVals.push_back( |
1901 | Elt: std::make_pair(x&: Shuffle, y: std::move(FinalIndices))); |
1902 | } |
1903 | |
1904 | // If we had an odd number of defined values, then append the last |
1905 | // element to the array of new vectors. |
1906 | if ((IntermedVals.size() & 1) != 0) |
1907 | NewIntermedVals.push_back(Elt: IntermedVals.back()); |
1908 | |
1909 | IntermedVals.swap(RHS&: NewIntermedVals); |
1910 | } |
1911 | |
1912 | assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 && |
1913 | "Invalid number of intermediate vectors" ); |
1914 | SDValue Vec1 = IntermedVals[0].first; |
1915 | SDValue Vec2; |
1916 | if (IntermedVals.size() > 1) |
1917 | Vec2 = IntermedVals[1].first; |
1918 | else if (Phase) |
1919 | Vec2 = DAG.getUNDEF(VT); |
1920 | |
1921 | SmallVector<int, 16> ShuffleVec(NumElems, -1); |
1922 | for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i) |
1923 | ShuffleVec[IntermedVals[0].second[i]] = i; |
1924 | for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i) |
1925 | ShuffleVec[IntermedVals[1].second[i]] = NumElems + i; |
1926 | |
1927 | if (Phase) |
1928 | Res = DAG.getVectorShuffle(VT, dl, N1: Vec1, N2: Vec2, Mask: ShuffleVec); |
1929 | else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT)) |
1930 | return false; |
1931 | } |
1932 | |
1933 | return true; |
1934 | } |
1935 | |
1936 | /// Expand a BUILD_VECTOR node on targets that don't |
1937 | /// support the operation, but do support the resultant vector type. |
1938 | SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { |
1939 | unsigned NumElems = Node->getNumOperands(); |
1940 | SDValue Value1, Value2; |
1941 | SDLoc dl(Node); |
1942 | EVT VT = Node->getValueType(ResNo: 0); |
1943 | EVT OpVT = Node->getOperand(Num: 0).getValueType(); |
1944 | EVT EltVT = VT.getVectorElementType(); |
1945 | |
1946 | // If the only non-undef value is the low element, turn this into a |
1947 | // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X. |
1948 | bool isOnlyLowElement = true; |
1949 | bool MoreThanTwoValues = false; |
1950 | bool isConstant = true; |
1951 | for (unsigned i = 0; i < NumElems; ++i) { |
1952 | SDValue V = Node->getOperand(Num: i); |
1953 | if (V.isUndef()) |
1954 | continue; |
1955 | if (i > 0) |
1956 | isOnlyLowElement = false; |
1957 | if (!isa<ConstantFPSDNode>(Val: V) && !isa<ConstantSDNode>(Val: V)) |
1958 | isConstant = false; |
1959 | |
1960 | if (!Value1.getNode()) { |
1961 | Value1 = V; |
1962 | } else if (!Value2.getNode()) { |
1963 | if (V != Value1) |
1964 | Value2 = V; |
1965 | } else if (V != Value1 && V != Value2) { |
1966 | MoreThanTwoValues = true; |
1967 | } |
1968 | } |
1969 | |
1970 | if (!Value1.getNode()) |
1971 | return DAG.getUNDEF(VT); |
1972 | |
1973 | if (isOnlyLowElement) |
1974 | return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Node->getOperand(Num: 0)); |
1975 | |
1976 | // If all elements are constants, create a load from the constant pool. |
1977 | if (isConstant) { |
1978 | SmallVector<Constant*, 16> CV; |
1979 | for (unsigned i = 0, e = NumElems; i != e; ++i) { |
1980 | if (ConstantFPSDNode *V = |
1981 | dyn_cast<ConstantFPSDNode>(Val: Node->getOperand(Num: i))) { |
1982 | CV.push_back(Elt: const_cast<ConstantFP *>(V->getConstantFPValue())); |
1983 | } else if (ConstantSDNode *V = |
1984 | dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: i))) { |
1985 | if (OpVT==EltVT) |
1986 | CV.push_back(Elt: const_cast<ConstantInt *>(V->getConstantIntValue())); |
1987 | else { |
1988 | // If OpVT and EltVT don't match, EltVT is not legal and the |
1989 | // element values have been promoted/truncated earlier. Undo this; |
1990 | // we don't want a v16i8 to become a v16i32 for example. |
1991 | const ConstantInt *CI = V->getConstantIntValue(); |
1992 | CV.push_back(Elt: ConstantInt::get(Ty: EltVT.getTypeForEVT(Context&: *DAG.getContext()), |
1993 | V: CI->getZExtValue())); |
1994 | } |
1995 | } else { |
1996 | assert(Node->getOperand(i).isUndef()); |
1997 | Type *OpNTy = EltVT.getTypeForEVT(Context&: *DAG.getContext()); |
1998 | CV.push_back(Elt: UndefValue::get(T: OpNTy)); |
1999 | } |
2000 | } |
2001 | Constant *CP = ConstantVector::get(V: CV); |
2002 | SDValue CPIdx = |
2003 | DAG.getConstantPool(C: CP, VT: TLI.getPointerTy(DL: DAG.getDataLayout())); |
2004 | Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign(); |
2005 | return DAG.getLoad( |
2006 | VT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx, |
2007 | PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), |
2008 | Alignment); |
2009 | } |
2010 | |
2011 | SmallSet<SDValue, 16> DefinedValues; |
2012 | for (unsigned i = 0; i < NumElems; ++i) { |
2013 | if (Node->getOperand(Num: i).isUndef()) |
2014 | continue; |
2015 | DefinedValues.insert(V: Node->getOperand(Num: i)); |
2016 | } |
2017 | |
2018 | if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues: DefinedValues.size())) { |
2019 | if (!MoreThanTwoValues) { |
2020 | SmallVector<int, 8> ShuffleVec(NumElems, -1); |
2021 | for (unsigned i = 0; i < NumElems; ++i) { |
2022 | SDValue V = Node->getOperand(Num: i); |
2023 | if (V.isUndef()) |
2024 | continue; |
2025 | ShuffleVec[i] = V == Value1 ? 0 : NumElems; |
2026 | } |
2027 | if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(ResNo: 0))) { |
2028 | // Get the splatted value into the low element of a vector register. |
2029 | SDValue Vec1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value1); |
2030 | SDValue Vec2; |
2031 | if (Value2.getNode()) |
2032 | Vec2 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value2); |
2033 | else |
2034 | Vec2 = DAG.getUNDEF(VT); |
2035 | |
2036 | // Return shuffle(LowValVec, undef, <0,0,0,0>) |
2037 | return DAG.getVectorShuffle(VT, dl, N1: Vec1, N2: Vec2, Mask: ShuffleVec); |
2038 | } |
2039 | } else { |
2040 | SDValue Res; |
2041 | if (ExpandBVWithShuffles(Node, DAG, TLI, Res)) |
2042 | return Res; |
2043 | } |
2044 | } |
2045 | |
2046 | // Otherwise, we can't handle this case efficiently. |
2047 | return ExpandVectorBuildThroughStack(Node); |
2048 | } |
2049 | |
2050 | SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) { |
2051 | SDLoc DL(Node); |
2052 | EVT VT = Node->getValueType(ResNo: 0); |
2053 | SDValue SplatVal = Node->getOperand(Num: 0); |
2054 | |
2055 | return DAG.getSplatBuildVector(VT, DL, Op: SplatVal); |
2056 | } |
2057 | |
2058 | // Expand a node into a call to a libcall, returning the value as the first |
2059 | // result and the chain as the second. If the result value does not fit into a |
2060 | // register, return the lo part and set the hi part to the by-reg argument in |
2061 | // the first. If it does fit into a single register, return the result and |
2062 | // leave the Hi part unset. |
2063 | std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, |
2064 | TargetLowering::ArgListTy &&Args, |
2065 | bool isSigned) { |
2066 | SDValue Callee = DAG.getExternalSymbol(Sym: TLI.getLibcallName(Call: LC), |
2067 | VT: TLI.getPointerTy(DL: DAG.getDataLayout())); |
2068 | |
2069 | EVT RetVT = Node->getValueType(ResNo: 0); |
2070 | Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext()); |
2071 | |
2072 | // By default, the input chain to this libcall is the entry node of the |
2073 | // function. If the libcall is going to be emitted as a tail call then |
2074 | // TLI.isUsedByReturnOnly will change it to the right chain if the return |
2075 | // node which is being folded has a non-entry input chain. |
2076 | SDValue InChain = DAG.getEntryNode(); |
2077 | |
2078 | // isTailCall may be true since the callee does not reference caller stack |
2079 | // frame. Check if it's in the right position and that the return types match. |
2080 | SDValue TCChain = InChain; |
2081 | const Function &F = DAG.getMachineFunction().getFunction(); |
2082 | bool isTailCall = |
2083 | TLI.isInTailCallPosition(DAG, Node, Chain&: TCChain) && |
2084 | (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy()); |
2085 | if (isTailCall) |
2086 | InChain = TCChain; |
2087 | |
2088 | TargetLowering::CallLoweringInfo CLI(DAG); |
2089 | bool signExtend = TLI.shouldSignExtendTypeInLibCall(Type: RetVT, IsSigned: isSigned); |
2090 | CLI.setDebugLoc(SDLoc(Node)) |
2091 | .setChain(InChain) |
2092 | .setLibCallee(CC: TLI.getLibcallCallingConv(Call: LC), ResultType: RetTy, Target: Callee, |
2093 | ArgsList: std::move(Args)) |
2094 | .setTailCall(isTailCall) |
2095 | .setSExtResult(signExtend) |
2096 | .setZExtResult(!signExtend) |
2097 | .setIsPostTypeLegalization(true); |
2098 | |
2099 | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); |
2100 | |
2101 | if (!CallInfo.second.getNode()) { |
2102 | LLVM_DEBUG(dbgs() << "Created tailcall: " ; DAG.getRoot().dump(&DAG)); |
2103 | // It's a tailcall, return the chain (which is the DAG root). |
2104 | return {DAG.getRoot(), DAG.getRoot()}; |
2105 | } |
2106 | |
2107 | LLVM_DEBUG(dbgs() << "Created libcall: " ; CallInfo.first.dump(&DAG)); |
2108 | return CallInfo; |
2109 | } |
2110 | |
2111 | std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, |
2112 | bool isSigned) { |
2113 | TargetLowering::ArgListTy Args; |
2114 | TargetLowering::ArgListEntry Entry; |
2115 | for (const SDValue &Op : Node->op_values()) { |
2116 | EVT ArgVT = Op.getValueType(); |
2117 | Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext()); |
2118 | Entry.Node = Op; |
2119 | Entry.Ty = ArgTy; |
2120 | Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(Type: ArgVT, IsSigned: isSigned); |
2121 | Entry.IsZExt = !Entry.IsSExt; |
2122 | Args.push_back(x: Entry); |
2123 | } |
2124 | |
2125 | return ExpandLibCall(LC, Node, Args: std::move(Args), isSigned); |
2126 | } |
2127 | |
2128 | void SelectionDAGLegalize::ExpandFrexpLibCall( |
2129 | SDNode *Node, SmallVectorImpl<SDValue> &Results) { |
2130 | SDLoc dl(Node); |
2131 | EVT VT = Node->getValueType(ResNo: 0); |
2132 | EVT ExpVT = Node->getValueType(ResNo: 1); |
2133 | |
2134 | SDValue FPOp = Node->getOperand(Num: 0); |
2135 | |
2136 | EVT ArgVT = FPOp.getValueType(); |
2137 | Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext()); |
2138 | |
2139 | TargetLowering::ArgListEntry FPArgEntry; |
2140 | FPArgEntry.Node = FPOp; |
2141 | FPArgEntry.Ty = ArgTy; |
2142 | |
2143 | SDValue StackSlot = DAG.CreateStackTemporary(VT: ExpVT); |
2144 | TargetLowering::ArgListEntry PtrArgEntry; |
2145 | PtrArgEntry.Node = StackSlot; |
2146 | PtrArgEntry.Ty = PointerType::get(C&: *DAG.getContext(), |
2147 | AddressSpace: DAG.getDataLayout().getAllocaAddrSpace()); |
2148 | |
2149 | TargetLowering::ArgListTy Args = {FPArgEntry, PtrArgEntry}; |
2150 | |
2151 | RTLIB::Libcall LC = RTLIB::getFREXP(RetVT: VT); |
2152 | auto [Call, Chain] = ExpandLibCall(LC, Node, Args: std::move(Args), isSigned: false); |
2153 | |
2154 | // FIXME: Get type of int for libcall declaration and cast |
2155 | |
2156 | int FrameIdx = cast<FrameIndexSDNode>(Val&: StackSlot)->getIndex(); |
2157 | auto PtrInfo = |
2158 | MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx); |
2159 | |
2160 | SDValue LoadExp = DAG.getLoad(VT: ExpVT, dl, Chain, Ptr: StackSlot, PtrInfo); |
2161 | SDValue OutputChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, |
2162 | LoadExp.getValue(R: 1), DAG.getRoot()); |
2163 | DAG.setRoot(OutputChain); |
2164 | |
2165 | Results.push_back(Elt: Call); |
2166 | Results.push_back(Elt: LoadExp); |
2167 | } |
2168 | |
2169 | void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, |
2170 | RTLIB::Libcall LC, |
2171 | SmallVectorImpl<SDValue> &Results) { |
2172 | if (LC == RTLIB::UNKNOWN_LIBCALL) |
2173 | llvm_unreachable("Can't create an unknown libcall!" ); |
2174 | |
2175 | if (Node->isStrictFPOpcode()) { |
2176 | EVT RetVT = Node->getValueType(ResNo: 0); |
2177 | SmallVector<SDValue, 4> Ops(drop_begin(RangeOrContainer: Node->ops())); |
2178 | TargetLowering::MakeLibCallOptions CallOptions; |
2179 | // FIXME: This doesn't support tail calls. |
2180 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT, |
2181 | Ops, CallOptions, |
2182 | dl: SDLoc(Node), |
2183 | Chain: Node->getOperand(Num: 0)); |
2184 | Results.push_back(Elt: Tmp.first); |
2185 | Results.push_back(Elt: Tmp.second); |
2186 | } else { |
2187 | SDValue Tmp = ExpandLibCall(LC, Node, isSigned: false).first; |
2188 | Results.push_back(Elt: Tmp); |
2189 | } |
2190 | } |
2191 | |
2192 | /// Expand the node to a libcall based on the result type. |
2193 | void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, |
2194 | RTLIB::Libcall Call_F32, |
2195 | RTLIB::Libcall Call_F64, |
2196 | RTLIB::Libcall Call_F80, |
2197 | RTLIB::Libcall Call_F128, |
2198 | RTLIB::Libcall Call_PPCF128, |
2199 | SmallVectorImpl<SDValue> &Results) { |
2200 | RTLIB::Libcall LC = RTLIB::getFPLibCall(VT: Node->getSimpleValueType(ResNo: 0), |
2201 | Call_F32, Call_F64, Call_F80, |
2202 | Call_F128, Call_PPCF128); |
2203 | ExpandFPLibCall(Node, LC, Results); |
2204 | } |
2205 | |
2206 | SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned, |
2207 | RTLIB::Libcall Call_I8, |
2208 | RTLIB::Libcall Call_I16, |
2209 | RTLIB::Libcall Call_I32, |
2210 | RTLIB::Libcall Call_I64, |
2211 | RTLIB::Libcall Call_I128) { |
2212 | RTLIB::Libcall LC; |
2213 | switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) { |
2214 | default: llvm_unreachable("Unexpected request for libcall!" ); |
2215 | case MVT::i8: LC = Call_I8; break; |
2216 | case MVT::i16: LC = Call_I16; break; |
2217 | case MVT::i32: LC = Call_I32; break; |
2218 | case MVT::i64: LC = Call_I64; break; |
2219 | case MVT::i128: LC = Call_I128; break; |
2220 | } |
2221 | return ExpandLibCall(LC, Node, isSigned).first; |
2222 | } |
2223 | |
2224 | /// Expand the node to a libcall based on first argument type (for instance |
2225 | /// lround and its variant). |
2226 | void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node, |
2227 | RTLIB::Libcall Call_F32, |
2228 | RTLIB::Libcall Call_F64, |
2229 | RTLIB::Libcall Call_F80, |
2230 | RTLIB::Libcall Call_F128, |
2231 | RTLIB::Libcall Call_PPCF128, |
2232 | SmallVectorImpl<SDValue> &Results) { |
2233 | EVT InVT = Node->getOperand(Num: Node->isStrictFPOpcode() ? 1 : 0).getValueType(); |
2234 | RTLIB::Libcall LC = RTLIB::getFPLibCall(VT: InVT.getSimpleVT(), |
2235 | Call_F32, Call_F64, Call_F80, |
2236 | Call_F128, Call_PPCF128); |
2237 | ExpandFPLibCall(Node, LC, Results); |
2238 | } |
2239 | |
2240 | /// Issue libcalls to __{u}divmod to compute div / rem pairs. |
2241 | void |
2242 | SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node, |
2243 | SmallVectorImpl<SDValue> &Results) { |
2244 | unsigned Opcode = Node->getOpcode(); |
2245 | bool isSigned = Opcode == ISD::SDIVREM; |
2246 | |
2247 | RTLIB::Libcall LC; |
2248 | switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) { |
2249 | default: llvm_unreachable("Unexpected request for libcall!" ); |
2250 | case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break; |
2251 | case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break; |
2252 | case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break; |
2253 | case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break; |
2254 | case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break; |
2255 | } |
2256 | |
2257 | // The input chain to this libcall is the entry node of the function. |
2258 | // Legalizing the call will automatically add the previous call to the |
2259 | // dependence. |
2260 | SDValue InChain = DAG.getEntryNode(); |
2261 | |
2262 | EVT RetVT = Node->getValueType(ResNo: 0); |
2263 | Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext()); |
2264 | |
2265 | TargetLowering::ArgListTy Args; |
2266 | TargetLowering::ArgListEntry Entry; |
2267 | for (const SDValue &Op : Node->op_values()) { |
2268 | EVT ArgVT = Op.getValueType(); |
2269 | Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext()); |
2270 | Entry.Node = Op; |
2271 | Entry.Ty = ArgTy; |
2272 | Entry.IsSExt = isSigned; |
2273 | Entry.IsZExt = !isSigned; |
2274 | Args.push_back(x: Entry); |
2275 | } |
2276 | |
2277 | // Also pass the return address of the remainder. |
2278 | SDValue FIPtr = DAG.CreateStackTemporary(VT: RetVT); |
2279 | Entry.Node = FIPtr; |
2280 | Entry.Ty = PointerType::getUnqual(C&: RetTy->getContext()); |
2281 | Entry.IsSExt = isSigned; |
2282 | Entry.IsZExt = !isSigned; |
2283 | Args.push_back(x: Entry); |
2284 | |
2285 | SDValue Callee = DAG.getExternalSymbol(Sym: TLI.getLibcallName(Call: LC), |
2286 | VT: TLI.getPointerTy(DL: DAG.getDataLayout())); |
2287 | |
2288 | SDLoc dl(Node); |
2289 | TargetLowering::CallLoweringInfo CLI(DAG); |
2290 | CLI.setDebugLoc(dl) |
2291 | .setChain(InChain) |
2292 | .setLibCallee(CC: TLI.getLibcallCallingConv(Call: LC), ResultType: RetTy, Target: Callee, |
2293 | ArgsList: std::move(Args)) |
2294 | .setSExtResult(isSigned) |
2295 | .setZExtResult(!isSigned); |
2296 | |
2297 | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); |
2298 | |
2299 | // Remainder is loaded back from the stack frame. |
2300 | SDValue Rem = |
2301 | DAG.getLoad(VT: RetVT, dl, Chain: CallInfo.second, Ptr: FIPtr, PtrInfo: MachinePointerInfo()); |
2302 | Results.push_back(Elt: CallInfo.first); |
2303 | Results.push_back(Elt: Rem); |
2304 | } |
2305 | |
2306 | /// Return true if sincos libcall is available. |
2307 | static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) { |
2308 | RTLIB::Libcall LC; |
2309 | switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) { |
2310 | default: llvm_unreachable("Unexpected request for libcall!" ); |
2311 | case MVT::f32: LC = RTLIB::SINCOS_F32; break; |
2312 | case MVT::f64: LC = RTLIB::SINCOS_F64; break; |
2313 | case MVT::f80: LC = RTLIB::SINCOS_F80; break; |
2314 | case MVT::f128: LC = RTLIB::SINCOS_F128; break; |
2315 | case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break; |
2316 | } |
2317 | return TLI.getLibcallName(Call: LC) != nullptr; |
2318 | } |
2319 | |
2320 | /// Only issue sincos libcall if both sin and cos are needed. |
2321 | static bool useSinCos(SDNode *Node) { |
2322 | unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN |
2323 | ? ISD::FCOS : ISD::FSIN; |
2324 | |
2325 | SDValue Op0 = Node->getOperand(Num: 0); |
2326 | for (const SDNode *User : Op0.getNode()->uses()) { |
2327 | if (User == Node) |
2328 | continue; |
2329 | // The other user might have been turned into sincos already. |
2330 | if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS) |
2331 | return true; |
2332 | } |
2333 | return false; |
2334 | } |
2335 | |
2336 | /// Issue libcalls to sincos to compute sin / cos pairs. |
2337 | void |
2338 | SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node, |
2339 | SmallVectorImpl<SDValue> &Results) { |
2340 | RTLIB::Libcall LC; |
2341 | switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) { |
2342 | default: llvm_unreachable("Unexpected request for libcall!" ); |
2343 | case MVT::f32: LC = RTLIB::SINCOS_F32; break; |
2344 | case MVT::f64: LC = RTLIB::SINCOS_F64; break; |
2345 | case MVT::f80: LC = RTLIB::SINCOS_F80; break; |
2346 | case MVT::f128: LC = RTLIB::SINCOS_F128; break; |
2347 | case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break; |
2348 | } |
2349 | |
2350 | // The input chain to this libcall is the entry node of the function. |
2351 | // Legalizing the call will automatically add the previous call to the |
2352 | // dependence. |
2353 | SDValue InChain = DAG.getEntryNode(); |
2354 | |
2355 | EVT RetVT = Node->getValueType(ResNo: 0); |
2356 | Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext()); |
2357 | |
2358 | TargetLowering::ArgListTy Args; |
2359 | TargetLowering::ArgListEntry Entry; |
2360 | |
2361 | // Pass the argument. |
2362 | Entry.Node = Node->getOperand(Num: 0); |
2363 | Entry.Ty = RetTy; |
2364 | Entry.IsSExt = false; |
2365 | Entry.IsZExt = false; |
2366 | Args.push_back(x: Entry); |
2367 | |
2368 | // Pass the return address of sin. |
2369 | SDValue SinPtr = DAG.CreateStackTemporary(VT: RetVT); |
2370 | Entry.Node = SinPtr; |
2371 | Entry.Ty = PointerType::getUnqual(C&: RetTy->getContext()); |
2372 | Entry.IsSExt = false; |
2373 | Entry.IsZExt = false; |
2374 | Args.push_back(x: Entry); |
2375 | |
2376 | // Also pass the return address of the cos. |
2377 | SDValue CosPtr = DAG.CreateStackTemporary(VT: RetVT); |
2378 | Entry.Node = CosPtr; |
2379 | Entry.Ty = PointerType::getUnqual(C&: RetTy->getContext()); |
2380 | Entry.IsSExt = false; |
2381 | Entry.IsZExt = false; |
2382 | Args.push_back(x: Entry); |
2383 | |
2384 | SDValue Callee = DAG.getExternalSymbol(Sym: TLI.getLibcallName(Call: LC), |
2385 | VT: TLI.getPointerTy(DL: DAG.getDataLayout())); |
2386 | |
2387 | SDLoc dl(Node); |
2388 | TargetLowering::CallLoweringInfo CLI(DAG); |
2389 | CLI.setDebugLoc(dl).setChain(InChain).setLibCallee( |
2390 | CC: TLI.getLibcallCallingConv(Call: LC), ResultType: Type::getVoidTy(C&: *DAG.getContext()), Target: Callee, |
2391 | ArgsList: std::move(Args)); |
2392 | |
2393 | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); |
2394 | |
2395 | Results.push_back( |
2396 | Elt: DAG.getLoad(VT: RetVT, dl, Chain: CallInfo.second, Ptr: SinPtr, PtrInfo: MachinePointerInfo())); |
2397 | Results.push_back( |
2398 | Elt: DAG.getLoad(VT: RetVT, dl, Chain: CallInfo.second, Ptr: CosPtr, PtrInfo: MachinePointerInfo())); |
2399 | } |
2400 | |
2401 | SDValue SelectionDAGLegalize::expandLdexp(SDNode *Node) const { |
2402 | SDLoc dl(Node); |
2403 | EVT VT = Node->getValueType(ResNo: 0); |
2404 | SDValue X = Node->getOperand(Num: 0); |
2405 | SDValue N = Node->getOperand(Num: 1); |
2406 | EVT ExpVT = N.getValueType(); |
2407 | EVT AsIntVT = VT.changeTypeToInteger(); |
2408 | if (AsIntVT == EVT()) // TODO: How to handle f80? |
2409 | return SDValue(); |
2410 | |
2411 | if (Node->getOpcode() == ISD::STRICT_FLDEXP) // TODO |
2412 | return SDValue(); |
2413 | |
2414 | SDNodeFlags NSW; |
2415 | NSW.setNoSignedWrap(true); |
2416 | SDNodeFlags NUW_NSW; |
2417 | NUW_NSW.setNoUnsignedWrap(true); |
2418 | NUW_NSW.setNoSignedWrap(true); |
2419 | |
2420 | EVT SetCCVT = |
2421 | TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: ExpVT); |
2422 | const fltSemantics &FltSem = SelectionDAG::EVTToAPFloatSemantics(VT); |
2423 | |
2424 | const APFloat::ExponentType MaxExpVal = APFloat::semanticsMaxExponent(FltSem); |
2425 | const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem); |
2426 | const int Precision = APFloat::semanticsPrecision(FltSem); |
2427 | |
2428 | const SDValue MaxExp = DAG.getConstant(Val: MaxExpVal, DL: dl, VT: ExpVT); |
2429 | const SDValue MinExp = DAG.getConstant(Val: MinExpVal, DL: dl, VT: ExpVT); |
2430 | |
2431 | const SDValue DoubleMaxExp = DAG.getConstant(Val: 2 * MaxExpVal, DL: dl, VT: ExpVT); |
2432 | |
2433 | const APFloat One(FltSem, "1.0" ); |
2434 | APFloat ScaleUpK = scalbn(X: One, Exp: MaxExpVal, RM: APFloat::rmNearestTiesToEven); |
2435 | |
2436 | // Offset by precision to avoid denormal range. |
2437 | APFloat ScaleDownK = |
2438 | scalbn(X: One, Exp: MinExpVal + Precision, RM: APFloat::rmNearestTiesToEven); |
2439 | |
2440 | // TODO: Should really introduce control flow and use a block for the > |
2441 | // MaxExp, < MinExp cases |
2442 | |
2443 | // First, handle exponents Exp > MaxExp and scale down. |
2444 | SDValue NGtMaxExp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: MaxExp, Cond: ISD::SETGT); |
2445 | |
2446 | SDValue DecN0 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ExpVT, N1: N, N2: MaxExp, Flags: NSW); |
2447 | SDValue ClampMaxVal = DAG.getConstant(Val: 3 * MaxExpVal, DL: dl, VT: ExpVT); |
2448 | SDValue ClampN_Big = DAG.getNode(Opcode: ISD::SMIN, DL: dl, VT: ExpVT, N1: N, N2: ClampMaxVal); |
2449 | SDValue DecN1 = |
2450 | DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ExpVT, N1: ClampN_Big, N2: DoubleMaxExp, Flags: NSW); |
2451 | |
2452 | SDValue ScaleUpTwice = |
2453 | DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: DoubleMaxExp, Cond: ISD::SETUGT); |
2454 | |
2455 | const SDValue ScaleUpVal = DAG.getConstantFP(Val: ScaleUpK, DL: dl, VT); |
2456 | SDValue ScaleUp0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: X, N2: ScaleUpVal); |
2457 | SDValue ScaleUp1 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: ScaleUp0, N2: ScaleUpVal); |
2458 | |
2459 | SDValue SelectN_Big = |
2460 | DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: ScaleUpTwice, N2: DecN1, N3: DecN0); |
2461 | SDValue SelectX_Big = |
2462 | DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: ScaleUpTwice, N2: ScaleUp1, N3: ScaleUp0); |
2463 | |
2464 | // Now handle exponents Exp < MinExp |
2465 | SDValue NLtMinExp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: MinExp, Cond: ISD::SETLT); |
2466 | |
2467 | SDValue Increment0 = DAG.getConstant(Val: -(MinExpVal + Precision), DL: dl, VT: ExpVT); |
2468 | SDValue Increment1 = DAG.getConstant(Val: -2 * (MinExpVal + Precision), DL: dl, VT: ExpVT); |
2469 | |
2470 | SDValue IncN0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: N, N2: Increment0, Flags: NUW_NSW); |
2471 | |
2472 | SDValue ClampMinVal = |
2473 | DAG.getConstant(Val: 3 * MinExpVal + 2 * Precision, DL: dl, VT: ExpVT); |
2474 | SDValue ClampN_Small = DAG.getNode(Opcode: ISD::SMAX, DL: dl, VT: ExpVT, N1: N, N2: ClampMinVal); |
2475 | SDValue IncN1 = |
2476 | DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: ClampN_Small, N2: Increment1, Flags: NSW); |
2477 | |
2478 | const SDValue ScaleDownVal = DAG.getConstantFP(Val: ScaleDownK, DL: dl, VT); |
2479 | SDValue ScaleDown0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: X, N2: ScaleDownVal); |
2480 | SDValue ScaleDown1 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: ScaleDown0, N2: ScaleDownVal); |
2481 | |
2482 | SDValue ScaleDownTwice = DAG.getSetCC( |
2483 | DL: dl, VT: SetCCVT, LHS: N, RHS: DAG.getConstant(Val: 2 * MinExpVal + Precision, DL: dl, VT: ExpVT), |
2484 | Cond: ISD::SETULT); |
2485 | |
2486 | SDValue SelectN_Small = |
2487 | DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: ScaleDownTwice, N2: IncN1, N3: IncN0); |
2488 | SDValue SelectX_Small = |
2489 | DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: ScaleDownTwice, N2: ScaleDown1, N3: ScaleDown0); |
2490 | |
2491 | // Now combine the two out of range exponent handling cases with the base |
2492 | // case. |
2493 | SDValue NewX = DAG.getNode( |
2494 | Opcode: ISD::SELECT, DL: dl, VT, N1: NGtMaxExp, N2: SelectX_Big, |
2495 | N3: DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: NLtMinExp, N2: SelectX_Small, N3: X)); |
2496 | |
2497 | SDValue NewN = DAG.getNode( |
2498 | Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: NGtMaxExp, N2: SelectN_Big, |
2499 | N3: DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: NLtMinExp, N2: SelectN_Small, N3: N)); |
2500 | |
2501 | SDValue BiasedN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: NewN, N2: MaxExp, Flags: NSW); |
2502 | |
2503 | SDValue ExponentShiftAmt = |
2504 | DAG.getShiftAmountConstant(Val: Precision - 1, VT: ExpVT, DL: dl); |
2505 | SDValue CastExpToValTy = DAG.getZExtOrTrunc(Op: BiasedN, DL: dl, VT: AsIntVT); |
2506 | |
2507 | SDValue AsInt = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: AsIntVT, N1: CastExpToValTy, |
2508 | N2: ExponentShiftAmt, Flags: NUW_NSW); |
2509 | SDValue AsFP = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: AsInt); |
2510 | return DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: NewX, N2: AsFP); |
2511 | } |
2512 | |
2513 | SDValue SelectionDAGLegalize::expandFrexp(SDNode *Node) const { |
2514 | SDLoc dl(Node); |
2515 | SDValue Val = Node->getOperand(Num: 0); |
2516 | EVT VT = Val.getValueType(); |
2517 | EVT ExpVT = Node->getValueType(ResNo: 1); |
2518 | EVT AsIntVT = VT.changeTypeToInteger(); |
2519 | if (AsIntVT == EVT()) // TODO: How to handle f80? |
2520 | return SDValue(); |
2521 | |
2522 | const fltSemantics &FltSem = SelectionDAG::EVTToAPFloatSemantics(VT); |
2523 | const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem); |
2524 | const unsigned Precision = APFloat::semanticsPrecision(FltSem); |
2525 | const unsigned BitSize = VT.getScalarSizeInBits(); |
2526 | |
2527 | // TODO: Could introduce control flow and skip over the denormal handling. |
2528 | |
2529 | // scale_up = fmul value, scalbn(1.0, precision + 1) |
2530 | // extracted_exp = (bitcast value to uint) >> precision - 1 |
2531 | // biased_exp = extracted_exp + min_exp |
2532 | // extracted_fract = (bitcast value to uint) & (fract_mask | sign_mask) |
2533 | // |
2534 | // is_denormal = val < smallest_normalized |
2535 | // computed_fract = is_denormal ? scale_up : extracted_fract |
2536 | // computed_exp = is_denormal ? biased_exp + (-precision - 1) : biased_exp |
2537 | // |
2538 | // result_0 = (!isfinite(val) || iszero(val)) ? val : computed_fract |
2539 | // result_1 = (!isfinite(val) || iszero(val)) ? 0 : computed_exp |
2540 | |
2541 | SDValue NegSmallestNormalizedInt = DAG.getConstant( |
2542 | Val: APFloat::getSmallestNormalized(Sem: FltSem, Negative: true).bitcastToAPInt(), DL: dl, |
2543 | VT: AsIntVT); |
2544 | |
2545 | SDValue SmallestNormalizedInt = DAG.getConstant( |
2546 | Val: APFloat::getSmallestNormalized(Sem: FltSem, Negative: false).bitcastToAPInt(), DL: dl, |
2547 | VT: AsIntVT); |
2548 | |
2549 | // Masks out the exponent bits. |
2550 | SDValue ExpMask = |
2551 | DAG.getConstant(Val: APFloat::getInf(Sem: FltSem).bitcastToAPInt(), DL: dl, VT: AsIntVT); |
2552 | |
2553 | // Mask out the exponent part of the value. |
2554 | // |
2555 | // e.g, for f32 FractSignMaskVal = 0x807fffff |
2556 | APInt FractSignMaskVal = APInt::getBitsSet(numBits: BitSize, loBit: 0, hiBit: Precision - 1); |
2557 | FractSignMaskVal.setBit(BitSize - 1); // Set the sign bit |
2558 | |
2559 | APInt SignMaskVal = APInt::getSignedMaxValue(numBits: BitSize); |
2560 | SDValue SignMask = DAG.getConstant(Val: SignMaskVal, DL: dl, VT: AsIntVT); |
2561 | |
2562 | SDValue FractSignMask = DAG.getConstant(Val: FractSignMaskVal, DL: dl, VT: AsIntVT); |
2563 | |
2564 | const APFloat One(FltSem, "1.0" ); |
2565 | // Scale a possible denormal input. |
2566 | // e.g., for f64, 0x1p+54 |
2567 | APFloat ScaleUpKVal = |
2568 | scalbn(X: One, Exp: Precision + 1, RM: APFloat::rmNearestTiesToEven); |
2569 | |
2570 | SDValue ScaleUpK = DAG.getConstantFP(Val: ScaleUpKVal, DL: dl, VT); |
2571 | SDValue ScaleUp = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: Val, N2: ScaleUpK); |
2572 | |
2573 | EVT SetCCVT = |
2574 | TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT); |
2575 | |
2576 | SDValue AsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AsIntVT, Operand: Val); |
2577 | |
2578 | SDValue Abs = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: AsInt, N2: SignMask); |
2579 | |
2580 | SDValue AddNegSmallestNormal = |
2581 | DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AsIntVT, N1: Abs, N2: NegSmallestNormalizedInt); |
2582 | SDValue DenormOrZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AddNegSmallestNormal, |
2583 | RHS: NegSmallestNormalizedInt, Cond: ISD::SETULE); |
2584 | |
2585 | SDValue IsDenormal = |
2586 | DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Abs, RHS: SmallestNormalizedInt, Cond: ISD::SETULT); |
2587 | |
2588 | SDValue MinExp = DAG.getConstant(Val: MinExpVal, DL: dl, VT: ExpVT); |
2589 | SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: ExpVT); |
2590 | |
2591 | SDValue ScaledAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AsIntVT, Operand: ScaleUp); |
2592 | SDValue ScaledSelect = |
2593 | DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: AsIntVT, N1: IsDenormal, N2: ScaledAsInt, N3: AsInt); |
2594 | |
2595 | SDValue ExpMaskScaled = |
2596 | DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: ScaledAsInt, N2: ExpMask); |
2597 | |
2598 | SDValue ScaledValue = |
2599 | DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: AsIntVT, N1: IsDenormal, N2: ExpMaskScaled, N3: Abs); |
2600 | |
2601 | // Extract the exponent bits. |
2602 | SDValue ExponentShiftAmt = |
2603 | DAG.getShiftAmountConstant(Val: Precision - 1, VT: AsIntVT, DL: dl); |
2604 | SDValue ShiftedExp = |
2605 | DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: AsIntVT, N1: ScaledValue, N2: ExponentShiftAmt); |
2606 | SDValue Exp = DAG.getSExtOrTrunc(Op: ShiftedExp, DL: dl, VT: ExpVT); |
2607 | |
2608 | SDValue NormalBiasedExp = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: Exp, N2: MinExp); |
2609 | SDValue DenormalOffset = DAG.getConstant(Val: -Precision - 1, DL: dl, VT: ExpVT); |
2610 | SDValue DenormalExpBias = |
2611 | DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: IsDenormal, N2: DenormalOffset, N3: Zero); |
2612 | |
2613 | SDValue MaskedFractAsInt = |
2614 | DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: ScaledSelect, N2: FractSignMask); |
2615 | const APFloat Half(FltSem, "0.5" ); |
2616 | SDValue FPHalf = DAG.getConstant(Val: Half.bitcastToAPInt(), DL: dl, VT: AsIntVT); |
2617 | SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: AsIntVT, N1: MaskedFractAsInt, N2: FPHalf); |
2618 | SDValue MaskedFract = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Or); |
2619 | |
2620 | SDValue ComputedExp = |
2621 | DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: NormalBiasedExp, N2: DenormalExpBias); |
2622 | |
2623 | SDValue Result0 = |
2624 | DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: DenormOrZero, N2: Val, N3: MaskedFract); |
2625 | |
2626 | SDValue Result1 = |
2627 | DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: DenormOrZero, N2: Zero, N3: ComputedExp); |
2628 | |
2629 | return DAG.getMergeValues(Ops: {Result0, Result1}, dl); |
2630 | } |
2631 | |
2632 | /// This function is responsible for legalizing a |
2633 | /// INT_TO_FP operation of the specified operand when the target requests that |
2634 | /// we expand it. At this point, we know that the result and operand types are |
2635 | /// legal for the target. |
2636 | SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node, |
2637 | SDValue &Chain) { |
2638 | bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP || |
2639 | Node->getOpcode() == ISD::SINT_TO_FP); |
2640 | EVT DestVT = Node->getValueType(ResNo: 0); |
2641 | SDLoc dl(Node); |
2642 | unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0; |
2643 | SDValue Op0 = Node->getOperand(Num: OpNo); |
2644 | EVT SrcVT = Op0.getValueType(); |
2645 | |
2646 | // TODO: Should any fast-math-flags be set for the created nodes? |
2647 | LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n" ); |
2648 | if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) && |
2649 | (DestVT.bitsLE(MVT::f64) || |
2650 | TLI.isOperationLegal(Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND |
2651 | : ISD::FP_EXTEND, |
2652 | DestVT))) { |
2653 | LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double " |
2654 | "expansion\n" ); |
2655 | |
2656 | // Get the stack frame index of a 8 byte buffer. |
2657 | SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64); |
2658 | |
2659 | SDValue Lo = Op0; |
2660 | // if signed map to unsigned space |
2661 | if (isSigned) { |
2662 | // Invert sign bit (signed to unsigned mapping). |
2663 | Lo = DAG.getNode(ISD::XOR, dl, MVT::i32, Lo, |
2664 | DAG.getConstant(0x80000000u, dl, MVT::i32)); |
2665 | } |
2666 | // Initial hi portion of constructed double. |
2667 | SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32); |
2668 | |
2669 | // If this a big endian target, swap the lo and high data. |
2670 | if (DAG.getDataLayout().isBigEndian()) |
2671 | std::swap(a&: Lo, b&: Hi); |
2672 | |
2673 | SDValue MemChain = DAG.getEntryNode(); |
2674 | |
2675 | // Store the lo of the constructed double. |
2676 | SDValue Store1 = DAG.getStore(Chain: MemChain, dl, Val: Lo, Ptr: StackSlot, |
2677 | PtrInfo: MachinePointerInfo()); |
2678 | // Store the hi of the constructed double. |
2679 | SDValue HiPtr = |
2680 | DAG.getMemBasePlusOffset(Base: StackSlot, Offset: TypeSize::getFixed(ExactSize: 4), DL: dl); |
2681 | SDValue Store2 = |
2682 | DAG.getStore(Chain: MemChain, dl, Val: Hi, Ptr: HiPtr, PtrInfo: MachinePointerInfo()); |
2683 | MemChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2); |
2684 | |
2685 | // load the constructed double |
2686 | SDValue Load = |
2687 | DAG.getLoad(MVT::f64, dl, MemChain, StackSlot, MachinePointerInfo()); |
2688 | // FP constant to bias correct the final result |
2689 | SDValue Bias = DAG.getConstantFP( |
2690 | isSigned ? llvm::bit_cast<double>(0x4330000080000000ULL) |
2691 | : llvm::bit_cast<double>(0x4330000000000000ULL), |
2692 | dl, MVT::f64); |
2693 | // Subtract the bias and get the final result. |
2694 | SDValue Sub; |
2695 | SDValue Result; |
2696 | if (Node->isStrictFPOpcode()) { |
2697 | Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other}, |
2698 | {Node->getOperand(0), Load, Bias}); |
2699 | Chain = Sub.getValue(R: 1); |
2700 | if (DestVT != Sub.getValueType()) { |
2701 | std::pair<SDValue, SDValue> ResultPair; |
2702 | ResultPair = |
2703 | DAG.getStrictFPExtendOrRound(Op: Sub, Chain, DL: dl, VT: DestVT); |
2704 | Result = ResultPair.first; |
2705 | Chain = ResultPair.second; |
2706 | } |
2707 | else |
2708 | Result = Sub; |
2709 | } else { |
2710 | Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias); |
2711 | Result = DAG.getFPExtendOrRound(Op: Sub, DL: dl, VT: DestVT); |
2712 | } |
2713 | return Result; |
2714 | } |
2715 | |
2716 | if (isSigned) |
2717 | return SDValue(); |
2718 | |
2719 | // TODO: Generalize this for use with other types. |
2720 | if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) || |
2721 | (SrcVT == MVT::i64 && DestVT == MVT::f64)) { |
2722 | LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n" ); |
2723 | // For unsigned conversions, convert them to signed conversions using the |
2724 | // algorithm from the x86_64 __floatundisf in compiler_rt. That method |
2725 | // should be valid for i32->f32 as well. |
2726 | |
2727 | // More generally this transform should be valid if there are 3 more bits |
2728 | // in the integer type than the significand. Rounding uses the first bit |
2729 | // after the width of the significand and the OR of all bits after that. So |
2730 | // we need to be able to OR the shifted out bit into one of the bits that |
2731 | // participate in the OR. |
2732 | |
2733 | // TODO: This really should be implemented using a branch rather than a |
2734 | // select. We happen to get lucky and machinesink does the right |
2735 | // thing most of the time. This would be a good candidate for a |
2736 | // pseudo-op, or, even better, for whole-function isel. |
2737 | EVT SetCCVT = getSetCCResultType(VT: SrcVT); |
2738 | |
2739 | SDValue SignBitTest = DAG.getSetCC( |
2740 | DL: dl, VT: SetCCVT, LHS: Op0, RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT), Cond: ISD::SETLT); |
2741 | |
2742 | EVT ShiftVT = TLI.getShiftAmountTy(LHSTy: SrcVT, DL: DAG.getDataLayout()); |
2743 | SDValue ShiftConst = DAG.getConstant(Val: 1, DL: dl, VT: ShiftVT); |
2744 | SDValue Shr = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: SrcVT, N1: Op0, N2: ShiftConst); |
2745 | SDValue AndConst = DAG.getConstant(Val: 1, DL: dl, VT: SrcVT); |
2746 | SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SrcVT, N1: Op0, N2: AndConst); |
2747 | SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SrcVT, N1: And, N2: Shr); |
2748 | |
2749 | SDValue Slow, Fast; |
2750 | if (Node->isStrictFPOpcode()) { |
2751 | // In strict mode, we must avoid spurious exceptions, and therefore |
2752 | // must make sure to only emit a single STRICT_SINT_TO_FP. |
2753 | SDValue InCvt = DAG.getSelect(DL: dl, VT: SrcVT, Cond: SignBitTest, LHS: Or, RHS: Op0); |
2754 | Fast = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other }, |
2755 | { Node->getOperand(0), InCvt }); |
2756 | Slow = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other }, |
2757 | { Fast.getValue(1), Fast, Fast }); |
2758 | Chain = Slow.getValue(R: 1); |
2759 | // The STRICT_SINT_TO_FP inherits the exception mode from the |
2760 | // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can |
2761 | // never raise any exception. |
2762 | SDNodeFlags Flags; |
2763 | Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept()); |
2764 | Fast->setFlags(Flags); |
2765 | Flags.setNoFPExcept(true); |
2766 | Slow->setFlags(Flags); |
2767 | } else { |
2768 | SDValue SignCvt = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Or); |
2769 | Slow = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DestVT, N1: SignCvt, N2: SignCvt); |
2770 | Fast = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Op0); |
2771 | } |
2772 | |
2773 | return DAG.getSelect(DL: dl, VT: DestVT, Cond: SignBitTest, LHS: Slow, RHS: Fast); |
2774 | } |
2775 | |
2776 | // Don't expand it if there isn't cheap fadd. |
2777 | if (!TLI.isOperationLegalOrCustom( |
2778 | Op: Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, VT: DestVT)) |
2779 | return SDValue(); |
2780 | |
2781 | // The following optimization is valid only if every value in SrcVT (when |
2782 | // treated as signed) is representable in DestVT. Check that the mantissa |
2783 | // size of DestVT is >= than the number of bits in SrcVT -1. |
2784 | assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >= |
2785 | SrcVT.getSizeInBits() - 1 && |
2786 | "Cannot perform lossless SINT_TO_FP!" ); |
2787 | |
2788 | SDValue Tmp1; |
2789 | if (Node->isStrictFPOpcode()) { |
2790 | Tmp1 = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other }, |
2791 | { Node->getOperand(0), Op0 }); |
2792 | } else |
2793 | Tmp1 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Op0); |
2794 | |
2795 | SDValue SignSet = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: SrcVT), LHS: Op0, |
2796 | RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT), Cond: ISD::SETLT); |
2797 | SDValue Zero = DAG.getIntPtrConstant(Val: 0, DL: dl), |
2798 | Four = DAG.getIntPtrConstant(Val: 4, DL: dl); |
2799 | SDValue CstOffset = DAG.getSelect(DL: dl, VT: Zero.getValueType(), |
2800 | Cond: SignSet, LHS: Four, RHS: Zero); |
2801 | |
2802 | // If the sign bit of the integer is set, the large number will be treated |
2803 | // as a negative number. To counteract this, the dynamic code adds an |
2804 | // offset depending on the data type. |
2805 | uint64_t FF; |
2806 | switch (SrcVT.getSimpleVT().SimpleTy) { |
2807 | default: |
2808 | return SDValue(); |
2809 | case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float) |
2810 | case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float) |
2811 | case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float) |
2812 | case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float) |
2813 | } |
2814 | if (DAG.getDataLayout().isLittleEndian()) |
2815 | FF <<= 32; |
2816 | Constant *FudgeFactor = ConstantInt::get( |
2817 | Ty: Type::getInt64Ty(C&: *DAG.getContext()), V: FF); |
2818 | |
2819 | SDValue CPIdx = |
2820 | DAG.getConstantPool(C: FudgeFactor, VT: TLI.getPointerTy(DL: DAG.getDataLayout())); |
2821 | Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign(); |
2822 | CPIdx = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: CPIdx.getValueType(), N1: CPIdx, N2: CstOffset); |
2823 | Alignment = commonAlignment(A: Alignment, Offset: 4); |
2824 | SDValue FudgeInReg; |
2825 | if (DestVT == MVT::f32) |
2826 | FudgeInReg = DAG.getLoad( |
2827 | MVT::f32, dl, DAG.getEntryNode(), CPIdx, |
2828 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), |
2829 | Alignment); |
2830 | else { |
2831 | SDValue Load = DAG.getExtLoad( |
2832 | ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx, |
2833 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32, |
2834 | Alignment); |
2835 | HandleSDNode Handle(Load); |
2836 | LegalizeOp(Node: Load.getNode()); |
2837 | FudgeInReg = Handle.getValue(); |
2838 | } |
2839 | |
2840 | if (Node->isStrictFPOpcode()) { |
2841 | SDValue Result = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other }, |
2842 | { Tmp1.getValue(1), Tmp1, FudgeInReg }); |
2843 | Chain = Result.getValue(R: 1); |
2844 | return Result; |
2845 | } |
2846 | |
2847 | return DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DestVT, N1: Tmp1, N2: FudgeInReg); |
2848 | } |
2849 | |
2850 | /// This function is responsible for legalizing a |
2851 | /// *INT_TO_FP operation of the specified operand when the target requests that |
2852 | /// we promote it. At this point, we know that the result and operand types are |
2853 | /// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP |
2854 | /// operation that takes a larger input. |
2855 | void SelectionDAGLegalize::PromoteLegalINT_TO_FP( |
2856 | SDNode *N, const SDLoc &dl, SmallVectorImpl<SDValue> &Results) { |
2857 | bool IsStrict = N->isStrictFPOpcode(); |
2858 | bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP || |
2859 | N->getOpcode() == ISD::STRICT_SINT_TO_FP; |
2860 | EVT DestVT = N->getValueType(ResNo: 0); |
2861 | SDValue LegalOp = N->getOperand(Num: IsStrict ? 1 : 0); |
2862 | unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP; |
2863 | unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP; |
2864 | |
2865 | // First step, figure out the appropriate *INT_TO_FP operation to use. |
2866 | EVT NewInTy = LegalOp.getValueType(); |
2867 | |
2868 | unsigned OpToUse = 0; |
2869 | |
2870 | // Scan for the appropriate larger type to use. |
2871 | while (true) { |
2872 | NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1); |
2873 | assert(NewInTy.isInteger() && "Ran out of possibilities!" ); |
2874 | |
2875 | // If the target supports SINT_TO_FP of this type, use it. |
2876 | if (TLI.isOperationLegalOrCustom(Op: SIntOp, VT: NewInTy)) { |
2877 | OpToUse = SIntOp; |
2878 | break; |
2879 | } |
2880 | if (IsSigned) |
2881 | continue; |
2882 | |
2883 | // If the target supports UINT_TO_FP of this type, use it. |
2884 | if (TLI.isOperationLegalOrCustom(Op: UIntOp, VT: NewInTy)) { |
2885 | OpToUse = UIntOp; |
2886 | break; |
2887 | } |
2888 | |
2889 | // Otherwise, try a larger type. |
2890 | } |
2891 | |
2892 | // Okay, we found the operation and type to use. Zero extend our input to the |
2893 | // desired type then run the operation on it. |
2894 | if (IsStrict) { |
2895 | SDValue Res = |
2896 | DAG.getNode(OpToUse, dl, {DestVT, MVT::Other}, |
2897 | {N->getOperand(0), |
2898 | DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, |
2899 | dl, NewInTy, LegalOp)}); |
2900 | Results.push_back(Elt: Res); |
2901 | Results.push_back(Elt: Res.getValue(R: 1)); |
2902 | return; |
2903 | } |
2904 | |
2905 | Results.push_back( |
2906 | Elt: DAG.getNode(Opcode: OpToUse, DL: dl, VT: DestVT, |
2907 | Operand: DAG.getNode(Opcode: IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, |
2908 | DL: dl, VT: NewInTy, Operand: LegalOp))); |
2909 | } |
2910 | |
2911 | /// This function is responsible for legalizing a |
2912 | /// FP_TO_*INT operation of the specified operand when the target requests that |
2913 | /// we promote it. At this point, we know that the result and operand types are |
2914 | /// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT |
2915 | /// operation that returns a larger result. |
2916 | void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl, |
2917 | SmallVectorImpl<SDValue> &Results) { |
2918 | bool IsStrict = N->isStrictFPOpcode(); |
2919 | bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT || |
2920 | N->getOpcode() == ISD::STRICT_FP_TO_SINT; |
2921 | EVT DestVT = N->getValueType(ResNo: 0); |
2922 | SDValue LegalOp = N->getOperand(Num: IsStrict ? 1 : 0); |
2923 | // First step, figure out the appropriate FP_TO*INT operation to use. |
2924 | EVT NewOutTy = DestVT; |
2925 | |
2926 | unsigned OpToUse = 0; |
2927 | |
2928 | // Scan for the appropriate larger type to use. |
2929 | while (true) { |
2930 | NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1); |
2931 | assert(NewOutTy.isInteger() && "Ran out of possibilities!" ); |
2932 | |
2933 | // A larger signed type can hold all unsigned values of the requested type, |
2934 | // so using FP_TO_SINT is valid |
2935 | OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT; |
2936 | if (TLI.isOperationLegalOrCustom(Op: OpToUse, VT: NewOutTy)) |
2937 | break; |
2938 | |
2939 | // However, if the value may be < 0.0, we *must* use some FP_TO_SINT. |
2940 | OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT; |
2941 | if (!IsSigned && TLI.isOperationLegalOrCustom(Op: OpToUse, VT: NewOutTy)) |
2942 | break; |
2943 | |
2944 | // Otherwise, try a larger type. |
2945 | } |
2946 | |
2947 | // Okay, we found the operation and type to use. |
2948 | SDValue Operation; |
2949 | if (IsStrict) { |
2950 | SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other); |
2951 | Operation = DAG.getNode(Opcode: OpToUse, DL: dl, VTList: VTs, N1: N->getOperand(Num: 0), N2: LegalOp); |
2952 | } else |
2953 | Operation = DAG.getNode(Opcode: OpToUse, DL: dl, VT: NewOutTy, Operand: LegalOp); |
2954 | |
2955 | // Truncate the result of the extended FP_TO_*INT operation to the desired |
2956 | // size. |
2957 | SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: DestVT, Operand: Operation); |
2958 | Results.push_back(Elt: Trunc); |
2959 | if (IsStrict) |
2960 | Results.push_back(Elt: Operation.getValue(R: 1)); |
2961 | } |
2962 | |
2963 | /// Promote FP_TO_*INT_SAT operation to a larger result type. At this point |
2964 | /// the result and operand types are legal and there must be a legal |
2965 | /// FP_TO_*INT_SAT operation for a larger result type. |
2966 | SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node, |
2967 | const SDLoc &dl) { |
2968 | unsigned Opcode = Node->getOpcode(); |
2969 | |
2970 | // Scan for the appropriate larger type to use. |
2971 | EVT NewOutTy = Node->getValueType(ResNo: 0); |
2972 | while (true) { |
2973 | NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1); |
2974 | assert(NewOutTy.isInteger() && "Ran out of possibilities!" ); |
2975 | |
2976 | if (TLI.isOperationLegalOrCustom(Op: Opcode, VT: NewOutTy)) |
2977 | break; |
2978 | } |
2979 | |
2980 | // Saturation width is determined by second operand, so we don't have to |
2981 | // perform any fixup and can directly truncate the result. |
2982 | SDValue Result = DAG.getNode(Opcode, DL: dl, VT: NewOutTy, N1: Node->getOperand(Num: 0), |
2983 | N2: Node->getOperand(Num: 1)); |
2984 | return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Result); |
2985 | } |
2986 | |
2987 | /// Open code the operations for PARITY of the specified operation. |
2988 | SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) { |
2989 | EVT VT = Op.getValueType(); |
2990 | EVT ShVT = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout()); |
2991 | unsigned Sz = VT.getScalarSizeInBits(); |
2992 | |
2993 | // If CTPOP is legal, use it. Otherwise use shifts and xor. |
2994 | SDValue Result; |
2995 | if (TLI.isOperationLegalOrPromote(Op: ISD::CTPOP, VT)) { |
2996 | Result = DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Op); |
2997 | } else { |
2998 | Result = Op; |
2999 | for (unsigned i = Log2_32_Ceil(Value: Sz); i != 0;) { |
3000 | SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Result, |
3001 | N2: DAG.getConstant(Val: 1ULL << (--i), DL: dl, VT: ShVT)); |
3002 | Result = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Result, N2: Shift); |
3003 | } |
3004 | } |
3005 | |
3006 | return DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Result, N2: DAG.getConstant(Val: 1, DL: dl, VT)); |
3007 | } |
3008 | |
3009 | bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { |
3010 | LLVM_DEBUG(dbgs() << "Trying to expand node\n" ); |
3011 | SmallVector<SDValue, 8> Results; |
3012 | SDLoc dl(Node); |
3013 | SDValue Tmp1, Tmp2, Tmp3, Tmp4; |
3014 | bool NeedInvert; |
3015 | switch (Node->getOpcode()) { |
3016 | case ISD::ABS: |
3017 | if ((Tmp1 = TLI.expandABS(N: Node, DAG))) |
3018 | Results.push_back(Elt: Tmp1); |
3019 | break; |
3020 | case ISD::ABDS: |
3021 | case ISD::ABDU: |
3022 | if ((Tmp1 = TLI.expandABD(N: Node, DAG))) |
3023 | Results.push_back(Elt: Tmp1); |
3024 | break; |
3025 | case ISD::CTPOP: |
3026 | if ((Tmp1 = TLI.expandCTPOP(N: Node, DAG))) |
3027 | Results.push_back(Elt: Tmp1); |
3028 | break; |
3029 | case ISD::CTLZ: |
3030 | case ISD::CTLZ_ZERO_UNDEF: |
3031 | if ((Tmp1 = TLI.expandCTLZ(N: Node, DAG))) |
3032 | Results.push_back(Elt: Tmp1); |
3033 | break; |
3034 | case ISD::CTTZ: |
3035 | case ISD::CTTZ_ZERO_UNDEF: |
3036 | if ((Tmp1 = TLI.expandCTTZ(N: Node, DAG))) |
3037 | Results.push_back(Elt: Tmp1); |
3038 | break; |
3039 | case ISD::BITREVERSE: |
3040 | if ((Tmp1 = TLI.expandBITREVERSE(N: Node, DAG))) |
3041 | Results.push_back(Elt: Tmp1); |
3042 | break; |
3043 | case ISD::BSWAP: |
3044 | if ((Tmp1 = TLI.expandBSWAP(N: Node, DAG))) |
3045 | Results.push_back(Elt: Tmp1); |
3046 | break; |
3047 | case ISD::PARITY: |
3048 | Results.push_back(Elt: ExpandPARITY(Op: Node->getOperand(Num: 0), dl)); |
3049 | break; |
3050 | case ISD::FRAMEADDR: |
3051 | case ISD::RETURNADDR: |
3052 | case ISD::FRAME_TO_ARGS_OFFSET: |
3053 | Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0))); |
3054 | break; |
3055 | case ISD::EH_DWARF_CFA: { |
3056 | SDValue CfaArg = DAG.getSExtOrTrunc(Op: Node->getOperand(Num: 0), DL: dl, |
3057 | VT: TLI.getPointerTy(DL: DAG.getDataLayout())); |
3058 | SDValue Offset = DAG.getNode(Opcode: ISD::ADD, DL: dl, |
3059 | VT: CfaArg.getValueType(), |
3060 | N1: DAG.getNode(Opcode: ISD::FRAME_TO_ARGS_OFFSET, DL: dl, |
3061 | VT: CfaArg.getValueType()), |
3062 | N2: CfaArg); |
3063 | SDValue FA = DAG.getNode( |
3064 | Opcode: ISD::FRAMEADDR, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()), |
3065 | Operand: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()))); |
3066 | Results.push_back(Elt: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: FA.getValueType(), |
3067 | N1: FA, N2: Offset)); |
3068 | break; |
3069 | } |
3070 | case ISD::GET_ROUNDING: |
3071 | Results.push_back(Elt: DAG.getConstant(Val: 1, DL: dl, VT: Node->getValueType(ResNo: 0))); |
3072 | Results.push_back(Elt: Node->getOperand(Num: 0)); |
3073 | break; |
3074 | case ISD::EH_RETURN: |
3075 | case ISD::EH_LABEL: |
3076 | case ISD::PREFETCH: |
3077 | case ISD::VAEND: |
3078 | case ISD::EH_SJLJ_LONGJMP: |
3079 | // If the target didn't expand these, there's nothing to do, so just |
3080 | // preserve the chain and be done. |
3081 | Results.push_back(Elt: Node->getOperand(Num: 0)); |
3082 | break; |
3083 | case ISD::READCYCLECOUNTER: |
3084 | case ISD::READSTEADYCOUNTER: |
3085 | // If the target didn't expand this, just return 'zero' and preserve the |
3086 | // chain. |
3087 | Results.append(NumInputs: Node->getNumValues() - 1, |
3088 | Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0))); |
3089 | Results.push_back(Elt: Node->getOperand(Num: 0)); |
3090 | break; |
3091 | case ISD::EH_SJLJ_SETJMP: |
3092 | // If the target didn't expand this, just return 'zero' and preserve the |
3093 | // chain. |
3094 | Results.push_back(DAG.getConstant(0, dl, MVT::i32)); |
3095 | Results.push_back(Elt: Node->getOperand(Num: 0)); |
3096 | break; |
3097 | case ISD::ATOMIC_LOAD: { |
3098 | // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP. |
3099 | SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)); |
3100 | SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); |
3101 | SDValue Swap = DAG.getAtomicCmpSwap( |
3102 | Opcode: ISD::ATOMIC_CMP_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(), VTs, |
3103 | Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1), Cmp: Zero, Swp: Zero, |
3104 | MMO: cast<AtomicSDNode>(Val: Node)->getMemOperand()); |
3105 | Results.push_back(Elt: Swap.getValue(R: 0)); |
3106 | Results.push_back(Elt: Swap.getValue(R: 1)); |
3107 | break; |
3108 | } |
3109 | case ISD::ATOMIC_STORE: { |
3110 | // There is no libcall for atomic store; fake it with ATOMIC_SWAP. |
3111 | SDValue Swap = DAG.getAtomic( |
3112 | Opcode: ISD::ATOMIC_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(), |
3113 | Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 2), Val: Node->getOperand(Num: 1), |
3114 | MMO: cast<AtomicSDNode>(Val: Node)->getMemOperand()); |
3115 | Results.push_back(Elt: Swap.getValue(R: 1)); |
3116 | break; |
3117 | } |
3118 | case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { |
3119 | // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and |
3120 | // splits out the success value as a comparison. Expanding the resulting |
3121 | // ATOMIC_CMP_SWAP will produce a libcall. |
3122 | SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); |
3123 | SDValue Res = DAG.getAtomicCmpSwap( |
3124 | Opcode: ISD::ATOMIC_CMP_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(), VTs, |
3125 | Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1), Cmp: Node->getOperand(Num: 2), |
3126 | Swp: Node->getOperand(Num: 3), MMO: cast<MemSDNode>(Val: Node)->getMemOperand()); |
3127 | |
3128 | SDValue ExtRes = Res; |
3129 | SDValue LHS = Res; |
3130 | SDValue RHS = Node->getOperand(Num: 1); |
3131 | |
3132 | EVT AtomicType = cast<AtomicSDNode>(Val: Node)->getMemoryVT(); |
3133 | EVT OuterType = Node->getValueType(ResNo: 0); |
3134 | switch (TLI.getExtendForAtomicOps()) { |
3135 | case ISD::SIGN_EXTEND: |
3136 | LHS = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: OuterType, N1: Res, |
3137 | N2: DAG.getValueType(AtomicType)); |
3138 | RHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: OuterType, |
3139 | N1: Node->getOperand(Num: 2), N2: DAG.getValueType(AtomicType)); |
3140 | ExtRes = LHS; |
3141 | break; |
3142 | case ISD::ZERO_EXTEND: |
3143 | LHS = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: OuterType, N1: Res, |
3144 | N2: DAG.getValueType(AtomicType)); |
3145 | RHS = DAG.getZeroExtendInReg(Op: Node->getOperand(Num: 2), DL: dl, VT: AtomicType); |
3146 | ExtRes = LHS; |
3147 | break; |
3148 | case ISD::ANY_EXTEND: |
3149 | LHS = DAG.getZeroExtendInReg(Op: Res, DL: dl, VT: AtomicType); |
3150 | RHS = DAG.getZeroExtendInReg(Op: Node->getOperand(Num: 2), DL: dl, VT: AtomicType); |
3151 | break; |
3152 | default: |
3153 | llvm_unreachable("Invalid atomic op extension" ); |
3154 | } |
3155 | |
3156 | SDValue Success = |
3157 | DAG.getSetCC(DL: dl, VT: Node->getValueType(ResNo: 1), LHS, RHS, Cond: ISD::SETEQ); |
3158 | |
3159 | Results.push_back(Elt: ExtRes.getValue(R: 0)); |
3160 | Results.push_back(Elt: Success); |
3161 | Results.push_back(Elt: Res.getValue(R: 1)); |
3162 | break; |
3163 | } |
3164 | case ISD::ATOMIC_LOAD_SUB: { |
3165 | SDLoc DL(Node); |
3166 | EVT VT = Node->getValueType(ResNo: 0); |
3167 | SDValue RHS = Node->getOperand(Num: 2); |
3168 | AtomicSDNode *AN = cast<AtomicSDNode>(Val: Node); |
3169 | if (RHS->getOpcode() == ISD::SIGN_EXTEND_INREG && |
3170 | cast<VTSDNode>(Val: RHS->getOperand(Num: 1))->getVT() == AN->getMemoryVT()) |
3171 | RHS = RHS->getOperand(Num: 0); |
3172 | SDValue NewRHS = |
3173 | DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: RHS); |
3174 | SDValue Res = DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD_ADD, dl: DL, MemVT: AN->getMemoryVT(), |
3175 | Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1), |
3176 | Val: NewRHS, MMO: AN->getMemOperand()); |
3177 | Results.push_back(Elt: Res); |
3178 | Results.push_back(Elt: Res.getValue(R: 1)); |
3179 | break; |
3180 | } |
3181 | case ISD::DYNAMIC_STACKALLOC: |
3182 | ExpandDYNAMIC_STACKALLOC(Node, Results); |
3183 | break; |
3184 | case ISD::MERGE_VALUES: |
3185 | for (unsigned i = 0; i < Node->getNumValues(); i++) |
3186 | Results.push_back(Elt: Node->getOperand(Num: i)); |
3187 | break; |
3188 | case ISD::UNDEF: { |
3189 | EVT VT = Node->getValueType(ResNo: 0); |
3190 | if (VT.isInteger()) |
3191 | Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT)); |
3192 | else { |
3193 | assert(VT.isFloatingPoint() && "Unknown value type!" ); |
3194 | Results.push_back(Elt: DAG.getConstantFP(Val: 0, DL: dl, VT)); |
3195 | } |
3196 | break; |
3197 | } |
3198 | case ISD::STRICT_FP_ROUND: |
3199 | // When strict mode is enforced we can't do expansion because it |
3200 | // does not honor the "strict" properties. Only libcall is allowed. |
3201 | if (TLI.isStrictFPEnabled()) |
3202 | break; |
3203 | // We might as well mutate to FP_ROUND when FP_ROUND operation is legal |
3204 | // since this operation is more efficient than stack operation. |
3205 | if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(), |
3206 | VT: Node->getValueType(ResNo: 0)) |
3207 | == TargetLowering::Legal) |
3208 | break; |
3209 | // We fall back to use stack operation when the FP_ROUND operation |
3210 | // isn't available. |
3211 | if ((Tmp1 = EmitStackConvert(SrcOp: Node->getOperand(Num: 1), SlotVT: Node->getValueType(ResNo: 0), |
3212 | DestVT: Node->getValueType(ResNo: 0), dl, |
3213 | Chain: Node->getOperand(Num: 0)))) { |
3214 | ReplaceNode(Old: Node, New: Tmp1.getNode()); |
3215 | LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n" ); |
3216 | return true; |
3217 | } |
3218 | break; |
3219 | case ISD::FP_ROUND: { |
3220 | EVT VT = Node->getValueType(ResNo: 0); |
3221 | if (VT.getScalarType() == MVT::bf16) { |
3222 | Results.push_back( |
3223 | Elt: DAG.getNode(Opcode: ISD::FP_TO_BF16, DL: SDLoc(Node), VT, Operand: Node->getOperand(Num: 0))); |
3224 | break; |
3225 | } |
3226 | |
3227 | LLVM_FALLTHROUGH; |
3228 | } |
3229 | case ISD::BITCAST: |
3230 | if ((Tmp1 = EmitStackConvert(SrcOp: Node->getOperand(Num: 0), SlotVT: Node->getValueType(ResNo: 0), |
3231 | DestVT: Node->getValueType(ResNo: 0), dl))) |
3232 | Results.push_back(Elt: Tmp1); |
3233 | break; |
3234 | case ISD::STRICT_FP_EXTEND: |
3235 | // When strict mode is enforced we can't do expansion because it |
3236 | // does not honor the "strict" properties. Only libcall is allowed. |
3237 | if (TLI.isStrictFPEnabled()) |
3238 | break; |
3239 | // We might as well mutate to FP_EXTEND when FP_EXTEND operation is legal |
3240 | // since this operation is more efficient than stack operation. |
3241 | if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(), |
3242 | VT: Node->getValueType(ResNo: 0)) |
3243 | == TargetLowering::Legal) |
3244 | break; |
3245 | // We fall back to use stack operation when the FP_EXTEND operation |
3246 | // isn't available. |
3247 | if ((Tmp1 = EmitStackConvert( |
3248 | SrcOp: Node->getOperand(Num: 1), SlotVT: Node->getOperand(Num: 1).getValueType(), |
3249 | DestVT: Node->getValueType(ResNo: 0), dl, Chain: Node->getOperand(Num: 0)))) { |
3250 | ReplaceNode(Old: Node, New: Tmp1.getNode()); |
3251 | LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n" ); |
3252 | return true; |
3253 | } |
3254 | break; |
3255 | case ISD::FP_EXTEND: { |
3256 | SDValue Op = Node->getOperand(Num: 0); |
3257 | EVT SrcVT = Op.getValueType(); |
3258 | EVT DstVT = Node->getValueType(ResNo: 0); |
3259 | if (SrcVT.getScalarType() == MVT::bf16) { |
3260 | Results.push_back(Elt: DAG.getNode(Opcode: ISD::BF16_TO_FP, DL: SDLoc(Node), VT: DstVT, Operand: Op)); |
3261 | break; |
3262 | } |
3263 | |
3264 | if ((Tmp1 = EmitStackConvert(SrcOp: Op, SlotVT: SrcVT, DestVT: DstVT, dl))) |
3265 | Results.push_back(Elt: Tmp1); |
3266 | break; |
3267 | } |
3268 | case ISD::BF16_TO_FP: { |
3269 | // Always expand bf16 to f32 casts, they lower to ext + shift. |
3270 | // |
3271 | // Note that the operand of this code can be bf16 or an integer type in case |
3272 | // bf16 is not supported on the target and was softened. |
3273 | SDValue Op = Node->getOperand(Num: 0); |
3274 | if (Op.getValueType() == MVT::bf16) { |
3275 | Op = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, |
3276 | DAG.getNode(ISD::BITCAST, dl, MVT::i16, Op)); |
3277 | } else { |
3278 | Op = DAG.getAnyExtOrTrunc(Op, dl, MVT::i32); |
3279 | } |
3280 | Op = DAG.getNode( |
3281 | ISD::SHL, dl, MVT::i32, Op, |
3282 | DAG.getConstant(16, dl, |
3283 | TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout()))); |
3284 | Op = DAG.getNode(ISD::BITCAST, dl, MVT::f32, Op); |
3285 | // Add fp_extend in case the output is bigger than f32. |
3286 | if (Node->getValueType(0) != MVT::f32) |
3287 | Op = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Op); |
3288 | Results.push_back(Elt: Op); |
3289 | break; |
3290 | } |
3291 | case ISD::FP_TO_BF16: { |
3292 | SDValue Op = Node->getOperand(Num: 0); |
3293 | if (Op.getValueType() != MVT::f32) |
3294 | Op = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op, |
3295 | DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)); |
3296 | Op = DAG.getNode( |
3297 | ISD::SRL, dl, MVT::i32, DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op), |
3298 | DAG.getConstant(16, dl, |
3299 | TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout()))); |
3300 | // The result of this node can be bf16 or an integer type in case bf16 is |
3301 | // not supported on the target and was softened to i16 for storage. |
3302 | if (Node->getValueType(0) == MVT::bf16) { |
3303 | Op = DAG.getNode(ISD::BITCAST, dl, MVT::bf16, |
3304 | DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Op)); |
3305 | } else { |
3306 | Op = DAG.getAnyExtOrTrunc(Op, DL: dl, VT: Node->getValueType(ResNo: 0)); |
3307 | } |
3308 | Results.push_back(Elt: Op); |
3309 | break; |
3310 | } |
3311 | case ISD::SIGN_EXTEND_INREG: { |
3312 | EVT = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT(); |
3313 | EVT VT = Node->getValueType(ResNo: 0); |
3314 | |
3315 | // An in-register sign-extend of a boolean is a negation: |
3316 | // 'true' (1) sign-extended is -1. |
3317 | // 'false' (0) sign-extended is 0. |
3318 | // However, we must mask the high bits of the source operand because the |
3319 | // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero. |
3320 | |
3321 | // TODO: Do this for vectors too? |
3322 | if (ExtraVT.isScalarInteger() && ExtraVT.getSizeInBits() == 1) { |
3323 | SDValue One = DAG.getConstant(Val: 1, DL: dl, VT); |
3324 | SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: One); |
3325 | SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT); |
3326 | SDValue Neg = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: And); |
3327 | Results.push_back(Elt: Neg); |
3328 | break; |
3329 | } |
3330 | |
3331 | // NOTE: we could fall back on load/store here too for targets without |
3332 | // SRA. However, it is doubtful that any exist. |
3333 | EVT ShiftAmountTy = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout()); |
3334 | unsigned BitsDiff = VT.getScalarSizeInBits() - |
3335 | ExtraVT.getScalarSizeInBits(); |
3336 | SDValue ShiftCst = DAG.getConstant(Val: BitsDiff, DL: dl, VT: ShiftAmountTy); |
3337 | Tmp1 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: Node->getValueType(ResNo: 0), |
3338 | N1: Node->getOperand(Num: 0), N2: ShiftCst); |
3339 | Tmp1 = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1, N2: ShiftCst); |
3340 | Results.push_back(Elt: Tmp1); |
3341 | break; |
3342 | } |
3343 | case ISD::UINT_TO_FP: |
3344 | case ISD::STRICT_UINT_TO_FP: |
3345 | if (TLI.expandUINT_TO_FP(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG)) { |
3346 | Results.push_back(Elt: Tmp1); |
3347 | if (Node->isStrictFPOpcode()) |
3348 | Results.push_back(Elt: Tmp2); |
3349 | break; |
3350 | } |
3351 | [[fallthrough]]; |
3352 | case ISD::SINT_TO_FP: |
3353 | case ISD::STRICT_SINT_TO_FP: |
3354 | if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Chain&: Tmp2))) { |
3355 | Results.push_back(Elt: Tmp1); |
3356 | if (Node->isStrictFPOpcode()) |
3357 | Results.push_back(Elt: Tmp2); |
3358 | } |
3359 | break; |
3360 | case ISD::FP_TO_SINT: |
3361 | if (TLI.expandFP_TO_SINT(N: Node, Result&: Tmp1, DAG)) |
3362 | Results.push_back(Elt: Tmp1); |
3363 | break; |
3364 | case ISD::STRICT_FP_TO_SINT: |
3365 | if (TLI.expandFP_TO_SINT(N: Node, Result&: Tmp1, DAG)) { |
3366 | ReplaceNode(Old: Node, New: Tmp1.getNode()); |
3367 | LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n" ); |
3368 | return true; |
3369 | } |
3370 | break; |
3371 | case ISD::FP_TO_UINT: |
3372 | if (TLI.expandFP_TO_UINT(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG)) |
3373 | Results.push_back(Elt: Tmp1); |
3374 | break; |
3375 | case ISD::STRICT_FP_TO_UINT: |
3376 | if (TLI.expandFP_TO_UINT(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG)) { |
3377 | // Relink the chain. |
3378 | DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node,1), To: Tmp2); |
3379 | // Replace the new UINT result. |
3380 | ReplaceNodeWithValue(Old: SDValue(Node, 0), New: Tmp1); |
3381 | LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n" ); |
3382 | return true; |
3383 | } |
3384 | break; |
3385 | case ISD::FP_TO_SINT_SAT: |
3386 | case ISD::FP_TO_UINT_SAT: |
3387 | Results.push_back(Elt: TLI.expandFP_TO_INT_SAT(N: Node, DAG)); |
3388 | break; |
3389 | case ISD::VAARG: |
3390 | Results.push_back(Elt: DAG.expandVAArg(Node)); |
3391 | Results.push_back(Elt: Results[0].getValue(R: 1)); |
3392 | break; |
3393 | case ISD::VACOPY: |
3394 | Results.push_back(Elt: DAG.expandVACopy(Node)); |
3395 | break; |
3396 | case ISD::EXTRACT_VECTOR_ELT: |
3397 | if (Node->getOperand(Num: 0).getValueType().getVectorElementCount().isScalar()) |
3398 | // This must be an access of the only element. Return it. |
3399 | Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Node->getValueType(ResNo: 0), |
3400 | Operand: Node->getOperand(Num: 0)); |
3401 | else |
3402 | Tmp1 = ExpandExtractFromVectorThroughStack(Op: SDValue(Node, 0)); |
3403 | Results.push_back(Elt: Tmp1); |
3404 | break; |
3405 | case ISD::EXTRACT_SUBVECTOR: |
3406 | Results.push_back(Elt: ExpandExtractFromVectorThroughStack(Op: SDValue(Node, 0))); |
3407 | break; |
3408 | case ISD::INSERT_SUBVECTOR: |
3409 | Results.push_back(Elt: ExpandInsertToVectorThroughStack(Op: SDValue(Node, 0))); |
3410 | break; |
3411 | case ISD::CONCAT_VECTORS: |
3412 | Results.push_back(Elt: ExpandVectorBuildThroughStack(Node)); |
3413 | break; |
3414 | case ISD::SCALAR_TO_VECTOR: |
3415 | Results.push_back(Elt: ExpandSCALAR_TO_VECTOR(Node)); |
3416 | break; |
3417 | case ISD::INSERT_VECTOR_ELT: |
3418 | Results.push_back(Elt: ExpandINSERT_VECTOR_ELT(Vec: Node->getOperand(Num: 0), |
3419 | Val: Node->getOperand(Num: 1), |
3420 | Idx: Node->getOperand(Num: 2), dl)); |
3421 | break; |
3422 | case ISD::VECTOR_SHUFFLE: { |
3423 | SmallVector<int, 32> NewMask; |
3424 | ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Node)->getMask(); |
3425 | |
3426 | EVT VT = Node->getValueType(ResNo: 0); |
3427 | EVT EltVT = VT.getVectorElementType(); |
3428 | SDValue Op0 = Node->getOperand(Num: 0); |
3429 | SDValue Op1 = Node->getOperand(Num: 1); |
3430 | if (!TLI.isTypeLegal(VT: EltVT)) { |
3431 | EVT NewEltVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: EltVT); |
3432 | |
3433 | // BUILD_VECTOR operands are allowed to be wider than the element type. |
3434 | // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept |
3435 | // it. |
3436 | if (NewEltVT.bitsLT(VT: EltVT)) { |
3437 | // Convert shuffle node. |
3438 | // If original node was v4i64 and the new EltVT is i32, |
3439 | // cast operands to v8i32 and re-build the mask. |
3440 | |
3441 | // Calculate new VT, the size of the new VT should be equal to original. |
3442 | EVT NewVT = |
3443 | EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewEltVT, |
3444 | NumElements: VT.getSizeInBits() / NewEltVT.getSizeInBits()); |
3445 | assert(NewVT.bitsEq(VT)); |
3446 | |
3447 | // cast operands to new VT |
3448 | Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: Op0); |
3449 | Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: Op1); |
3450 | |
3451 | // Convert the shuffle mask |
3452 | unsigned int factor = |
3453 | NewVT.getVectorNumElements()/VT.getVectorNumElements(); |
3454 | |
3455 | // EltVT gets smaller |
3456 | assert(factor > 0); |
3457 | |
3458 | for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) { |
3459 | if (Mask[i] < 0) { |
3460 | for (unsigned fi = 0; fi < factor; ++fi) |
3461 | NewMask.push_back(Elt: Mask[i]); |
3462 | } |
3463 | else { |
3464 | for (unsigned fi = 0; fi < factor; ++fi) |
3465 | NewMask.push_back(Elt: Mask[i]*factor+fi); |
3466 | } |
3467 | } |
3468 | Mask = NewMask; |
3469 | VT = NewVT; |
3470 | } |
3471 | EltVT = NewEltVT; |
3472 | } |
3473 | unsigned NumElems = VT.getVectorNumElements(); |
3474 | SmallVector<SDValue, 16> Ops; |
3475 | for (unsigned i = 0; i != NumElems; ++i) { |
3476 | if (Mask[i] < 0) { |
3477 | Ops.push_back(Elt: DAG.getUNDEF(VT: EltVT)); |
3478 | continue; |
3479 | } |
3480 | unsigned Idx = Mask[i]; |
3481 | if (Idx < NumElems) |
3482 | Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0, |
3483 | N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl))); |
3484 | else |
3485 | Ops.push_back( |
3486 | Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op1, |
3487 | N2: DAG.getVectorIdxConstant(Val: Idx - NumElems, DL: dl))); |
3488 | } |
3489 | |
3490 | Tmp1 = DAG.getBuildVector(VT, DL: dl, Ops); |
3491 | // We may have changed the BUILD_VECTOR type. Cast it back to the Node type. |
3492 | Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1); |
3493 | Results.push_back(Elt: Tmp1); |
3494 | break; |
3495 | } |
3496 | case ISD::VECTOR_SPLICE: { |
3497 | Results.push_back(Elt: TLI.expandVectorSplice(Node, DAG)); |
3498 | break; |
3499 | } |
3500 | case ISD::EXTRACT_ELEMENT: { |
3501 | EVT OpTy = Node->getOperand(Num: 0).getValueType(); |
3502 | if (Node->getConstantOperandVal(Num: 1)) { |
3503 | // 1 -> Hi |
3504 | Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: OpTy, N1: Node->getOperand(Num: 0), |
3505 | N2: DAG.getConstant(Val: OpTy.getSizeInBits() / 2, DL: dl, |
3506 | VT: TLI.getShiftAmountTy( |
3507 | LHSTy: Node->getOperand(Num: 0).getValueType(), |
3508 | DL: DAG.getDataLayout()))); |
3509 | Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1); |
3510 | } else { |
3511 | // 0 -> Lo |
3512 | Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0), |
3513 | Operand: Node->getOperand(Num: 0)); |
3514 | } |
3515 | Results.push_back(Elt: Tmp1); |
3516 | break; |
3517 | } |
3518 | case ISD::STACKSAVE: |
3519 | // Expand to CopyFromReg if the target set |
3520 | // StackPointerRegisterToSaveRestore. |
3521 | if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) { |
3522 | Results.push_back(Elt: DAG.getCopyFromReg(Chain: Node->getOperand(Num: 0), dl, Reg: SP, |
3523 | VT: Node->getValueType(ResNo: 0))); |
3524 | Results.push_back(Elt: Results[0].getValue(R: 1)); |
3525 | } else { |
3526 | Results.push_back(Elt: DAG.getUNDEF(VT: Node->getValueType(ResNo: 0))); |
3527 | Results.push_back(Elt: Node->getOperand(Num: 0)); |
3528 | } |
3529 | break; |
3530 | case ISD::STACKRESTORE: |
3531 | // Expand to CopyToReg if the target set |
3532 | // StackPointerRegisterToSaveRestore. |
3533 | if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) { |
3534 | Results.push_back(Elt: DAG.getCopyToReg(Chain: Node->getOperand(Num: 0), dl, Reg: SP, |
3535 | N: Node->getOperand(Num: 1))); |
3536 | } else { |
3537 | Results.push_back(Elt: Node->getOperand(Num: 0)); |
3538 | } |
3539 | break; |
3540 | case ISD::GET_DYNAMIC_AREA_OFFSET: |
3541 | Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0))); |
3542 | Results.push_back(Elt: Results[0].getValue(R: 0)); |
3543 | break; |
3544 | case ISD::FCOPYSIGN: |
3545 | Results.push_back(Elt: ExpandFCOPYSIGN(Node)); |
3546 | break; |
3547 | case ISD::FNEG: |
3548 | Results.push_back(Elt: ExpandFNEG(Node)); |
3549 | break; |
3550 | case ISD::FABS: |
3551 | Results.push_back(Elt: ExpandFABS(Node)); |
3552 | break; |
3553 | case ISD::IS_FPCLASS: { |
3554 | auto Test = static_cast<FPClassTest>(Node->getConstantOperandVal(Num: 1)); |
3555 | if (SDValue Expanded = |
3556 | TLI.expandIS_FPCLASS(ResultVT: Node->getValueType(ResNo: 0), Op: Node->getOperand(Num: 0), |
3557 | Test, Flags: Node->getFlags(), DL: SDLoc(Node), DAG)) |
3558 | Results.push_back(Elt: Expanded); |
3559 | break; |
3560 | } |
3561 | case ISD::SMIN: |
3562 | case ISD::SMAX: |
3563 | case ISD::UMIN: |
3564 | case ISD::UMAX: { |
3565 | // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B |
3566 | ISD::CondCode Pred; |
3567 | switch (Node->getOpcode()) { |
3568 | default: llvm_unreachable("How did we get here?" ); |
3569 | case ISD::SMAX: Pred = ISD::SETGT; break; |
3570 | case ISD::SMIN: Pred = ISD::SETLT; break; |
3571 | case ISD::UMAX: Pred = ISD::SETUGT; break; |
3572 | case ISD::UMIN: Pred = ISD::SETULT; break; |
3573 | } |
3574 | Tmp1 = Node->getOperand(Num: 0); |
3575 | Tmp2 = Node->getOperand(Num: 1); |
3576 | Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: Tmp2, True: Tmp1, False: Tmp2, Cond: Pred); |
3577 | Results.push_back(Elt: Tmp1); |
3578 | break; |
3579 | } |
3580 | case ISD::FMINNUM: |
3581 | case ISD::FMAXNUM: { |
3582 | if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(N: Node, DAG)) |
3583 | Results.push_back(Elt: Expanded); |
3584 | break; |
3585 | } |
3586 | case ISD::FSIN: |
3587 | case ISD::FCOS: { |
3588 | EVT VT = Node->getValueType(ResNo: 0); |
3589 | // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin / |
3590 | // fcos which share the same operand and both are used. |
3591 | if ((TLI.isOperationLegalOrCustom(Op: ISD::FSINCOS, VT) || |
3592 | isSinCosLibcallAvailable(Node, TLI)) |
3593 | && useSinCos(Node)) { |
3594 | SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT); |
3595 | Tmp1 = DAG.getNode(Opcode: ISD::FSINCOS, DL: dl, VTList: VTs, N: Node->getOperand(Num: 0)); |
3596 | if (Node->getOpcode() == ISD::FCOS) |
3597 | Tmp1 = Tmp1.getValue(R: 1); |
3598 | Results.push_back(Elt: Tmp1); |
3599 | } |
3600 | break; |
3601 | } |
3602 | case ISD::FLDEXP: |
3603 | case ISD::STRICT_FLDEXP: { |
3604 | EVT VT = Node->getValueType(ResNo: 0); |
3605 | RTLIB::Libcall LC = RTLIB::getLDEXP(RetVT: VT); |
3606 | // Use the LibCall instead, it is very likely faster |
3607 | // FIXME: Use separate LibCall action. |
3608 | if (TLI.getLibcallName(Call: LC)) |
3609 | break; |
3610 | |
3611 | if (SDValue Expanded = expandLdexp(Node)) { |
3612 | Results.push_back(Elt: Expanded); |
3613 | if (Node->getOpcode() == ISD::STRICT_FLDEXP) |
3614 | Results.push_back(Elt: Expanded.getValue(R: 1)); |
3615 | } |
3616 | |
3617 | break; |
3618 | } |
3619 | case ISD::FFREXP: { |
3620 | RTLIB::Libcall LC = RTLIB::getFREXP(RetVT: Node->getValueType(ResNo: 0)); |
3621 | // Use the LibCall instead, it is very likely faster |
3622 | // FIXME: Use separate LibCall action. |
3623 | if (TLI.getLibcallName(Call: LC)) |
3624 | break; |
3625 | |
3626 | if (SDValue Expanded = expandFrexp(Node)) { |
3627 | Results.push_back(Elt: Expanded); |
3628 | Results.push_back(Elt: Expanded.getValue(R: 1)); |
3629 | } |
3630 | break; |
3631 | } |
3632 | case ISD::FMAD: |
3633 | llvm_unreachable("Illegal fmad should never be formed" ); |
3634 | |
3635 | case ISD::FP16_TO_FP: |
3636 | if (Node->getValueType(0) != MVT::f32) { |
3637 | // We can extend to types bigger than f32 in two steps without changing |
3638 | // the result. Since "f16 -> f32" is much more commonly available, give |
3639 | // CodeGen the option of emitting that before resorting to a libcall. |
3640 | SDValue Res = |
3641 | DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0)); |
3642 | Results.push_back( |
3643 | Elt: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Res)); |
3644 | } |
3645 | break; |
3646 | case ISD::STRICT_FP16_TO_FP: |
3647 | if (Node->getValueType(0) != MVT::f32) { |
3648 | // We can extend to types bigger than f32 in two steps without changing |
3649 | // the result. Since "f16 -> f32" is much more commonly available, give |
3650 | // CodeGen the option of emitting that before resorting to a libcall. |
3651 | SDValue Res = |
3652 | DAG.getNode(ISD::STRICT_FP16_TO_FP, dl, {MVT::f32, MVT::Other}, |
3653 | {Node->getOperand(0), Node->getOperand(1)}); |
3654 | Res = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, |
3655 | {Node->getValueType(0), MVT::Other}, |
3656 | {Res.getValue(1), Res}); |
3657 | Results.push_back(Elt: Res); |
3658 | Results.push_back(Elt: Res.getValue(R: 1)); |
3659 | } |
3660 | break; |
3661 | case ISD::FP_TO_FP16: |
3662 | LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n" ); |
3663 | if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) { |
3664 | SDValue Op = Node->getOperand(Num: 0); |
3665 | MVT SVT = Op.getSimpleValueType(); |
3666 | if ((SVT == MVT::f64 || SVT == MVT::f80) && |
3667 | TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) { |
3668 | // Under fastmath, we can expand this node into a fround followed by |
3669 | // a float-half conversion. |
3670 | SDValue FloatVal = |
3671 | DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op, |
3672 | DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)); |
3673 | Results.push_back( |
3674 | Elt: DAG.getNode(Opcode: ISD::FP_TO_FP16, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: FloatVal)); |
3675 | } |
3676 | } |
3677 | break; |
3678 | case ISD::ConstantFP: { |
3679 | ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Val: Node); |
3680 | // Check to see if this FP immediate is already legal. |
3681 | // If this is a legal constant, turn it into a TargetConstantFP node. |
3682 | if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(ResNo: 0), |
3683 | ForCodeSize: DAG.shouldOptForSize())) |
3684 | Results.push_back(Elt: ExpandConstantFP(CFP, UseCP: true)); |
3685 | break; |
3686 | } |
3687 | case ISD::Constant: { |
3688 | ConstantSDNode *CP = cast<ConstantSDNode>(Val: Node); |
3689 | Results.push_back(Elt: ExpandConstant(CP)); |
3690 | break; |
3691 | } |
3692 | case ISD::FSUB: { |
3693 | EVT VT = Node->getValueType(ResNo: 0); |
3694 | if (TLI.isOperationLegalOrCustom(Op: ISD::FADD, VT) && |
3695 | TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT)) { |
3696 | const SDNodeFlags Flags = Node->getFlags(); |
3697 | Tmp1 = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Node->getOperand(Num: 1)); |
3698 | Tmp1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: Tmp1, Flags); |
3699 | Results.push_back(Elt: Tmp1); |
3700 | } |
3701 | break; |
3702 | } |
3703 | case ISD::SUB: { |
3704 | EVT VT = Node->getValueType(ResNo: 0); |
3705 | assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) && |
3706 | TLI.isOperationLegalOrCustom(ISD::XOR, VT) && |
3707 | "Don't know how to expand this subtraction!" ); |
3708 | Tmp1 = DAG.getNOT(DL: dl, Val: Node->getOperand(Num: 1), VT); |
3709 | Tmp1 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Tmp1, N2: DAG.getConstant(Val: 1, DL: dl, VT)); |
3710 | Results.push_back(Elt: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: Tmp1)); |
3711 | break; |
3712 | } |
3713 | case ISD::UREM: |
3714 | case ISD::SREM: |
3715 | if (TLI.expandREM(Node, Result&: Tmp1, DAG)) |
3716 | Results.push_back(Elt: Tmp1); |
3717 | break; |
3718 | case ISD::UDIV: |
3719 | case ISD::SDIV: { |
3720 | bool isSigned = Node->getOpcode() == ISD::SDIV; |
3721 | unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; |
3722 | EVT VT = Node->getValueType(ResNo: 0); |
3723 | if (TLI.isOperationLegalOrCustom(Op: DivRemOpc, VT)) { |
3724 | SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT); |
3725 | Tmp1 = DAG.getNode(Opcode: DivRemOpc, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0), |
3726 | N2: Node->getOperand(Num: 1)); |
3727 | Results.push_back(Elt: Tmp1); |
3728 | } |
3729 | break; |
3730 | } |
3731 | case ISD::MULHU: |
3732 | case ISD::MULHS: { |
3733 | unsigned ExpandOpcode = |
3734 | Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI; |
3735 | EVT VT = Node->getValueType(ResNo: 0); |
3736 | SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT); |
3737 | |
3738 | Tmp1 = DAG.getNode(Opcode: ExpandOpcode, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0), |
3739 | N2: Node->getOperand(Num: 1)); |
3740 | Results.push_back(Elt: Tmp1.getValue(R: 1)); |
3741 | break; |
3742 | } |
3743 | case ISD::UMUL_LOHI: |
3744 | case ISD::SMUL_LOHI: { |
3745 | SDValue LHS = Node->getOperand(Num: 0); |
3746 | SDValue RHS = Node->getOperand(Num: 1); |
3747 | MVT VT = LHS.getSimpleValueType(); |
3748 | unsigned MULHOpcode = |
3749 | Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS; |
3750 | |
3751 | if (TLI.isOperationLegalOrCustom(Op: MULHOpcode, VT)) { |
3752 | Results.push_back(Elt: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS)); |
3753 | Results.push_back(Elt: DAG.getNode(Opcode: MULHOpcode, DL: dl, VT, N1: LHS, N2: RHS)); |
3754 | break; |
3755 | } |
3756 | |
3757 | SmallVector<SDValue, 4> Halves; |
3758 | EVT HalfType = EVT(VT).getHalfSizedIntegerVT(Context&: *DAG.getContext()); |
3759 | assert(TLI.isTypeLegal(HalfType)); |
3760 | if (TLI.expandMUL_LOHI(Opcode: Node->getOpcode(), VT, dl, LHS, RHS, Result&: Halves, |
3761 | HiLoVT: HalfType, DAG, |
3762 | Kind: TargetLowering::MulExpansionKind::Always)) { |
3763 | for (unsigned i = 0; i < 2; ++i) { |
3764 | SDValue Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Halves[2 * i]); |
3765 | SDValue Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Halves[2 * i + 1]); |
3766 | SDValue Shift = DAG.getConstant( |
3767 | Val: HalfType.getScalarSizeInBits(), DL: dl, |
3768 | VT: TLI.getShiftAmountTy(LHSTy: HalfType, DL: DAG.getDataLayout())); |
3769 | Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift); |
3770 | Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi)); |
3771 | } |
3772 | break; |
3773 | } |
3774 | break; |
3775 | } |
3776 | case ISD::MUL: { |
3777 | EVT VT = Node->getValueType(ResNo: 0); |
3778 | SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT); |
3779 | // See if multiply or divide can be lowered using two-result operations. |
3780 | // We just need the low half of the multiply; try both the signed |
3781 | // and unsigned forms. If the target supports both SMUL_LOHI and |
3782 | // UMUL_LOHI, form a preference by checking which forms of plain |
3783 | // MULH it supports. |
3784 | bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(Op: ISD::SMUL_LOHI, VT); |
3785 | bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT); |
3786 | bool HasMULHS = TLI.isOperationLegalOrCustom(Op: ISD::MULHS, VT); |
3787 | bool HasMULHU = TLI.isOperationLegalOrCustom(Op: ISD::MULHU, VT); |
3788 | unsigned OpToUse = 0; |
3789 | if (HasSMUL_LOHI && !HasMULHS) { |
3790 | OpToUse = ISD::SMUL_LOHI; |
3791 | } else if (HasUMUL_LOHI && !HasMULHU) { |
3792 | OpToUse = ISD::UMUL_LOHI; |
3793 | } else if (HasSMUL_LOHI) { |
3794 | OpToUse = ISD::SMUL_LOHI; |
3795 | } else if (HasUMUL_LOHI) { |
3796 | OpToUse = ISD::UMUL_LOHI; |
3797 | } |
3798 | if (OpToUse) { |
3799 | Results.push_back(Elt: DAG.getNode(Opcode: OpToUse, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0), |
3800 | N2: Node->getOperand(Num: 1))); |
3801 | break; |
3802 | } |
3803 | |
3804 | SDValue Lo, Hi; |
3805 | EVT HalfType = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext()); |
3806 | if (TLI.isOperationLegalOrCustom(Op: ISD::ZERO_EXTEND, VT) && |
3807 | TLI.isOperationLegalOrCustom(Op: ISD::ANY_EXTEND, VT) && |
3808 | TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT) && |
3809 | TLI.isOperationLegalOrCustom(Op: ISD::OR, VT) && |
3810 | TLI.expandMUL(N: Node, Lo, Hi, HiLoVT: HalfType, DAG, |
3811 | Kind: TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) { |
3812 | Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Lo); |
3813 | Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Hi); |
3814 | SDValue Shift = |
3815 | DAG.getConstant(Val: HalfType.getSizeInBits(), DL: dl, |
3816 | VT: TLI.getShiftAmountTy(LHSTy: HalfType, DL: DAG.getDataLayout())); |
3817 | Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift); |
3818 | Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi)); |
3819 | } |
3820 | break; |
3821 | } |
3822 | case ISD::FSHL: |
3823 | case ISD::FSHR: |
3824 | if (SDValue Expanded = TLI.expandFunnelShift(N: Node, DAG)) |
3825 | Results.push_back(Elt: Expanded); |
3826 | break; |
3827 | case ISD::ROTL: |
3828 | case ISD::ROTR: |
3829 | if (SDValue Expanded = TLI.expandROT(N: Node, AllowVectorOps: true /*AllowVectorOps*/, DAG)) |
3830 | Results.push_back(Elt: Expanded); |
3831 | break; |
3832 | case ISD::SADDSAT: |
3833 | case ISD::UADDSAT: |
3834 | case ISD::SSUBSAT: |
3835 | case ISD::USUBSAT: |
3836 | Results.push_back(Elt: TLI.expandAddSubSat(Node, DAG)); |
3837 | break; |
3838 | case ISD::SSHLSAT: |
3839 | case ISD::USHLSAT: |
3840 | Results.push_back(Elt: TLI.expandShlSat(Node, DAG)); |
3841 | break; |
3842 | case ISD::SMULFIX: |
3843 | case ISD::SMULFIXSAT: |
3844 | case ISD::UMULFIX: |
3845 | case ISD::UMULFIXSAT: |
3846 | Results.push_back(Elt: TLI.expandFixedPointMul(Node, DAG)); |
3847 | break; |
3848 | case ISD::SDIVFIX: |
3849 | case ISD::SDIVFIXSAT: |
3850 | case ISD::UDIVFIX: |
3851 | case ISD::UDIVFIXSAT: |
3852 | if (SDValue V = TLI.expandFixedPointDiv(Opcode: Node->getOpcode(), dl: SDLoc(Node), |
3853 | LHS: Node->getOperand(Num: 0), |
3854 | RHS: Node->getOperand(Num: 1), |
3855 | Scale: Node->getConstantOperandVal(Num: 2), |
3856 | DAG)) { |
3857 | Results.push_back(Elt: V); |
3858 | break; |
3859 | } |
3860 | // FIXME: We might want to retry here with a wider type if we fail, if that |
3861 | // type is legal. |
3862 | // FIXME: Technically, so long as we only have sdivfixes where BW+Scale is |
3863 | // <= 128 (which is the case for all of the default Embedded-C types), |
3864 | // we will only get here with types and scales that we could always expand |
3865 | // if we were allowed to generate libcalls to division functions of illegal |
3866 | // type. But we cannot do that. |
3867 | llvm_unreachable("Cannot expand DIVFIX!" ); |
3868 | case ISD::UADDO_CARRY: |
3869 | case ISD::USUBO_CARRY: { |
3870 | SDValue LHS = Node->getOperand(Num: 0); |
3871 | SDValue RHS = Node->getOperand(Num: 1); |
3872 | SDValue Carry = Node->getOperand(Num: 2); |
3873 | |
3874 | bool IsAdd = Node->getOpcode() == ISD::UADDO_CARRY; |
3875 | |
3876 | // Initial add of the 2 operands. |
3877 | unsigned Op = IsAdd ? ISD::ADD : ISD::SUB; |
3878 | EVT VT = LHS.getValueType(); |
3879 | SDValue Sum = DAG.getNode(Opcode: Op, DL: dl, VT, N1: LHS, N2: RHS); |
3880 | |
3881 | // Initial check for overflow. |
3882 | EVT CarryType = Node->getValueType(ResNo: 1); |
3883 | EVT SetCCType = getSetCCResultType(VT: Node->getValueType(ResNo: 0)); |
3884 | ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT; |
3885 | SDValue Overflow = DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: LHS, Cond: CC); |
3886 | |
3887 | // Add of the sum and the carry. |
3888 | SDValue One = DAG.getConstant(Val: 1, DL: dl, VT); |
3889 | SDValue CarryExt = |
3890 | DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT), N2: One); |
3891 | SDValue Sum2 = DAG.getNode(Opcode: Op, DL: dl, VT, N1: Sum, N2: CarryExt); |
3892 | |
3893 | // Second check for overflow. If we are adding, we can only overflow if the |
3894 | // initial sum is all 1s ang the carry is set, resulting in a new sum of 0. |
3895 | // If we are subtracting, we can only overflow if the initial sum is 0 and |
3896 | // the carry is set, resulting in a new sum of all 1s. |
3897 | SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT); |
3898 | SDValue Overflow2 = |
3899 | IsAdd ? DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum2, RHS: Zero, Cond: ISD::SETEQ) |
3900 | : DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: Zero, Cond: ISD::SETEQ); |
3901 | Overflow2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCType, N1: Overflow2, |
3902 | N2: DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT: SetCCType)); |
3903 | |
3904 | SDValue ResultCarry = |
3905 | DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SetCCType, N1: Overflow, N2: Overflow2); |
3906 | |
3907 | Results.push_back(Elt: Sum2); |
3908 | Results.push_back(Elt: DAG.getBoolExtOrTrunc(Op: ResultCarry, SL: dl, VT: CarryType, OpVT: VT)); |
3909 | break; |
3910 | } |
3911 | case ISD::SADDO: |
3912 | case ISD::SSUBO: { |
3913 | SDValue Result, Overflow; |
3914 | TLI.expandSADDSUBO(Node, Result, Overflow, DAG); |
3915 | Results.push_back(Elt: Result); |
3916 | Results.push_back(Elt: Overflow); |
3917 | break; |
3918 | } |
3919 | case ISD::UADDO: |
3920 | case ISD::USUBO: { |
3921 | SDValue Result, Overflow; |
3922 | TLI.expandUADDSUBO(Node, Result, Overflow, DAG); |
3923 | Results.push_back(Elt: Result); |
3924 | Results.push_back(Elt: Overflow); |
3925 | break; |
3926 | } |
3927 | case ISD::UMULO: |
3928 | case ISD::SMULO: { |
3929 | SDValue Result, Overflow; |
3930 | if (TLI.expandMULO(Node, Result, Overflow, DAG)) { |
3931 | Results.push_back(Elt: Result); |
3932 | Results.push_back(Elt: Overflow); |
3933 | } |
3934 | break; |
3935 | } |
3936 | case ISD::BUILD_PAIR: { |
3937 | EVT PairTy = Node->getValueType(ResNo: 0); |
3938 | Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: PairTy, Operand: Node->getOperand(Num: 0)); |
3939 | Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: PairTy, Operand: Node->getOperand(Num: 1)); |
3940 | Tmp2 = DAG.getNode( |
3941 | Opcode: ISD::SHL, DL: dl, VT: PairTy, N1: Tmp2, |
3942 | N2: DAG.getConstant(Val: PairTy.getSizeInBits() / 2, DL: dl, |
3943 | VT: TLI.getShiftAmountTy(LHSTy: PairTy, DL: DAG.getDataLayout()))); |
3944 | Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: PairTy, N1: Tmp1, N2: Tmp2)); |
3945 | break; |
3946 | } |
3947 | case ISD::SELECT: |
3948 | Tmp1 = Node->getOperand(Num: 0); |
3949 | Tmp2 = Node->getOperand(Num: 1); |
3950 | Tmp3 = Node->getOperand(Num: 2); |
3951 | if (Tmp1.getOpcode() == ISD::SETCC) { |
3952 | Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1.getOperand(i: 0), RHS: Tmp1.getOperand(i: 1), |
3953 | True: Tmp2, False: Tmp3, |
3954 | Cond: cast<CondCodeSDNode>(Val: Tmp1.getOperand(i: 2))->get()); |
3955 | } else { |
3956 | Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1, |
3957 | RHS: DAG.getConstant(Val: 0, DL: dl, VT: Tmp1.getValueType()), |
3958 | True: Tmp2, False: Tmp3, Cond: ISD::SETNE); |
3959 | } |
3960 | Tmp1->setFlags(Node->getFlags()); |
3961 | Results.push_back(Elt: Tmp1); |
3962 | break; |
3963 | case ISD::BR_JT: { |
3964 | SDValue Chain = Node->getOperand(Num: 0); |
3965 | SDValue Table = Node->getOperand(Num: 1); |
3966 | SDValue Index = Node->getOperand(Num: 2); |
3967 | int JTI = cast<JumpTableSDNode>(Val: Table.getNode())->getIndex(); |
3968 | |
3969 | const DataLayout &TD = DAG.getDataLayout(); |
3970 | EVT PTy = TLI.getPointerTy(DL: TD); |
3971 | |
3972 | unsigned EntrySize = |
3973 | DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD); |
3974 | |
3975 | // For power-of-two jumptable entry sizes convert multiplication to a shift. |
3976 | // This transformation needs to be done here since otherwise the MIPS |
3977 | // backend will end up emitting a three instruction multiply sequence |
3978 | // instead of a single shift and MSP430 will call a runtime function. |
3979 | if (llvm::isPowerOf2_32(Value: EntrySize)) |
3980 | Index = DAG.getNode( |
3981 | Opcode: ISD::SHL, DL: dl, VT: Index.getValueType(), N1: Index, |
3982 | N2: DAG.getConstant(Val: llvm::Log2_32(Value: EntrySize), DL: dl, VT: Index.getValueType())); |
3983 | else |
3984 | Index = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: Index.getValueType(), N1: Index, |
3985 | N2: DAG.getConstant(Val: EntrySize, DL: dl, VT: Index.getValueType())); |
3986 | SDValue Addr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Index.getValueType(), |
3987 | N1: Index, N2: Table); |
3988 | |
3989 | EVT MemVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: EntrySize * 8); |
3990 | SDValue LD = DAG.getExtLoad( |
3991 | ExtType: ISD::SEXTLOAD, dl, VT: PTy, Chain, Ptr: Addr, |
3992 | PtrInfo: MachinePointerInfo::getJumpTable(MF&: DAG.getMachineFunction()), MemVT); |
3993 | Addr = LD; |
3994 | if (TLI.isJumpTableRelative()) { |
3995 | // For PIC, the sequence is: |
3996 | // BRIND(load(Jumptable + index) + RelocBase) |
3997 | // RelocBase can be JumpTable, GOT or some sort of global base. |
3998 | Addr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PTy, N1: Addr, |
3999 | N2: TLI.getPICJumpTableRelocBase(Table, DAG)); |
4000 | } |
4001 | |
4002 | Tmp1 = TLI.expandIndirectJTBranch(dl, Value: LD.getValue(R: 1), Addr, JTI, DAG); |
4003 | Results.push_back(Elt: Tmp1); |
4004 | break; |
4005 | } |
4006 | case ISD::BRCOND: |
4007 | // Expand brcond's setcc into its constituent parts and create a BR_CC |
4008 | // Node. |
4009 | Tmp1 = Node->getOperand(Num: 0); |
4010 | Tmp2 = Node->getOperand(Num: 1); |
4011 | if (Tmp2.getOpcode() == ISD::SETCC && |
4012 | TLI.isOperationLegalOrCustom(Op: ISD::BR_CC, |
4013 | VT: Tmp2.getOperand(i: 0).getValueType())) { |
4014 | Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, Tmp2.getOperand(2), |
4015 | Tmp2.getOperand(0), Tmp2.getOperand(1), |
4016 | Node->getOperand(2)); |
4017 | } else { |
4018 | // We test only the i1 bit. Skip the AND if UNDEF or another AND. |
4019 | if (Tmp2.isUndef() || |
4020 | (Tmp2.getOpcode() == ISD::AND && isOneConstant(V: Tmp2.getOperand(i: 1)))) |
4021 | Tmp3 = Tmp2; |
4022 | else |
4023 | Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp2.getValueType(), N1: Tmp2, |
4024 | N2: DAG.getConstant(Val: 1, DL: dl, VT: Tmp2.getValueType())); |
4025 | Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, |
4026 | DAG.getCondCode(ISD::SETNE), Tmp3, |
4027 | DAG.getConstant(0, dl, Tmp3.getValueType()), |
4028 | Node->getOperand(2)); |
4029 | } |
4030 | Results.push_back(Elt: Tmp1); |
4031 | break; |
4032 | case ISD::SETCC: |
4033 | case ISD::VP_SETCC: |
4034 | case ISD::STRICT_FSETCC: |
4035 | case ISD::STRICT_FSETCCS: { |
4036 | bool IsVP = Node->getOpcode() == ISD::VP_SETCC; |
4037 | bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC || |
4038 | Node->getOpcode() == ISD::STRICT_FSETCCS; |
4039 | bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS; |
4040 | SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue(); |
4041 | unsigned Offset = IsStrict ? 1 : 0; |
4042 | Tmp1 = Node->getOperand(Num: 0 + Offset); |
4043 | Tmp2 = Node->getOperand(Num: 1 + Offset); |
4044 | Tmp3 = Node->getOperand(Num: 2 + Offset); |
4045 | SDValue Mask, EVL; |
4046 | if (IsVP) { |
4047 | Mask = Node->getOperand(Num: 3 + Offset); |
4048 | EVL = Node->getOperand(Num: 4 + Offset); |
4049 | } |
4050 | bool Legalized = TLI.LegalizeSetCCCondCode( |
4051 | DAG, VT: Node->getValueType(ResNo: 0), LHS&: Tmp1, RHS&: Tmp2, CC&: Tmp3, Mask, EVL, NeedInvert, dl, |
4052 | Chain, IsSignaling); |
4053 | |
4054 | if (Legalized) { |
4055 | // If we expanded the SETCC by swapping LHS and RHS, or by inverting the |
4056 | // condition code, create a new SETCC node. |
4057 | if (Tmp3.getNode()) { |
4058 | if (IsStrict) { |
4059 | Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: Node->getVTList(), |
4060 | Ops: {Chain, Tmp1, Tmp2, Tmp3}, Flags: Node->getFlags()); |
4061 | Chain = Tmp1.getValue(R: 1); |
4062 | } else if (IsVP) { |
4063 | Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), |
4064 | Ops: {Tmp1, Tmp2, Tmp3, Mask, EVL}, Flags: Node->getFlags()); |
4065 | } else { |
4066 | Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1, |
4067 | N2: Tmp2, N3: Tmp3, Flags: Node->getFlags()); |
4068 | } |
4069 | } |
4070 | |
4071 | // If we expanded the SETCC by inverting the condition code, then wrap |
4072 | // the existing SETCC in a NOT to restore the intended condition. |
4073 | if (NeedInvert) { |
4074 | if (!IsVP) |
4075 | Tmp1 = DAG.getLogicalNOT(DL: dl, Val: Tmp1, VT: Tmp1->getValueType(ResNo: 0)); |
4076 | else |
4077 | Tmp1 = |
4078 | DAG.getVPLogicalNOT(DL: dl, Val: Tmp1, Mask, EVL, VT: Tmp1->getValueType(ResNo: 0)); |
4079 | } |
4080 | |
4081 | Results.push_back(Elt: Tmp1); |
4082 | if (IsStrict) |
4083 | Results.push_back(Elt: Chain); |
4084 | |
4085 | break; |
4086 | } |
4087 | |
4088 | // FIXME: It seems Legalized is false iff CCCode is Legal. I don't |
4089 | // understand if this code is useful for strict nodes. |
4090 | assert(!IsStrict && "Don't know how to expand for strict nodes." ); |
4091 | |
4092 | // Otherwise, SETCC for the given comparison type must be completely |
4093 | // illegal; expand it into a SELECT_CC. |
4094 | // FIXME: This drops the mask/evl for VP_SETCC. |
4095 | EVT VT = Node->getValueType(ResNo: 0); |
4096 | EVT Tmp1VT = Tmp1.getValueType(); |
4097 | Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT, N1: Tmp1, N2: Tmp2, |
4098 | N3: DAG.getBoolConstant(V: true, DL: dl, VT, OpVT: Tmp1VT), |
4099 | N4: DAG.getBoolConstant(V: false, DL: dl, VT, OpVT: Tmp1VT), N5: Tmp3); |
4100 | Tmp1->setFlags(Node->getFlags()); |
4101 | Results.push_back(Elt: Tmp1); |
4102 | break; |
4103 | } |
4104 | case ISD::SELECT_CC: { |
4105 | // TODO: need to add STRICT_SELECT_CC and STRICT_SELECT_CCS |
4106 | Tmp1 = Node->getOperand(Num: 0); // LHS |
4107 | Tmp2 = Node->getOperand(Num: 1); // RHS |
4108 | Tmp3 = Node->getOperand(Num: 2); // True |
4109 | Tmp4 = Node->getOperand(Num: 3); // False |
4110 | EVT VT = Node->getValueType(ResNo: 0); |
4111 | SDValue Chain; |
4112 | SDValue CC = Node->getOperand(Num: 4); |
4113 | ISD::CondCode CCOp = cast<CondCodeSDNode>(Val&: CC)->get(); |
4114 | |
4115 | if (TLI.isCondCodeLegalOrCustom(CC: CCOp, VT: Tmp1.getSimpleValueType())) { |
4116 | // If the condition code is legal, then we need to expand this |
4117 | // node using SETCC and SELECT. |
4118 | EVT CmpVT = Tmp1.getValueType(); |
4119 | assert(!TLI.isOperationExpand(ISD::SELECT, VT) && |
4120 | "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be " |
4121 | "expanded." ); |
4122 | EVT CCVT = getSetCCResultType(VT: CmpVT); |
4123 | SDValue Cond = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: CCVT, N1: Tmp1, N2: Tmp2, N3: CC, Flags: Node->getFlags()); |
4124 | Results.push_back(Elt: DAG.getSelect(DL: dl, VT, Cond, LHS: Tmp3, RHS: Tmp4)); |
4125 | break; |
4126 | } |
4127 | |
4128 | // SELECT_CC is legal, so the condition code must not be. |
4129 | bool Legalized = false; |
4130 | // Try to legalize by inverting the condition. This is for targets that |
4131 | // might support an ordered version of a condition, but not the unordered |
4132 | // version (or vice versa). |
4133 | ISD::CondCode InvCC = ISD::getSetCCInverse(Operation: CCOp, Type: Tmp1.getValueType()); |
4134 | if (TLI.isCondCodeLegalOrCustom(CC: InvCC, VT: Tmp1.getSimpleValueType())) { |
4135 | // Use the new condition code and swap true and false |
4136 | Legalized = true; |
4137 | Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: Tmp2, True: Tmp4, False: Tmp3, Cond: InvCC); |
4138 | Tmp1->setFlags(Node->getFlags()); |
4139 | } else { |
4140 | // If The inverse is not legal, then try to swap the arguments using |
4141 | // the inverse condition code. |
4142 | ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(Operation: InvCC); |
4143 | if (TLI.isCondCodeLegalOrCustom(CC: SwapInvCC, VT: Tmp1.getSimpleValueType())) { |
4144 | // The swapped inverse condition is legal, so swap true and false, |
4145 | // lhs and rhs. |
4146 | Legalized = true; |
4147 | Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp2, RHS: Tmp1, True: Tmp4, False: Tmp3, Cond: SwapInvCC); |
4148 | Tmp1->setFlags(Node->getFlags()); |
4149 | } |
4150 | } |
4151 | |
4152 | if (!Legalized) { |
4153 | Legalized = TLI.LegalizeSetCCCondCode( |
4154 | DAG, VT: getSetCCResultType(VT: Tmp1.getValueType()), LHS&: Tmp1, RHS&: Tmp2, CC, |
4155 | /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain); |
4156 | |
4157 | assert(Legalized && "Can't legalize SELECT_CC with legal condition!" ); |
4158 | |
4159 | // If we expanded the SETCC by inverting the condition code, then swap |
4160 | // the True/False operands to match. |
4161 | if (NeedInvert) |
4162 | std::swap(a&: Tmp3, b&: Tmp4); |
4163 | |
4164 | // If we expanded the SETCC by swapping LHS and RHS, or by inverting the |
4165 | // condition code, create a new SELECT_CC node. |
4166 | if (CC.getNode()) { |
4167 | Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: Node->getValueType(ResNo: 0), |
4168 | N1: Tmp1, N2: Tmp2, N3: Tmp3, N4: Tmp4, N5: CC); |
4169 | } else { |
4170 | Tmp2 = DAG.getConstant(Val: 0, DL: dl, VT: Tmp1.getValueType()); |
4171 | CC = DAG.getCondCode(Cond: ISD::SETNE); |
4172 | Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1, |
4173 | N2: Tmp2, N3: Tmp3, N4: Tmp4, N5: CC); |
4174 | } |
4175 | Tmp1->setFlags(Node->getFlags()); |
4176 | } |
4177 | Results.push_back(Elt: Tmp1); |
4178 | break; |
4179 | } |
4180 | case ISD::BR_CC: { |
4181 | // TODO: need to add STRICT_BR_CC and STRICT_BR_CCS |
4182 | SDValue Chain; |
4183 | Tmp1 = Node->getOperand(Num: 0); // Chain |
4184 | Tmp2 = Node->getOperand(Num: 2); // LHS |
4185 | Tmp3 = Node->getOperand(Num: 3); // RHS |
4186 | Tmp4 = Node->getOperand(Num: 1); // CC |
4187 | |
4188 | bool Legalized = TLI.LegalizeSetCCCondCode( |
4189 | DAG, VT: getSetCCResultType(VT: Tmp2.getValueType()), LHS&: Tmp2, RHS&: Tmp3, CC&: Tmp4, |
4190 | /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain); |
4191 | (void)Legalized; |
4192 | assert(Legalized && "Can't legalize BR_CC with legal condition!" ); |
4193 | |
4194 | // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC |
4195 | // node. |
4196 | if (Tmp4.getNode()) { |
4197 | assert(!NeedInvert && "Don't know how to invert BR_CC!" ); |
4198 | |
4199 | Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1, |
4200 | N2: Tmp4, N3: Tmp2, N4: Tmp3, N5: Node->getOperand(Num: 4)); |
4201 | } else { |
4202 | Tmp3 = DAG.getConstant(Val: 0, DL: dl, VT: Tmp2.getValueType()); |
4203 | Tmp4 = DAG.getCondCode(Cond: NeedInvert ? ISD::SETEQ : ISD::SETNE); |
4204 | Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1, N2: Tmp4, |
4205 | N3: Tmp2, N4: Tmp3, N5: Node->getOperand(Num: 4)); |
4206 | } |
4207 | Results.push_back(Elt: Tmp1); |
4208 | break; |
4209 | } |
4210 | case ISD::BUILD_VECTOR: |
4211 | Results.push_back(Elt: ExpandBUILD_VECTOR(Node)); |
4212 | break; |
4213 | case ISD::SPLAT_VECTOR: |
4214 | Results.push_back(Elt: ExpandSPLAT_VECTOR(Node)); |
4215 | break; |
4216 | case ISD::SRA: |
4217 | case ISD::SRL: |
4218 | case ISD::SHL: { |
4219 | // Scalarize vector SRA/SRL/SHL. |
4220 | EVT VT = Node->getValueType(ResNo: 0); |
4221 | assert(VT.isVector() && "Unable to legalize non-vector shift" ); |
4222 | assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal" ); |
4223 | unsigned NumElem = VT.getVectorNumElements(); |
4224 | |
4225 | SmallVector<SDValue, 8> Scalars; |
4226 | for (unsigned Idx = 0; Idx < NumElem; Idx++) { |
4227 | SDValue Ex = |
4228 | DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: VT.getScalarType(), |
4229 | N1: Node->getOperand(Num: 0), N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl)); |
4230 | SDValue Sh = |
4231 | DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: VT.getScalarType(), |
4232 | N1: Node->getOperand(Num: 1), N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl)); |
4233 | Scalars.push_back(Elt: DAG.getNode(Opcode: Node->getOpcode(), DL: dl, |
4234 | VT: VT.getScalarType(), N1: Ex, N2: Sh)); |
4235 | } |
4236 | |
4237 | SDValue Result = DAG.getBuildVector(VT: Node->getValueType(ResNo: 0), DL: dl, Ops: Scalars); |
4238 | Results.push_back(Elt: Result); |
4239 | break; |
4240 | } |
4241 | case ISD::VECREDUCE_FADD: |
4242 | case ISD::VECREDUCE_FMUL: |
4243 | case ISD::VECREDUCE_ADD: |
4244 | case ISD::VECREDUCE_MUL: |
4245 | case ISD::VECREDUCE_AND: |
4246 | case ISD::VECREDUCE_OR: |
4247 | case ISD::VECREDUCE_XOR: |
4248 | case ISD::VECREDUCE_SMAX: |
4249 | case ISD::VECREDUCE_SMIN: |
4250 | case ISD::VECREDUCE_UMAX: |
4251 | case ISD::VECREDUCE_UMIN: |
4252 | case ISD::VECREDUCE_FMAX: |
4253 | case ISD::VECREDUCE_FMIN: |
4254 | case ISD::VECREDUCE_FMAXIMUM: |
4255 | case ISD::VECREDUCE_FMINIMUM: |
4256 | Results.push_back(Elt: TLI.expandVecReduce(Node, DAG)); |
4257 | break; |
4258 | case ISD::GLOBAL_OFFSET_TABLE: |
4259 | case ISD::GlobalAddress: |
4260 | case ISD::GlobalTLSAddress: |
4261 | case ISD::ExternalSymbol: |
4262 | case ISD::ConstantPool: |
4263 | case ISD::JumpTable: |
4264 | case ISD::INTRINSIC_W_CHAIN: |
4265 | case ISD::INTRINSIC_WO_CHAIN: |
4266 | case ISD::INTRINSIC_VOID: |
4267 | // FIXME: Custom lowering for these operations shouldn't return null! |
4268 | // Return true so that we don't call ConvertNodeToLibcall which also won't |
4269 | // do anything. |
4270 | return true; |
4271 | } |
4272 | |
4273 | if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) { |
4274 | // FIXME: We were asked to expand a strict floating-point operation, |
4275 | // but there is currently no expansion implemented that would preserve |
4276 | // the "strict" properties. For now, we just fall back to the non-strict |
4277 | // version if that is legal on the target. The actual mutation of the |
4278 | // operation will happen in SelectionDAGISel::DoInstructionSelection. |
4279 | switch (Node->getOpcode()) { |
4280 | default: |
4281 | if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(), |
4282 | VT: Node->getValueType(ResNo: 0)) |
4283 | == TargetLowering::Legal) |
4284 | return true; |
4285 | break; |
4286 | case ISD::STRICT_FSUB: { |
4287 | if (TLI.getStrictFPOperationAction( |
4288 | Op: ISD::STRICT_FSUB, VT: Node->getValueType(ResNo: 0)) == TargetLowering::Legal) |
4289 | return true; |
4290 | if (TLI.getStrictFPOperationAction( |
4291 | Op: ISD::STRICT_FADD, VT: Node->getValueType(ResNo: 0)) != TargetLowering::Legal) |
4292 | break; |
4293 | |
4294 | EVT VT = Node->getValueType(ResNo: 0); |
4295 | const SDNodeFlags Flags = Node->getFlags(); |
4296 | SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Node->getOperand(Num: 2), Flags); |
4297 | SDValue Fadd = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, VTList: Node->getVTList(), |
4298 | Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1), Neg}, |
4299 | Flags); |
4300 | |
4301 | Results.push_back(Elt: Fadd); |
4302 | Results.push_back(Elt: Fadd.getValue(R: 1)); |
4303 | break; |
4304 | } |
4305 | case ISD::STRICT_SINT_TO_FP: |
4306 | case ISD::STRICT_UINT_TO_FP: |
4307 | case ISD::STRICT_LRINT: |
4308 | case ISD::STRICT_LLRINT: |
4309 | case ISD::STRICT_LROUND: |
4310 | case ISD::STRICT_LLROUND: |
4311 | // These are registered by the operand type instead of the value |
4312 | // type. Reflect that here. |
4313 | if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(), |
4314 | VT: Node->getOperand(Num: 1).getValueType()) |
4315 | == TargetLowering::Legal) |
4316 | return true; |
4317 | break; |
4318 | } |
4319 | } |
4320 | |
4321 | // Replace the original node with the legalized result. |
4322 | if (Results.empty()) { |
4323 | LLVM_DEBUG(dbgs() << "Cannot expand node\n" ); |
4324 | return false; |
4325 | } |
4326 | |
4327 | LLVM_DEBUG(dbgs() << "Successfully expanded node\n" ); |
4328 | ReplaceNode(Old: Node, New: Results.data()); |
4329 | return true; |
4330 | } |
4331 | |
4332 | void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) { |
4333 | LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n" ); |
4334 | SmallVector<SDValue, 8> Results; |
4335 | SDLoc dl(Node); |
4336 | // FIXME: Check flags on the node to see if we can use a finite call. |
4337 | unsigned Opc = Node->getOpcode(); |
4338 | switch (Opc) { |
4339 | case ISD::ATOMIC_FENCE: { |
4340 | // If the target didn't lower this, lower it to '__sync_synchronize()' call |
4341 | // FIXME: handle "fence singlethread" more efficiently. |
4342 | TargetLowering::ArgListTy Args; |
4343 | |
4344 | TargetLowering::CallLoweringInfo CLI(DAG); |
4345 | CLI.setDebugLoc(dl) |
4346 | .setChain(Node->getOperand(Num: 0)) |
4347 | .setLibCallee( |
4348 | CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()), |
4349 | Target: DAG.getExternalSymbol(Sym: "__sync_synchronize" , |
4350 | VT: TLI.getPointerTy(DL: DAG.getDataLayout())), |
4351 | ArgsList: std::move(Args)); |
4352 | |
4353 | std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); |
4354 | |
4355 | Results.push_back(Elt: CallResult.second); |
4356 | break; |
4357 | } |
4358 | // By default, atomic intrinsics are marked Legal and lowered. Targets |
4359 | // which don't support them directly, however, may want libcalls, in which |
4360 | // case they mark them Expand, and we get here. |
4361 | case ISD::ATOMIC_SWAP: |
4362 | case ISD::ATOMIC_LOAD_ADD: |
4363 | case ISD::ATOMIC_LOAD_SUB: |
4364 | case ISD::ATOMIC_LOAD_AND: |
4365 | case ISD::ATOMIC_LOAD_CLR: |
4366 | case ISD::ATOMIC_LOAD_OR: |
4367 | case ISD::ATOMIC_LOAD_XOR: |
4368 | case ISD::ATOMIC_LOAD_NAND: |
4369 | case ISD::ATOMIC_LOAD_MIN: |
4370 | case ISD::ATOMIC_LOAD_MAX: |
4371 | case ISD::ATOMIC_LOAD_UMIN: |
4372 | case ISD::ATOMIC_LOAD_UMAX: |
4373 | case ISD::ATOMIC_CMP_SWAP: { |
4374 | MVT VT = cast<AtomicSDNode>(Val: Node)->getMemoryVT().getSimpleVT(); |
4375 | AtomicOrdering Order = cast<AtomicSDNode>(Val: Node)->getMergedOrdering(); |
4376 | RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT); |
4377 | EVT RetVT = Node->getValueType(ResNo: 0); |
4378 | TargetLowering::MakeLibCallOptions CallOptions; |
4379 | SmallVector<SDValue, 4> Ops; |
4380 | if (TLI.getLibcallName(Call: LC)) { |
4381 | // If outline atomic available, prepare its arguments and expand. |
4382 | Ops.append(in_start: Node->op_begin() + 2, in_end: Node->op_end()); |
4383 | Ops.push_back(Elt: Node->getOperand(Num: 1)); |
4384 | |
4385 | } else { |
4386 | LC = RTLIB::getSYNC(Opc, VT); |
4387 | assert(LC != RTLIB::UNKNOWN_LIBCALL && |
4388 | "Unexpected atomic op or value type!" ); |
4389 | // Arguments for expansion to sync libcall |
4390 | Ops.append(in_start: Node->op_begin() + 1, in_end: Node->op_end()); |
4391 | } |
4392 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT, |
4393 | Ops, CallOptions, |
4394 | dl: SDLoc(Node), |
4395 | Chain: Node->getOperand(Num: 0)); |
4396 | Results.push_back(Elt: Tmp.first); |
4397 | Results.push_back(Elt: Tmp.second); |
4398 | break; |
4399 | } |
4400 | case ISD::TRAP: { |
4401 | // If this operation is not supported, lower it to 'abort()' call |
4402 | TargetLowering::ArgListTy Args; |
4403 | TargetLowering::CallLoweringInfo CLI(DAG); |
4404 | CLI.setDebugLoc(dl) |
4405 | .setChain(Node->getOperand(Num: 0)) |
4406 | .setLibCallee(CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()), |
4407 | Target: DAG.getExternalSymbol( |
4408 | Sym: "abort" , VT: TLI.getPointerTy(DL: DAG.getDataLayout())), |
4409 | ArgsList: std::move(Args)); |
4410 | std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); |
4411 | |
4412 | Results.push_back(Elt: CallResult.second); |
4413 | break; |
4414 | } |
4415 | case ISD::FMINNUM: |
4416 | case ISD::STRICT_FMINNUM: |
4417 | ExpandFPLibCall(Node, Call_F32: RTLIB::FMIN_F32, Call_F64: RTLIB::FMIN_F64, |
4418 | Call_F80: RTLIB::FMIN_F80, Call_F128: RTLIB::FMIN_F128, |
4419 | Call_PPCF128: RTLIB::FMIN_PPCF128, Results); |
4420 | break; |
4421 | // FIXME: We do not have libcalls for FMAXIMUM and FMINIMUM. So, we cannot use |
4422 | // libcall legalization for these nodes, but there is no default expasion for |
4423 | // these nodes either (see PR63267 for example). |
4424 | case ISD::FMAXNUM: |
4425 | case ISD::STRICT_FMAXNUM: |
4426 | ExpandFPLibCall(Node, Call_F32: RTLIB::FMAX_F32, Call_F64: RTLIB::FMAX_F64, |
4427 | Call_F80: RTLIB::FMAX_F80, Call_F128: RTLIB::FMAX_F128, |
4428 | Call_PPCF128: RTLIB::FMAX_PPCF128, Results); |
4429 | break; |
4430 | case ISD::FSQRT: |
4431 | case ISD::STRICT_FSQRT: |
4432 | ExpandFPLibCall(Node, Call_F32: RTLIB::SQRT_F32, Call_F64: RTLIB::SQRT_F64, |
4433 | Call_F80: RTLIB::SQRT_F80, Call_F128: RTLIB::SQRT_F128, |
4434 | Call_PPCF128: RTLIB::SQRT_PPCF128, Results); |
4435 | break; |
4436 | case ISD::FCBRT: |
4437 | ExpandFPLibCall(Node, Call_F32: RTLIB::CBRT_F32, Call_F64: RTLIB::CBRT_F64, |
4438 | Call_F80: RTLIB::CBRT_F80, Call_F128: RTLIB::CBRT_F128, |
4439 | Call_PPCF128: RTLIB::CBRT_PPCF128, Results); |
4440 | break; |
4441 | case ISD::FSIN: |
4442 | case ISD::STRICT_FSIN: |
4443 | ExpandFPLibCall(Node, Call_F32: RTLIB::SIN_F32, Call_F64: RTLIB::SIN_F64, |
4444 | Call_F80: RTLIB::SIN_F80, Call_F128: RTLIB::SIN_F128, |
4445 | Call_PPCF128: RTLIB::SIN_PPCF128, Results); |
4446 | break; |
4447 | case ISD::FCOS: |
4448 | case ISD::STRICT_FCOS: |
4449 | ExpandFPLibCall(Node, Call_F32: RTLIB::COS_F32, Call_F64: RTLIB::COS_F64, |
4450 | Call_F80: RTLIB::COS_F80, Call_F128: RTLIB::COS_F128, |
4451 | Call_PPCF128: RTLIB::COS_PPCF128, Results); |
4452 | break; |
4453 | case ISD::FSINCOS: |
4454 | // Expand into sincos libcall. |
4455 | ExpandSinCosLibCall(Node, Results); |
4456 | break; |
4457 | case ISD::FLOG: |
4458 | case ISD::STRICT_FLOG: |
4459 | ExpandFPLibCall(Node, Call_F32: RTLIB::LOG_F32, Call_F64: RTLIB::LOG_F64, Call_F80: RTLIB::LOG_F80, |
4460 | Call_F128: RTLIB::LOG_F128, Call_PPCF128: RTLIB::LOG_PPCF128, Results); |
4461 | break; |
4462 | case ISD::FLOG2: |
4463 | case ISD::STRICT_FLOG2: |
4464 | ExpandFPLibCall(Node, Call_F32: RTLIB::LOG2_F32, Call_F64: RTLIB::LOG2_F64, Call_F80: RTLIB::LOG2_F80, |
4465 | Call_F128: RTLIB::LOG2_F128, Call_PPCF128: RTLIB::LOG2_PPCF128, Results); |
4466 | break; |
4467 | case ISD::FLOG10: |
4468 | case ISD::STRICT_FLOG10: |
4469 | ExpandFPLibCall(Node, Call_F32: RTLIB::LOG10_F32, Call_F64: RTLIB::LOG10_F64, Call_F80: RTLIB::LOG10_F80, |
4470 | Call_F128: RTLIB::LOG10_F128, Call_PPCF128: RTLIB::LOG10_PPCF128, Results); |
4471 | break; |
4472 | case ISD::FEXP: |
4473 | case ISD::STRICT_FEXP: |
4474 | ExpandFPLibCall(Node, Call_F32: RTLIB::EXP_F32, Call_F64: RTLIB::EXP_F64, Call_F80: RTLIB::EXP_F80, |
4475 | Call_F128: RTLIB::EXP_F128, Call_PPCF128: RTLIB::EXP_PPCF128, Results); |
4476 | break; |
4477 | case ISD::FEXP2: |
4478 | case ISD::STRICT_FEXP2: |
4479 | ExpandFPLibCall(Node, Call_F32: RTLIB::EXP2_F32, Call_F64: RTLIB::EXP2_F64, Call_F80: RTLIB::EXP2_F80, |
4480 | Call_F128: RTLIB::EXP2_F128, Call_PPCF128: RTLIB::EXP2_PPCF128, Results); |
4481 | break; |
4482 | case ISD::FEXP10: |
4483 | ExpandFPLibCall(Node, Call_F32: RTLIB::EXP10_F32, Call_F64: RTLIB::EXP10_F64, Call_F80: RTLIB::EXP10_F80, |
4484 | Call_F128: RTLIB::EXP10_F128, Call_PPCF128: RTLIB::EXP10_PPCF128, Results); |
4485 | break; |
4486 | case ISD::FTRUNC: |
4487 | case ISD::STRICT_FTRUNC: |
4488 | ExpandFPLibCall(Node, Call_F32: RTLIB::TRUNC_F32, Call_F64: RTLIB::TRUNC_F64, |
4489 | Call_F80: RTLIB::TRUNC_F80, Call_F128: RTLIB::TRUNC_F128, |
4490 | Call_PPCF128: RTLIB::TRUNC_PPCF128, Results); |
4491 | break; |
4492 | case ISD::FFLOOR: |
4493 | case ISD::STRICT_FFLOOR: |
4494 | ExpandFPLibCall(Node, Call_F32: RTLIB::FLOOR_F32, Call_F64: RTLIB::FLOOR_F64, |
4495 | Call_F80: RTLIB::FLOOR_F80, Call_F128: RTLIB::FLOOR_F128, |
4496 | Call_PPCF128: RTLIB::FLOOR_PPCF128, Results); |
4497 | break; |
4498 | case ISD::FCEIL: |
4499 | case ISD::STRICT_FCEIL: |
4500 | ExpandFPLibCall(Node, Call_F32: RTLIB::CEIL_F32, Call_F64: RTLIB::CEIL_F64, |
4501 | Call_F80: RTLIB::CEIL_F80, Call_F128: RTLIB::CEIL_F128, |
4502 | Call_PPCF128: RTLIB::CEIL_PPCF128, Results); |
4503 | break; |
4504 | case ISD::FRINT: |
4505 | case ISD::STRICT_FRINT: |
4506 | ExpandFPLibCall(Node, Call_F32: RTLIB::RINT_F32, Call_F64: RTLIB::RINT_F64, |
4507 | Call_F80: RTLIB::RINT_F80, Call_F128: RTLIB::RINT_F128, |
4508 | Call_PPCF128: RTLIB::RINT_PPCF128, Results); |
4509 | break; |
4510 | case ISD::FNEARBYINT: |
4511 | case ISD::STRICT_FNEARBYINT: |
4512 | ExpandFPLibCall(Node, Call_F32: RTLIB::NEARBYINT_F32, |
4513 | Call_F64: RTLIB::NEARBYINT_F64, |
4514 | Call_F80: RTLIB::NEARBYINT_F80, |
4515 | Call_F128: RTLIB::NEARBYINT_F128, |
4516 | Call_PPCF128: RTLIB::NEARBYINT_PPCF128, Results); |
4517 | break; |
4518 | case ISD::FROUND: |
4519 | case ISD::STRICT_FROUND: |
4520 | ExpandFPLibCall(Node, Call_F32: RTLIB::ROUND_F32, |
4521 | Call_F64: RTLIB::ROUND_F64, |
4522 | Call_F80: RTLIB::ROUND_F80, |
4523 | Call_F128: RTLIB::ROUND_F128, |
4524 | Call_PPCF128: RTLIB::ROUND_PPCF128, Results); |
4525 | break; |
4526 | case ISD::FROUNDEVEN: |
4527 | case ISD::STRICT_FROUNDEVEN: |
4528 | ExpandFPLibCall(Node, Call_F32: RTLIB::ROUNDEVEN_F32, |
4529 | Call_F64: RTLIB::ROUNDEVEN_F64, |
4530 | Call_F80: RTLIB::ROUNDEVEN_F80, |
4531 | Call_F128: RTLIB::ROUNDEVEN_F128, |
4532 | Call_PPCF128: RTLIB::ROUNDEVEN_PPCF128, Results); |
4533 | break; |
4534 | case ISD::FLDEXP: |
4535 | case ISD::STRICT_FLDEXP: |
4536 | ExpandFPLibCall(Node, Call_F32: RTLIB::LDEXP_F32, Call_F64: RTLIB::LDEXP_F64, Call_F80: RTLIB::LDEXP_F80, |
4537 | Call_F128: RTLIB::LDEXP_F128, Call_PPCF128: RTLIB::LDEXP_PPCF128, Results); |
4538 | break; |
4539 | case ISD::FFREXP: { |
4540 | ExpandFrexpLibCall(Node, Results); |
4541 | break; |
4542 | } |
4543 | case ISD::FPOWI: |
4544 | case ISD::STRICT_FPOWI: { |
4545 | RTLIB::Libcall LC = RTLIB::getPOWI(RetVT: Node->getSimpleValueType(ResNo: 0)); |
4546 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi." ); |
4547 | if (!TLI.getLibcallName(Call: LC)) { |
4548 | // Some targets don't have a powi libcall; use pow instead. |
4549 | if (Node->isStrictFPOpcode()) { |
4550 | SDValue Exponent = |
4551 | DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: SDLoc(Node), |
4552 | ResultTys: {Node->getValueType(ResNo: 0), Node->getValueType(ResNo: 1)}, |
4553 | Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)}); |
4554 | SDValue FPOW = |
4555 | DAG.getNode(Opcode: ISD::STRICT_FPOW, DL: SDLoc(Node), |
4556 | ResultTys: {Node->getValueType(ResNo: 0), Node->getValueType(ResNo: 1)}, |
4557 | Ops: {Exponent.getValue(R: 1), Node->getOperand(Num: 1), Exponent}); |
4558 | Results.push_back(Elt: FPOW); |
4559 | Results.push_back(Elt: FPOW.getValue(R: 1)); |
4560 | } else { |
4561 | SDValue Exponent = |
4562 | DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0), |
4563 | Operand: Node->getOperand(Num: 1)); |
4564 | Results.push_back(Elt: DAG.getNode(Opcode: ISD::FPOW, DL: SDLoc(Node), |
4565 | VT: Node->getValueType(ResNo: 0), |
4566 | N1: Node->getOperand(Num: 0), N2: Exponent)); |
4567 | } |
4568 | break; |
4569 | } |
4570 | unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0; |
4571 | bool ExponentHasSizeOfInt = |
4572 | DAG.getLibInfo().getIntSize() == |
4573 | Node->getOperand(Num: 1 + Offset).getValueType().getSizeInBits(); |
4574 | if (!ExponentHasSizeOfInt) { |
4575 | // If the exponent does not match with sizeof(int) a libcall to |
4576 | // RTLIB::POWI would use the wrong type for the argument. |
4577 | DAG.getContext()->emitError(ErrorStr: "POWI exponent does not match sizeof(int)" ); |
4578 | Results.push_back(Elt: DAG.getUNDEF(VT: Node->getValueType(ResNo: 0))); |
4579 | break; |
4580 | } |
4581 | ExpandFPLibCall(Node, LC, Results); |
4582 | break; |
4583 | } |
4584 | case ISD::FPOW: |
4585 | case ISD::STRICT_FPOW: |
4586 | ExpandFPLibCall(Node, Call_F32: RTLIB::POW_F32, Call_F64: RTLIB::POW_F64, Call_F80: RTLIB::POW_F80, |
4587 | Call_F128: RTLIB::POW_F128, Call_PPCF128: RTLIB::POW_PPCF128, Results); |
4588 | break; |
4589 | case ISD::LROUND: |
4590 | case ISD::STRICT_LROUND: |
4591 | ExpandArgFPLibCall(Node, Call_F32: RTLIB::LROUND_F32, |
4592 | Call_F64: RTLIB::LROUND_F64, Call_F80: RTLIB::LROUND_F80, |
4593 | Call_F128: RTLIB::LROUND_F128, |
4594 | Call_PPCF128: RTLIB::LROUND_PPCF128, Results); |
4595 | break; |
4596 | case ISD::LLROUND: |
4597 | case ISD::STRICT_LLROUND: |
4598 | ExpandArgFPLibCall(Node, Call_F32: RTLIB::LLROUND_F32, |
4599 | Call_F64: RTLIB::LLROUND_F64, Call_F80: RTLIB::LLROUND_F80, |
4600 | Call_F128: RTLIB::LLROUND_F128, |
4601 | Call_PPCF128: RTLIB::LLROUND_PPCF128, Results); |
4602 | break; |
4603 | case ISD::LRINT: |
4604 | case ISD::STRICT_LRINT: |
4605 | ExpandArgFPLibCall(Node, Call_F32: RTLIB::LRINT_F32, |
4606 | Call_F64: RTLIB::LRINT_F64, Call_F80: RTLIB::LRINT_F80, |
4607 | Call_F128: RTLIB::LRINT_F128, |
4608 | Call_PPCF128: RTLIB::LRINT_PPCF128, Results); |
4609 | break; |
4610 | case ISD::LLRINT: |
4611 | case ISD::STRICT_LLRINT: |
4612 | ExpandArgFPLibCall(Node, Call_F32: RTLIB::LLRINT_F32, |
4613 | Call_F64: RTLIB::LLRINT_F64, Call_F80: RTLIB::LLRINT_F80, |
4614 | Call_F128: RTLIB::LLRINT_F128, |
4615 | Call_PPCF128: RTLIB::LLRINT_PPCF128, Results); |
4616 | break; |
4617 | case ISD::FDIV: |
4618 | case ISD::STRICT_FDIV: |
4619 | ExpandFPLibCall(Node, Call_F32: RTLIB::DIV_F32, Call_F64: RTLIB::DIV_F64, |
4620 | Call_F80: RTLIB::DIV_F80, Call_F128: RTLIB::DIV_F128, |
4621 | Call_PPCF128: RTLIB::DIV_PPCF128, Results); |
4622 | break; |
4623 | case ISD::FREM: |
4624 | case ISD::STRICT_FREM: |
4625 | ExpandFPLibCall(Node, Call_F32: RTLIB::REM_F32, Call_F64: RTLIB::REM_F64, |
4626 | Call_F80: RTLIB::REM_F80, Call_F128: RTLIB::REM_F128, |
4627 | Call_PPCF128: RTLIB::REM_PPCF128, Results); |
4628 | break; |
4629 | case ISD::FMA: |
4630 | case ISD::STRICT_FMA: |
4631 | ExpandFPLibCall(Node, Call_F32: RTLIB::FMA_F32, Call_F64: RTLIB::FMA_F64, |
4632 | Call_F80: RTLIB::FMA_F80, Call_F128: RTLIB::FMA_F128, |
4633 | Call_PPCF128: RTLIB::FMA_PPCF128, Results); |
4634 | break; |
4635 | case ISD::FADD: |
4636 | case ISD::STRICT_FADD: |
4637 | ExpandFPLibCall(Node, Call_F32: RTLIB::ADD_F32, Call_F64: RTLIB::ADD_F64, |
4638 | Call_F80: RTLIB::ADD_F80, Call_F128: RTLIB::ADD_F128, |
4639 | Call_PPCF128: RTLIB::ADD_PPCF128, Results); |
4640 | break; |
4641 | case ISD::FMUL: |
4642 | case ISD::STRICT_FMUL: |
4643 | ExpandFPLibCall(Node, Call_F32: RTLIB::MUL_F32, Call_F64: RTLIB::MUL_F64, |
4644 | Call_F80: RTLIB::MUL_F80, Call_F128: RTLIB::MUL_F128, |
4645 | Call_PPCF128: RTLIB::MUL_PPCF128, Results); |
4646 | break; |
4647 | case ISD::FP16_TO_FP: |
4648 | if (Node->getValueType(0) == MVT::f32) { |
4649 | Results.push_back(Elt: ExpandLibCall(LC: RTLIB::FPEXT_F16_F32, Node, isSigned: false).first); |
4650 | } |
4651 | break; |
4652 | case ISD::STRICT_FP16_TO_FP: { |
4653 | if (Node->getValueType(0) == MVT::f32) { |
4654 | TargetLowering::MakeLibCallOptions CallOptions; |
4655 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall( |
4656 | DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Node->getOperand(1), CallOptions, |
4657 | SDLoc(Node), Node->getOperand(0)); |
4658 | Results.push_back(Elt: Tmp.first); |
4659 | Results.push_back(Elt: Tmp.second); |
4660 | } |
4661 | break; |
4662 | } |
4663 | case ISD::FP_TO_FP16: { |
4664 | RTLIB::Libcall LC = |
4665 | RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16); |
4666 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16" ); |
4667 | Results.push_back(Elt: ExpandLibCall(LC, Node, isSigned: false).first); |
4668 | break; |
4669 | } |
4670 | case ISD::FP_TO_BF16: { |
4671 | RTLIB::Libcall LC = |
4672 | RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::bf16); |
4673 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_bf16" ); |
4674 | Results.push_back(Elt: ExpandLibCall(LC, Node, isSigned: false).first); |
4675 | break; |
4676 | } |
4677 | case ISD::STRICT_SINT_TO_FP: |
4678 | case ISD::STRICT_UINT_TO_FP: |
4679 | case ISD::SINT_TO_FP: |
4680 | case ISD::UINT_TO_FP: { |
4681 | // TODO - Common the code with DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP |
4682 | bool IsStrict = Node->isStrictFPOpcode(); |
4683 | bool Signed = Node->getOpcode() == ISD::SINT_TO_FP || |
4684 | Node->getOpcode() == ISD::STRICT_SINT_TO_FP; |
4685 | EVT SVT = Node->getOperand(Num: IsStrict ? 1 : 0).getValueType(); |
4686 | EVT RVT = Node->getValueType(ResNo: 0); |
4687 | EVT NVT = EVT(); |
4688 | SDLoc dl(Node); |
4689 | |
4690 | // Even if the input is legal, no libcall may exactly match, eg. we don't |
4691 | // have i1 -> fp conversions. So, it needs to be promoted to a larger type, |
4692 | // eg: i13 -> fp. Then, look for an appropriate libcall. |
4693 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
4694 | for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE; |
4695 | t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; |
4696 | ++t) { |
4697 | NVT = (MVT::SimpleValueType)t; |
4698 | // The source needs to big enough to hold the operand. |
4699 | if (NVT.bitsGE(SVT)) |
4700 | LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT) |
4701 | : RTLIB::getUINTTOFP(NVT, RVT); |
4702 | } |
4703 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall" ); |
4704 | |
4705 | SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue(); |
4706 | // Sign/zero extend the argument if the libcall takes a larger type. |
4707 | SDValue Op = DAG.getNode(Opcode: Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl, |
4708 | VT: NVT, Operand: Node->getOperand(Num: IsStrict ? 1 : 0)); |
4709 | TargetLowering::MakeLibCallOptions CallOptions; |
4710 | CallOptions.setSExt(Signed); |
4711 | std::pair<SDValue, SDValue> Tmp = |
4712 | TLI.makeLibCall(DAG, LC, RetVT: RVT, Ops: Op, CallOptions, dl, Chain); |
4713 | Results.push_back(Elt: Tmp.first); |
4714 | if (IsStrict) |
4715 | Results.push_back(Elt: Tmp.second); |
4716 | break; |
4717 | } |
4718 | case ISD::FP_TO_SINT: |
4719 | case ISD::FP_TO_UINT: |
4720 | case ISD::STRICT_FP_TO_SINT: |
4721 | case ISD::STRICT_FP_TO_UINT: { |
4722 | // TODO - Common the code with DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT. |
4723 | bool IsStrict = Node->isStrictFPOpcode(); |
4724 | bool Signed = Node->getOpcode() == ISD::FP_TO_SINT || |
4725 | Node->getOpcode() == ISD::STRICT_FP_TO_SINT; |
4726 | |
4727 | SDValue Op = Node->getOperand(Num: IsStrict ? 1 : 0); |
4728 | EVT SVT = Op.getValueType(); |
4729 | EVT RVT = Node->getValueType(ResNo: 0); |
4730 | EVT NVT = EVT(); |
4731 | SDLoc dl(Node); |
4732 | |
4733 | // Even if the result is legal, no libcall may exactly match, eg. we don't |
4734 | // have fp -> i1 conversions. So, it needs to be promoted to a larger type, |
4735 | // eg: fp -> i32. Then, look for an appropriate libcall. |
4736 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
4737 | for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE; |
4738 | IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; |
4739 | ++IntVT) { |
4740 | NVT = (MVT::SimpleValueType)IntVT; |
4741 | // The type needs to big enough to hold the result. |
4742 | if (NVT.bitsGE(RVT)) |
4743 | LC = Signed ? RTLIB::getFPTOSINT(SVT, NVT) |
4744 | : RTLIB::getFPTOUINT(SVT, NVT); |
4745 | } |
4746 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall" ); |
4747 | |
4748 | SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue(); |
4749 | TargetLowering::MakeLibCallOptions CallOptions; |
4750 | std::pair<SDValue, SDValue> Tmp = |
4751 | TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op, CallOptions, dl, Chain); |
4752 | |
4753 | // Truncate the result if the libcall returns a larger type. |
4754 | Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RVT, Operand: Tmp.first)); |
4755 | if (IsStrict) |
4756 | Results.push_back(Elt: Tmp.second); |
4757 | break; |
4758 | } |
4759 | |
4760 | case ISD::FP_ROUND: |
4761 | case ISD::STRICT_FP_ROUND: { |
4762 | // X = FP_ROUND(Y, TRUNC) |
4763 | // TRUNC is a flag, which is always an integer that is zero or one. |
4764 | // If TRUNC is 0, this is a normal rounding, if it is 1, this FP_ROUND |
4765 | // is known to not change the value of Y. |
4766 | // We can only expand it into libcall if the TRUNC is 0. |
4767 | bool IsStrict = Node->isStrictFPOpcode(); |
4768 | SDValue Op = Node->getOperand(Num: IsStrict ? 1 : 0); |
4769 | SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue(); |
4770 | EVT VT = Node->getValueType(ResNo: 0); |
4771 | assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))->isZero() && |
4772 | "Unable to expand as libcall if it is not normal rounding" ); |
4773 | |
4774 | RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op.getValueType(), RetVT: VT); |
4775 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall" ); |
4776 | |
4777 | TargetLowering::MakeLibCallOptions CallOptions; |
4778 | std::pair<SDValue, SDValue> Tmp = |
4779 | TLI.makeLibCall(DAG, LC, RetVT: VT, Ops: Op, CallOptions, dl: SDLoc(Node), Chain); |
4780 | Results.push_back(Elt: Tmp.first); |
4781 | if (IsStrict) |
4782 | Results.push_back(Elt: Tmp.second); |
4783 | break; |
4784 | } |
4785 | case ISD::FP_EXTEND: { |
4786 | Results.push_back( |
4787 | Elt: ExpandLibCall(LC: RTLIB::getFPEXT(OpVT: Node->getOperand(Num: 0).getValueType(), |
4788 | RetVT: Node->getValueType(ResNo: 0)), |
4789 | Node, isSigned: false).first); |
4790 | break; |
4791 | } |
4792 | case ISD::STRICT_FP_EXTEND: |
4793 | case ISD::STRICT_FP_TO_FP16: { |
4794 | RTLIB::Libcall LC = |
4795 | Node->getOpcode() == ISD::STRICT_FP_TO_FP16 |
4796 | ? RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::f16) |
4797 | : RTLIB::getFPEXT(Node->getOperand(1).getValueType(), |
4798 | Node->getValueType(0)); |
4799 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall" ); |
4800 | |
4801 | TargetLowering::MakeLibCallOptions CallOptions; |
4802 | std::pair<SDValue, SDValue> Tmp = |
4803 | TLI.makeLibCall(DAG, LC, RetVT: Node->getValueType(ResNo: 0), Ops: Node->getOperand(Num: 1), |
4804 | CallOptions, dl: SDLoc(Node), Chain: Node->getOperand(Num: 0)); |
4805 | Results.push_back(Elt: Tmp.first); |
4806 | Results.push_back(Elt: Tmp.second); |
4807 | break; |
4808 | } |
4809 | case ISD::FSUB: |
4810 | case ISD::STRICT_FSUB: |
4811 | ExpandFPLibCall(Node, Call_F32: RTLIB::SUB_F32, Call_F64: RTLIB::SUB_F64, |
4812 | Call_F80: RTLIB::SUB_F80, Call_F128: RTLIB::SUB_F128, |
4813 | Call_PPCF128: RTLIB::SUB_PPCF128, Results); |
4814 | break; |
4815 | case ISD::SREM: |
4816 | Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: true, |
4817 | Call_I8: RTLIB::SREM_I8, |
4818 | Call_I16: RTLIB::SREM_I16, Call_I32: RTLIB::SREM_I32, |
4819 | Call_I64: RTLIB::SREM_I64, Call_I128: RTLIB::SREM_I128)); |
4820 | break; |
4821 | case ISD::UREM: |
4822 | Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false, |
4823 | Call_I8: RTLIB::UREM_I8, |
4824 | Call_I16: RTLIB::UREM_I16, Call_I32: RTLIB::UREM_I32, |
4825 | Call_I64: RTLIB::UREM_I64, Call_I128: RTLIB::UREM_I128)); |
4826 | break; |
4827 | case ISD::SDIV: |
4828 | Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: true, |
4829 | Call_I8: RTLIB::SDIV_I8, |
4830 | Call_I16: RTLIB::SDIV_I16, Call_I32: RTLIB::SDIV_I32, |
4831 | Call_I64: RTLIB::SDIV_I64, Call_I128: RTLIB::SDIV_I128)); |
4832 | break; |
4833 | case ISD::UDIV: |
4834 | Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false, |
4835 | Call_I8: RTLIB::UDIV_I8, |
4836 | Call_I16: RTLIB::UDIV_I16, Call_I32: RTLIB::UDIV_I32, |
4837 | Call_I64: RTLIB::UDIV_I64, Call_I128: RTLIB::UDIV_I128)); |
4838 | break; |
4839 | case ISD::SDIVREM: |
4840 | case ISD::UDIVREM: |
4841 | // Expand into divrem libcall |
4842 | ExpandDivRemLibCall(Node, Results); |
4843 | break; |
4844 | case ISD::MUL: |
4845 | Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false, |
4846 | Call_I8: RTLIB::MUL_I8, |
4847 | Call_I16: RTLIB::MUL_I16, Call_I32: RTLIB::MUL_I32, |
4848 | Call_I64: RTLIB::MUL_I64, Call_I128: RTLIB::MUL_I128)); |
4849 | break; |
4850 | case ISD::CTLZ_ZERO_UNDEF: |
4851 | switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) { |
4852 | default: |
4853 | llvm_unreachable("LibCall explicitly requested, but not available" ); |
4854 | case MVT::i32: |
4855 | Results.push_back(Elt: ExpandLibCall(LC: RTLIB::CTLZ_I32, Node, isSigned: false).first); |
4856 | break; |
4857 | case MVT::i64: |
4858 | Results.push_back(Elt: ExpandLibCall(LC: RTLIB::CTLZ_I64, Node, isSigned: false).first); |
4859 | break; |
4860 | case MVT::i128: |
4861 | Results.push_back(Elt: ExpandLibCall(LC: RTLIB::CTLZ_I128, Node, isSigned: false).first); |
4862 | break; |
4863 | } |
4864 | break; |
4865 | case ISD::RESET_FPENV: { |
4866 | // It is legalized to call 'fesetenv(FE_DFL_ENV)'. On most targets |
4867 | // FE_DFL_ENV is defined as '((const fenv_t *) -1)' in glibc. |
4868 | SDValue Ptr = DAG.getIntPtrConstant(Val: -1LL, DL: dl); |
4869 | SDValue Chain = Node->getOperand(Num: 0); |
4870 | Results.push_back( |
4871 | Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETENV, Ptr, InChain: Chain, DLoc: dl)); |
4872 | break; |
4873 | } |
4874 | case ISD::GET_FPENV_MEM: { |
4875 | SDValue Chain = Node->getOperand(Num: 0); |
4876 | SDValue EnvPtr = Node->getOperand(Num: 1); |
4877 | Results.push_back( |
4878 | Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FEGETENV, Ptr: EnvPtr, InChain: Chain, DLoc: dl)); |
4879 | break; |
4880 | } |
4881 | case ISD::SET_FPENV_MEM: { |
4882 | SDValue Chain = Node->getOperand(Num: 0); |
4883 | SDValue EnvPtr = Node->getOperand(Num: 1); |
4884 | Results.push_back( |
4885 | Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETENV, Ptr: EnvPtr, InChain: Chain, DLoc: dl)); |
4886 | break; |
4887 | } |
4888 | case ISD::GET_FPMODE: { |
4889 | // Call fegetmode, which saves control modes into a stack slot. Then load |
4890 | // the value to return from the stack. |
4891 | EVT ModeVT = Node->getValueType(ResNo: 0); |
4892 | SDValue StackPtr = DAG.CreateStackTemporary(VT: ModeVT); |
4893 | int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex(); |
4894 | SDValue Chain = DAG.makeStateFunctionCall(LibFunc: RTLIB::FEGETMODE, Ptr: StackPtr, |
4895 | InChain: Node->getOperand(Num: 0), DLoc: dl); |
4896 | SDValue LdInst = DAG.getLoad( |
4897 | VT: ModeVT, dl, Chain, Ptr: StackPtr, |
4898 | PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI)); |
4899 | Results.push_back(Elt: LdInst); |
4900 | Results.push_back(Elt: LdInst.getValue(R: 1)); |
4901 | break; |
4902 | } |
4903 | case ISD::SET_FPMODE: { |
4904 | // Move control modes to stack slot and then call fesetmode with the pointer |
4905 | // to the slot as argument. |
4906 | SDValue Mode = Node->getOperand(Num: 1); |
4907 | EVT ModeVT = Mode.getValueType(); |
4908 | SDValue StackPtr = DAG.CreateStackTemporary(VT: ModeVT); |
4909 | int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex(); |
4910 | SDValue StInst = DAG.getStore( |
4911 | Chain: Node->getOperand(Num: 0), dl, Val: Mode, Ptr: StackPtr, |
4912 | PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI)); |
4913 | Results.push_back( |
4914 | Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETMODE, Ptr: StackPtr, InChain: StInst, DLoc: dl)); |
4915 | break; |
4916 | } |
4917 | case ISD::RESET_FPMODE: { |
4918 | // It is legalized to a call 'fesetmode(FE_DFL_MODE)'. On most targets |
4919 | // FE_DFL_MODE is defined as '((const femode_t *) -1)' in glibc. If not, the |
4920 | // target must provide custom lowering. |
4921 | const DataLayout &DL = DAG.getDataLayout(); |
4922 | EVT PtrTy = TLI.getPointerTy(DL); |
4923 | SDValue Mode = DAG.getConstant(Val: -1LL, DL: dl, VT: PtrTy); |
4924 | Results.push_back(Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETMODE, Ptr: Mode, |
4925 | InChain: Node->getOperand(Num: 0), DLoc: dl)); |
4926 | break; |
4927 | } |
4928 | } |
4929 | |
4930 | // Replace the original node with the legalized result. |
4931 | if (!Results.empty()) { |
4932 | LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n" ); |
4933 | ReplaceNode(Old: Node, New: Results.data()); |
4934 | } else |
4935 | LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n" ); |
4936 | } |
4937 | |
4938 | // Determine the vector type to use in place of an original scalar element when |
4939 | // promoting equally sized vectors. |
4940 | static MVT getPromotedVectorElementType(const TargetLowering &TLI, |
4941 | MVT EltVT, MVT NewEltVT) { |
4942 | unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits(); |
4943 | MVT MidVT = OldEltsPerNewElt == 1 |
4944 | ? NewEltVT |
4945 | : MVT::getVectorVT(VT: NewEltVT, NumElements: OldEltsPerNewElt); |
4946 | assert(TLI.isTypeLegal(MidVT) && "unexpected" ); |
4947 | return MidVT; |
4948 | } |
4949 | |
4950 | void SelectionDAGLegalize::PromoteNode(SDNode *Node) { |
4951 | LLVM_DEBUG(dbgs() << "Trying to promote node\n" ); |
4952 | SmallVector<SDValue, 8> Results; |
4953 | MVT OVT = Node->getSimpleValueType(ResNo: 0); |
4954 | if (Node->getOpcode() == ISD::UINT_TO_FP || |
4955 | Node->getOpcode() == ISD::SINT_TO_FP || |
4956 | Node->getOpcode() == ISD::SETCC || |
4957 | Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT || |
4958 | Node->getOpcode() == ISD::INSERT_VECTOR_ELT) { |
4959 | OVT = Node->getOperand(Num: 0).getSimpleValueType(); |
4960 | } |
4961 | if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP || |
4962 | Node->getOpcode() == ISD::STRICT_SINT_TO_FP || |
4963 | Node->getOpcode() == ISD::STRICT_FSETCC || |
4964 | Node->getOpcode() == ISD::STRICT_FSETCCS) |
4965 | OVT = Node->getOperand(Num: 1).getSimpleValueType(); |
4966 | if (Node->getOpcode() == ISD::BR_CC || |
4967 | Node->getOpcode() == ISD::SELECT_CC) |
4968 | OVT = Node->getOperand(Num: 2).getSimpleValueType(); |
4969 | MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: OVT); |
4970 | SDLoc dl(Node); |
4971 | SDValue Tmp1, Tmp2, Tmp3, Tmp4; |
4972 | switch (Node->getOpcode()) { |
4973 | case ISD::CTTZ: |
4974 | case ISD::CTTZ_ZERO_UNDEF: |
4975 | case ISD::CTLZ: |
4976 | case ISD::CTLZ_ZERO_UNDEF: |
4977 | case ISD::CTPOP: |
4978 | // Zero extend the argument unless its cttz, then use any_extend. |
4979 | if (Node->getOpcode() == ISD::CTTZ || |
4980 | Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF) |
4981 | Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0)); |
4982 | else |
4983 | Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0)); |
4984 | |
4985 | if (Node->getOpcode() == ISD::CTTZ) { |
4986 | // The count is the same in the promoted type except if the original |
4987 | // value was zero. This can be handled by setting the bit just off |
4988 | // the top of the original type. |
4989 | auto TopBit = APInt::getOneBitSet(numBits: NVT.getSizeInBits(), |
4990 | BitNo: OVT.getSizeInBits()); |
4991 | Tmp1 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, N1: Tmp1, |
4992 | N2: DAG.getConstant(Val: TopBit, DL: dl, VT: NVT)); |
4993 | } |
4994 | // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is |
4995 | // already the correct result. |
4996 | Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1); |
4997 | if (Node->getOpcode() == ISD::CTLZ || |
4998 | Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) { |
4999 | // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT)) |
5000 | Tmp1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, N1: Tmp1, |
5001 | N2: DAG.getConstant(Val: NVT.getSizeInBits() - |
5002 | OVT.getSizeInBits(), DL: dl, VT: NVT)); |
5003 | } |
5004 | Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1)); |
5005 | break; |
5006 | case ISD::BITREVERSE: |
5007 | case ISD::BSWAP: { |
5008 | unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits(); |
5009 | Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0)); |
5010 | Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1); |
5011 | Tmp1 = DAG.getNode( |
5012 | Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Tmp1, |
5013 | N2: DAG.getConstant(Val: DiffBits, DL: dl, |
5014 | VT: TLI.getShiftAmountTy(LHSTy: NVT, DL: DAG.getDataLayout()))); |
5015 | |
5016 | Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1)); |
5017 | break; |
5018 | } |
5019 | case ISD::FP_TO_UINT: |
5020 | case ISD::STRICT_FP_TO_UINT: |
5021 | case ISD::FP_TO_SINT: |
5022 | case ISD::STRICT_FP_TO_SINT: |
5023 | PromoteLegalFP_TO_INT(N: Node, dl, Results); |
5024 | break; |
5025 | case ISD::FP_TO_UINT_SAT: |
5026 | case ISD::FP_TO_SINT_SAT: |
5027 | Results.push_back(Elt: PromoteLegalFP_TO_INT_SAT(Node, dl)); |
5028 | break; |
5029 | case ISD::UINT_TO_FP: |
5030 | case ISD::STRICT_UINT_TO_FP: |
5031 | case ISD::SINT_TO_FP: |
5032 | case ISD::STRICT_SINT_TO_FP: |
5033 | PromoteLegalINT_TO_FP(N: Node, dl, Results); |
5034 | break; |
5035 | case ISD::VAARG: { |
5036 | SDValue Chain = Node->getOperand(Num: 0); // Get the chain. |
5037 | SDValue Ptr = Node->getOperand(Num: 1); // Get the pointer. |
5038 | |
5039 | unsigned TruncOp; |
5040 | if (OVT.isVector()) { |
5041 | TruncOp = ISD::BITCAST; |
5042 | } else { |
5043 | assert(OVT.isInteger() |
5044 | && "VAARG promotion is supported only for vectors or integer types" ); |
5045 | TruncOp = ISD::TRUNCATE; |
5046 | } |
5047 | |
5048 | // Perform the larger operation, then convert back |
5049 | Tmp1 = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV: Node->getOperand(Num: 2), |
5050 | Align: Node->getConstantOperandVal(Num: 3)); |
5051 | Chain = Tmp1.getValue(R: 1); |
5052 | |
5053 | Tmp2 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: OVT, Operand: Tmp1); |
5054 | |
5055 | // Modified the chain result - switch anything that used the old chain to |
5056 | // use the new one. |
5057 | DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: Tmp2); |
5058 | DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: Chain); |
5059 | if (UpdatedNodes) { |
5060 | UpdatedNodes->insert(X: Tmp2.getNode()); |
5061 | UpdatedNodes->insert(X: Chain.getNode()); |
5062 | } |
5063 | ReplacedNode(N: Node); |
5064 | break; |
5065 | } |
5066 | case ISD::MUL: |
5067 | case ISD::SDIV: |
5068 | case ISD::SREM: |
5069 | case ISD::UDIV: |
5070 | case ISD::UREM: |
5071 | case ISD::SMIN: |
5072 | case ISD::SMAX: |
5073 | case ISD::UMIN: |
5074 | case ISD::UMAX: |
5075 | case ISD::AND: |
5076 | case ISD::OR: |
5077 | case ISD::XOR: { |
5078 | unsigned ExtOp, TruncOp; |
5079 | if (OVT.isVector()) { |
5080 | ExtOp = ISD::BITCAST; |
5081 | TruncOp = ISD::BITCAST; |
5082 | } else { |
5083 | assert(OVT.isInteger() && "Cannot promote logic operation" ); |
5084 | |
5085 | switch (Node->getOpcode()) { |
5086 | default: |
5087 | ExtOp = ISD::ANY_EXTEND; |
5088 | break; |
5089 | case ISD::SDIV: |
5090 | case ISD::SREM: |
5091 | case ISD::SMIN: |
5092 | case ISD::SMAX: |
5093 | ExtOp = ISD::SIGN_EXTEND; |
5094 | break; |
5095 | case ISD::UDIV: |
5096 | case ISD::UREM: |
5097 | ExtOp = ISD::ZERO_EXTEND; |
5098 | break; |
5099 | case ISD::UMIN: |
5100 | case ISD::UMAX: |
5101 | if (TLI.isSExtCheaperThanZExt(FromTy: OVT, ToTy: NVT)) |
5102 | ExtOp = ISD::SIGN_EXTEND; |
5103 | else |
5104 | ExtOp = ISD::ZERO_EXTEND; |
5105 | break; |
5106 | } |
5107 | TruncOp = ISD::TRUNCATE; |
5108 | } |
5109 | // Promote each of the values to the new type. |
5110 | Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0)); |
5111 | Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1)); |
5112 | // Perform the larger operation, then convert back |
5113 | Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2); |
5114 | Results.push_back(Elt: DAG.getNode(Opcode: TruncOp, DL: dl, VT: OVT, Operand: Tmp1)); |
5115 | break; |
5116 | } |
5117 | case ISD::UMUL_LOHI: |
5118 | case ISD::SMUL_LOHI: { |
5119 | // Promote to a multiply in a wider integer type. |
5120 | unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND |
5121 | : ISD::SIGN_EXTEND; |
5122 | Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0)); |
5123 | Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1)); |
5124 | Tmp1 = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2); |
5125 | |
5126 | auto &DL = DAG.getDataLayout(); |
5127 | unsigned OriginalSize = OVT.getScalarSizeInBits(); |
5128 | Tmp2 = DAG.getNode( |
5129 | Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Tmp1, |
5130 | N2: DAG.getConstant(Val: OriginalSize, DL: dl, VT: TLI.getScalarShiftAmountTy(DL, NVT))); |
5131 | Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1)); |
5132 | Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp2)); |
5133 | break; |
5134 | } |
5135 | case ISD::SELECT: { |
5136 | unsigned ExtOp, TruncOp; |
5137 | if (Node->getValueType(ResNo: 0).isVector() || |
5138 | Node->getValueType(ResNo: 0).getSizeInBits() == NVT.getSizeInBits()) { |
5139 | ExtOp = ISD::BITCAST; |
5140 | TruncOp = ISD::BITCAST; |
5141 | } else if (Node->getValueType(ResNo: 0).isInteger()) { |
5142 | ExtOp = ISD::ANY_EXTEND; |
5143 | TruncOp = ISD::TRUNCATE; |
5144 | } else { |
5145 | ExtOp = ISD::FP_EXTEND; |
5146 | TruncOp = ISD::FP_ROUND; |
5147 | } |
5148 | Tmp1 = Node->getOperand(Num: 0); |
5149 | // Promote each of the values to the new type. |
5150 | Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1)); |
5151 | Tmp3 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2)); |
5152 | // Perform the larger operation, then round down. |
5153 | Tmp1 = DAG.getSelect(DL: dl, VT: NVT, Cond: Tmp1, LHS: Tmp2, RHS: Tmp3); |
5154 | Tmp1->setFlags(Node->getFlags()); |
5155 | if (TruncOp != ISD::FP_ROUND) |
5156 | Tmp1 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1); |
5157 | else |
5158 | Tmp1 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1, |
5159 | N2: DAG.getIntPtrConstant(Val: 0, DL: dl)); |
5160 | Results.push_back(Elt: Tmp1); |
5161 | break; |
5162 | } |
5163 | case ISD::VECTOR_SHUFFLE: { |
5164 | ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Node)->getMask(); |
5165 | |
5166 | // Cast the two input vectors. |
5167 | Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0)); |
5168 | Tmp2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1)); |
5169 | |
5170 | // Convert the shuffle mask to the right # elements. |
5171 | Tmp1 = ShuffleWithNarrowerEltType(NVT, VT: OVT, dl, N1: Tmp1, N2: Tmp2, Mask); |
5172 | Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OVT, Operand: Tmp1); |
5173 | Results.push_back(Elt: Tmp1); |
5174 | break; |
5175 | } |
5176 | case ISD::VECTOR_SPLICE: { |
5177 | Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0)); |
5178 | Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1)); |
5179 | Tmp3 = DAG.getNode(Opcode: ISD::VECTOR_SPLICE, DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2, |
5180 | N3: Node->getOperand(Num: 2)); |
5181 | Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp3)); |
5182 | break; |
5183 | } |
5184 | case ISD::SELECT_CC: { |
5185 | SDValue Cond = Node->getOperand(Num: 4); |
5186 | ISD::CondCode CCCode = cast<CondCodeSDNode>(Val&: Cond)->get(); |
5187 | // Type of the comparison operands. |
5188 | MVT CVT = Node->getSimpleValueType(ResNo: 0); |
5189 | assert(CVT == OVT && "not handled" ); |
5190 | |
5191 | unsigned ExtOp = ISD::FP_EXTEND; |
5192 | if (NVT.isInteger()) { |
5193 | ExtOp = isSignedIntSetCC(Code: CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; |
5194 | } |
5195 | |
5196 | // Promote the comparison operands, if needed. |
5197 | if (TLI.isCondCodeLegal(CC: CCCode, VT: CVT)) { |
5198 | Tmp1 = Node->getOperand(Num: 0); |
5199 | Tmp2 = Node->getOperand(Num: 1); |
5200 | } else { |
5201 | Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0)); |
5202 | Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1)); |
5203 | } |
5204 | // Cast the true/false operands. |
5205 | Tmp3 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2)); |
5206 | Tmp4 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 3)); |
5207 | |
5208 | Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: NVT, Ops: {Tmp1, Tmp2, Tmp3, Tmp4, Cond}, |
5209 | Flags: Node->getFlags()); |
5210 | |
5211 | // Cast the result back to the original type. |
5212 | if (ExtOp != ISD::FP_EXTEND) |
5213 | Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1); |
5214 | else |
5215 | Tmp1 = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp1, |
5216 | N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)); |
5217 | |
5218 | Results.push_back(Elt: Tmp1); |
5219 | break; |
5220 | } |
5221 | case ISD::SETCC: |
5222 | case ISD::STRICT_FSETCC: |
5223 | case ISD::STRICT_FSETCCS: { |
5224 | unsigned ExtOp = ISD::FP_EXTEND; |
5225 | if (NVT.isInteger()) { |
5226 | ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Node->getOperand(Num: 2))->get(); |
5227 | if (isSignedIntSetCC(Code: CCCode) || |
5228 | TLI.isSExtCheaperThanZExt(FromTy: Node->getOperand(Num: 0).getValueType(), ToTy: NVT)) |
5229 | ExtOp = ISD::SIGN_EXTEND; |
5230 | else |
5231 | ExtOp = ISD::ZERO_EXTEND; |
5232 | } |
5233 | if (Node->isStrictFPOpcode()) { |
5234 | SDValue InChain = Node->getOperand(Num: 0); |
5235 | std::tie(args&: Tmp1, args: std::ignore) = |
5236 | DAG.getStrictFPExtendOrRound(Op: Node->getOperand(Num: 1), Chain: InChain, DL: dl, VT: NVT); |
5237 | std::tie(args&: Tmp2, args: std::ignore) = |
5238 | DAG.getStrictFPExtendOrRound(Op: Node->getOperand(Num: 2), Chain: InChain, DL: dl, VT: NVT); |
5239 | SmallVector<SDValue, 2> TmpChains = {Tmp1.getValue(R: 1), Tmp2.getValue(R: 1)}; |
5240 | SDValue OutChain = DAG.getTokenFactor(DL: dl, Vals&: TmpChains); |
5241 | SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); |
5242 | Results.push_back(Elt: DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: VTs, |
5243 | Ops: {OutChain, Tmp1, Tmp2, Node->getOperand(Num: 3)}, |
5244 | Flags: Node->getFlags())); |
5245 | Results.push_back(Elt: Results.back().getValue(R: 1)); |
5246 | break; |
5247 | } |
5248 | Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0)); |
5249 | Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1)); |
5250 | Results.push_back(Elt: DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1, |
5251 | N2: Tmp2, N3: Node->getOperand(Num: 2), Flags: Node->getFlags())); |
5252 | break; |
5253 | } |
5254 | case ISD::BR_CC: { |
5255 | unsigned ExtOp = ISD::FP_EXTEND; |
5256 | if (NVT.isInteger()) { |
5257 | ISD::CondCode CCCode = |
5258 | cast<CondCodeSDNode>(Val: Node->getOperand(Num: 1))->get(); |
5259 | ExtOp = isSignedIntSetCC(Code: CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; |
5260 | } |
5261 | Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2)); |
5262 | Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 3)); |
5263 | Results.push_back(Elt: DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0), |
5264 | N1: Node->getOperand(Num: 0), N2: Node->getOperand(Num: 1), |
5265 | N3: Tmp1, N4: Tmp2, N5: Node->getOperand(Num: 4))); |
5266 | break; |
5267 | } |
5268 | case ISD::FADD: |
5269 | case ISD::FSUB: |
5270 | case ISD::FMUL: |
5271 | case ISD::FDIV: |
5272 | case ISD::FREM: |
5273 | case ISD::FMINNUM: |
5274 | case ISD::FMAXNUM: |
5275 | case ISD::FMINIMUM: |
5276 | case ISD::FMAXIMUM: |
5277 | case ISD::FPOW: |
5278 | Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0)); |
5279 | Tmp2 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1)); |
5280 | Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2, |
5281 | Flags: Node->getFlags()); |
5282 | Results.push_back( |
5283 | Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp3, |
5284 | N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true))); |
5285 | break; |
5286 | case ISD::STRICT_FADD: |
5287 | case ISD::STRICT_FSUB: |
5288 | case ISD::STRICT_FMUL: |
5289 | case ISD::STRICT_FDIV: |
5290 | case ISD::STRICT_FMINNUM: |
5291 | case ISD::STRICT_FMAXNUM: |
5292 | case ISD::STRICT_FREM: |
5293 | case ISD::STRICT_FPOW: |
5294 | Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, |
5295 | {Node->getOperand(0), Node->getOperand(1)}); |
5296 | Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, |
5297 | {Node->getOperand(0), Node->getOperand(2)}); |
5298 | Tmp3 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1), |
5299 | Tmp2.getValue(1)); |
5300 | Tmp1 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, |
5301 | {Tmp3, Tmp1, Tmp2}); |
5302 | Tmp1 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other}, |
5303 | {Tmp1.getValue(1), Tmp1, DAG.getIntPtrConstant(0, dl)}); |
5304 | Results.push_back(Elt: Tmp1); |
5305 | Results.push_back(Elt: Tmp1.getValue(R: 1)); |
5306 | break; |
5307 | case ISD::FMA: |
5308 | Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0)); |
5309 | Tmp2 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1)); |
5310 | Tmp3 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2)); |
5311 | Results.push_back( |
5312 | Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, |
5313 | N1: DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2, N3: Tmp3), |
5314 | N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true))); |
5315 | break; |
5316 | case ISD::STRICT_FMA: |
5317 | Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, |
5318 | {Node->getOperand(0), Node->getOperand(1)}); |
5319 | Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, |
5320 | {Node->getOperand(0), Node->getOperand(2)}); |
5321 | Tmp3 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, |
5322 | {Node->getOperand(0), Node->getOperand(3)}); |
5323 | Tmp4 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1), |
5324 | Tmp2.getValue(1), Tmp3.getValue(1)); |
5325 | Tmp4 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, |
5326 | {Tmp4, Tmp1, Tmp2, Tmp3}); |
5327 | Tmp4 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other}, |
5328 | {Tmp4.getValue(1), Tmp4, DAG.getIntPtrConstant(0, dl)}); |
5329 | Results.push_back(Elt: Tmp4); |
5330 | Results.push_back(Elt: Tmp4.getValue(R: 1)); |
5331 | break; |
5332 | case ISD::FCOPYSIGN: |
5333 | case ISD::FLDEXP: |
5334 | case ISD::FPOWI: { |
5335 | Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0)); |
5336 | Tmp2 = Node->getOperand(Num: 1); |
5337 | Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2); |
5338 | |
5339 | // fcopysign doesn't change anything but the sign bit, so |
5340 | // (fp_round (fcopysign (fpext a), b)) |
5341 | // is as precise as |
5342 | // (fp_round (fpext a)) |
5343 | // which is a no-op. Mark it as a TRUNCating FP_ROUND. |
5344 | const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN); |
5345 | Results.push_back( |
5346 | Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp3, |
5347 | N2: DAG.getIntPtrConstant(Val: isTrunc, DL: dl, /*isTarget=*/true))); |
5348 | break; |
5349 | } |
5350 | case ISD::STRICT_FPOWI: |
5351 | Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, |
5352 | {Node->getOperand(0), Node->getOperand(1)}); |
5353 | Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, |
5354 | {Tmp1.getValue(1), Tmp1, Node->getOperand(2)}); |
5355 | Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other}, |
5356 | {Tmp2.getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)}); |
5357 | Results.push_back(Elt: Tmp3); |
5358 | Results.push_back(Elt: Tmp3.getValue(R: 1)); |
5359 | break; |
5360 | case ISD::FFREXP: { |
5361 | Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0)); |
5362 | Tmp2 = DAG.getNode(Opcode: ISD::FFREXP, DL: dl, ResultTys: {NVT, Node->getValueType(ResNo: 1)}, Ops: Tmp1); |
5363 | |
5364 | Results.push_back( |
5365 | Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2, |
5366 | N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true))); |
5367 | |
5368 | Results.push_back(Elt: Tmp2.getValue(R: 1)); |
5369 | break; |
5370 | } |
5371 | case ISD::FFLOOR: |
5372 | case ISD::FCEIL: |
5373 | case ISD::FRINT: |
5374 | case ISD::FNEARBYINT: |
5375 | case ISD::FROUND: |
5376 | case ISD::FROUNDEVEN: |
5377 | case ISD::FTRUNC: |
5378 | case ISD::FNEG: |
5379 | case ISD::FSQRT: |
5380 | case ISD::FSIN: |
5381 | case ISD::FCOS: |
5382 | case ISD::FLOG: |
5383 | case ISD::FLOG2: |
5384 | case ISD::FLOG10: |
5385 | case ISD::FABS: |
5386 | case ISD::FEXP: |
5387 | case ISD::FEXP2: |
5388 | case ISD::FEXP10: |
5389 | case ISD::FCANONICALIZE: |
5390 | Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0)); |
5391 | Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1); |
5392 | Results.push_back( |
5393 | Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2, |
5394 | N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true))); |
5395 | break; |
5396 | case ISD::STRICT_FFLOOR: |
5397 | case ISD::STRICT_FCEIL: |
5398 | case ISD::STRICT_FRINT: |
5399 | case ISD::STRICT_FNEARBYINT: |
5400 | case ISD::STRICT_FROUND: |
5401 | case ISD::STRICT_FROUNDEVEN: |
5402 | case ISD::STRICT_FTRUNC: |
5403 | case ISD::STRICT_FSQRT: |
5404 | case ISD::STRICT_FSIN: |
5405 | case ISD::STRICT_FCOS: |
5406 | case ISD::STRICT_FLOG: |
5407 | case ISD::STRICT_FLOG2: |
5408 | case ISD::STRICT_FLOG10: |
5409 | case ISD::STRICT_FEXP: |
5410 | case ISD::STRICT_FEXP2: |
5411 | Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, |
5412 | {Node->getOperand(0), Node->getOperand(1)}); |
5413 | Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, |
5414 | {Tmp1.getValue(1), Tmp1}); |
5415 | Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other}, |
5416 | {Tmp2.getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)}); |
5417 | Results.push_back(Elt: Tmp3); |
5418 | Results.push_back(Elt: Tmp3.getValue(R: 1)); |
5419 | break; |
5420 | case ISD::BUILD_VECTOR: { |
5421 | MVT EltVT = OVT.getVectorElementType(); |
5422 | MVT NewEltVT = NVT.getVectorElementType(); |
5423 | |
5424 | // Handle bitcasts to a different vector type with the same total bit size |
5425 | // |
5426 | // e.g. v2i64 = build_vector i64:x, i64:y => v4i32 |
5427 | // => |
5428 | // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y)) |
5429 | |
5430 | assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && |
5431 | "Invalid promote type for build_vector" ); |
5432 | assert(NewEltVT.bitsLE(EltVT) && "not handled" ); |
5433 | |
5434 | MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); |
5435 | |
5436 | SmallVector<SDValue, 8> NewOps; |
5437 | for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) { |
5438 | SDValue Op = Node->getOperand(Num: I); |
5439 | NewOps.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(Op), VT: MidVT, Operand: Op)); |
5440 | } |
5441 | |
5442 | SDLoc SL(Node); |
5443 | SDValue Concat = |
5444 | DAG.getNode(Opcode: MidVT == NewEltVT ? ISD::BUILD_VECTOR : ISD::CONCAT_VECTORS, |
5445 | DL: SL, VT: NVT, Ops: NewOps); |
5446 | SDValue CvtVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: Concat); |
5447 | Results.push_back(Elt: CvtVec); |
5448 | break; |
5449 | } |
5450 | case ISD::EXTRACT_VECTOR_ELT: { |
5451 | MVT EltVT = OVT.getVectorElementType(); |
5452 | MVT NewEltVT = NVT.getVectorElementType(); |
5453 | |
5454 | // Handle bitcasts to a different vector type with the same total bit size. |
5455 | // |
5456 | // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32 |
5457 | // => |
5458 | // v4i32:castx = bitcast x:v2i64 |
5459 | // |
5460 | // i64 = bitcast |
5461 | // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))), |
5462 | // (i32 (extract_vector_elt castx, (2 * y + 1))) |
5463 | // |
5464 | |
5465 | assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && |
5466 | "Invalid promote type for extract_vector_elt" ); |
5467 | assert(NewEltVT.bitsLT(EltVT) && "not handled" ); |
5468 | |
5469 | MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); |
5470 | unsigned NewEltsPerOldElt = MidVT.getVectorNumElements(); |
5471 | |
5472 | SDValue Idx = Node->getOperand(Num: 1); |
5473 | EVT IdxVT = Idx.getValueType(); |
5474 | SDLoc SL(Node); |
5475 | SDValue Factor = DAG.getConstant(Val: NewEltsPerOldElt, DL: SL, VT: IdxVT); |
5476 | SDValue NewBaseIdx = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: IdxVT, N1: Idx, N2: Factor); |
5477 | |
5478 | SDValue CastVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: Node->getOperand(Num: 0)); |
5479 | |
5480 | SmallVector<SDValue, 8> NewOps; |
5481 | for (unsigned I = 0; I < NewEltsPerOldElt; ++I) { |
5482 | SDValue IdxOffset = DAG.getConstant(Val: I, DL: SL, VT: IdxVT); |
5483 | SDValue TmpIdx = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: IdxVT, N1: NewBaseIdx, N2: IdxOffset); |
5484 | |
5485 | SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: NewEltVT, |
5486 | N1: CastVec, N2: TmpIdx); |
5487 | NewOps.push_back(Elt); |
5488 | } |
5489 | |
5490 | SDValue NewVec = DAG.getBuildVector(VT: MidVT, DL: SL, Ops: NewOps); |
5491 | Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: EltVT, Operand: NewVec)); |
5492 | break; |
5493 | } |
5494 | case ISD::INSERT_VECTOR_ELT: { |
5495 | MVT EltVT = OVT.getVectorElementType(); |
5496 | MVT NewEltVT = NVT.getVectorElementType(); |
5497 | |
5498 | // Handle bitcasts to a different vector type with the same total bit size |
5499 | // |
5500 | // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32 |
5501 | // => |
5502 | // v4i32:castx = bitcast x:v2i64 |
5503 | // v2i32:casty = bitcast y:i64 |
5504 | // |
5505 | // v2i64 = bitcast |
5506 | // (v4i32 insert_vector_elt |
5507 | // (v4i32 insert_vector_elt v4i32:castx, |
5508 | // (extract_vector_elt casty, 0), 2 * z), |
5509 | // (extract_vector_elt casty, 1), (2 * z + 1)) |
5510 | |
5511 | assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && |
5512 | "Invalid promote type for insert_vector_elt" ); |
5513 | assert(NewEltVT.bitsLT(EltVT) && "not handled" ); |
5514 | |
5515 | MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); |
5516 | unsigned NewEltsPerOldElt = MidVT.getVectorNumElements(); |
5517 | |
5518 | SDValue Val = Node->getOperand(Num: 1); |
5519 | SDValue Idx = Node->getOperand(Num: 2); |
5520 | EVT IdxVT = Idx.getValueType(); |
5521 | SDLoc SL(Node); |
5522 | |
5523 | SDValue Factor = DAG.getConstant(Val: NewEltsPerOldElt, DL: SDLoc(), VT: IdxVT); |
5524 | SDValue NewBaseIdx = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: IdxVT, N1: Idx, N2: Factor); |
5525 | |
5526 | SDValue CastVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: Node->getOperand(Num: 0)); |
5527 | SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MidVT, Operand: Val); |
5528 | |
5529 | SDValue NewVec = CastVec; |
5530 | for (unsigned I = 0; I < NewEltsPerOldElt; ++I) { |
5531 | SDValue IdxOffset = DAG.getConstant(Val: I, DL: SL, VT: IdxVT); |
5532 | SDValue InEltIdx = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: IdxVT, N1: NewBaseIdx, N2: IdxOffset); |
5533 | |
5534 | SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: NewEltVT, |
5535 | N1: CastVal, N2: IdxOffset); |
5536 | |
5537 | NewVec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: NVT, |
5538 | N1: NewVec, N2: Elt, N3: InEltIdx); |
5539 | } |
5540 | |
5541 | Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewVec)); |
5542 | break; |
5543 | } |
5544 | case ISD::SCALAR_TO_VECTOR: { |
5545 | MVT EltVT = OVT.getVectorElementType(); |
5546 | MVT NewEltVT = NVT.getVectorElementType(); |
5547 | |
5548 | // Handle bitcasts to different vector type with the same total bit size. |
5549 | // |
5550 | // e.g. v2i64 = scalar_to_vector x:i64 |
5551 | // => |
5552 | // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef) |
5553 | // |
5554 | |
5555 | MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); |
5556 | SDValue Val = Node->getOperand(Num: 0); |
5557 | SDLoc SL(Node); |
5558 | |
5559 | SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MidVT, Operand: Val); |
5560 | SDValue Undef = DAG.getUNDEF(VT: MidVT); |
5561 | |
5562 | SmallVector<SDValue, 8> NewElts; |
5563 | NewElts.push_back(Elt: CastVal); |
5564 | for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I) |
5565 | NewElts.push_back(Elt: Undef); |
5566 | |
5567 | SDValue Concat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT: NVT, Ops: NewElts); |
5568 | SDValue CvtVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: Concat); |
5569 | Results.push_back(Elt: CvtVec); |
5570 | break; |
5571 | } |
5572 | case ISD::ATOMIC_SWAP: { |
5573 | AtomicSDNode *AM = cast<AtomicSDNode>(Val: Node); |
5574 | SDLoc SL(Node); |
5575 | SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: AM->getVal()); |
5576 | assert(NVT.getSizeInBits() == OVT.getSizeInBits() && |
5577 | "unexpected promotion type" ); |
5578 | assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() && |
5579 | "unexpected atomic_swap with illegal type" ); |
5580 | |
5581 | SDValue NewAtomic |
5582 | = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT, |
5583 | DAG.getVTList(NVT, MVT::Other), |
5584 | { AM->getChain(), AM->getBasePtr(), CastVal }, |
5585 | AM->getMemOperand()); |
5586 | Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewAtomic)); |
5587 | Results.push_back(Elt: NewAtomic.getValue(R: 1)); |
5588 | break; |
5589 | } |
5590 | case ISD::SPLAT_VECTOR: { |
5591 | SDValue Scalar = Node->getOperand(Num: 0); |
5592 | MVT ScalarType = Scalar.getSimpleValueType(); |
5593 | MVT NewScalarType = NVT.getVectorElementType(); |
5594 | if (ScalarType.isInteger()) { |
5595 | Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NewScalarType, Operand: Scalar); |
5596 | Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1); |
5597 | Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp2)); |
5598 | break; |
5599 | } |
5600 | Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NewScalarType, Operand: Scalar); |
5601 | Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1); |
5602 | Results.push_back( |
5603 | Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2, |
5604 | N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true))); |
5605 | break; |
5606 | } |
5607 | } |
5608 | |
5609 | // Replace the original node with the legalized result. |
5610 | if (!Results.empty()) { |
5611 | LLVM_DEBUG(dbgs() << "Successfully promoted node\n" ); |
5612 | ReplaceNode(Old: Node, New: Results.data()); |
5613 | } else |
5614 | LLVM_DEBUG(dbgs() << "Could not promote node\n" ); |
5615 | } |
5616 | |
5617 | /// This is the entry point for the file. |
5618 | void SelectionDAG::Legalize() { |
5619 | AssignTopologicalOrder(); |
5620 | |
5621 | SmallPtrSet<SDNode *, 16> LegalizedNodes; |
5622 | // Use a delete listener to remove nodes which were deleted during |
5623 | // legalization from LegalizeNodes. This is needed to handle the situation |
5624 | // where a new node is allocated by the object pool to the same address of a |
5625 | // previously deleted node. |
5626 | DAGNodeDeletedListener DeleteListener( |
5627 | *this, |
5628 | [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(Ptr: N); }); |
5629 | |
5630 | SelectionDAGLegalize Legalizer(*this, LegalizedNodes); |
5631 | |
5632 | // Visit all the nodes. We start in topological order, so that we see |
5633 | // nodes with their original operands intact. Legalization can produce |
5634 | // new nodes which may themselves need to be legalized. Iterate until all |
5635 | // nodes have been legalized. |
5636 | while (true) { |
5637 | bool AnyLegalized = false; |
5638 | for (auto NI = allnodes_end(); NI != allnodes_begin();) { |
5639 | --NI; |
5640 | |
5641 | SDNode *N = &*NI; |
5642 | if (N->use_empty() && N != getRoot().getNode()) { |
5643 | ++NI; |
5644 | DeleteNode(N); |
5645 | continue; |
5646 | } |
5647 | |
5648 | if (LegalizedNodes.insert(Ptr: N).second) { |
5649 | AnyLegalized = true; |
5650 | Legalizer.LegalizeOp(Node: N); |
5651 | |
5652 | if (N->use_empty() && N != getRoot().getNode()) { |
5653 | ++NI; |
5654 | DeleteNode(N); |
5655 | } |
5656 | } |
5657 | } |
5658 | if (!AnyLegalized) |
5659 | break; |
5660 | |
5661 | } |
5662 | |
5663 | // Remove dead nodes now. |
5664 | RemoveDeadNodes(); |
5665 | } |
5666 | |
5667 | bool SelectionDAG::LegalizeOp(SDNode *N, |
5668 | SmallSetVector<SDNode *, 16> &UpdatedNodes) { |
5669 | SmallPtrSet<SDNode *, 16> LegalizedNodes; |
5670 | SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes); |
5671 | |
5672 | // Directly insert the node in question, and legalize it. This will recurse |
5673 | // as needed through operands. |
5674 | LegalizedNodes.insert(Ptr: N); |
5675 | Legalizer.LegalizeOp(Node: N); |
5676 | |
5677 | return LegalizedNodes.count(Ptr: N); |
5678 | } |
5679 | |