1 | //===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file implements float type expansion and softening for LegalizeTypes. |
10 | // Softening is the act of turning a computation in an illegal floating point |
11 | // type into a computation in an integer type of the same size; also known as |
12 | // "soft float". For example, turning f32 arithmetic into operations using i32. |
13 | // The resulting integer value is the same as what you would get by performing |
14 | // the floating point operation and bitcasting the result to the integer type. |
15 | // Expansion is the act of changing a computation in an illegal type to be a |
16 | // computation in two identical registers of a smaller type. For example, |
17 | // implementing ppcf128 arithmetic in two f64 registers. |
18 | // |
19 | //===----------------------------------------------------------------------===// |
20 | |
21 | #include "LegalizeTypes.h" |
22 | #include "llvm/Analysis/TargetLibraryInfo.h" |
23 | #include "llvm/Support/ErrorHandling.h" |
24 | #include "llvm/Support/raw_ostream.h" |
25 | using namespace llvm; |
26 | |
27 | #define DEBUG_TYPE "legalize-types" |
28 | |
29 | /// GetFPLibCall - Return the right libcall for the given floating point type. |
30 | /// FIXME: This is a local version of RTLIB::getFPLibCall that should be |
31 | /// refactored away (see RTLIB::getPOWI for an example). |
32 | static RTLIB::Libcall GetFPLibCall(EVT VT, |
33 | RTLIB::Libcall Call_F32, |
34 | RTLIB::Libcall Call_F64, |
35 | RTLIB::Libcall Call_F80, |
36 | RTLIB::Libcall Call_F128, |
37 | RTLIB::Libcall Call_PPCF128) { |
38 | return |
39 | VT == MVT::f32 ? Call_F32 : |
40 | VT == MVT::f64 ? Call_F64 : |
41 | VT == MVT::f80 ? Call_F80 : |
42 | VT == MVT::f128 ? Call_F128 : |
43 | VT == MVT::ppcf128 ? Call_PPCF128 : |
44 | RTLIB::UNKNOWN_LIBCALL; |
45 | } |
46 | |
47 | //===----------------------------------------------------------------------===// |
48 | // Convert Float Results to Integer |
49 | //===----------------------------------------------------------------------===// |
50 | |
51 | void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) { |
52 | LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": " ; N->dump(&DAG)); |
53 | SDValue R = SDValue(); |
54 | |
55 | switch (N->getOpcode()) { |
56 | // clang-format off |
57 | default: |
58 | #ifndef NDEBUG |
59 | dbgs() << "SoftenFloatResult #" << ResNo << ": " ; |
60 | N->dump(G: &DAG); dbgs() << "\n" ; |
61 | #endif |
62 | report_fatal_error(reason: "Do not know how to soften the result of this " |
63 | "operator!" ); |
64 | case ISD::EXTRACT_ELEMENT: R = SoftenFloatRes_EXTRACT_ELEMENT(N); break; |
65 | case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break; |
66 | case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break; |
67 | case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break; |
68 | case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break; |
69 | case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break; |
70 | case ISD::EXTRACT_VECTOR_ELT: |
71 | R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break; |
72 | case ISD::FABS: R = SoftenFloatRes_FABS(N); break; |
73 | case ISD::STRICT_FMINNUM: |
74 | case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break; |
75 | case ISD::STRICT_FMAXNUM: |
76 | case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break; |
77 | case ISD::STRICT_FADD: |
78 | case ISD::FADD: R = SoftenFloatRes_FADD(N); break; |
79 | case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break; |
80 | case ISD::STRICT_FCEIL: |
81 | case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break; |
82 | case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break; |
83 | case ISD::STRICT_FCOS: |
84 | case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break; |
85 | case ISD::STRICT_FDIV: |
86 | case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break; |
87 | case ISD::STRICT_FEXP: |
88 | case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break; |
89 | case ISD::STRICT_FEXP2: |
90 | case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break; |
91 | case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break; |
92 | case ISD::STRICT_FFLOOR: |
93 | case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break; |
94 | case ISD::STRICT_FLOG: |
95 | case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break; |
96 | case ISD::STRICT_FLOG2: |
97 | case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break; |
98 | case ISD::STRICT_FLOG10: |
99 | case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break; |
100 | case ISD::STRICT_FMA: |
101 | case ISD::FMA: R = SoftenFloatRes_FMA(N); break; |
102 | case ISD::STRICT_FMUL: |
103 | case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break; |
104 | case ISD::STRICT_FNEARBYINT: |
105 | case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break; |
106 | case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break; |
107 | case ISD::STRICT_FP_EXTEND: |
108 | case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break; |
109 | case ISD::STRICT_FP_ROUND: |
110 | case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break; |
111 | case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break; |
112 | case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break; |
113 | case ISD::STRICT_FPOW: |
114 | case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break; |
115 | case ISD::STRICT_FPOWI: |
116 | case ISD::FPOWI: |
117 | case ISD::FLDEXP: |
118 | case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break; |
119 | case ISD::FFREXP: R = SoftenFloatRes_FFREXP(N); break; |
120 | case ISD::STRICT_FREM: |
121 | case ISD::FREM: R = SoftenFloatRes_FREM(N); break; |
122 | case ISD::STRICT_FRINT: |
123 | case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break; |
124 | case ISD::STRICT_FROUND: |
125 | case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break; |
126 | case ISD::STRICT_FROUNDEVEN: |
127 | case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break; |
128 | case ISD::STRICT_FSIN: |
129 | case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break; |
130 | case ISD::STRICT_FSQRT: |
131 | case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break; |
132 | case ISD::STRICT_FSUB: |
133 | case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break; |
134 | case ISD::STRICT_FTRUNC: |
135 | case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break; |
136 | case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break; |
137 | case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; |
138 | case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break; |
139 | case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break; |
140 | case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break; |
141 | case ISD::STRICT_SINT_TO_FP: |
142 | case ISD::STRICT_UINT_TO_FP: |
143 | case ISD::SINT_TO_FP: |
144 | case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break; |
145 | case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break; |
146 | case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break; |
147 | case ISD::VECREDUCE_FADD: |
148 | case ISD::VECREDUCE_FMUL: |
149 | case ISD::VECREDUCE_FMIN: |
150 | case ISD::VECREDUCE_FMAX: |
151 | case ISD::VECREDUCE_FMAXIMUM: |
152 | case ISD::VECREDUCE_FMINIMUM: R = SoftenFloatRes_VECREDUCE(N); break; |
153 | case ISD::VECREDUCE_SEQ_FADD: |
154 | case ISD::VECREDUCE_SEQ_FMUL: R = SoftenFloatRes_VECREDUCE_SEQ(N); break; |
155 | // clang-format on |
156 | } |
157 | |
158 | // If R is null, the sub-method took care of registering the result. |
159 | if (R.getNode()) { |
160 | assert(R.getNode() != N); |
161 | SetSoftenedFloat(Op: SDValue(N, ResNo), Result: R); |
162 | } |
163 | } |
164 | |
165 | SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) { |
166 | bool IsStrict = N->isStrictFPOpcode(); |
167 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
168 | unsigned Offset = IsStrict ? 1 : 0; |
169 | assert(N->getNumOperands() == (1 + Offset) && |
170 | "Unexpected number of operands!" ); |
171 | SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)); |
172 | SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue(); |
173 | TargetLowering::MakeLibCallOptions CallOptions; |
174 | EVT OpVT = N->getOperand(Num: 0 + Offset).getValueType(); |
175 | CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0), Value: true); |
176 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op, |
177 | CallOptions, dl: SDLoc(N), |
178 | Chain); |
179 | if (IsStrict) |
180 | ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second); |
181 | return Tmp.first; |
182 | } |
183 | |
184 | SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) { |
185 | bool IsStrict = N->isStrictFPOpcode(); |
186 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
187 | unsigned Offset = IsStrict ? 1 : 0; |
188 | assert(N->getNumOperands() == (2 + Offset) && |
189 | "Unexpected number of operands!" ); |
190 | SDValue Ops[2] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)), |
191 | GetSoftenedFloat(Op: N->getOperand(Num: 1 + Offset)) }; |
192 | SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue(); |
193 | TargetLowering::MakeLibCallOptions CallOptions; |
194 | EVT OpsVT[2] = { N->getOperand(Num: 0 + Offset).getValueType(), |
195 | N->getOperand(Num: 1 + Offset).getValueType() }; |
196 | CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0), Value: true); |
197 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops, |
198 | CallOptions, dl: SDLoc(N), |
199 | Chain); |
200 | if (IsStrict) |
201 | ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second); |
202 | return Tmp.first; |
203 | } |
204 | |
205 | SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) { |
206 | return BitConvertToInteger(Op: N->getOperand(Num: 0)); |
207 | } |
208 | |
209 | SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) { |
210 | EVT Ty = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
211 | return DAG.getNode(Opcode: ISD::FREEZE, DL: SDLoc(N), VT: Ty, |
212 | Operand: GetSoftenedFloat(Op: N->getOperand(Num: 0))); |
213 | } |
214 | |
215 | SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) { |
216 | EVT Ty = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
217 | SDValue NewFence = DAG.getNode(Opcode: ISD::ARITH_FENCE, DL: SDLoc(N), VT: Ty, |
218 | Operand: GetSoftenedFloat(Op: N->getOperand(Num: 0))); |
219 | return NewFence; |
220 | } |
221 | |
222 | SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N, |
223 | unsigned ResNo) { |
224 | SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); |
225 | return BitConvertToInteger(Op); |
226 | } |
227 | |
228 | SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) { |
229 | // Convert the inputs to integers, and build a new pair out of them. |
230 | return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SDLoc(N), |
231 | VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), |
232 | VT: N->getValueType(ResNo: 0)), |
233 | N1: BitConvertToInteger(Op: N->getOperand(Num: 0)), |
234 | N2: BitConvertToInteger(Op: N->getOperand(Num: 1))); |
235 | } |
236 | |
237 | SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) { |
238 | ConstantFPSDNode *CN = cast<ConstantFPSDNode>(Val: N); |
239 | // In ppcf128, the high 64 bits are always first in memory regardless |
240 | // of Endianness. LLVM's APFloat representation is not Endian sensitive, |
241 | // and so always converts into a 128-bit APInt in a non-Endian-sensitive |
242 | // way. However, APInt's are serialized in an Endian-sensitive fashion, |
243 | // so on big-Endian targets, the two doubles are output in the wrong |
244 | // order. Fix this by manually flipping the order of the high 64 bits |
245 | // and the low 64 bits here. |
246 | if (DAG.getDataLayout().isBigEndian() && |
247 | CN->getValueType(ResNo: 0).getSimpleVT() == llvm::MVT::ppcf128) { |
248 | uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1], |
249 | CN->getValueAPF().bitcastToAPInt().getRawData()[0] }; |
250 | APInt Val(128, words); |
251 | return DAG.getConstant(Val, DL: SDLoc(CN), |
252 | VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), |
253 | VT: CN->getValueType(ResNo: 0))); |
254 | } else { |
255 | return DAG.getConstant(Val: CN->getValueAPF().bitcastToAPInt(), DL: SDLoc(CN), |
256 | VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), |
257 | VT: CN->getValueType(ResNo: 0))); |
258 | } |
259 | } |
260 | |
261 | SDValue DAGTypeLegalizer::(SDNode *N) { |
262 | SDValue Src = N->getOperand(Num: 0); |
263 | assert(Src.getValueType() == MVT::ppcf128 && |
264 | "In floats only ppcf128 can be extracted by element!" ); |
265 | return DAG.getNode(ISD::EXTRACT_ELEMENT, SDLoc(N), |
266 | N->getValueType(ResNo: 0).changeTypeToInteger(), |
267 | DAG.getBitcast(MVT::VT: i128, V: Src), N->getOperand(Num: 1)); |
268 | } |
269 | |
270 | SDValue DAGTypeLegalizer::(SDNode *N, unsigned ResNo) { |
271 | SDValue NewOp = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0)); |
272 | return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N), |
273 | VT: NewOp.getValueType().getVectorElementType(), |
274 | N1: NewOp, N2: N->getOperand(Num: 1)); |
275 | } |
276 | |
277 | SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) { |
278 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
279 | unsigned Size = NVT.getSizeInBits(); |
280 | |
281 | // Mask = ~(1 << (Size-1)) |
282 | APInt API = APInt::getAllOnes(numBits: Size); |
283 | API.clearBit(BitPosition: Size - 1); |
284 | SDValue Mask = DAG.getConstant(Val: API, DL: SDLoc(N), VT: NVT); |
285 | SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0)); |
286 | return DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: NVT, N1: Op, N2: Mask); |
287 | } |
288 | |
289 | SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) { |
290 | if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(Node: N, DAG)) |
291 | return SoftenFloatRes_SELECT_CC(N: SelCC.getNode()); |
292 | return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
293 | Call_F32: RTLIB::FMIN_F32, |
294 | Call_F64: RTLIB::FMIN_F64, |
295 | Call_F80: RTLIB::FMIN_F80, |
296 | Call_F128: RTLIB::FMIN_F128, |
297 | Call_PPCF128: RTLIB::FMIN_PPCF128)); |
298 | } |
299 | |
300 | SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) { |
301 | if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(Node: N, DAG)) |
302 | return SoftenFloatRes_SELECT_CC(N: SelCC.getNode()); |
303 | return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
304 | Call_F32: RTLIB::FMAX_F32, |
305 | Call_F64: RTLIB::FMAX_F64, |
306 | Call_F80: RTLIB::FMAX_F80, |
307 | Call_F128: RTLIB::FMAX_F128, |
308 | Call_PPCF128: RTLIB::FMAX_PPCF128)); |
309 | } |
310 | |
311 | SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) { |
312 | return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
313 | Call_F32: RTLIB::ADD_F32, |
314 | Call_F64: RTLIB::ADD_F64, |
315 | Call_F80: RTLIB::ADD_F80, |
316 | Call_F128: RTLIB::ADD_F128, |
317 | Call_PPCF128: RTLIB::ADD_PPCF128)); |
318 | } |
319 | |
320 | SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) { |
321 | return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
322 | Call_F32: RTLIB::CBRT_F32, |
323 | Call_F64: RTLIB::CBRT_F64, |
324 | Call_F80: RTLIB::CBRT_F80, |
325 | Call_F128: RTLIB::CBRT_F128, |
326 | Call_PPCF128: RTLIB::CBRT_PPCF128)); |
327 | } |
328 | |
329 | SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) { |
330 | return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
331 | Call_F32: RTLIB::CEIL_F32, |
332 | Call_F64: RTLIB::CEIL_F64, |
333 | Call_F80: RTLIB::CEIL_F80, |
334 | Call_F128: RTLIB::CEIL_F128, |
335 | Call_PPCF128: RTLIB::CEIL_PPCF128)); |
336 | } |
337 | |
338 | SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) { |
339 | SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 0)); |
340 | SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1)); |
341 | SDLoc dl(N); |
342 | |
343 | EVT LVT = LHS.getValueType(); |
344 | EVT RVT = RHS.getValueType(); |
345 | |
346 | unsigned LSize = LVT.getSizeInBits(); |
347 | unsigned RSize = RVT.getSizeInBits(); |
348 | |
349 | // First get the sign bit of second operand. |
350 | SDValue SignBit = DAG.getNode( |
351 | Opcode: ISD::SHL, DL: dl, VT: RVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: RVT), |
352 | N2: DAG.getConstant(Val: RSize - 1, DL: dl, |
353 | VT: TLI.getShiftAmountTy(LHSTy: RVT, DL: DAG.getDataLayout()))); |
354 | SignBit = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: RVT, N1: RHS, N2: SignBit); |
355 | |
356 | // Shift right or sign-extend it if the two operands have different types. |
357 | int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits(); |
358 | if (SizeDiff > 0) { |
359 | SignBit = |
360 | DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: SignBit, |
361 | N2: DAG.getConstant(Val: SizeDiff, DL: dl, |
362 | VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(), |
363 | DL: DAG.getDataLayout()))); |
364 | SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LVT, Operand: SignBit); |
365 | } else if (SizeDiff < 0) { |
366 | SignBit = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: SignBit); |
367 | SignBit = |
368 | DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: LVT, N1: SignBit, |
369 | N2: DAG.getConstant(Val: -SizeDiff, DL: dl, |
370 | VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(), |
371 | DL: DAG.getDataLayout()))); |
372 | } |
373 | |
374 | // Clear the sign bit of the first operand. |
375 | SDValue Mask = DAG.getNode( |
376 | Opcode: ISD::SHL, DL: dl, VT: LVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: LVT), |
377 | N2: DAG.getConstant(Val: LSize - 1, DL: dl, |
378 | VT: TLI.getShiftAmountTy(LHSTy: LVT, DL: DAG.getDataLayout()))); |
379 | Mask = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: LVT, N1: Mask, N2: DAG.getConstant(Val: 1, DL: dl, VT: LVT)); |
380 | LHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: LVT, N1: LHS, N2: Mask); |
381 | |
382 | // Or the value with the sign bit. |
383 | return DAG.getNode(Opcode: ISD::OR, DL: dl, VT: LVT, N1: LHS, N2: SignBit); |
384 | } |
385 | |
386 | SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) { |
387 | return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
388 | Call_F32: RTLIB::COS_F32, |
389 | Call_F64: RTLIB::COS_F64, |
390 | Call_F80: RTLIB::COS_F80, |
391 | Call_F128: RTLIB::COS_F128, |
392 | Call_PPCF128: RTLIB::COS_PPCF128)); |
393 | } |
394 | |
395 | SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) { |
396 | return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
397 | Call_F32: RTLIB::DIV_F32, |
398 | Call_F64: RTLIB::DIV_F64, |
399 | Call_F80: RTLIB::DIV_F80, |
400 | Call_F128: RTLIB::DIV_F128, |
401 | Call_PPCF128: RTLIB::DIV_PPCF128)); |
402 | } |
403 | |
404 | SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) { |
405 | return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
406 | Call_F32: RTLIB::EXP_F32, |
407 | Call_F64: RTLIB::EXP_F64, |
408 | Call_F80: RTLIB::EXP_F80, |
409 | Call_F128: RTLIB::EXP_F128, |
410 | Call_PPCF128: RTLIB::EXP_PPCF128)); |
411 | } |
412 | |
413 | SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) { |
414 | return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
415 | Call_F32: RTLIB::EXP2_F32, |
416 | Call_F64: RTLIB::EXP2_F64, |
417 | Call_F80: RTLIB::EXP2_F80, |
418 | Call_F128: RTLIB::EXP2_F128, |
419 | Call_PPCF128: RTLIB::EXP2_PPCF128)); |
420 | } |
421 | |
422 | SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) { |
423 | return SoftenFloatRes_Unary( |
424 | N, |
425 | LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::EXP10_F32, Call_F64: RTLIB::EXP10_F64, |
426 | Call_F80: RTLIB::EXP10_F80, Call_F128: RTLIB::EXP10_F128, Call_PPCF128: RTLIB::EXP10_PPCF128)); |
427 | } |
428 | |
429 | SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) { |
430 | return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
431 | Call_F32: RTLIB::FLOOR_F32, |
432 | Call_F64: RTLIB::FLOOR_F64, |
433 | Call_F80: RTLIB::FLOOR_F80, |
434 | Call_F128: RTLIB::FLOOR_F128, |
435 | Call_PPCF128: RTLIB::FLOOR_PPCF128)); |
436 | } |
437 | |
438 | SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) { |
439 | return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
440 | Call_F32: RTLIB::LOG_F32, |
441 | Call_F64: RTLIB::LOG_F64, |
442 | Call_F80: RTLIB::LOG_F80, |
443 | Call_F128: RTLIB::LOG_F128, |
444 | Call_PPCF128: RTLIB::LOG_PPCF128)); |
445 | } |
446 | |
447 | SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) { |
448 | return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
449 | Call_F32: RTLIB::LOG2_F32, |
450 | Call_F64: RTLIB::LOG2_F64, |
451 | Call_F80: RTLIB::LOG2_F80, |
452 | Call_F128: RTLIB::LOG2_F128, |
453 | Call_PPCF128: RTLIB::LOG2_PPCF128)); |
454 | } |
455 | |
456 | SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) { |
457 | return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
458 | Call_F32: RTLIB::LOG10_F32, |
459 | Call_F64: RTLIB::LOG10_F64, |
460 | Call_F80: RTLIB::LOG10_F80, |
461 | Call_F128: RTLIB::LOG10_F128, |
462 | Call_PPCF128: RTLIB::LOG10_PPCF128)); |
463 | } |
464 | |
465 | SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) { |
466 | bool IsStrict = N->isStrictFPOpcode(); |
467 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
468 | unsigned Offset = IsStrict ? 1 : 0; |
469 | SDValue Ops[3] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)), |
470 | GetSoftenedFloat(Op: N->getOperand(Num: 1 + Offset)), |
471 | GetSoftenedFloat(Op: N->getOperand(Num: 2 + Offset)) }; |
472 | SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue(); |
473 | TargetLowering::MakeLibCallOptions CallOptions; |
474 | EVT OpsVT[3] = { N->getOperand(Num: 0 + Offset).getValueType(), |
475 | N->getOperand(Num: 1 + Offset).getValueType(), |
476 | N->getOperand(Num: 2 + Offset).getValueType() }; |
477 | CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0), Value: true); |
478 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, |
479 | LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
480 | Call_F32: RTLIB::FMA_F32, |
481 | Call_F64: RTLIB::FMA_F64, |
482 | Call_F80: RTLIB::FMA_F80, |
483 | Call_F128: RTLIB::FMA_F128, |
484 | Call_PPCF128: RTLIB::FMA_PPCF128), |
485 | RetVT: NVT, Ops, CallOptions, dl: SDLoc(N), Chain); |
486 | if (IsStrict) |
487 | ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second); |
488 | return Tmp.first; |
489 | } |
490 | |
491 | SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) { |
492 | return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
493 | Call_F32: RTLIB::MUL_F32, |
494 | Call_F64: RTLIB::MUL_F64, |
495 | Call_F80: RTLIB::MUL_F80, |
496 | Call_F128: RTLIB::MUL_F128, |
497 | Call_PPCF128: RTLIB::MUL_PPCF128)); |
498 | } |
499 | |
500 | SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) { |
501 | return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
502 | Call_F32: RTLIB::NEARBYINT_F32, |
503 | Call_F64: RTLIB::NEARBYINT_F64, |
504 | Call_F80: RTLIB::NEARBYINT_F80, |
505 | Call_F128: RTLIB::NEARBYINT_F128, |
506 | Call_PPCF128: RTLIB::NEARBYINT_PPCF128)); |
507 | } |
508 | |
509 | SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) { |
510 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
511 | SDLoc dl(N); |
512 | |
513 | // Expand Y = FNEG(X) -> Y = X ^ sign mask |
514 | APInt SignMask = APInt::getSignMask(BitWidth: NVT.getSizeInBits()); |
515 | return DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: NVT, N1: GetSoftenedFloat(Op: N->getOperand(Num: 0)), |
516 | N2: DAG.getConstant(Val: SignMask, DL: dl, VT: NVT)); |
517 | } |
518 | |
519 | SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) { |
520 | bool IsStrict = N->isStrictFPOpcode(); |
521 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
522 | SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0); |
523 | |
524 | SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue(); |
525 | |
526 | if (getTypeAction(VT: Op.getValueType()) == TargetLowering::TypePromoteFloat) { |
527 | Op = GetPromotedFloat(Op); |
528 | // If the promotion did the FP_EXTEND to the destination type for us, |
529 | // there's nothing left to do here. |
530 | if (Op.getValueType() == N->getValueType(ResNo: 0)) { |
531 | if (IsStrict) |
532 | ReplaceValueWith(From: SDValue(N, 1), To: Chain); |
533 | return BitConvertToInteger(Op); |
534 | } |
535 | } |
536 | |
537 | // There's only a libcall for f16 -> f32 and shifting is only valid for bf16 |
538 | // -> f32, so proceed in two stages. Also, it's entirely possible for both |
539 | // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather |
540 | // than FP16_TO_FP. |
541 | if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) && |
542 | N->getValueType(ResNo: 0) != MVT::f32) { |
543 | if (IsStrict) { |
544 | Op = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), |
545 | { MVT::f32, MVT::Other }, { Chain, Op }); |
546 | Chain = Op.getValue(R: 1); |
547 | } else { |
548 | Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op); |
549 | } |
550 | } |
551 | |
552 | if (Op.getValueType() == MVT::bf16) { |
553 | // FIXME: Need ReplaceValueWith on chain in strict case |
554 | return SoftenFloatRes_BF16_TO_FP(N); |
555 | } |
556 | |
557 | RTLIB::Libcall LC = RTLIB::getFPEXT(OpVT: Op.getValueType(), RetVT: N->getValueType(ResNo: 0)); |
558 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!" ); |
559 | TargetLowering::MakeLibCallOptions CallOptions; |
560 | EVT OpVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType(); |
561 | CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0), Value: true); |
562 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op, |
563 | CallOptions, dl: SDLoc(N), |
564 | Chain); |
565 | if (IsStrict) |
566 | ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second); |
567 | return Tmp.first; |
568 | } |
569 | |
570 | // FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special |
571 | // nodes? |
572 | SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) { |
573 | EVT MidVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), MVT::VT: f32); |
574 | SDValue Op = N->getOperand(Num: 0); |
575 | TargetLowering::MakeLibCallOptions CallOptions; |
576 | EVT OpsVT[1] = { N->getOperand(Num: 0).getValueType() }; |
577 | CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0), Value: true); |
578 | SDValue Res32 = TLI.makeLibCall(DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MidVT, Ops: Op, |
579 | CallOptions, dl: SDLoc(N)).first; |
580 | if (N->getValueType(ResNo: 0) == MVT::f32) |
581 | return Res32; |
582 | |
583 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
584 | RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::OpVT: f32, RetVT: N->getValueType(ResNo: 0)); |
585 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!" ); |
586 | return TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Res32, CallOptions, dl: SDLoc(N)).first; |
587 | } |
588 | |
589 | // FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special |
590 | // nodes? |
591 | SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) { |
592 | assert(N->getValueType(0) == MVT::f32 && |
593 | "Can only soften BF16_TO_FP with f32 result" ); |
594 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), MVT::VT: f32); |
595 | SDValue Op = N->getOperand(Num: 0); |
596 | SDLoc DL(N); |
597 | Op = DAG.getNode(ISD::ANY_EXTEND, DL, NVT, |
598 | DAG.getNode(ISD::BITCAST, DL, MVT::i16, Op)); |
599 | SDValue Res = DAG.getNode(Opcode: ISD::SHL, DL, VT: NVT, N1: Op, |
600 | N2: DAG.getShiftAmountConstant(Val: 16, VT: NVT, DL)); |
601 | return Res; |
602 | } |
603 | |
604 | SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) { |
605 | bool IsStrict = N->isStrictFPOpcode(); |
606 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
607 | SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0); |
608 | SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue(); |
609 | RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op.getValueType(), RetVT: N->getValueType(ResNo: 0)); |
610 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!" ); |
611 | TargetLowering::MakeLibCallOptions CallOptions; |
612 | EVT OpVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType(); |
613 | CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0), Value: true); |
614 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op, |
615 | CallOptions, dl: SDLoc(N), |
616 | Chain); |
617 | if (IsStrict) |
618 | ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second); |
619 | return Tmp.first; |
620 | } |
621 | |
622 | SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) { |
623 | return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
624 | Call_F32: RTLIB::POW_F32, |
625 | Call_F64: RTLIB::POW_F64, |
626 | Call_F80: RTLIB::POW_F80, |
627 | Call_F128: RTLIB::POW_F128, |
628 | Call_PPCF128: RTLIB::POW_PPCF128)); |
629 | } |
630 | |
631 | SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) { |
632 | bool IsStrict = N->isStrictFPOpcode(); |
633 | unsigned Offset = IsStrict ? 1 : 0; |
634 | assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 || |
635 | N->getOperand(1 + Offset).getValueType() == MVT::i32) && |
636 | "Unsupported power type!" ); |
637 | bool IsPowI = |
638 | N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI; |
639 | |
640 | RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(RetVT: N->getValueType(ResNo: 0)) |
641 | : RTLIB::getLDEXP(RetVT: N->getValueType(ResNo: 0)); |
642 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi." ); |
643 | if (!TLI.getLibcallName(Call: LC)) { |
644 | // Some targets don't have a powi libcall; use pow instead. |
645 | // FIXME: Implement this if some target needs it. |
646 | DAG.getContext()->emitError(ErrorStr: "Don't know how to soften fpowi to fpow" ); |
647 | return DAG.getUNDEF(VT: N->getValueType(ResNo: 0)); |
648 | } |
649 | |
650 | if (DAG.getLibInfo().getIntSize() != |
651 | N->getOperand(Num: 1 + Offset).getValueType().getSizeInBits()) { |
652 | // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI |
653 | // would use the wrong type for the argument. |
654 | DAG.getContext()->emitError(ErrorStr: "POWI exponent does not match sizeof(int)" ); |
655 | return DAG.getUNDEF(VT: N->getValueType(ResNo: 0)); |
656 | } |
657 | |
658 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
659 | SDValue Ops[2] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)), |
660 | N->getOperand(Num: 1 + Offset) }; |
661 | SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue(); |
662 | TargetLowering::MakeLibCallOptions CallOptions; |
663 | EVT OpsVT[2] = { N->getOperand(Num: 0 + Offset).getValueType(), |
664 | N->getOperand(Num: 1 + Offset).getValueType() }; |
665 | CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0), Value: true); |
666 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops, |
667 | CallOptions, dl: SDLoc(N), |
668 | Chain); |
669 | if (IsStrict) |
670 | ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second); |
671 | return Tmp.first; |
672 | } |
673 | |
674 | SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) { |
675 | assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp" ); |
676 | EVT VT0 = N->getValueType(ResNo: 0); |
677 | EVT VT1 = N->getValueType(ResNo: 1); |
678 | RTLIB::Libcall LC = RTLIB::getFREXP(RetVT: VT0); |
679 | |
680 | if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) { |
681 | // If the exponent does not match with sizeof(int) a libcall would use the |
682 | // wrong type for the argument. |
683 | // TODO: Should be able to handle mismatches. |
684 | DAG.getContext()->emitError(ErrorStr: "ffrexp exponent does not match sizeof(int)" ); |
685 | return DAG.getUNDEF(VT: N->getValueType(ResNo: 0)); |
686 | } |
687 | |
688 | EVT NVT0 = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: VT0); |
689 | SDValue StackSlot = DAG.CreateStackTemporary(VT: VT1); |
690 | |
691 | SDLoc DL(N); |
692 | |
693 | TargetLowering::MakeLibCallOptions CallOptions; |
694 | SDValue Ops[2] = {GetSoftenedFloat(Op: N->getOperand(Num: 0)), StackSlot}; |
695 | EVT OpsVT[2] = {VT0, StackSlot.getValueType()}; |
696 | |
697 | // TODO: setTypeListBeforeSoften can't properly express multiple return types, |
698 | // but we only really need to handle the 0th one for softening anyway. |
699 | CallOptions.setTypeListBeforeSoften(OpsVT: {OpsVT}, RetVT: VT0, Value: true); |
700 | |
701 | auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, RetVT: NVT0, Ops, CallOptions, dl: DL, |
702 | /*Chain=*/SDValue()); |
703 | int FrameIdx = cast<FrameIndexSDNode>(Val&: StackSlot)->getIndex(); |
704 | auto PtrInfo = |
705 | MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx); |
706 | |
707 | SDValue LoadExp = DAG.getLoad(VT: VT1, dl: DL, Chain, Ptr: StackSlot, PtrInfo); |
708 | |
709 | ReplaceValueWith(From: SDValue(N, 1), To: LoadExp); |
710 | return ReturnVal; |
711 | } |
712 | |
713 | SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) { |
714 | return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
715 | Call_F32: RTLIB::REM_F32, |
716 | Call_F64: RTLIB::REM_F64, |
717 | Call_F80: RTLIB::REM_F80, |
718 | Call_F128: RTLIB::REM_F128, |
719 | Call_PPCF128: RTLIB::REM_PPCF128)); |
720 | } |
721 | |
722 | SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) { |
723 | return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
724 | Call_F32: RTLIB::RINT_F32, |
725 | Call_F64: RTLIB::RINT_F64, |
726 | Call_F80: RTLIB::RINT_F80, |
727 | Call_F128: RTLIB::RINT_F128, |
728 | Call_PPCF128: RTLIB::RINT_PPCF128)); |
729 | } |
730 | |
731 | SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) { |
732 | return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
733 | Call_F32: RTLIB::ROUND_F32, |
734 | Call_F64: RTLIB::ROUND_F64, |
735 | Call_F80: RTLIB::ROUND_F80, |
736 | Call_F128: RTLIB::ROUND_F128, |
737 | Call_PPCF128: RTLIB::ROUND_PPCF128)); |
738 | } |
739 | |
740 | SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) { |
741 | return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
742 | Call_F32: RTLIB::ROUNDEVEN_F32, |
743 | Call_F64: RTLIB::ROUNDEVEN_F64, |
744 | Call_F80: RTLIB::ROUNDEVEN_F80, |
745 | Call_F128: RTLIB::ROUNDEVEN_F128, |
746 | Call_PPCF128: RTLIB::ROUNDEVEN_PPCF128)); |
747 | } |
748 | |
749 | SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) { |
750 | return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
751 | Call_F32: RTLIB::SIN_F32, |
752 | Call_F64: RTLIB::SIN_F64, |
753 | Call_F80: RTLIB::SIN_F80, |
754 | Call_F128: RTLIB::SIN_F128, |
755 | Call_PPCF128: RTLIB::SIN_PPCF128)); |
756 | } |
757 | |
758 | SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) { |
759 | return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
760 | Call_F32: RTLIB::SQRT_F32, |
761 | Call_F64: RTLIB::SQRT_F64, |
762 | Call_F80: RTLIB::SQRT_F80, |
763 | Call_F128: RTLIB::SQRT_F128, |
764 | Call_PPCF128: RTLIB::SQRT_PPCF128)); |
765 | } |
766 | |
767 | SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) { |
768 | return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
769 | Call_F32: RTLIB::SUB_F32, |
770 | Call_F64: RTLIB::SUB_F64, |
771 | Call_F80: RTLIB::SUB_F80, |
772 | Call_F128: RTLIB::SUB_F128, |
773 | Call_PPCF128: RTLIB::SUB_PPCF128)); |
774 | } |
775 | |
776 | SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) { |
777 | return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
778 | Call_F32: RTLIB::TRUNC_F32, |
779 | Call_F64: RTLIB::TRUNC_F64, |
780 | Call_F80: RTLIB::TRUNC_F80, |
781 | Call_F128: RTLIB::TRUNC_F128, |
782 | Call_PPCF128: RTLIB::TRUNC_PPCF128)); |
783 | } |
784 | |
785 | SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) { |
786 | LoadSDNode *L = cast<LoadSDNode>(Val: N); |
787 | EVT VT = N->getValueType(ResNo: 0); |
788 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
789 | SDLoc dl(N); |
790 | |
791 | auto MMOFlags = |
792 | L->getMemOperand()->getFlags() & |
793 | ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); |
794 | SDValue NewL; |
795 | if (L->getExtensionType() == ISD::NON_EXTLOAD) { |
796 | NewL = DAG.getLoad(AM: L->getAddressingMode(), ExtType: L->getExtensionType(), VT: NVT, dl, |
797 | Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(), |
798 | PtrInfo: L->getPointerInfo(), MemVT: NVT, Alignment: L->getOriginalAlign(), |
799 | MMOFlags, AAInfo: L->getAAInfo()); |
800 | // Legalized the chain result - switch anything that used the old chain to |
801 | // use the new one. |
802 | ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1)); |
803 | return NewL; |
804 | } |
805 | |
806 | // Do a non-extending load followed by FP_EXTEND. |
807 | NewL = DAG.getLoad(AM: L->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: L->getMemoryVT(), |
808 | dl, Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(), |
809 | PtrInfo: L->getPointerInfo(), MemVT: L->getMemoryVT(), |
810 | Alignment: L->getOriginalAlign(), MMOFlags, AAInfo: L->getAAInfo()); |
811 | // Legalized the chain result - switch anything that used the old chain to |
812 | // use the new one. |
813 | ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1)); |
814 | auto ExtendNode = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT, Operand: NewL); |
815 | return BitConvertToInteger(Op: ExtendNode); |
816 | } |
817 | |
818 | SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) { |
819 | SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 1)); |
820 | SDValue RHS = GetSoftenedFloat(Op: N->getOperand(Num: 2)); |
821 | return DAG.getSelect(DL: SDLoc(N), |
822 | VT: LHS.getValueType(), Cond: N->getOperand(Num: 0), LHS, RHS); |
823 | } |
824 | |
825 | SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) { |
826 | SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 2)); |
827 | SDValue RHS = GetSoftenedFloat(Op: N->getOperand(Num: 3)); |
828 | return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), |
829 | VT: LHS.getValueType(), N1: N->getOperand(Num: 0), |
830 | N2: N->getOperand(Num: 1), N3: LHS, N4: RHS, N5: N->getOperand(Num: 4)); |
831 | } |
832 | |
833 | SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) { |
834 | return DAG.getUNDEF(VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), |
835 | VT: N->getValueType(ResNo: 0))); |
836 | } |
837 | |
838 | SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) { |
839 | SDValue Chain = N->getOperand(Num: 0); // Get the chain. |
840 | SDValue Ptr = N->getOperand(Num: 1); // Get the pointer. |
841 | EVT VT = N->getValueType(ResNo: 0); |
842 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
843 | SDLoc dl(N); |
844 | |
845 | SDValue NewVAARG; |
846 | NewVAARG = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV: N->getOperand(Num: 2), |
847 | Align: N->getConstantOperandVal(Num: 3)); |
848 | |
849 | // Legalized the chain result - switch anything that used the old chain to |
850 | // use the new one. |
851 | if (N != NewVAARG.getValue(R: 1).getNode()) |
852 | ReplaceValueWith(From: SDValue(N, 1), To: NewVAARG.getValue(R: 1)); |
853 | return NewVAARG; |
854 | } |
855 | |
856 | SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) { |
857 | bool IsStrict = N->isStrictFPOpcode(); |
858 | bool Signed = N->getOpcode() == ISD::SINT_TO_FP || |
859 | N->getOpcode() == ISD::STRICT_SINT_TO_FP; |
860 | EVT SVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType(); |
861 | EVT RVT = N->getValueType(ResNo: 0); |
862 | EVT NVT = EVT(); |
863 | SDLoc dl(N); |
864 | |
865 | // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to |
866 | // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly |
867 | // match. Look for an appropriate libcall. |
868 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
869 | for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE; |
870 | t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) { |
871 | NVT = (MVT::SimpleValueType)t; |
872 | // The source needs to big enough to hold the operand. |
873 | if (NVT.bitsGE(VT: SVT)) |
874 | LC = Signed ? RTLIB::getSINTTOFP(OpVT: NVT, RetVT: RVT):RTLIB::getUINTTOFP (OpVT: NVT, RetVT: RVT); |
875 | } |
876 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!" ); |
877 | |
878 | SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue(); |
879 | // Sign/zero extend the argument if the libcall takes a larger type. |
880 | SDValue Op = DAG.getNode(Opcode: Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl, |
881 | VT: NVT, Operand: N->getOperand(Num: IsStrict ? 1 : 0)); |
882 | TargetLowering::MakeLibCallOptions CallOptions; |
883 | CallOptions.setSExt(Signed); |
884 | CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT, Value: true); |
885 | std::pair<SDValue, SDValue> Tmp = |
886 | TLI.makeLibCall(DAG, LC, RetVT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: RVT), |
887 | Ops: Op, CallOptions, dl, Chain); |
888 | |
889 | if (IsStrict) |
890 | ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second); |
891 | return Tmp.first; |
892 | } |
893 | |
894 | SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) { |
895 | // Expand and soften recursively. |
896 | ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduce(Node: N, DAG)); |
897 | return SDValue(); |
898 | } |
899 | |
900 | SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) { |
901 | ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduceSeq(Node: N, DAG)); |
902 | return SDValue(); |
903 | } |
904 | |
905 | //===----------------------------------------------------------------------===// |
906 | // Convert Float Operand to Integer |
907 | //===----------------------------------------------------------------------===// |
908 | |
909 | bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) { |
910 | LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": " ; N->dump(&DAG)); |
911 | SDValue Res = SDValue(); |
912 | |
913 | switch (N->getOpcode()) { |
914 | default: |
915 | #ifndef NDEBUG |
916 | dbgs() << "SoftenFloatOperand Op #" << OpNo << ": " ; |
917 | N->dump(G: &DAG); dbgs() << "\n" ; |
918 | #endif |
919 | report_fatal_error(reason: "Do not know how to soften this operator's operand!" ); |
920 | |
921 | case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break; |
922 | case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break; |
923 | case ISD::STRICT_FP_TO_FP16: |
924 | case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes |
925 | case ISD::FP_TO_BF16: |
926 | case ISD::STRICT_FP_TO_BF16: |
927 | case ISD::STRICT_FP_ROUND: |
928 | case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break; |
929 | case ISD::STRICT_FP_TO_SINT: |
930 | case ISD::STRICT_FP_TO_UINT: |
931 | case ISD::FP_TO_SINT: |
932 | case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break; |
933 | case ISD::FP_TO_SINT_SAT: |
934 | case ISD::FP_TO_UINT_SAT: |
935 | Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break; |
936 | case ISD::STRICT_LROUND: |
937 | case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break; |
938 | case ISD::STRICT_LLROUND: |
939 | case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break; |
940 | case ISD::STRICT_LRINT: |
941 | case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break; |
942 | case ISD::STRICT_LLRINT: |
943 | case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break; |
944 | case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break; |
945 | case ISD::STRICT_FSETCC: |
946 | case ISD::STRICT_FSETCCS: |
947 | case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break; |
948 | case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break; |
949 | case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break; |
950 | } |
951 | |
952 | // If the result is null, the sub-method took care of registering results etc. |
953 | if (!Res.getNode()) return false; |
954 | |
955 | // If the result is N, the sub-method updated N in place. Tell the legalizer |
956 | // core about this to re-analyze. |
957 | if (Res.getNode() == N) |
958 | return true; |
959 | |
960 | assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && |
961 | "Invalid operand softening" ); |
962 | |
963 | ReplaceValueWith(From: SDValue(N, 0), To: Res); |
964 | return false; |
965 | } |
966 | |
967 | SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) { |
968 | SDValue Op0 = GetSoftenedFloat(Op: N->getOperand(Num: 0)); |
969 | |
970 | return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op0); |
971 | } |
972 | |
973 | SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) { |
974 | // We actually deal with the partially-softened FP_TO_FP16 node too, which |
975 | // returns an i16 so doesn't meet the constraints necessary for FP_ROUND. |
976 | assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 || |
977 | N->getOpcode() == ISD::STRICT_FP_TO_FP16 || |
978 | N->getOpcode() == ISD::FP_TO_BF16 || |
979 | N->getOpcode() == ISD::STRICT_FP_TO_BF16 || |
980 | N->getOpcode() == ISD::STRICT_FP_ROUND); |
981 | |
982 | bool IsStrict = N->isStrictFPOpcode(); |
983 | SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0); |
984 | EVT SVT = Op.getValueType(); |
985 | EVT RVT = N->getValueType(ResNo: 0); |
986 | EVT FloatRVT = RVT; |
987 | if (N->getOpcode() == ISD::FP_TO_FP16 || |
988 | N->getOpcode() == ISD::STRICT_FP_TO_FP16) |
989 | FloatRVT = MVT::f16; |
990 | else if (N->getOpcode() == ISD::FP_TO_BF16 || |
991 | N->getOpcode() == ISD::STRICT_FP_TO_BF16) |
992 | FloatRVT = MVT::bf16; |
993 | |
994 | RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: SVT, RetVT: FloatRVT); |
995 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall" ); |
996 | |
997 | SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue(); |
998 | Op = GetSoftenedFloat(Op); |
999 | TargetLowering::MakeLibCallOptions CallOptions; |
1000 | CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT, Value: true); |
1001 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: RVT, Ops: Op, |
1002 | CallOptions, dl: SDLoc(N), |
1003 | Chain); |
1004 | if (IsStrict) { |
1005 | ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second); |
1006 | ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first); |
1007 | return SDValue(); |
1008 | } |
1009 | return Tmp.first; |
1010 | } |
1011 | |
1012 | SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) { |
1013 | SDValue NewLHS = N->getOperand(Num: 2), NewRHS = N->getOperand(Num: 3); |
1014 | ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get(); |
1015 | |
1016 | EVT VT = NewLHS.getValueType(); |
1017 | NewLHS = GetSoftenedFloat(Op: NewLHS); |
1018 | NewRHS = GetSoftenedFloat(Op: NewRHS); |
1019 | TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N), |
1020 | OldLHS: N->getOperand(Num: 2), OldRHS: N->getOperand(Num: 3)); |
1021 | |
1022 | // If softenSetCCOperands returned a scalar, we need to compare the result |
1023 | // against zero to select between true and false values. |
1024 | if (!NewRHS.getNode()) { |
1025 | NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType()); |
1026 | CCCode = ISD::SETNE; |
1027 | } |
1028 | |
1029 | // Update N to have the operands specified. |
1030 | return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), |
1031 | Op2: DAG.getCondCode(Cond: CCCode), Op3: NewLHS, Op4: NewRHS, |
1032 | Op5: N->getOperand(Num: 4)), |
1033 | 0); |
1034 | } |
1035 | |
1036 | // Even if the result type is legal, no libcall may exactly match. (e.g. We |
1037 | // don't have FP-i8 conversions) This helper method looks for an appropriate |
1038 | // promoted libcall. |
1039 | static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted, |
1040 | bool Signed) { |
1041 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
1042 | for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE; |
1043 | IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; |
1044 | ++IntVT) { |
1045 | Promoted = (MVT::SimpleValueType)IntVT; |
1046 | // The type needs to big enough to hold the result. |
1047 | if (Promoted.bitsGE(VT: RetVT)) |
1048 | LC = Signed ? RTLIB::getFPTOSINT(OpVT: SrcVT, RetVT: Promoted) |
1049 | : RTLIB::getFPTOUINT(OpVT: SrcVT, RetVT: Promoted); |
1050 | } |
1051 | return LC; |
1052 | } |
1053 | |
1054 | SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) { |
1055 | bool IsStrict = N->isStrictFPOpcode(); |
1056 | bool Signed = N->getOpcode() == ISD::FP_TO_SINT || |
1057 | N->getOpcode() == ISD::STRICT_FP_TO_SINT; |
1058 | |
1059 | SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0); |
1060 | EVT SVT = Op.getValueType(); |
1061 | EVT RVT = N->getValueType(ResNo: 0); |
1062 | EVT NVT = EVT(); |
1063 | SDLoc dl(N); |
1064 | |
1065 | // If the result is not legal, eg: fp -> i1, then it needs to be promoted to |
1066 | // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly |
1067 | // match, eg. we don't have fp -> i8 conversions. |
1068 | // Look for an appropriate libcall. |
1069 | RTLIB::Libcall LC = findFPToIntLibcall(SrcVT: SVT, RetVT: RVT, Promoted&: NVT, Signed); |
1070 | assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() && |
1071 | "Unsupported FP_TO_XINT!" ); |
1072 | |
1073 | Op = GetSoftenedFloat(Op); |
1074 | SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue(); |
1075 | TargetLowering::MakeLibCallOptions CallOptions; |
1076 | CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT, Value: true); |
1077 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op, |
1078 | CallOptions, dl, Chain); |
1079 | |
1080 | // Truncate the result if the libcall returns a larger type. |
1081 | SDValue Res = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RVT, Operand: Tmp.first); |
1082 | |
1083 | if (!IsStrict) |
1084 | return Res; |
1085 | |
1086 | ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second); |
1087 | ReplaceValueWith(From: SDValue(N, 0), To: Res); |
1088 | return SDValue(); |
1089 | } |
1090 | |
1091 | SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) { |
1092 | SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG); |
1093 | return Res; |
1094 | } |
1095 | |
1096 | SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) { |
1097 | SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1); |
1098 | ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get(); |
1099 | |
1100 | EVT VT = NewLHS.getValueType(); |
1101 | NewLHS = GetSoftenedFloat(Op: NewLHS); |
1102 | NewRHS = GetSoftenedFloat(Op: NewRHS); |
1103 | TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N), |
1104 | OldLHS: N->getOperand(Num: 0), OldRHS: N->getOperand(Num: 1)); |
1105 | |
1106 | // If softenSetCCOperands returned a scalar, we need to compare the result |
1107 | // against zero to select between true and false values. |
1108 | if (!NewRHS.getNode()) { |
1109 | NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType()); |
1110 | CCCode = ISD::SETNE; |
1111 | } |
1112 | |
1113 | // Update N to have the operands specified. |
1114 | return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS, |
1115 | Op3: N->getOperand(Num: 2), Op4: N->getOperand(Num: 3), |
1116 | Op5: DAG.getCondCode(Cond: CCCode)), |
1117 | 0); |
1118 | } |
1119 | |
1120 | SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) { |
1121 | bool IsStrict = N->isStrictFPOpcode(); |
1122 | SDValue Op0 = N->getOperand(Num: IsStrict ? 1 : 0); |
1123 | SDValue Op1 = N->getOperand(Num: IsStrict ? 2 : 1); |
1124 | SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue(); |
1125 | ISD::CondCode CCCode = |
1126 | cast<CondCodeSDNode>(Val: N->getOperand(Num: IsStrict ? 3 : 2))->get(); |
1127 | |
1128 | EVT VT = Op0.getValueType(); |
1129 | SDValue NewLHS = GetSoftenedFloat(Op: Op0); |
1130 | SDValue NewRHS = GetSoftenedFloat(Op: Op1); |
1131 | TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N), OldLHS: Op0, OldRHS: Op1, |
1132 | Chain, IsSignaling: N->getOpcode() == ISD::STRICT_FSETCCS); |
1133 | |
1134 | // Update N to have the operands specified. |
1135 | if (NewRHS.getNode()) { |
1136 | if (IsStrict) |
1137 | NewLHS = DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: NewLHS, |
1138 | N2: NewRHS, N3: DAG.getCondCode(Cond: CCCode)); |
1139 | else |
1140 | return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS, |
1141 | Op3: DAG.getCondCode(Cond: CCCode)), 0); |
1142 | } |
1143 | |
1144 | // Otherwise, softenSetCCOperands returned a scalar, use it. |
1145 | assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) && |
1146 | "Unexpected setcc expansion!" ); |
1147 | |
1148 | if (IsStrict) { |
1149 | ReplaceValueWith(From: SDValue(N, 0), To: NewLHS); |
1150 | ReplaceValueWith(From: SDValue(N, 1), To: Chain); |
1151 | return SDValue(); |
1152 | } |
1153 | return NewLHS; |
1154 | } |
1155 | |
1156 | SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) { |
1157 | assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!" ); |
1158 | assert(OpNo == 1 && "Can only soften the stored value!" ); |
1159 | StoreSDNode *ST = cast<StoreSDNode>(Val: N); |
1160 | SDValue Val = ST->getValue(); |
1161 | SDLoc dl(N); |
1162 | |
1163 | if (ST->isTruncatingStore()) |
1164 | // Do an FP_ROUND followed by a non-truncating store. |
1165 | Val = BitConvertToInteger( |
1166 | Op: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: ST->getMemoryVT(), N1: Val, |
1167 | N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true))); |
1168 | else |
1169 | Val = GetSoftenedFloat(Op: Val); |
1170 | |
1171 | return DAG.getStore(Chain: ST->getChain(), dl, Val, Ptr: ST->getBasePtr(), |
1172 | MMO: ST->getMemOperand()); |
1173 | } |
1174 | |
1175 | SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) { |
1176 | SDValue LHS = N->getOperand(Num: 0); |
1177 | SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1)); |
1178 | SDLoc dl(N); |
1179 | |
1180 | EVT LVT = LHS.getValueType(); |
1181 | EVT ILVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LVT.getSizeInBits()); |
1182 | EVT RVT = RHS.getValueType(); |
1183 | |
1184 | unsigned LSize = LVT.getSizeInBits(); |
1185 | unsigned RSize = RVT.getSizeInBits(); |
1186 | |
1187 | // Shift right or sign-extend it if the two operands have different types. |
1188 | int SizeDiff = RSize - LSize; |
1189 | if (SizeDiff > 0) { |
1190 | RHS = |
1191 | DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: RHS, |
1192 | N2: DAG.getConstant(Val: SizeDiff, DL: dl, |
1193 | VT: TLI.getShiftAmountTy(LHSTy: RHS.getValueType(), |
1194 | DL: DAG.getDataLayout()))); |
1195 | RHS = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ILVT, Operand: RHS); |
1196 | } else if (SizeDiff < 0) { |
1197 | RHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: RHS); |
1198 | RHS = |
1199 | DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: ILVT, N1: RHS, |
1200 | N2: DAG.getConstant(Val: -SizeDiff, DL: dl, |
1201 | VT: TLI.getShiftAmountTy(LHSTy: RHS.getValueType(), |
1202 | DL: DAG.getDataLayout()))); |
1203 | } |
1204 | |
1205 | RHS = DAG.getBitcast(VT: LVT, V: RHS); |
1206 | return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: dl, VT: LVT, N1: LHS, N2: RHS); |
1207 | } |
1208 | |
1209 | SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) { |
1210 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
1211 | bool IsStrict = N->isStrictFPOpcode(); |
1212 | unsigned Offset = IsStrict ? 1 : 0; |
1213 | SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)); |
1214 | SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue(); |
1215 | TargetLowering::MakeLibCallOptions CallOptions; |
1216 | EVT OpVT = N->getOperand(Num: 0 + Offset).getValueType(); |
1217 | CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0), Value: true); |
1218 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op, |
1219 | CallOptions, dl: SDLoc(N), |
1220 | Chain); |
1221 | if (IsStrict) { |
1222 | ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second); |
1223 | ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first); |
1224 | return SDValue(); |
1225 | } |
1226 | |
1227 | return Tmp.first; |
1228 | } |
1229 | |
1230 | SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) { |
1231 | EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType(); |
1232 | return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT, |
1233 | Call_F32: RTLIB::LROUND_F32, |
1234 | Call_F64: RTLIB::LROUND_F64, |
1235 | Call_F80: RTLIB::LROUND_F80, |
1236 | Call_F128: RTLIB::LROUND_F128, |
1237 | Call_PPCF128: RTLIB::LROUND_PPCF128)); |
1238 | } |
1239 | |
1240 | SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) { |
1241 | EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType(); |
1242 | return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT, |
1243 | Call_F32: RTLIB::LLROUND_F32, |
1244 | Call_F64: RTLIB::LLROUND_F64, |
1245 | Call_F80: RTLIB::LLROUND_F80, |
1246 | Call_F128: RTLIB::LLROUND_F128, |
1247 | Call_PPCF128: RTLIB::LLROUND_PPCF128)); |
1248 | } |
1249 | |
1250 | SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) { |
1251 | EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType(); |
1252 | return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT, |
1253 | Call_F32: RTLIB::LRINT_F32, |
1254 | Call_F64: RTLIB::LRINT_F64, |
1255 | Call_F80: RTLIB::LRINT_F80, |
1256 | Call_F128: RTLIB::LRINT_F128, |
1257 | Call_PPCF128: RTLIB::LRINT_PPCF128)); |
1258 | } |
1259 | |
1260 | SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) { |
1261 | EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType(); |
1262 | return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT, |
1263 | Call_F32: RTLIB::LLRINT_F32, |
1264 | Call_F64: RTLIB::LLRINT_F64, |
1265 | Call_F80: RTLIB::LLRINT_F80, |
1266 | Call_F128: RTLIB::LLRINT_F128, |
1267 | Call_PPCF128: RTLIB::LLRINT_PPCF128)); |
1268 | } |
1269 | |
1270 | //===----------------------------------------------------------------------===// |
1271 | // Float Result Expansion |
1272 | //===----------------------------------------------------------------------===// |
1273 | |
1274 | /// ExpandFloatResult - This method is called when the specified result of the |
1275 | /// specified node is found to need expansion. At this point, the node may also |
1276 | /// have invalid operands or may have other results that need promotion, we just |
1277 | /// know that (at least) one result needs expansion. |
1278 | void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) { |
1279 | LLVM_DEBUG(dbgs() << "Expand float result: " ; N->dump(&DAG)); |
1280 | SDValue Lo, Hi; |
1281 | Lo = Hi = SDValue(); |
1282 | |
1283 | // See if the target wants to custom expand this node. |
1284 | if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true)) |
1285 | return; |
1286 | |
1287 | switch (N->getOpcode()) { |
1288 | default: |
1289 | #ifndef NDEBUG |
1290 | dbgs() << "ExpandFloatResult #" << ResNo << ": " ; |
1291 | N->dump(G: &DAG); dbgs() << "\n" ; |
1292 | #endif |
1293 | report_fatal_error(reason: "Do not know how to expand the result of this " |
1294 | "operator!" ); |
1295 | |
1296 | case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; |
1297 | case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break; |
1298 | case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; |
1299 | |
1300 | case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; |
1301 | case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; |
1302 | case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; |
1303 | case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; |
1304 | case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; |
1305 | case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; |
1306 | |
1307 | case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break; |
1308 | case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break; |
1309 | case ISD::STRICT_FMINNUM: |
1310 | case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break; |
1311 | case ISD::STRICT_FMAXNUM: |
1312 | case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break; |
1313 | case ISD::STRICT_FADD: |
1314 | case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break; |
1315 | case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break; |
1316 | case ISD::STRICT_FCEIL: |
1317 | case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break; |
1318 | case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break; |
1319 | case ISD::STRICT_FCOS: |
1320 | case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break; |
1321 | case ISD::STRICT_FDIV: |
1322 | case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break; |
1323 | case ISD::STRICT_FEXP: |
1324 | case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break; |
1325 | case ISD::STRICT_FEXP2: |
1326 | case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break; |
1327 | case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break; |
1328 | case ISD::STRICT_FFLOOR: |
1329 | case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break; |
1330 | case ISD::STRICT_FLOG: |
1331 | case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break; |
1332 | case ISD::STRICT_FLOG2: |
1333 | case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break; |
1334 | case ISD::STRICT_FLOG10: |
1335 | case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break; |
1336 | case ISD::STRICT_FMA: |
1337 | case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break; |
1338 | case ISD::STRICT_FMUL: |
1339 | case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break; |
1340 | case ISD::STRICT_FNEARBYINT: |
1341 | case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break; |
1342 | case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break; |
1343 | case ISD::STRICT_FP_EXTEND: |
1344 | case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break; |
1345 | case ISD::STRICT_FPOW: |
1346 | case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break; |
1347 | case ISD::STRICT_FPOWI: |
1348 | case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break; |
1349 | case ISD::FLDEXP: |
1350 | case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break; |
1351 | case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break; |
1352 | case ISD::STRICT_FRINT: |
1353 | case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break; |
1354 | case ISD::STRICT_FROUND: |
1355 | case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break; |
1356 | case ISD::STRICT_FROUNDEVEN: |
1357 | case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break; |
1358 | case ISD::STRICT_FSIN: |
1359 | case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break; |
1360 | case ISD::STRICT_FSQRT: |
1361 | case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break; |
1362 | case ISD::STRICT_FSUB: |
1363 | case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break; |
1364 | case ISD::STRICT_FTRUNC: |
1365 | case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break; |
1366 | case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break; |
1367 | case ISD::STRICT_SINT_TO_FP: |
1368 | case ISD::STRICT_UINT_TO_FP: |
1369 | case ISD::SINT_TO_FP: |
1370 | case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break; |
1371 | case ISD::STRICT_FREM: |
1372 | case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break; |
1373 | } |
1374 | |
1375 | // If Lo/Hi is null, the sub-method took care of registering results etc. |
1376 | if (Lo.getNode()) |
1377 | SetExpandedFloat(Op: SDValue(N, ResNo), Lo, Hi); |
1378 | } |
1379 | |
1380 | void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo, |
1381 | SDValue &Hi) { |
1382 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
1383 | assert(NVT.getSizeInBits() == 64 && |
1384 | "Do not know how to expand this float constant!" ); |
1385 | APInt C = cast<ConstantFPSDNode>(Val: N)->getValueAPF().bitcastToAPInt(); |
1386 | SDLoc dl(N); |
1387 | Lo = DAG.getConstantFP(Val: APFloat(DAG.EVTToAPFloatSemantics(VT: NVT), |
1388 | APInt(64, C.getRawData()[1])), |
1389 | DL: dl, VT: NVT); |
1390 | Hi = DAG.getConstantFP(Val: APFloat(DAG.EVTToAPFloatSemantics(VT: NVT), |
1391 | APInt(64, C.getRawData()[0])), |
1392 | DL: dl, VT: NVT); |
1393 | } |
1394 | |
1395 | void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC, |
1396 | SDValue &Lo, SDValue &Hi) { |
1397 | bool IsStrict = N->isStrictFPOpcode(); |
1398 | unsigned Offset = IsStrict ? 1 : 0; |
1399 | SDValue Op = N->getOperand(Num: 0 + Offset); |
1400 | SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue(); |
1401 | TargetLowering::MakeLibCallOptions CallOptions; |
1402 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: N->getValueType(ResNo: 0), |
1403 | Ops: Op, CallOptions, dl: SDLoc(N), |
1404 | Chain); |
1405 | if (IsStrict) |
1406 | ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second); |
1407 | GetPairElements(Pair: Tmp.first, Lo, Hi); |
1408 | } |
1409 | |
1410 | void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC, |
1411 | SDValue &Lo, SDValue &Hi) { |
1412 | bool IsStrict = N->isStrictFPOpcode(); |
1413 | unsigned Offset = IsStrict ? 1 : 0; |
1414 | SDValue Ops[] = { N->getOperand(Num: 0 + Offset), N->getOperand(Num: 1 + Offset) }; |
1415 | SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue(); |
1416 | TargetLowering::MakeLibCallOptions CallOptions; |
1417 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: N->getValueType(ResNo: 0), |
1418 | Ops, CallOptions, dl: SDLoc(N), |
1419 | Chain); |
1420 | if (IsStrict) |
1421 | ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second); |
1422 | GetPairElements(Pair: Tmp.first, Lo, Hi); |
1423 | } |
1424 | |
1425 | void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo, |
1426 | SDValue &Hi) { |
1427 | assert(N->getValueType(0) == MVT::ppcf128 && |
1428 | "Logic only correct for ppcf128!" ); |
1429 | SDLoc dl(N); |
1430 | SDValue Tmp; |
1431 | GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi&: Tmp); |
1432 | Hi = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT: Tmp.getValueType(), Operand: Tmp); |
1433 | // Lo = Hi==fabs(Hi) ? Lo : -Lo; |
1434 | Lo = DAG.getSelectCC(DL: dl, LHS: Tmp, RHS: Hi, True: Lo, |
1435 | False: DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Lo.getValueType(), Operand: Lo), |
1436 | Cond: ISD::SETEQ); |
1437 | } |
1438 | |
1439 | void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo, |
1440 | SDValue &Hi) { |
1441 | ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
1442 | Call_F32: RTLIB::FMIN_F32, Call_F64: RTLIB::FMIN_F64, |
1443 | Call_F80: RTLIB::FMIN_F80, Call_F128: RTLIB::FMIN_F128, |
1444 | Call_PPCF128: RTLIB::FMIN_PPCF128), Lo, Hi); |
1445 | } |
1446 | |
1447 | void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo, |
1448 | SDValue &Hi) { |
1449 | ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
1450 | Call_F32: RTLIB::FMAX_F32, Call_F64: RTLIB::FMAX_F64, |
1451 | Call_F80: RTLIB::FMAX_F80, Call_F128: RTLIB::FMAX_F128, |
1452 | Call_PPCF128: RTLIB::FMAX_PPCF128), Lo, Hi); |
1453 | } |
1454 | |
1455 | void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo, |
1456 | SDValue &Hi) { |
1457 | ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
1458 | Call_F32: RTLIB::ADD_F32, Call_F64: RTLIB::ADD_F64, |
1459 | Call_F80: RTLIB::ADD_F80, Call_F128: RTLIB::ADD_F128, |
1460 | Call_PPCF128: RTLIB::ADD_PPCF128), Lo, Hi); |
1461 | } |
1462 | |
1463 | void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo, |
1464 | SDValue &Hi) { |
1465 | ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::CBRT_F32, |
1466 | Call_F64: RTLIB::CBRT_F64, Call_F80: RTLIB::CBRT_F80, |
1467 | Call_F128: RTLIB::CBRT_F128, |
1468 | Call_PPCF128: RTLIB::CBRT_PPCF128), Lo, Hi); |
1469 | } |
1470 | |
1471 | void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N, |
1472 | SDValue &Lo, SDValue &Hi) { |
1473 | ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
1474 | Call_F32: RTLIB::CEIL_F32, Call_F64: RTLIB::CEIL_F64, |
1475 | Call_F80: RTLIB::CEIL_F80, Call_F128: RTLIB::CEIL_F128, |
1476 | Call_PPCF128: RTLIB::CEIL_PPCF128), Lo, Hi); |
1477 | } |
1478 | |
1479 | void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N, |
1480 | SDValue &Lo, SDValue &Hi) { |
1481 | ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
1482 | Call_F32: RTLIB::COPYSIGN_F32, |
1483 | Call_F64: RTLIB::COPYSIGN_F64, |
1484 | Call_F80: RTLIB::COPYSIGN_F80, |
1485 | Call_F128: RTLIB::COPYSIGN_F128, |
1486 | Call_PPCF128: RTLIB::COPYSIGN_PPCF128), Lo, Hi); |
1487 | } |
1488 | |
1489 | void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N, |
1490 | SDValue &Lo, SDValue &Hi) { |
1491 | ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
1492 | Call_F32: RTLIB::COS_F32, Call_F64: RTLIB::COS_F64, |
1493 | Call_F80: RTLIB::COS_F80, Call_F128: RTLIB::COS_F128, |
1494 | Call_PPCF128: RTLIB::COS_PPCF128), Lo, Hi); |
1495 | } |
1496 | |
1497 | void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo, |
1498 | SDValue &Hi) { |
1499 | ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
1500 | Call_F32: RTLIB::DIV_F32, |
1501 | Call_F64: RTLIB::DIV_F64, |
1502 | Call_F80: RTLIB::DIV_F80, |
1503 | Call_F128: RTLIB::DIV_F128, |
1504 | Call_PPCF128: RTLIB::DIV_PPCF128), Lo, Hi); |
1505 | } |
1506 | |
1507 | void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N, |
1508 | SDValue &Lo, SDValue &Hi) { |
1509 | ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
1510 | Call_F32: RTLIB::EXP_F32, Call_F64: RTLIB::EXP_F64, |
1511 | Call_F80: RTLIB::EXP_F80, Call_F128: RTLIB::EXP_F128, |
1512 | Call_PPCF128: RTLIB::EXP_PPCF128), Lo, Hi); |
1513 | } |
1514 | |
1515 | void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N, |
1516 | SDValue &Lo, SDValue &Hi) { |
1517 | ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
1518 | Call_F32: RTLIB::EXP2_F32, Call_F64: RTLIB::EXP2_F64, |
1519 | Call_F80: RTLIB::EXP2_F80, Call_F128: RTLIB::EXP2_F128, |
1520 | Call_PPCF128: RTLIB::EXP2_PPCF128), Lo, Hi); |
1521 | } |
1522 | |
1523 | void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo, |
1524 | SDValue &Hi) { |
1525 | ExpandFloatRes_Unary(N, |
1526 | LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::EXP10_F32, |
1527 | Call_F64: RTLIB::EXP10_F64, Call_F80: RTLIB::EXP10_F80, |
1528 | Call_F128: RTLIB::EXP10_F128, Call_PPCF128: RTLIB::EXP10_PPCF128), |
1529 | Lo, Hi); |
1530 | } |
1531 | |
1532 | void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N, |
1533 | SDValue &Lo, SDValue &Hi) { |
1534 | ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
1535 | Call_F32: RTLIB::FLOOR_F32, Call_F64: RTLIB::FLOOR_F64, |
1536 | Call_F80: RTLIB::FLOOR_F80, Call_F128: RTLIB::FLOOR_F128, |
1537 | Call_PPCF128: RTLIB::FLOOR_PPCF128), Lo, Hi); |
1538 | } |
1539 | |
1540 | void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N, |
1541 | SDValue &Lo, SDValue &Hi) { |
1542 | ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
1543 | Call_F32: RTLIB::LOG_F32, Call_F64: RTLIB::LOG_F64, |
1544 | Call_F80: RTLIB::LOG_F80, Call_F128: RTLIB::LOG_F128, |
1545 | Call_PPCF128: RTLIB::LOG_PPCF128), Lo, Hi); |
1546 | } |
1547 | |
1548 | void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N, |
1549 | SDValue &Lo, SDValue &Hi) { |
1550 | ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
1551 | Call_F32: RTLIB::LOG2_F32, Call_F64: RTLIB::LOG2_F64, |
1552 | Call_F80: RTLIB::LOG2_F80, Call_F128: RTLIB::LOG2_F128, |
1553 | Call_PPCF128: RTLIB::LOG2_PPCF128), Lo, Hi); |
1554 | } |
1555 | |
1556 | void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N, |
1557 | SDValue &Lo, SDValue &Hi) { |
1558 | ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
1559 | Call_F32: RTLIB::LOG10_F32, Call_F64: RTLIB::LOG10_F64, |
1560 | Call_F80: RTLIB::LOG10_F80, Call_F128: RTLIB::LOG10_F128, |
1561 | Call_PPCF128: RTLIB::LOG10_PPCF128), Lo, Hi); |
1562 | } |
1563 | |
1564 | void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo, |
1565 | SDValue &Hi) { |
1566 | bool IsStrict = N->isStrictFPOpcode(); |
1567 | unsigned Offset = IsStrict ? 1 : 0; |
1568 | SDValue Ops[3] = { N->getOperand(Num: 0 + Offset), N->getOperand(Num: 1 + Offset), |
1569 | N->getOperand(Num: 2 + Offset) }; |
1570 | SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue(); |
1571 | TargetLowering::MakeLibCallOptions CallOptions; |
1572 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
1573 | Call_F32: RTLIB::FMA_F32, |
1574 | Call_F64: RTLIB::FMA_F64, |
1575 | Call_F80: RTLIB::FMA_F80, |
1576 | Call_F128: RTLIB::FMA_F128, |
1577 | Call_PPCF128: RTLIB::FMA_PPCF128), |
1578 | RetVT: N->getValueType(ResNo: 0), Ops, CallOptions, |
1579 | dl: SDLoc(N), Chain); |
1580 | if (IsStrict) |
1581 | ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second); |
1582 | GetPairElements(Pair: Tmp.first, Lo, Hi); |
1583 | } |
1584 | |
1585 | void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo, |
1586 | SDValue &Hi) { |
1587 | ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
1588 | Call_F32: RTLIB::MUL_F32, |
1589 | Call_F64: RTLIB::MUL_F64, |
1590 | Call_F80: RTLIB::MUL_F80, |
1591 | Call_F128: RTLIB::MUL_F128, |
1592 | Call_PPCF128: RTLIB::MUL_PPCF128), Lo, Hi); |
1593 | } |
1594 | |
1595 | void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N, |
1596 | SDValue &Lo, SDValue &Hi) { |
1597 | ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
1598 | Call_F32: RTLIB::NEARBYINT_F32, |
1599 | Call_F64: RTLIB::NEARBYINT_F64, |
1600 | Call_F80: RTLIB::NEARBYINT_F80, |
1601 | Call_F128: RTLIB::NEARBYINT_F128, |
1602 | Call_PPCF128: RTLIB::NEARBYINT_PPCF128), Lo, Hi); |
1603 | } |
1604 | |
1605 | void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo, |
1606 | SDValue &Hi) { |
1607 | SDLoc dl(N); |
1608 | GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi); |
1609 | Lo = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Lo.getValueType(), Operand: Lo); |
1610 | Hi = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Hi.getValueType(), Operand: Hi); |
1611 | } |
1612 | |
1613 | void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo, |
1614 | SDValue &Hi) { |
1615 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
1616 | SDLoc dl(N); |
1617 | bool IsStrict = N->isStrictFPOpcode(); |
1618 | |
1619 | SDValue Chain; |
1620 | if (IsStrict) { |
1621 | // If the expanded type is the same as the input type, just bypass the node. |
1622 | if (NVT == N->getOperand(Num: 1).getValueType()) { |
1623 | Hi = N->getOperand(Num: 1); |
1624 | Chain = N->getOperand(Num: 0); |
1625 | } else { |
1626 | // Other we need to extend. |
1627 | Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other }, |
1628 | { N->getOperand(0), N->getOperand(1) }); |
1629 | Chain = Hi.getValue(R: 1); |
1630 | } |
1631 | } else { |
1632 | Hi = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: N->getOperand(Num: 0)); |
1633 | } |
1634 | |
1635 | Lo = DAG.getConstantFP(Val: APFloat(DAG.EVTToAPFloatSemantics(VT: NVT), |
1636 | APInt(NVT.getSizeInBits(), 0)), DL: dl, VT: NVT); |
1637 | |
1638 | if (IsStrict) |
1639 | ReplaceValueWith(From: SDValue(N, 1), To: Chain); |
1640 | } |
1641 | |
1642 | void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N, |
1643 | SDValue &Lo, SDValue &Hi) { |
1644 | ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
1645 | Call_F32: RTLIB::POW_F32, Call_F64: RTLIB::POW_F64, |
1646 | Call_F80: RTLIB::POW_F80, Call_F128: RTLIB::POW_F128, |
1647 | Call_PPCF128: RTLIB::POW_PPCF128), Lo, Hi); |
1648 | } |
1649 | |
1650 | void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N, |
1651 | SDValue &Lo, SDValue &Hi) { |
1652 | ExpandFloatRes_Binary(N, LC: RTLIB::getPOWI(RetVT: N->getValueType(ResNo: 0)), Lo, Hi); |
1653 | } |
1654 | |
1655 | void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo, |
1656 | SDValue &Hi) { |
1657 | ExpandFloatRes_Binary(N, LC: RTLIB::getLDEXP(RetVT: N->getValueType(ResNo: 0)), Lo, Hi); |
1658 | } |
1659 | |
1660 | void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N, |
1661 | SDValue &Lo, SDValue &Hi) { |
1662 | assert(N->getValueType(0) == MVT::ppcf128 && |
1663 | "Logic only correct for ppcf128!" ); |
1664 | |
1665 | SDLoc dl(N); |
1666 | GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi); |
1667 | Lo = DAG.getNode(Opcode: ISD::FREEZE, DL: dl, VT: Lo.getValueType(), Operand: Lo); |
1668 | Hi = DAG.getNode(Opcode: ISD::FREEZE, DL: dl, VT: Hi.getValueType(), Operand: Hi); |
1669 | } |
1670 | |
1671 | void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N, |
1672 | SDValue &Lo, SDValue &Hi) { |
1673 | ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
1674 | Call_F32: RTLIB::REM_F32, Call_F64: RTLIB::REM_F64, |
1675 | Call_F80: RTLIB::REM_F80, Call_F128: RTLIB::REM_F128, |
1676 | Call_PPCF128: RTLIB::REM_PPCF128), Lo, Hi); |
1677 | } |
1678 | |
1679 | void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N, |
1680 | SDValue &Lo, SDValue &Hi) { |
1681 | ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
1682 | Call_F32: RTLIB::RINT_F32, Call_F64: RTLIB::RINT_F64, |
1683 | Call_F80: RTLIB::RINT_F80, Call_F128: RTLIB::RINT_F128, |
1684 | Call_PPCF128: RTLIB::RINT_PPCF128), Lo, Hi); |
1685 | } |
1686 | |
1687 | void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N, |
1688 | SDValue &Lo, SDValue &Hi) { |
1689 | ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
1690 | Call_F32: RTLIB::ROUND_F32, |
1691 | Call_F64: RTLIB::ROUND_F64, |
1692 | Call_F80: RTLIB::ROUND_F80, |
1693 | Call_F128: RTLIB::ROUND_F128, |
1694 | Call_PPCF128: RTLIB::ROUND_PPCF128), Lo, Hi); |
1695 | } |
1696 | |
1697 | void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N, |
1698 | SDValue &Lo, SDValue &Hi) { |
1699 | ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
1700 | Call_F32: RTLIB::ROUNDEVEN_F32, |
1701 | Call_F64: RTLIB::ROUNDEVEN_F64, |
1702 | Call_F80: RTLIB::ROUNDEVEN_F80, |
1703 | Call_F128: RTLIB::ROUNDEVEN_F128, |
1704 | Call_PPCF128: RTLIB::ROUNDEVEN_PPCF128), Lo, Hi); |
1705 | } |
1706 | |
1707 | void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N, |
1708 | SDValue &Lo, SDValue &Hi) { |
1709 | ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
1710 | Call_F32: RTLIB::SIN_F32, Call_F64: RTLIB::SIN_F64, |
1711 | Call_F80: RTLIB::SIN_F80, Call_F128: RTLIB::SIN_F128, |
1712 | Call_PPCF128: RTLIB::SIN_PPCF128), Lo, Hi); |
1713 | } |
1714 | |
1715 | void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N, |
1716 | SDValue &Lo, SDValue &Hi) { |
1717 | ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
1718 | Call_F32: RTLIB::SQRT_F32, Call_F64: RTLIB::SQRT_F64, |
1719 | Call_F80: RTLIB::SQRT_F80, Call_F128: RTLIB::SQRT_F128, |
1720 | Call_PPCF128: RTLIB::SQRT_PPCF128), Lo, Hi); |
1721 | } |
1722 | |
1723 | void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo, |
1724 | SDValue &Hi) { |
1725 | ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
1726 | Call_F32: RTLIB::SUB_F32, |
1727 | Call_F64: RTLIB::SUB_F64, |
1728 | Call_F80: RTLIB::SUB_F80, |
1729 | Call_F128: RTLIB::SUB_F128, |
1730 | Call_PPCF128: RTLIB::SUB_PPCF128), Lo, Hi); |
1731 | } |
1732 | |
1733 | void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N, |
1734 | SDValue &Lo, SDValue &Hi) { |
1735 | ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), |
1736 | Call_F32: RTLIB::TRUNC_F32, Call_F64: RTLIB::TRUNC_F64, |
1737 | Call_F80: RTLIB::TRUNC_F80, Call_F128: RTLIB::TRUNC_F128, |
1738 | Call_PPCF128: RTLIB::TRUNC_PPCF128), Lo, Hi); |
1739 | } |
1740 | |
1741 | void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo, |
1742 | SDValue &Hi) { |
1743 | if (ISD::isNormalLoad(N)) { |
1744 | ExpandRes_NormalLoad(N, Lo, Hi); |
1745 | return; |
1746 | } |
1747 | |
1748 | assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!" ); |
1749 | LoadSDNode *LD = cast<LoadSDNode>(Val: N); |
1750 | SDValue Chain = LD->getChain(); |
1751 | SDValue Ptr = LD->getBasePtr(); |
1752 | SDLoc dl(N); |
1753 | |
1754 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: LD->getValueType(ResNo: 0)); |
1755 | assert(NVT.isByteSized() && "Expanded type not byte sized!" ); |
1756 | assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?" ); |
1757 | |
1758 | Hi = DAG.getExtLoad(ExtType: LD->getExtensionType(), dl, VT: NVT, Chain, Ptr, |
1759 | MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand()); |
1760 | |
1761 | // Remember the chain. |
1762 | Chain = Hi.getValue(R: 1); |
1763 | |
1764 | // The low part is zero. |
1765 | Lo = DAG.getConstantFP(Val: APFloat(DAG.EVTToAPFloatSemantics(VT: NVT), |
1766 | APInt(NVT.getSizeInBits(), 0)), DL: dl, VT: NVT); |
1767 | |
1768 | // Modified the chain - switch anything that used the old chain to use the |
1769 | // new one. |
1770 | ReplaceValueWith(From: SDValue(LD, 1), To: Chain); |
1771 | } |
1772 | |
1773 | void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo, |
1774 | SDValue &Hi) { |
1775 | assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!" ); |
1776 | EVT VT = N->getValueType(ResNo: 0); |
1777 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
1778 | bool Strict = N->isStrictFPOpcode(); |
1779 | SDValue Src = N->getOperand(Num: Strict ? 1 : 0); |
1780 | EVT SrcVT = Src.getValueType(); |
1781 | bool isSigned = N->getOpcode() == ISD::SINT_TO_FP || |
1782 | N->getOpcode() == ISD::STRICT_SINT_TO_FP; |
1783 | SDLoc dl(N); |
1784 | SDValue Chain = Strict ? N->getOperand(Num: 0) : DAG.getEntryNode(); |
1785 | |
1786 | // TODO: Any other flags to propagate? |
1787 | SDNodeFlags Flags; |
1788 | Flags.setNoFPExcept(N->getFlags().hasNoFPExcept()); |
1789 | |
1790 | // First do an SINT_TO_FP, whether the original was signed or unsigned. |
1791 | // When promoting partial word types to i32 we must honor the signedness, |
1792 | // though. |
1793 | if (SrcVT.bitsLE(MVT::VT: i32)) { |
1794 | // The integer can be represented exactly in an f64. |
1795 | Lo = DAG.getConstantFP(Val: APFloat(DAG.EVTToAPFloatSemantics(VT: NVT), |
1796 | APInt(NVT.getSizeInBits(), 0)), DL: dl, VT: NVT); |
1797 | if (Strict) { |
1798 | Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other), |
1799 | {Chain, Src}, Flags); |
1800 | Chain = Hi.getValue(R: 1); |
1801 | } else |
1802 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Src); |
1803 | } else { |
1804 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
1805 | if (SrcVT.bitsLE(MVT::VT: i64)) { |
1806 | Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, |
1807 | MVT::i64, Src); |
1808 | LC = RTLIB::SINTTOFP_I64_PPCF128; |
1809 | } else if (SrcVT.bitsLE(MVT::VT: i128)) { |
1810 | Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src); |
1811 | LC = RTLIB::SINTTOFP_I128_PPCF128; |
1812 | } |
1813 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!" ); |
1814 | |
1815 | TargetLowering::MakeLibCallOptions CallOptions; |
1816 | CallOptions.setSExt(true); |
1817 | std::pair<SDValue, SDValue> Tmp = |
1818 | TLI.makeLibCall(DAG, LC, RetVT: VT, Ops: Src, CallOptions, dl, Chain); |
1819 | if (Strict) |
1820 | Chain = Tmp.second; |
1821 | GetPairElements(Pair: Tmp.first, Lo, Hi); |
1822 | } |
1823 | |
1824 | // No need to complement for unsigned 32-bit integers |
1825 | if (isSigned || SrcVT.bitsLE(MVT::VT: i32)) { |
1826 | if (Strict) |
1827 | ReplaceValueWith(From: SDValue(N, 1), To: Chain); |
1828 | |
1829 | return; |
1830 | } |
1831 | |
1832 | // Unsigned - fix up the SINT_TO_FP value just calculated. |
1833 | // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully |
1834 | // keep semantics correctness if the integer is not exactly representable |
1835 | // here. See ExpandLegalINT_TO_FP. |
1836 | Hi = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT, N1: Lo, N2: Hi); |
1837 | SrcVT = Src.getValueType(); |
1838 | |
1839 | // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128. |
1840 | static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 }; |
1841 | static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 }; |
1842 | static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 }; |
1843 | ArrayRef<uint64_t> Parts; |
1844 | |
1845 | switch (SrcVT.getSimpleVT().SimpleTy) { |
1846 | default: |
1847 | llvm_unreachable("Unsupported UINT_TO_FP!" ); |
1848 | case MVT::i32: |
1849 | Parts = TwoE32; |
1850 | break; |
1851 | case MVT::i64: |
1852 | Parts = TwoE64; |
1853 | break; |
1854 | case MVT::i128: |
1855 | Parts = TwoE128; |
1856 | break; |
1857 | } |
1858 | |
1859 | // TODO: Are there other fast-math-flags to propagate to this FADD? |
1860 | SDValue NewLo = DAG.getConstantFP( |
1861 | APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128); |
1862 | if (Strict) { |
1863 | Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other), |
1864 | {Chain, Hi, NewLo}, Flags); |
1865 | Chain = Lo.getValue(R: 1); |
1866 | ReplaceValueWith(From: SDValue(N, 1), To: Chain); |
1867 | } else |
1868 | Lo = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Hi, N2: NewLo); |
1869 | Lo = DAG.getSelectCC(DL: dl, LHS: Src, RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT), |
1870 | True: Lo, False: Hi, Cond: ISD::SETLT); |
1871 | GetPairElements(Pair: Lo, Lo, Hi); |
1872 | } |
1873 | |
1874 | |
1875 | //===----------------------------------------------------------------------===// |
1876 | // Float Operand Expansion |
1877 | //===----------------------------------------------------------------------===// |
1878 | |
1879 | /// ExpandFloatOperand - This method is called when the specified operand of the |
1880 | /// specified node is found to need expansion. At this point, all of the result |
1881 | /// types of the node are known to be legal, but other operands of the node may |
1882 | /// need promotion or expansion as well as the specified one. |
1883 | bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) { |
1884 | LLVM_DEBUG(dbgs() << "Expand float operand: " ; N->dump(&DAG)); |
1885 | SDValue Res = SDValue(); |
1886 | |
1887 | // See if the target wants to custom expand this node. |
1888 | if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false)) |
1889 | return false; |
1890 | |
1891 | switch (N->getOpcode()) { |
1892 | default: |
1893 | #ifndef NDEBUG |
1894 | dbgs() << "ExpandFloatOperand Op #" << OpNo << ": " ; |
1895 | N->dump(G: &DAG); dbgs() << "\n" ; |
1896 | #endif |
1897 | report_fatal_error(reason: "Do not know how to expand this operator's operand!" ); |
1898 | |
1899 | case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; |
1900 | case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; |
1901 | case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; |
1902 | |
1903 | case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break; |
1904 | case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break; |
1905 | case ISD::STRICT_FP_ROUND: |
1906 | case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break; |
1907 | case ISD::STRICT_FP_TO_SINT: |
1908 | case ISD::STRICT_FP_TO_UINT: |
1909 | case ISD::FP_TO_SINT: |
1910 | case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break; |
1911 | case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break; |
1912 | case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break; |
1913 | case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break; |
1914 | case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break; |
1915 | case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break; |
1916 | case ISD::STRICT_FSETCC: |
1917 | case ISD::STRICT_FSETCCS: |
1918 | case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break; |
1919 | case ISD::STORE: Res = ExpandFloatOp_STORE(N: cast<StoreSDNode>(Val: N), |
1920 | OpNo); break; |
1921 | } |
1922 | |
1923 | // If the result is null, the sub-method took care of registering results etc. |
1924 | if (!Res.getNode()) return false; |
1925 | |
1926 | // If the result is N, the sub-method updated N in place. Tell the legalizer |
1927 | // core about this. |
1928 | if (Res.getNode() == N) |
1929 | return true; |
1930 | |
1931 | assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && |
1932 | "Invalid operand expansion" ); |
1933 | |
1934 | ReplaceValueWith(From: SDValue(N, 0), To: Res); |
1935 | return false; |
1936 | } |
1937 | |
1938 | /// FloatExpandSetCCOperands - Expand the operands of a comparison. This code |
1939 | /// is shared among BR_CC, SELECT_CC, and SETCC handlers. |
1940 | void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS, |
1941 | SDValue &NewRHS, |
1942 | ISD::CondCode &CCCode, |
1943 | const SDLoc &dl, SDValue &Chain, |
1944 | bool IsSignaling) { |
1945 | SDValue LHSLo, LHSHi, RHSLo, RHSHi; |
1946 | GetExpandedFloat(Op: NewLHS, Lo&: LHSLo, Hi&: LHSHi); |
1947 | GetExpandedFloat(Op: NewRHS, Lo&: RHSLo, Hi&: RHSHi); |
1948 | |
1949 | assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!" ); |
1950 | |
1951 | // FIXME: This generated code sucks. We want to generate |
1952 | // FCMPU crN, hi1, hi2 |
1953 | // BNE crN, L: |
1954 | // FCMPU crN, lo1, lo2 |
1955 | // The following can be improved, but not that much. |
1956 | SDValue Tmp1, Tmp2, Tmp3, OutputChain; |
1957 | Tmp1 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi, |
1958 | RHS: RHSHi, Cond: ISD::SETOEQ, Chain, IsSignaling); |
1959 | OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(R: 1) : SDValue(); |
1960 | Tmp2 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSLo.getValueType()), LHS: LHSLo, |
1961 | RHS: RHSLo, Cond: CCCode, Chain: OutputChain, IsSignaling); |
1962 | OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(R: 1) : SDValue(); |
1963 | Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp2); |
1964 | Tmp1 = |
1965 | DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi, RHS: RHSHi, |
1966 | Cond: ISD::SETUNE, Chain: OutputChain, IsSignaling); |
1967 | OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(R: 1) : SDValue(); |
1968 | Tmp2 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi, |
1969 | RHS: RHSHi, Cond: CCCode, Chain: OutputChain, IsSignaling); |
1970 | OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(R: 1) : SDValue(); |
1971 | Tmp1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp2); |
1972 | NewLHS = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp3); |
1973 | NewRHS = SDValue(); // LHS is the result, not a compare. |
1974 | Chain = OutputChain; |
1975 | } |
1976 | |
1977 | SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) { |
1978 | SDValue NewLHS = N->getOperand(Num: 2), NewRHS = N->getOperand(Num: 3); |
1979 | ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get(); |
1980 | SDValue Chain; |
1981 | FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain); |
1982 | |
1983 | // If ExpandSetCCOperands returned a scalar, we need to compare the result |
1984 | // against zero to select between true and false values. |
1985 | if (!NewRHS.getNode()) { |
1986 | NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType()); |
1987 | CCCode = ISD::SETNE; |
1988 | } |
1989 | |
1990 | // Update N to have the operands specified. |
1991 | return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), |
1992 | Op2: DAG.getCondCode(Cond: CCCode), Op3: NewLHS, Op4: NewRHS, |
1993 | Op5: N->getOperand(Num: 4)), 0); |
1994 | } |
1995 | |
1996 | SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) { |
1997 | assert(N->getOperand(1).getValueType() == MVT::ppcf128 && |
1998 | "Logic only correct for ppcf128!" ); |
1999 | SDValue Lo, Hi; |
2000 | GetExpandedFloat(Op: N->getOperand(Num: 1), Lo, Hi); |
2001 | // The ppcf128 value is providing only the sign; take it from the |
2002 | // higher-order double (which must have the larger magnitude). |
2003 | return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SDLoc(N), |
2004 | VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0), N2: Hi); |
2005 | } |
2006 | |
2007 | SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) { |
2008 | bool IsStrict = N->isStrictFPOpcode(); |
2009 | assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 && |
2010 | "Logic only correct for ppcf128!" ); |
2011 | SDValue Lo, Hi; |
2012 | GetExpandedFloat(Op: N->getOperand(Num: IsStrict ? 1 : 0), Lo, Hi); |
2013 | |
2014 | if (!IsStrict) |
2015 | // Round it the rest of the way (e.g. to f32) if needed. |
2016 | return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N), |
2017 | VT: N->getValueType(ResNo: 0), N1: Hi, N2: N->getOperand(Num: 1)); |
2018 | |
2019 | // Eliminate the node if the input float type is the same as the output float |
2020 | // type. |
2021 | if (Hi.getValueType() == N->getValueType(ResNo: 0)) { |
2022 | // Connect the output chain to the input chain, unlinking the node. |
2023 | ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0)); |
2024 | ReplaceValueWith(From: SDValue(N, 0), To: Hi); |
2025 | return SDValue(); |
2026 | } |
2027 | |
2028 | SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N), |
2029 | {N->getValueType(0), MVT::Other}, |
2030 | {N->getOperand(0), Hi, N->getOperand(2)}); |
2031 | ReplaceValueWith(From: SDValue(N, 1), To: Expansion.getValue(R: 1)); |
2032 | ReplaceValueWith(From: SDValue(N, 0), To: Expansion); |
2033 | return SDValue(); |
2034 | } |
2035 | |
2036 | SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) { |
2037 | EVT RVT = N->getValueType(ResNo: 0); |
2038 | SDLoc dl(N); |
2039 | |
2040 | bool IsStrict = N->isStrictFPOpcode(); |
2041 | bool Signed = N->getOpcode() == ISD::FP_TO_SINT || |
2042 | N->getOpcode() == ISD::STRICT_FP_TO_SINT; |
2043 | SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0); |
2044 | SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue(); |
2045 | |
2046 | EVT NVT; |
2047 | RTLIB::Libcall LC = findFPToIntLibcall(SrcVT: Op.getValueType(), RetVT: RVT, Promoted&: NVT, Signed); |
2048 | assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() && |
2049 | "Unsupported FP_TO_XINT!" ); |
2050 | TargetLowering::MakeLibCallOptions CallOptions; |
2051 | std::pair<SDValue, SDValue> Tmp = |
2052 | TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op, CallOptions, dl, Chain); |
2053 | if (!IsStrict) |
2054 | return Tmp.first; |
2055 | |
2056 | ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second); |
2057 | ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first); |
2058 | return SDValue(); |
2059 | } |
2060 | |
2061 | SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) { |
2062 | SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1); |
2063 | ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get(); |
2064 | SDValue Chain; |
2065 | FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain); |
2066 | |
2067 | // If ExpandSetCCOperands returned a scalar, we need to compare the result |
2068 | // against zero to select between true and false values. |
2069 | if (!NewRHS.getNode()) { |
2070 | NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType()); |
2071 | CCCode = ISD::SETNE; |
2072 | } |
2073 | |
2074 | // Update N to have the operands specified. |
2075 | return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS, |
2076 | Op3: N->getOperand(Num: 2), Op4: N->getOperand(Num: 3), |
2077 | Op5: DAG.getCondCode(Cond: CCCode)), 0); |
2078 | } |
2079 | |
2080 | SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) { |
2081 | bool IsStrict = N->isStrictFPOpcode(); |
2082 | SDValue NewLHS = N->getOperand(Num: IsStrict ? 1 : 0); |
2083 | SDValue NewRHS = N->getOperand(Num: IsStrict ? 2 : 1); |
2084 | SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue(); |
2085 | ISD::CondCode CCCode = |
2086 | cast<CondCodeSDNode>(Val: N->getOperand(Num: IsStrict ? 3 : 2))->get(); |
2087 | FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain, |
2088 | IsSignaling: N->getOpcode() == ISD::STRICT_FSETCCS); |
2089 | |
2090 | // FloatExpandSetCCOperands always returned a scalar. |
2091 | assert(!NewRHS.getNode() && "Expect to return scalar" ); |
2092 | assert(NewLHS.getValueType() == N->getValueType(0) && |
2093 | "Unexpected setcc expansion!" ); |
2094 | if (Chain) { |
2095 | ReplaceValueWith(From: SDValue(N, 0), To: NewLHS); |
2096 | ReplaceValueWith(From: SDValue(N, 1), To: Chain); |
2097 | return SDValue(); |
2098 | } |
2099 | return NewLHS; |
2100 | } |
2101 | |
2102 | SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) { |
2103 | if (ISD::isNormalStore(N)) |
2104 | return ExpandOp_NormalStore(N, OpNo); |
2105 | |
2106 | assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!" ); |
2107 | assert(OpNo == 1 && "Can only expand the stored value so far" ); |
2108 | StoreSDNode *ST = cast<StoreSDNode>(Val: N); |
2109 | |
2110 | SDValue Chain = ST->getChain(); |
2111 | SDValue Ptr = ST->getBasePtr(); |
2112 | |
2113 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), |
2114 | VT: ST->getValue().getValueType()); |
2115 | assert(NVT.isByteSized() && "Expanded type not byte sized!" ); |
2116 | assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?" ); |
2117 | (void)NVT; |
2118 | |
2119 | SDValue Lo, Hi; |
2120 | GetExpandedOp(Op: ST->getValue(), Lo, Hi); |
2121 | |
2122 | return DAG.getTruncStore(Chain, dl: SDLoc(N), Val: Hi, Ptr, |
2123 | SVT: ST->getMemoryVT(), MMO: ST->getMemOperand()); |
2124 | } |
2125 | |
2126 | SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) { |
2127 | EVT RVT = N->getValueType(ResNo: 0); |
2128 | EVT RetVT = N->getOperand(Num: 0).getValueType(); |
2129 | TargetLowering::MakeLibCallOptions CallOptions; |
2130 | return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT, |
2131 | Call_F32: RTLIB::LROUND_F32, |
2132 | Call_F64: RTLIB::LROUND_F64, |
2133 | Call_F80: RTLIB::LROUND_F80, |
2134 | Call_F128: RTLIB::LROUND_F128, |
2135 | Call_PPCF128: RTLIB::LROUND_PPCF128), |
2136 | RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first; |
2137 | } |
2138 | |
2139 | SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) { |
2140 | EVT RVT = N->getValueType(ResNo: 0); |
2141 | EVT RetVT = N->getOperand(Num: 0).getValueType(); |
2142 | TargetLowering::MakeLibCallOptions CallOptions; |
2143 | return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT, |
2144 | Call_F32: RTLIB::LLROUND_F32, |
2145 | Call_F64: RTLIB::LLROUND_F64, |
2146 | Call_F80: RTLIB::LLROUND_F80, |
2147 | Call_F128: RTLIB::LLROUND_F128, |
2148 | Call_PPCF128: RTLIB::LLROUND_PPCF128), |
2149 | RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first; |
2150 | } |
2151 | |
2152 | SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) { |
2153 | EVT RVT = N->getValueType(ResNo: 0); |
2154 | EVT RetVT = N->getOperand(Num: 0).getValueType(); |
2155 | TargetLowering::MakeLibCallOptions CallOptions; |
2156 | return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT, |
2157 | Call_F32: RTLIB::LRINT_F32, |
2158 | Call_F64: RTLIB::LRINT_F64, |
2159 | Call_F80: RTLIB::LRINT_F80, |
2160 | Call_F128: RTLIB::LRINT_F128, |
2161 | Call_PPCF128: RTLIB::LRINT_PPCF128), |
2162 | RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first; |
2163 | } |
2164 | |
2165 | SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) { |
2166 | EVT RVT = N->getValueType(ResNo: 0); |
2167 | EVT RetVT = N->getOperand(Num: 0).getValueType(); |
2168 | TargetLowering::MakeLibCallOptions CallOptions; |
2169 | return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT, |
2170 | Call_F32: RTLIB::LLRINT_F32, |
2171 | Call_F64: RTLIB::LLRINT_F64, |
2172 | Call_F80: RTLIB::LLRINT_F80, |
2173 | Call_F128: RTLIB::LLRINT_F128, |
2174 | Call_PPCF128: RTLIB::LLRINT_PPCF128), |
2175 | RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first; |
2176 | } |
2177 | |
2178 | //===----------------------------------------------------------------------===// |
2179 | // Float Operand Promotion |
2180 | //===----------------------------------------------------------------------===// |
2181 | // |
2182 | |
2183 | static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) { |
2184 | if (OpVT == MVT::f16) { |
2185 | return ISD::FP16_TO_FP; |
2186 | } else if (RetVT == MVT::f16) { |
2187 | return ISD::FP_TO_FP16; |
2188 | } else if (OpVT == MVT::bf16) { |
2189 | return ISD::BF16_TO_FP; |
2190 | } else if (RetVT == MVT::bf16) { |
2191 | return ISD::FP_TO_BF16; |
2192 | } |
2193 | |
2194 | report_fatal_error(reason: "Attempt at an invalid promotion-related conversion" ); |
2195 | } |
2196 | |
2197 | static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT) { |
2198 | if (OpVT == MVT::f16) |
2199 | return ISD::STRICT_FP16_TO_FP; |
2200 | |
2201 | if (RetVT == MVT::f16) |
2202 | return ISD::STRICT_FP_TO_FP16; |
2203 | |
2204 | if (OpVT == MVT::bf16) |
2205 | return ISD::STRICT_BF16_TO_FP; |
2206 | |
2207 | if (RetVT == MVT::bf16) |
2208 | return ISD::STRICT_FP_TO_BF16; |
2209 | |
2210 | report_fatal_error(reason: "Attempt at an invalid promotion-related conversion" ); |
2211 | } |
2212 | |
2213 | bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) { |
2214 | LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": " ; N->dump(&DAG)); |
2215 | SDValue R = SDValue(); |
2216 | |
2217 | if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false)) { |
2218 | LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n" ); |
2219 | return false; |
2220 | } |
2221 | |
2222 | // Nodes that use a promotion-requiring floating point operand, but doesn't |
2223 | // produce a promotion-requiring floating point result, need to be legalized |
2224 | // to use the promoted float operand. Nodes that produce at least one |
2225 | // promotion-requiring floating point result have their operands legalized as |
2226 | // a part of PromoteFloatResult. |
2227 | // clang-format off |
2228 | switch (N->getOpcode()) { |
2229 | default: |
2230 | #ifndef NDEBUG |
2231 | dbgs() << "PromoteFloatOperand Op #" << OpNo << ": " ; |
2232 | N->dump(G: &DAG); dbgs() << "\n" ; |
2233 | #endif |
2234 | report_fatal_error(reason: "Do not know how to promote this operator's operand!" ); |
2235 | |
2236 | case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break; |
2237 | case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break; |
2238 | case ISD::FP_TO_SINT: |
2239 | case ISD::FP_TO_UINT: |
2240 | case ISD::LRINT: |
2241 | case ISD::LLRINT: R = PromoteFloatOp_UnaryOp(N, OpNo); break; |
2242 | case ISD::FP_TO_SINT_SAT: |
2243 | case ISD::FP_TO_UINT_SAT: |
2244 | R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break; |
2245 | case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break; |
2246 | case ISD::STRICT_FP_EXTEND: |
2247 | R = PromoteFloatOp_STRICT_FP_EXTEND(N, OpNo); |
2248 | break; |
2249 | case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break; |
2250 | case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break; |
2251 | case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break; |
2252 | } |
2253 | // clang-format on |
2254 | |
2255 | if (R.getNode()) |
2256 | ReplaceValueWith(From: SDValue(N, 0), To: R); |
2257 | return false; |
2258 | } |
2259 | |
2260 | SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) { |
2261 | SDValue Op = N->getOperand(Num: 0); |
2262 | EVT OpVT = Op->getValueType(ResNo: 0); |
2263 | |
2264 | SDValue Promoted = GetPromotedFloat(Op: N->getOperand(Num: 0)); |
2265 | EVT PromotedVT = Promoted->getValueType(ResNo: 0); |
2266 | |
2267 | // Convert the promoted float value to the desired IVT. |
2268 | EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: OpVT.getSizeInBits()); |
2269 | SDValue Convert = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: PromotedVT, RetVT: OpVT), DL: SDLoc(N), |
2270 | VT: IVT, Operand: Promoted); |
2271 | // The final result type might not be an scalar so we need a bitcast. The |
2272 | // bitcast will be further legalized if needed. |
2273 | return DAG.getBitcast(VT: N->getValueType(ResNo: 0), V: Convert); |
2274 | } |
2275 | |
2276 | // Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by |
2277 | // PromoteFloatRes_FCOPYSIGN. |
2278 | SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) { |
2279 | assert (OpNo == 1 && "Only Operand 1 must need promotion here" ); |
2280 | SDValue Op1 = GetPromotedFloat(Op: N->getOperand(Num: 1)); |
2281 | |
2282 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0), |
2283 | N1: N->getOperand(Num: 0), N2: Op1); |
2284 | } |
2285 | |
2286 | // Convert the promoted float value to the desired integer type |
2287 | SDValue DAGTypeLegalizer::PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo) { |
2288 | SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 0)); |
2289 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op); |
2290 | } |
2291 | |
2292 | SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N, |
2293 | unsigned OpNo) { |
2294 | SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 0)); |
2295 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Op, |
2296 | N2: N->getOperand(Num: 1)); |
2297 | } |
2298 | |
2299 | SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) { |
2300 | SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 0)); |
2301 | EVT VT = N->getValueType(ResNo: 0); |
2302 | |
2303 | // Desired VT is same as promoted type. Use promoted float directly. |
2304 | if (VT == Op->getValueType(ResNo: 0)) |
2305 | return Op; |
2306 | |
2307 | // Else, extend the promoted float value to the desired VT. |
2308 | return DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), VT, Operand: Op); |
2309 | } |
2310 | |
2311 | SDValue DAGTypeLegalizer::PromoteFloatOp_STRICT_FP_EXTEND(SDNode *N, |
2312 | unsigned OpNo) { |
2313 | assert(OpNo == 1 && "Promoting unpromotable operand" ); |
2314 | |
2315 | SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 1)); |
2316 | EVT VT = N->getValueType(ResNo: 0); |
2317 | |
2318 | // Desired VT is same as promoted type. Use promoted float directly. |
2319 | if (VT == Op->getValueType(ResNo: 0)) { |
2320 | ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0)); |
2321 | return Op; |
2322 | } |
2323 | |
2324 | // Else, extend the promoted float value to the desired VT. |
2325 | SDValue Res = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: SDLoc(N), VTList: N->getVTList(), |
2326 | N1: N->getOperand(Num: 0), N2: Op); |
2327 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
2328 | return Res; |
2329 | } |
2330 | |
2331 | // Promote the float operands used for comparison. The true- and false- |
2332 | // operands have the same type as the result and are promoted, if needed, by |
2333 | // PromoteFloatRes_SELECT_CC |
2334 | SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) { |
2335 | SDValue LHS = GetPromotedFloat(Op: N->getOperand(Num: 0)); |
2336 | SDValue RHS = GetPromotedFloat(Op: N->getOperand(Num: 1)); |
2337 | |
2338 | return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), |
2339 | N1: LHS, N2: RHS, N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3), |
2340 | N5: N->getOperand(Num: 4)); |
2341 | } |
2342 | |
2343 | // Construct a SETCC that compares the promoted values and sets the conditional |
2344 | // code. |
2345 | SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) { |
2346 | EVT VT = N->getValueType(ResNo: 0); |
2347 | SDValue Op0 = GetPromotedFloat(Op: N->getOperand(Num: 0)); |
2348 | SDValue Op1 = GetPromotedFloat(Op: N->getOperand(Num: 1)); |
2349 | ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get(); |
2350 | |
2351 | return DAG.getSetCC(DL: SDLoc(N), VT, LHS: Op0, RHS: Op1, Cond: CCCode); |
2352 | |
2353 | } |
2354 | |
2355 | // Lower the promoted Float down to the integer value of same size and construct |
2356 | // a STORE of the integer value. |
2357 | SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) { |
2358 | StoreSDNode *ST = cast<StoreSDNode>(Val: N); |
2359 | SDValue Val = ST->getValue(); |
2360 | SDLoc DL(N); |
2361 | |
2362 | SDValue Promoted = GetPromotedFloat(Op: Val); |
2363 | EVT VT = ST->getOperand(Num: 1).getValueType(); |
2364 | EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits()); |
2365 | |
2366 | SDValue NewVal; |
2367 | NewVal = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: Promoted.getValueType(), RetVT: VT), DL, |
2368 | VT: IVT, Operand: Promoted); |
2369 | |
2370 | return DAG.getStore(Chain: ST->getChain(), dl: DL, Val: NewVal, Ptr: ST->getBasePtr(), |
2371 | MMO: ST->getMemOperand()); |
2372 | } |
2373 | |
2374 | //===----------------------------------------------------------------------===// |
2375 | // Float Result Promotion |
2376 | //===----------------------------------------------------------------------===// |
2377 | |
2378 | void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) { |
2379 | LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": " ; N->dump(&DAG)); |
2380 | SDValue R = SDValue(); |
2381 | |
2382 | // See if the target wants to custom expand this node. |
2383 | if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true)) { |
2384 | LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n" ); |
2385 | return; |
2386 | } |
2387 | |
2388 | switch (N->getOpcode()) { |
2389 | // These opcodes cannot appear if promotion of FP16 is done in the backend |
2390 | // instead of Clang |
2391 | case ISD::FP16_TO_FP: |
2392 | case ISD::FP_TO_FP16: |
2393 | default: |
2394 | #ifndef NDEBUG |
2395 | dbgs() << "PromoteFloatResult #" << ResNo << ": " ; |
2396 | N->dump(G: &DAG); dbgs() << "\n" ; |
2397 | #endif |
2398 | report_fatal_error(reason: "Do not know how to promote this operator's result!" ); |
2399 | |
2400 | case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break; |
2401 | case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break; |
2402 | case ISD::EXTRACT_VECTOR_ELT: |
2403 | R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break; |
2404 | case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break; |
2405 | |
2406 | // Unary FP Operations |
2407 | case ISD::FABS: |
2408 | case ISD::FCBRT: |
2409 | case ISD::FCEIL: |
2410 | case ISD::FCOS: |
2411 | case ISD::FEXP: |
2412 | case ISD::FEXP2: |
2413 | case ISD::FEXP10: |
2414 | case ISD::FFLOOR: |
2415 | case ISD::FLOG: |
2416 | case ISD::FLOG2: |
2417 | case ISD::FLOG10: |
2418 | case ISD::FNEARBYINT: |
2419 | case ISD::FNEG: |
2420 | case ISD::FRINT: |
2421 | case ISD::FROUND: |
2422 | case ISD::FROUNDEVEN: |
2423 | case ISD::FSIN: |
2424 | case ISD::FSQRT: |
2425 | case ISD::FTRUNC: |
2426 | case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break; |
2427 | |
2428 | // Binary FP Operations |
2429 | case ISD::FADD: |
2430 | case ISD::FDIV: |
2431 | case ISD::FMAXIMUM: |
2432 | case ISD::FMINIMUM: |
2433 | case ISD::FMAXNUM: |
2434 | case ISD::FMINNUM: |
2435 | case ISD::FMUL: |
2436 | case ISD::FPOW: |
2437 | case ISD::FREM: |
2438 | case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break; |
2439 | |
2440 | case ISD::FMA: // FMA is same as FMAD |
2441 | case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break; |
2442 | |
2443 | case ISD::FPOWI: |
2444 | case ISD::FLDEXP: R = PromoteFloatRes_ExpOp(N); break; |
2445 | case ISD::FFREXP: R = PromoteFloatRes_FFREXP(N); break; |
2446 | |
2447 | case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break; |
2448 | case ISD::STRICT_FP_ROUND: |
2449 | R = PromoteFloatRes_STRICT_FP_ROUND(N); |
2450 | break; |
2451 | case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break; |
2452 | case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break; |
2453 | case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break; |
2454 | |
2455 | case ISD::SINT_TO_FP: |
2456 | case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break; |
2457 | case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break; |
2458 | case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; |
2459 | case ISD::VECREDUCE_FADD: |
2460 | case ISD::VECREDUCE_FMUL: |
2461 | case ISD::VECREDUCE_FMIN: |
2462 | case ISD::VECREDUCE_FMAX: |
2463 | case ISD::VECREDUCE_FMAXIMUM: |
2464 | case ISD::VECREDUCE_FMINIMUM: |
2465 | R = PromoteFloatRes_VECREDUCE(N); |
2466 | break; |
2467 | case ISD::VECREDUCE_SEQ_FADD: |
2468 | case ISD::VECREDUCE_SEQ_FMUL: |
2469 | R = PromoteFloatRes_VECREDUCE_SEQ(N); |
2470 | break; |
2471 | } |
2472 | |
2473 | if (R.getNode()) |
2474 | SetPromotedFloat(Op: SDValue(N, ResNo), Result: R); |
2475 | } |
2476 | |
2477 | // Bitcast from i16 to f16: convert the i16 to a f32 value instead. |
2478 | // At this point, it is not possible to determine if the bitcast value is |
2479 | // eventually stored to memory or promoted to f32 or promoted to a floating |
2480 | // point at a higher precision. Some of these cases are handled by FP_EXTEND, |
2481 | // STORE promotion handlers. |
2482 | SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) { |
2483 | EVT VT = N->getValueType(ResNo: 0); |
2484 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
2485 | // Input type isn't guaranteed to be a scalar int so bitcast if not. The |
2486 | // bitcast will be legalized further if necessary. |
2487 | EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), |
2488 | BitWidth: N->getOperand(Num: 0).getValueType().getSizeInBits()); |
2489 | SDValue Cast = DAG.getBitcast(VT: IVT, V: N->getOperand(Num: 0)); |
2490 | return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NVT), DL: SDLoc(N), VT: NVT, Operand: Cast); |
2491 | } |
2492 | |
2493 | SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) { |
2494 | ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(Val: N); |
2495 | EVT VT = N->getValueType(ResNo: 0); |
2496 | SDLoc DL(N); |
2497 | |
2498 | // Get the (bit-cast) APInt of the APFloat and build an integer constant |
2499 | EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits()); |
2500 | SDValue C = DAG.getConstant(Val: CFPNode->getValueAPF().bitcastToAPInt(), DL, |
2501 | VT: IVT); |
2502 | |
2503 | // Convert the Constant to the desired FP type |
2504 | // FIXME We might be able to do the conversion during compilation and get rid |
2505 | // of it from the object code |
2506 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
2507 | return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NVT), DL, VT: NVT, Operand: C); |
2508 | } |
2509 | |
2510 | // If the Index operand is a constant, try to redirect the extract operation to |
2511 | // the correct legalized vector. If not, bit-convert the input vector to |
2512 | // equivalent integer vector. Extract the element as an (bit-cast) integer |
2513 | // value and convert it to the promoted type. |
2514 | SDValue DAGTypeLegalizer::(SDNode *N) { |
2515 | SDLoc DL(N); |
2516 | |
2517 | // If the index is constant, try to extract the value from the legalized |
2518 | // vector type. |
2519 | if (isa<ConstantSDNode>(Val: N->getOperand(Num: 1))) { |
2520 | SDValue Vec = N->getOperand(Num: 0); |
2521 | SDValue Idx = N->getOperand(Num: 1); |
2522 | EVT VecVT = Vec->getValueType(ResNo: 0); |
2523 | EVT EltVT = VecVT.getVectorElementType(); |
2524 | |
2525 | uint64_t IdxVal = Idx->getAsZExtVal(); |
2526 | |
2527 | switch (getTypeAction(VT: VecVT)) { |
2528 | default: break; |
2529 | case TargetLowering::TypeScalarizeVector: { |
2530 | SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0)); |
2531 | ReplaceValueWith(From: SDValue(N, 0), To: Res); |
2532 | return SDValue(); |
2533 | } |
2534 | case TargetLowering::TypeWidenVector: { |
2535 | Vec = GetWidenedVector(Op: Vec); |
2536 | SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL, VT: EltVT, N1: Vec, N2: Idx); |
2537 | ReplaceValueWith(From: SDValue(N, 0), To: Res); |
2538 | return SDValue(); |
2539 | } |
2540 | case TargetLowering::TypeSplitVector: { |
2541 | SDValue Lo, Hi; |
2542 | GetSplitVector(Op: Vec, Lo, Hi); |
2543 | |
2544 | uint64_t LoElts = Lo.getValueType().getVectorNumElements(); |
2545 | SDValue Res; |
2546 | if (IdxVal < LoElts) |
2547 | Res = DAG.getNode(Opcode: N->getOpcode(), DL, VT: EltVT, N1: Lo, N2: Idx); |
2548 | else |
2549 | Res = DAG.getNode(Opcode: N->getOpcode(), DL, VT: EltVT, N1: Hi, |
2550 | N2: DAG.getConstant(Val: IdxVal - LoElts, DL, |
2551 | VT: Idx.getValueType())); |
2552 | ReplaceValueWith(From: SDValue(N, 0), To: Res); |
2553 | return SDValue(); |
2554 | } |
2555 | |
2556 | } |
2557 | } |
2558 | |
2559 | // Bit-convert the input vector to the equivalent integer vector |
2560 | SDValue NewOp = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0)); |
2561 | EVT IVT = NewOp.getValueType().getVectorElementType(); |
2562 | |
2563 | // Extract the element as an (bit-cast) integer value |
2564 | SDValue NewVal = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: IVT, |
2565 | N1: NewOp, N2: N->getOperand(Num: 1)); |
2566 | |
2567 | // Convert the element to the desired FP type |
2568 | EVT VT = N->getValueType(ResNo: 0); |
2569 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
2570 | return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NVT), DL: SDLoc(N), VT: NVT, Operand: NewVal); |
2571 | } |
2572 | |
2573 | // FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result |
2574 | // needs promotion, so does the argument X. Note that Y, if needed, will be |
2575 | // handled during operand promotion. |
2576 | SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) { |
2577 | EVT VT = N->getValueType(ResNo: 0); |
2578 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
2579 | SDValue Op0 = GetPromotedFloat(Op: N->getOperand(Num: 0)); |
2580 | |
2581 | SDValue Op1 = N->getOperand(Num: 1); |
2582 | |
2583 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NVT, N1: Op0, N2: Op1); |
2584 | } |
2585 | |
2586 | // Unary operation where the result and the operand have PromoteFloat type |
2587 | // action. Construct a new SDNode with the promoted float value of the old |
2588 | // operand. |
2589 | SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) { |
2590 | EVT VT = N->getValueType(ResNo: 0); |
2591 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
2592 | SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 0)); |
2593 | |
2594 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NVT, Operand: Op); |
2595 | } |
2596 | |
2597 | // Binary operations where the result and both operands have PromoteFloat type |
2598 | // action. Construct a new SDNode with the promoted float values of the old |
2599 | // operands. |
2600 | SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) { |
2601 | EVT VT = N->getValueType(ResNo: 0); |
2602 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
2603 | SDValue Op0 = GetPromotedFloat(Op: N->getOperand(Num: 0)); |
2604 | SDValue Op1 = GetPromotedFloat(Op: N->getOperand(Num: 1)); |
2605 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NVT, N1: Op0, N2: Op1, Flags: N->getFlags()); |
2606 | } |
2607 | |
2608 | SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) { |
2609 | EVT VT = N->getValueType(ResNo: 0); |
2610 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
2611 | SDValue Op0 = GetPromotedFloat(Op: N->getOperand(Num: 0)); |
2612 | SDValue Op1 = GetPromotedFloat(Op: N->getOperand(Num: 1)); |
2613 | SDValue Op2 = GetPromotedFloat(Op: N->getOperand(Num: 2)); |
2614 | |
2615 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NVT, N1: Op0, N2: Op1, N3: Op2); |
2616 | } |
2617 | |
2618 | // Promote the Float (first) operand and retain the Integer (second) operand |
2619 | SDValue DAGTypeLegalizer::PromoteFloatRes_ExpOp(SDNode *N) { |
2620 | EVT VT = N->getValueType(ResNo: 0); |
2621 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
2622 | SDValue Op0 = GetPromotedFloat(Op: N->getOperand(Num: 0)); |
2623 | SDValue Op1 = N->getOperand(Num: 1); |
2624 | |
2625 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NVT, N1: Op0, N2: Op1); |
2626 | } |
2627 | |
2628 | SDValue DAGTypeLegalizer::PromoteFloatRes_FFREXP(SDNode *N) { |
2629 | EVT VT = N->getValueType(ResNo: 0); |
2630 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
2631 | SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 0)); |
2632 | SDValue Res = |
2633 | DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), ResultTys: {NVT, N->getValueType(ResNo: 1)}, Ops: Op); |
2634 | |
2635 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
2636 | return Res; |
2637 | } |
2638 | |
2639 | // Explicit operation to reduce precision. Reduce the value to half precision |
2640 | // and promote it back to the legal type. |
2641 | SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) { |
2642 | SDLoc DL(N); |
2643 | |
2644 | SDValue Op = N->getOperand(Num: 0); |
2645 | EVT VT = N->getValueType(ResNo: 0); |
2646 | EVT OpVT = Op->getValueType(ResNo: 0); |
2647 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
2648 | EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits()); |
2649 | |
2650 | // Round promoted float to desired precision |
2651 | SDValue Round = DAG.getNode(Opcode: GetPromotionOpcode(OpVT, RetVT: VT), DL, VT: IVT, Operand: Op); |
2652 | // Promote it back to the legal output type |
2653 | return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NVT), DL, VT: NVT, Operand: Round); |
2654 | } |
2655 | |
2656 | // Explicit operation to reduce precision. Reduce the value to half precision |
2657 | // and promote it back to the legal type. |
2658 | SDValue DAGTypeLegalizer::PromoteFloatRes_STRICT_FP_ROUND(SDNode *N) { |
2659 | SDLoc DL(N); |
2660 | |
2661 | SDValue Chain = N->getOperand(Num: 0); |
2662 | SDValue Op = N->getOperand(Num: 1); |
2663 | EVT VT = N->getValueType(ResNo: 0); |
2664 | EVT OpVT = Op->getValueType(ResNo: 0); |
2665 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
2666 | EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits()); |
2667 | |
2668 | // Round promoted float to desired precision |
2669 | SDValue Round = DAG.getNode(GetPromotionOpcodeStrict(OpVT, VT), DL, |
2670 | DAG.getVTList(IVT, MVT::Other), Chain, Op); |
2671 | // Promote it back to the legal output type |
2672 | SDValue Res = |
2673 | DAG.getNode(GetPromotionOpcodeStrict(VT, NVT), DL, |
2674 | DAG.getVTList(NVT, MVT::Other), Round.getValue(1), Round); |
2675 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
2676 | return Res; |
2677 | } |
2678 | |
2679 | SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) { |
2680 | LoadSDNode *L = cast<LoadSDNode>(Val: N); |
2681 | EVT VT = N->getValueType(ResNo: 0); |
2682 | |
2683 | // Load the value as an integer value with the same number of bits. |
2684 | EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits()); |
2685 | SDValue newL = DAG.getLoad( |
2686 | AM: L->getAddressingMode(), ExtType: L->getExtensionType(), VT: IVT, dl: SDLoc(N), |
2687 | Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(), PtrInfo: L->getPointerInfo(), MemVT: IVT, |
2688 | Alignment: L->getOriginalAlign(), MMOFlags: L->getMemOperand()->getFlags(), AAInfo: L->getAAInfo()); |
2689 | // Legalize the chain result by replacing uses of the old value chain with the |
2690 | // new one |
2691 | ReplaceValueWith(From: SDValue(N, 1), To: newL.getValue(R: 1)); |
2692 | |
2693 | // Convert the integer value to the desired FP type |
2694 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
2695 | return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NVT), DL: SDLoc(N), VT: NVT, Operand: newL); |
2696 | } |
2697 | |
2698 | // Construct a new SELECT node with the promoted true- and false- values. |
2699 | SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) { |
2700 | SDValue TrueVal = GetPromotedFloat(Op: N->getOperand(Num: 1)); |
2701 | SDValue FalseVal = GetPromotedFloat(Op: N->getOperand(Num: 2)); |
2702 | |
2703 | return DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT: TrueVal->getValueType(ResNo: 0), |
2704 | N1: N->getOperand(Num: 0), N2: TrueVal, N3: FalseVal); |
2705 | } |
2706 | |
2707 | // Construct a new SELECT_CC node with the promoted true- and false- values. |
2708 | // The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC. |
2709 | SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) { |
2710 | SDValue TrueVal = GetPromotedFloat(Op: N->getOperand(Num: 2)); |
2711 | SDValue FalseVal = GetPromotedFloat(Op: N->getOperand(Num: 3)); |
2712 | |
2713 | return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), |
2714 | VT: TrueVal.getNode()->getValueType(ResNo: 0), N1: N->getOperand(Num: 0), |
2715 | N2: N->getOperand(Num: 1), N3: TrueVal, N4: FalseVal, N5: N->getOperand(Num: 4)); |
2716 | } |
2717 | |
2718 | // Construct a SDNode that transforms the SINT or UINT operand to the promoted |
2719 | // float type. |
2720 | SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) { |
2721 | SDLoc DL(N); |
2722 | EVT VT = N->getValueType(ResNo: 0); |
2723 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
2724 | SDValue NV = DAG.getNode(Opcode: N->getOpcode(), DL, VT: NVT, Operand: N->getOperand(Num: 0)); |
2725 | // Round the value to the desired precision (that of the source type). |
2726 | return DAG.getNode( |
2727 | Opcode: ISD::FP_EXTEND, DL, VT: NVT, |
2728 | Operand: DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: NV, |
2729 | N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true))); |
2730 | } |
2731 | |
2732 | SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) { |
2733 | return DAG.getUNDEF(VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), |
2734 | VT: N->getValueType(ResNo: 0))); |
2735 | } |
2736 | |
2737 | SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) { |
2738 | // Expand and promote recursively. |
2739 | // TODO: This is non-optimal, but dealing with the concurrently happening |
2740 | // vector-legalization is non-trivial. We could do something similar to |
2741 | // PromoteFloatRes_EXTRACT_VECTOR_ELT here. |
2742 | ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduce(Node: N, DAG)); |
2743 | return SDValue(); |
2744 | } |
2745 | |
2746 | SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) { |
2747 | ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduceSeq(Node: N, DAG)); |
2748 | return SDValue(); |
2749 | } |
2750 | |
2751 | SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) { |
2752 | EVT VT = N->getValueType(ResNo: 0); |
2753 | |
2754 | AtomicSDNode *AM = cast<AtomicSDNode>(Val: N); |
2755 | SDLoc SL(N); |
2756 | |
2757 | SDValue CastVal = BitConvertToInteger(Op: AM->getVal()); |
2758 | EVT CastVT = CastVal.getValueType(); |
2759 | |
2760 | SDValue NewAtomic |
2761 | = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT, |
2762 | DAG.getVTList(CastVT, MVT::Other), |
2763 | { AM->getChain(), AM->getBasePtr(), CastVal }, |
2764 | AM->getMemOperand()); |
2765 | |
2766 | SDValue Result = NewAtomic; |
2767 | |
2768 | if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) { |
2769 | EVT NFPVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
2770 | Result = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NFPVT), DL: SL, VT: NFPVT, |
2771 | Operand: NewAtomic); |
2772 | } |
2773 | |
2774 | // Legalize the chain result by replacing uses of the old value chain with the |
2775 | // new one |
2776 | ReplaceValueWith(From: SDValue(N, 1), To: NewAtomic.getValue(R: 1)); |
2777 | |
2778 | return Result; |
2779 | |
2780 | } |
2781 | |
2782 | //===----------------------------------------------------------------------===// |
2783 | // Half Result Soft Promotion |
2784 | //===----------------------------------------------------------------------===// |
2785 | |
2786 | void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) { |
2787 | LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": " ; |
2788 | N->dump(&DAG)); |
2789 | SDValue R = SDValue(); |
2790 | |
2791 | // See if the target wants to custom expand this node. |
2792 | if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true)) { |
2793 | LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n" ); |
2794 | return; |
2795 | } |
2796 | |
2797 | switch (N->getOpcode()) { |
2798 | default: |
2799 | #ifndef NDEBUG |
2800 | dbgs() << "SoftPromoteHalfResult #" << ResNo << ": " ; |
2801 | N->dump(G: &DAG); dbgs() << "\n" ; |
2802 | #endif |
2803 | report_fatal_error(reason: "Do not know how to soft promote this operator's " |
2804 | "result!" ); |
2805 | |
2806 | case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break; |
2807 | case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break; |
2808 | case ISD::EXTRACT_VECTOR_ELT: |
2809 | R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break; |
2810 | case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break; |
2811 | case ISD::STRICT_FP_ROUND: |
2812 | case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break; |
2813 | |
2814 | // Unary FP Operations |
2815 | case ISD::FABS: |
2816 | case ISD::FCBRT: |
2817 | case ISD::FCEIL: |
2818 | case ISD::FCOS: |
2819 | case ISD::FEXP: |
2820 | case ISD::FEXP2: |
2821 | case ISD::FEXP10: |
2822 | case ISD::FFLOOR: |
2823 | case ISD::FLOG: |
2824 | case ISD::FLOG2: |
2825 | case ISD::FLOG10: |
2826 | case ISD::FNEARBYINT: |
2827 | case ISD::FNEG: |
2828 | case ISD::FREEZE: |
2829 | case ISD::FRINT: |
2830 | case ISD::FROUND: |
2831 | case ISD::FROUNDEVEN: |
2832 | case ISD::FSIN: |
2833 | case ISD::FSQRT: |
2834 | case ISD::FTRUNC: |
2835 | case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break; |
2836 | |
2837 | // Binary FP Operations |
2838 | case ISD::FADD: |
2839 | case ISD::FDIV: |
2840 | case ISD::FMAXIMUM: |
2841 | case ISD::FMINIMUM: |
2842 | case ISD::FMAXNUM: |
2843 | case ISD::FMINNUM: |
2844 | case ISD::FMUL: |
2845 | case ISD::FPOW: |
2846 | case ISD::FREM: |
2847 | case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break; |
2848 | |
2849 | case ISD::FMA: // FMA is same as FMAD |
2850 | case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break; |
2851 | |
2852 | case ISD::FPOWI: |
2853 | case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break; |
2854 | |
2855 | case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break; |
2856 | |
2857 | case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break; |
2858 | case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break; |
2859 | case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break; |
2860 | case ISD::SINT_TO_FP: |
2861 | case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break; |
2862 | case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break; |
2863 | case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; |
2864 | case ISD::VECREDUCE_FADD: |
2865 | case ISD::VECREDUCE_FMUL: |
2866 | case ISD::VECREDUCE_FMIN: |
2867 | case ISD::VECREDUCE_FMAX: |
2868 | case ISD::VECREDUCE_FMAXIMUM: |
2869 | case ISD::VECREDUCE_FMINIMUM: |
2870 | R = SoftPromoteHalfRes_VECREDUCE(N); |
2871 | break; |
2872 | case ISD::VECREDUCE_SEQ_FADD: |
2873 | case ISD::VECREDUCE_SEQ_FMUL: |
2874 | R = SoftPromoteHalfRes_VECREDUCE_SEQ(N); |
2875 | break; |
2876 | } |
2877 | |
2878 | if (R.getNode()) |
2879 | SetSoftPromotedHalf(Op: SDValue(N, ResNo), Result: R); |
2880 | } |
2881 | |
2882 | SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) { |
2883 | return BitConvertToInteger(Op: N->getOperand(Num: 0)); |
2884 | } |
2885 | |
2886 | SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) { |
2887 | ConstantFPSDNode *CN = cast<ConstantFPSDNode>(Val: N); |
2888 | |
2889 | // Get the (bit-cast) APInt of the APFloat and build an integer constant |
2890 | return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN), |
2891 | MVT::i16); |
2892 | } |
2893 | |
2894 | SDValue DAGTypeLegalizer::(SDNode *N) { |
2895 | SDValue NewOp = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0)); |
2896 | return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N), |
2897 | VT: NewOp.getValueType().getVectorElementType(), N1: NewOp, |
2898 | N2: N->getOperand(Num: 1)); |
2899 | } |
2900 | |
2901 | SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) { |
2902 | SDValue LHS = GetSoftPromotedHalf(Op: N->getOperand(Num: 0)); |
2903 | SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1)); |
2904 | SDLoc dl(N); |
2905 | |
2906 | EVT LVT = LHS.getValueType(); |
2907 | EVT RVT = RHS.getValueType(); |
2908 | |
2909 | unsigned LSize = LVT.getSizeInBits(); |
2910 | unsigned RSize = RVT.getSizeInBits(); |
2911 | |
2912 | // First get the sign bit of second operand. |
2913 | SDValue SignBit = DAG.getNode( |
2914 | Opcode: ISD::SHL, DL: dl, VT: RVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: RVT), |
2915 | N2: DAG.getConstant(Val: RSize - 1, DL: dl, |
2916 | VT: TLI.getShiftAmountTy(LHSTy: RVT, DL: DAG.getDataLayout()))); |
2917 | SignBit = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: RVT, N1: RHS, N2: SignBit); |
2918 | |
2919 | // Shift right or sign-extend it if the two operands have different types. |
2920 | int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits(); |
2921 | if (SizeDiff > 0) { |
2922 | SignBit = |
2923 | DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: SignBit, |
2924 | N2: DAG.getConstant(Val: SizeDiff, DL: dl, |
2925 | VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(), |
2926 | DL: DAG.getDataLayout()))); |
2927 | SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LVT, Operand: SignBit); |
2928 | } else if (SizeDiff < 0) { |
2929 | SignBit = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: SignBit); |
2930 | SignBit = |
2931 | DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: LVT, N1: SignBit, |
2932 | N2: DAG.getConstant(Val: -SizeDiff, DL: dl, |
2933 | VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(), |
2934 | DL: DAG.getDataLayout()))); |
2935 | } |
2936 | |
2937 | // Clear the sign bit of the first operand. |
2938 | SDValue Mask = DAG.getNode( |
2939 | Opcode: ISD::SHL, DL: dl, VT: LVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: LVT), |
2940 | N2: DAG.getConstant(Val: LSize - 1, DL: dl, |
2941 | VT: TLI.getShiftAmountTy(LHSTy: LVT, DL: DAG.getDataLayout()))); |
2942 | Mask = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: LVT, N1: Mask, N2: DAG.getConstant(Val: 1, DL: dl, VT: LVT)); |
2943 | LHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: LVT, N1: LHS, N2: Mask); |
2944 | |
2945 | // Or the value with the sign bit. |
2946 | return DAG.getNode(Opcode: ISD::OR, DL: dl, VT: LVT, N1: LHS, N2: SignBit); |
2947 | } |
2948 | |
2949 | SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) { |
2950 | EVT OVT = N->getValueType(ResNo: 0); |
2951 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT); |
2952 | SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0)); |
2953 | SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1)); |
2954 | SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2)); |
2955 | SDLoc dl(N); |
2956 | |
2957 | // Promote to the larger FP type. |
2958 | auto PromotionOpcode = GetPromotionOpcode(OpVT: OVT, RetVT: NVT); |
2959 | Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0); |
2960 | Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1); |
2961 | Op2 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op2); |
2962 | |
2963 | SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1, N3: Op2); |
2964 | |
2965 | // Convert back to FP16 as an integer. |
2966 | return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); |
2967 | } |
2968 | |
2969 | SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) { |
2970 | EVT OVT = N->getValueType(ResNo: 0); |
2971 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT); |
2972 | SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0)); |
2973 | SDValue Op1 = N->getOperand(Num: 1); |
2974 | SDLoc dl(N); |
2975 | |
2976 | // Promote to the larger FP type. |
2977 | Op0 = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op0); |
2978 | |
2979 | SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1); |
2980 | |
2981 | // Convert back to FP16 as an integer. |
2982 | return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); |
2983 | } |
2984 | |
2985 | SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) { |
2986 | EVT OVT = N->getValueType(ResNo: 0); |
2987 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT); |
2988 | SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0)); |
2989 | SDLoc dl(N); |
2990 | |
2991 | // Promote to the larger FP type. |
2992 | Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op); |
2993 | |
2994 | SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, |
2995 | VTList: DAG.getVTList(VT1: NVT, VT2: N->getValueType(ResNo: 1)), N: Op); |
2996 | |
2997 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
2998 | |
2999 | // Convert back to FP16 as an integer. |
3000 | return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); |
3001 | } |
3002 | |
3003 | SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) { |
3004 | EVT RVT = N->getValueType(ResNo: 0); |
3005 | EVT SVT = N->getOperand(Num: 0).getValueType(); |
3006 | |
3007 | if (N->isStrictFPOpcode()) { |
3008 | // FIXME: assume we only have two f16 variants for now. |
3009 | unsigned Opcode; |
3010 | if (RVT == MVT::f16) |
3011 | Opcode = ISD::STRICT_FP_TO_FP16; |
3012 | else if (RVT == MVT::bf16) |
3013 | Opcode = ISD::STRICT_FP_TO_BF16; |
3014 | else |
3015 | llvm_unreachable("unknown half type" ); |
3016 | SDValue Res = DAG.getNode(Opcode, SDLoc(N), {MVT::i16, MVT::Other}, |
3017 | {N->getOperand(0), N->getOperand(1)}); |
3018 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
3019 | return Res; |
3020 | } |
3021 | |
3022 | return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), MVT::i16, |
3023 | N->getOperand(0)); |
3024 | } |
3025 | |
3026 | SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) { |
3027 | LoadSDNode *L = cast<LoadSDNode>(Val: N); |
3028 | |
3029 | // Load the value as an integer value with the same number of bits. |
3030 | assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!" ); |
3031 | SDValue NewL = |
3032 | DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16, |
3033 | SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(), |
3034 | L->getPointerInfo(), MVT::i16, L->getOriginalAlign(), |
3035 | L->getMemOperand()->getFlags(), L->getAAInfo()); |
3036 | // Legalize the chain result by replacing uses of the old value chain with the |
3037 | // new one |
3038 | ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1)); |
3039 | return NewL; |
3040 | } |
3041 | |
3042 | SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) { |
3043 | SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1)); |
3044 | SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2)); |
3045 | return DAG.getSelect(DL: SDLoc(N), VT: Op1.getValueType(), Cond: N->getOperand(Num: 0), LHS: Op1, |
3046 | RHS: Op2); |
3047 | } |
3048 | |
3049 | SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) { |
3050 | SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2)); |
3051 | SDValue Op3 = GetSoftPromotedHalf(Op: N->getOperand(Num: 3)); |
3052 | return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: Op2.getValueType(), |
3053 | N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1), N3: Op2, N4: Op3, |
3054 | N5: N->getOperand(Num: 4)); |
3055 | } |
3056 | |
3057 | SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) { |
3058 | EVT OVT = N->getValueType(ResNo: 0); |
3059 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT); |
3060 | SDLoc dl(N); |
3061 | |
3062 | SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: N->getOperand(Num: 0)); |
3063 | |
3064 | // Round the value to the softened type. |
3065 | return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); |
3066 | } |
3067 | |
3068 | SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) { |
3069 | return DAG.getUNDEF(MVT::i16); |
3070 | } |
3071 | |
3072 | SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) { |
3073 | EVT OVT = N->getValueType(ResNo: 0); |
3074 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT); |
3075 | SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0)); |
3076 | SDLoc dl(N); |
3077 | |
3078 | // Promote to the larger FP type. |
3079 | Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op); |
3080 | |
3081 | SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Op); |
3082 | |
3083 | // Convert back to FP16 as an integer. |
3084 | return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); |
3085 | } |
3086 | |
3087 | SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) { |
3088 | EVT OVT = N->getValueType(ResNo: 0); |
3089 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT); |
3090 | SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0)); |
3091 | SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1)); |
3092 | SDLoc dl(N); |
3093 | |
3094 | // Promote to the larger FP type. |
3095 | auto PromotionOpcode = GetPromotionOpcode(OpVT: OVT, RetVT: NVT); |
3096 | Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0); |
3097 | Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1); |
3098 | |
3099 | SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1); |
3100 | |
3101 | // Convert back to FP16 as an integer. |
3102 | return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); |
3103 | } |
3104 | |
3105 | SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) { |
3106 | // Expand and soften recursively. |
3107 | ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduce(Node: N, DAG)); |
3108 | return SDValue(); |
3109 | } |
3110 | |
3111 | SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) { |
3112 | // Expand and soften. |
3113 | ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduceSeq(Node: N, DAG)); |
3114 | return SDValue(); |
3115 | } |
3116 | |
3117 | //===----------------------------------------------------------------------===// |
3118 | // Half Operand Soft Promotion |
3119 | //===----------------------------------------------------------------------===// |
3120 | |
3121 | bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) { |
3122 | LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": " ; |
3123 | N->dump(&DAG)); |
3124 | SDValue Res = SDValue(); |
3125 | |
3126 | if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false)) { |
3127 | LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n" ); |
3128 | return false; |
3129 | } |
3130 | |
3131 | // Nodes that use a promotion-requiring floating point operand, but doesn't |
3132 | // produce a soft promotion-requiring floating point result, need to be |
3133 | // legalized to use the soft promoted float operand. Nodes that produce at |
3134 | // least one soft promotion-requiring floating point result have their |
3135 | // operands legalized as a part of PromoteFloatResult. |
3136 | switch (N->getOpcode()) { |
3137 | default: |
3138 | #ifndef NDEBUG |
3139 | dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": " ; |
3140 | N->dump(G: &DAG); dbgs() << "\n" ; |
3141 | #endif |
3142 | report_fatal_error(reason: "Do not know how to soft promote this operator's " |
3143 | "operand!" ); |
3144 | |
3145 | case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break; |
3146 | case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break; |
3147 | case ISD::FP_TO_SINT: |
3148 | case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break; |
3149 | case ISD::FP_TO_SINT_SAT: |
3150 | case ISD::FP_TO_UINT_SAT: |
3151 | Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break; |
3152 | case ISD::STRICT_FP_EXTEND: |
3153 | case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break; |
3154 | case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break; |
3155 | case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break; |
3156 | case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break; |
3157 | case ISD::STACKMAP: |
3158 | Res = SoftPromoteHalfOp_STACKMAP(N, OpNo); |
3159 | break; |
3160 | case ISD::PATCHPOINT: |
3161 | Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo); |
3162 | break; |
3163 | } |
3164 | |
3165 | if (!Res.getNode()) |
3166 | return false; |
3167 | |
3168 | assert(Res.getNode() != N && "Expected a new node!" ); |
3169 | |
3170 | assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && |
3171 | "Invalid operand expansion" ); |
3172 | |
3173 | ReplaceValueWith(From: SDValue(N, 0), To: Res); |
3174 | return false; |
3175 | } |
3176 | |
3177 | SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) { |
3178 | SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0)); |
3179 | |
3180 | return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op0); |
3181 | } |
3182 | |
3183 | SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N, |
3184 | unsigned OpNo) { |
3185 | assert(OpNo == 1 && "Only Operand 1 must need promotion here" ); |
3186 | SDValue Op1 = N->getOperand(Num: 1); |
3187 | EVT RVT = Op1.getValueType(); |
3188 | SDLoc dl(N); |
3189 | |
3190 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op1.getValueType()); |
3191 | |
3192 | Op1 = GetSoftPromotedHalf(Op: Op1); |
3193 | Op1 = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: RVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op1); |
3194 | |
3195 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0), |
3196 | N2: Op1); |
3197 | } |
3198 | |
3199 | SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) { |
3200 | EVT RVT = N->getValueType(ResNo: 0); |
3201 | bool IsStrict = N->isStrictFPOpcode(); |
3202 | SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0); |
3203 | EVT SVT = Op.getValueType(); |
3204 | Op = GetSoftPromotedHalf(Op: N->getOperand(Num: IsStrict ? 1 : 0)); |
3205 | |
3206 | if (IsStrict) { |
3207 | unsigned Opcode; |
3208 | if (SVT == MVT::f16) |
3209 | Opcode = ISD::STRICT_FP16_TO_FP; |
3210 | else if (SVT == MVT::bf16) |
3211 | Opcode = ISD::STRICT_BF16_TO_FP; |
3212 | else |
3213 | llvm_unreachable("unknown half type" ); |
3214 | SDValue Res = |
3215 | DAG.getNode(Opcode, SDLoc(N), {N->getValueType(0), MVT::Other}, |
3216 | {N->getOperand(0), Op}); |
3217 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
3218 | ReplaceValueWith(From: SDValue(N, 0), To: Res); |
3219 | return SDValue(); |
3220 | } |
3221 | |
3222 | return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: SDLoc(N), VT: RVT, Operand: Op); |
3223 | } |
3224 | |
3225 | SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) { |
3226 | EVT RVT = N->getValueType(ResNo: 0); |
3227 | SDValue Op = N->getOperand(Num: 0); |
3228 | EVT SVT = Op.getValueType(); |
3229 | SDLoc dl(N); |
3230 | |
3231 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op.getValueType()); |
3232 | |
3233 | Op = GetSoftPromotedHalf(Op); |
3234 | |
3235 | SDValue Res = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: dl, VT: NVT, Operand: Op); |
3236 | |
3237 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), Operand: Res); |
3238 | } |
3239 | |
3240 | SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) { |
3241 | EVT RVT = N->getValueType(ResNo: 0); |
3242 | SDValue Op = N->getOperand(Num: 0); |
3243 | EVT SVT = Op.getValueType(); |
3244 | SDLoc dl(N); |
3245 | |
3246 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op.getValueType()); |
3247 | |
3248 | Op = GetSoftPromotedHalf(Op); |
3249 | |
3250 | SDValue Res = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: dl, VT: NVT, Operand: Op); |
3251 | |
3252 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), N1: Res, |
3253 | N2: N->getOperand(Num: 1)); |
3254 | } |
3255 | |
3256 | SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N, |
3257 | unsigned OpNo) { |
3258 | assert(OpNo == 0 && "Can only soften the comparison values" ); |
3259 | SDValue Op0 = N->getOperand(Num: 0); |
3260 | SDValue Op1 = N->getOperand(Num: 1); |
3261 | SDLoc dl(N); |
3262 | |
3263 | EVT SVT = Op0.getValueType(); |
3264 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT); |
3265 | |
3266 | Op0 = GetSoftPromotedHalf(Op: Op0); |
3267 | Op1 = GetSoftPromotedHalf(Op: Op1); |
3268 | |
3269 | // Promote to the larger FP type. |
3270 | auto PromotionOpcode = GetPromotionOpcode(OpVT: SVT, RetVT: NVT); |
3271 | Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0); |
3272 | Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1); |
3273 | |
3274 | return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Op0, N2: Op1, |
3275 | N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3), N5: N->getOperand(Num: 4)); |
3276 | } |
3277 | |
3278 | SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) { |
3279 | SDValue Op0 = N->getOperand(Num: 0); |
3280 | SDValue Op1 = N->getOperand(Num: 1); |
3281 | ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get(); |
3282 | SDLoc dl(N); |
3283 | |
3284 | EVT SVT = Op0.getValueType(); |
3285 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op0.getValueType()); |
3286 | |
3287 | Op0 = GetSoftPromotedHalf(Op: Op0); |
3288 | Op1 = GetSoftPromotedHalf(Op: Op1); |
3289 | |
3290 | // Promote to the larger FP type. |
3291 | auto PromotionOpcode = GetPromotionOpcode(OpVT: SVT, RetVT: NVT); |
3292 | Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0); |
3293 | Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1); |
3294 | |
3295 | return DAG.getSetCC(DL: SDLoc(N), VT: N->getValueType(ResNo: 0), LHS: Op0, RHS: Op1, Cond: CCCode); |
3296 | } |
3297 | |
3298 | SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) { |
3299 | assert(OpNo == 1 && "Can only soften the stored value!" ); |
3300 | StoreSDNode *ST = cast<StoreSDNode>(Val: N); |
3301 | SDValue Val = ST->getValue(); |
3302 | SDLoc dl(N); |
3303 | |
3304 | assert(!ST->isTruncatingStore() && "Unexpected truncating store." ); |
3305 | SDValue Promoted = GetSoftPromotedHalf(Op: Val); |
3306 | return DAG.getStore(Chain: ST->getChain(), dl, Val: Promoted, Ptr: ST->getBasePtr(), |
3307 | MMO: ST->getMemOperand()); |
3308 | } |
3309 | |
3310 | SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) { |
3311 | assert(OpNo > 1); // Because the first two arguments are guaranteed legal. |
3312 | SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end()); |
3313 | SDValue Op = N->getOperand(Num: OpNo); |
3314 | NewOps[OpNo] = GetSoftPromotedHalf(Op); |
3315 | SDValue NewNode = |
3316 | DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: N->getVTList(), Ops: NewOps); |
3317 | |
3318 | for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) |
3319 | ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum)); |
3320 | |
3321 | return SDValue(); // Signal that we replaced the node ourselves. |
3322 | } |
3323 | |
3324 | SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N, |
3325 | unsigned OpNo) { |
3326 | assert(OpNo >= 7); |
3327 | SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end()); |
3328 | SDValue Op = N->getOperand(Num: OpNo); |
3329 | NewOps[OpNo] = GetSoftPromotedHalf(Op); |
3330 | SDValue NewNode = |
3331 | DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: N->getVTList(), Ops: NewOps); |
3332 | |
3333 | for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) |
3334 | ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum)); |
3335 | |
3336 | return SDValue(); // Signal that we replaced the node ourselves. |
3337 | } |
3338 | |