1	//===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file implements float type expansion and softening for LegalizeTypes.
10	// Softening is the act of turning a computation in an illegal floating point
11	// type into a computation in an integer type of the same size; also known as
12	// "soft float". For example, turning f32 arithmetic into operations using i32.
13	// The resulting integer value is the same as what you would get by performing
14	// the floating point operation and bitcasting the result to the integer type.
15	// Expansion is the act of changing a computation in an illegal type to be a
16	// computation in two identical registers of a smaller type. For example,
17	// implementing ppcf128 arithmetic in two f64 registers.
18	//
19	//===----------------------------------------------------------------------===//
20
21	#include "LegalizeTypes.h"
22	#include "llvm/Analysis/TargetLibraryInfo.h"
23	#include "llvm/Support/ErrorHandling.h"
24	#include "llvm/Support/raw_ostream.h"
25	using namespace llvm;
26
27	#define DEBUG_TYPE "legalize-types"
28
29	/// GetFPLibCall - Return the right libcall for the given floating point type.
30	/// FIXME: This is a local version of RTLIB::getFPLibCall that should be
31	/// refactored away (see RTLIB::getPOWI for an example).
32	static RTLIB::Libcall GetFPLibCall(EVT VT,
33	RTLIB::Libcall Call_F32,
34	RTLIB::Libcall Call_F64,
35	RTLIB::Libcall Call_F80,
36	RTLIB::Libcall Call_F128,
37	RTLIB::Libcall Call_PPCF128) {
38	return
39	VT == MVT::f32 ? Call_F32 :
40	VT == MVT::f64 ? Call_F64 :
41	VT == MVT::f80 ? Call_F80 :
42	VT == MVT::f128 ? Call_F128 :
43	VT == MVT::ppcf128 ? Call_PPCF128 :
44	RTLIB::UNKNOWN_LIBCALL;
45	}
46
47	//===----------------------------------------------------------------------===//
48	// Convert Float Results to Integer
49	//===----------------------------------------------------------------------===//
50
51	void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
52	LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG));
53	SDValue R = SDValue();
54
55	switch (N->getOpcode()) {
56	default:
57	#ifndef NDEBUG
58	dbgs() << "SoftenFloatResult #" << ResNo << ": ";
59	N->dump(G: &DAG); dbgs() << "\n";
60	#endif
61	report_fatal_error(reason: "Do not know how to soften the result of this "
62	"operator!");
63
64	case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break;
65	case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
66	case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
67	case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
68	case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
69	case ISD::EXTRACT_VECTOR_ELT:
70	R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
71	case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
72	case ISD::STRICT_FMINNUM:
73	case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
74	case ISD::STRICT_FMAXNUM:
75	case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
76	case ISD::STRICT_FADD:
77	case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
78	case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
79	case ISD::STRICT_FCEIL:
80	case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
81	case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
82	case ISD::STRICT_FCOS:
83	case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
84	case ISD::STRICT_FDIV:
85	case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
86	case ISD::STRICT_FEXP:
87	case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
88	case ISD::STRICT_FEXP2:
89	case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
90	case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break;
91	case ISD::STRICT_FFLOOR:
92	case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
93	case ISD::STRICT_FLOG:
94	case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
95	case ISD::STRICT_FLOG2:
96	case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
97	case ISD::STRICT_FLOG10:
98	case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
99	case ISD::STRICT_FMA:
100	case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
101	case ISD::STRICT_FMUL:
102	case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
103	case ISD::STRICT_FNEARBYINT:
104	case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
105	case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
106	case ISD::STRICT_FP_EXTEND:
107	case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
108	case ISD::STRICT_FP_ROUND:
109	case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
110	case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
111	case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break;
112	case ISD::STRICT_FPOW:
113	case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
114	case ISD::STRICT_FPOWI:
115	case ISD::FPOWI:
116	case ISD::FLDEXP:
117	case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break;
118	case ISD::FFREXP:
119	R = SoftenFloatRes_FFREXP(N);
120	break;
121	case ISD::STRICT_FREM:
122	case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
123	case ISD::STRICT_FRINT:
124	case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
125	case ISD::STRICT_FROUND:
126	case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
127	case ISD::STRICT_FROUNDEVEN:
128	case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
129	case ISD::STRICT_FSIN:
130	case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
131	case ISD::STRICT_FSQRT:
132	case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
133	case ISD::STRICT_FSUB:
134	case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
135	case ISD::STRICT_FTRUNC:
136	case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
137	case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
138	case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
139	case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
140	case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
141	case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
142	case ISD::STRICT_SINT_TO_FP:
143	case ISD::STRICT_UINT_TO_FP:
144	case ISD::SINT_TO_FP:
145	case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
146	case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
147	case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
148	case ISD::VECREDUCE_FADD:
149	case ISD::VECREDUCE_FMUL:
150	case ISD::VECREDUCE_FMIN:
151	case ISD::VECREDUCE_FMAX:
152	case ISD::VECREDUCE_FMAXIMUM:
153	case ISD::VECREDUCE_FMINIMUM:
154	R = SoftenFloatRes_VECREDUCE(N);
155	break;
156	case ISD::VECREDUCE_SEQ_FADD:
157	case ISD::VECREDUCE_SEQ_FMUL:
158	R = SoftenFloatRes_VECREDUCE_SEQ(N);
159	break;
160	}
161
162	// If R is null, the sub-method took care of registering the result.
163	if (R.getNode()) {
164	assert(R.getNode() != N);
165	SetSoftenedFloat(Op: SDValue(N, ResNo), Result: R);
166	}
167	}
168
169	SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
170	bool IsStrict = N->isStrictFPOpcode();
171	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
172	unsigned Offset = IsStrict ? 1 : 0;
173	assert(N->getNumOperands() == (1 + Offset) &&
174	"Unexpected number of operands!");
175	SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset));
176	SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
177	TargetLowering::MakeLibCallOptions CallOptions;
178	EVT OpVT = N->getOperand(Num: 0 + Offset).getValueType();
179	CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0), Value: true);
180	std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
181	CallOptions, dl: SDLoc(N),
182	Chain);
183	if (IsStrict)
184	ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
185	return Tmp.first;
186	}
187
188	SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
189	bool IsStrict = N->isStrictFPOpcode();
190	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
191	unsigned Offset = IsStrict ? 1 : 0;
192	assert(N->getNumOperands() == (2 + Offset) &&
193	"Unexpected number of operands!");
194	SDValue Ops[2] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
195	GetSoftenedFloat(Op: N->getOperand(Num: 1 + Offset)) };
196	SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
197	TargetLowering::MakeLibCallOptions CallOptions;
198	EVT OpsVT[2] = { N->getOperand(Num: 0 + Offset).getValueType(),
199	N->getOperand(Num: 1 + Offset).getValueType() };
200	CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0), Value: true);
201	std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops,
202	CallOptions, dl: SDLoc(N),
203	Chain);
204	if (IsStrict)
205	ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
206	return Tmp.first;
207	}
208
209	SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
210	return BitConvertToInteger(Op: N->getOperand(Num: 0));
211	}
212
213	SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
214	EVT Ty = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
215	return DAG.getNode(Opcode: ISD::FREEZE, DL: SDLoc(N), VT: Ty,
216	Operand: GetSoftenedFloat(Op: N->getOperand(Num: 0)));
217	}
218
219	SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) {
220	EVT Ty = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
221	SDValue NewFence = DAG.getNode(Opcode: ISD::ARITH_FENCE, DL: SDLoc(N), VT: Ty,
222	Operand: GetSoftenedFloat(Op: N->getOperand(Num: 0)));
223	return NewFence;
224	}
225
226	SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
227	unsigned ResNo) {
228	SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
229	return BitConvertToInteger(Op);
230	}
231
232	SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
233	// Convert the inputs to integers, and build a new pair out of them.
234	return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SDLoc(N),
235	VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
236	VT: N->getValueType(ResNo: 0)),
237	N1: BitConvertToInteger(Op: N->getOperand(Num: 0)),
238	N2: BitConvertToInteger(Op: N->getOperand(Num: 1)));
239	}
240
241	SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
242	ConstantFPSDNode *CN = cast<ConstantFPSDNode>(Val: N);
243	// In ppcf128, the high 64 bits are always first in memory regardless
244	// of Endianness. LLVM's APFloat representation is not Endian sensitive,
245	// and so always converts into a 128-bit APInt in a non-Endian-sensitive
246	// way. However, APInt's are serialized in an Endian-sensitive fashion,
247	// so on big-Endian targets, the two doubles are output in the wrong
248	// order. Fix this by manually flipping the order of the high 64 bits
249	// and the low 64 bits here.
250	if (DAG.getDataLayout().isBigEndian() &&
251	CN->getValueType(ResNo: 0).getSimpleVT() == llvm::MVT::ppcf128) {
252	uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
253	CN->getValueAPF().bitcastToAPInt().getRawData()[0] };
254	APInt Val(128, words);
255	return DAG.getConstant(Val, DL: SDLoc(CN),
256	VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
257	VT: CN->getValueType(ResNo: 0)));
258	} else {
259	return DAG.getConstant(Val: CN->getValueAPF().bitcastToAPInt(), DL: SDLoc(CN),
260	VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
261	VT: CN->getValueType(ResNo: 0)));
262	}
263	}
264
265	SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
266	SDValue NewOp = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0));
267	return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
268	VT: NewOp.getValueType().getVectorElementType(),
269	N1: NewOp, N2: N->getOperand(Num: 1));
270	}
271
272	SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
273	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
274	unsigned Size = NVT.getSizeInBits();
275
276	// Mask = ~(1 << (Size-1))
277	APInt API = APInt::getAllOnes(numBits: Size);
278	API.clearBit(BitPosition: Size - 1);
279	SDValue Mask = DAG.getConstant(Val: API, DL: SDLoc(N), VT: NVT);
280	SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0));
281	return DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: NVT, N1: Op, N2: Mask);
282	}
283
284	SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
285	if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(Node: N, DAG))
286	return SoftenFloatRes_SELECT_CC(N: SelCC.getNode());
287	return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
288	Call_F32: RTLIB::FMIN_F32,
289	Call_F64: RTLIB::FMIN_F64,
290	Call_F80: RTLIB::FMIN_F80,
291	Call_F128: RTLIB::FMIN_F128,
292	Call_PPCF128: RTLIB::FMIN_PPCF128));
293	}
294
295	SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
296	if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(Node: N, DAG))
297	return SoftenFloatRes_SELECT_CC(N: SelCC.getNode());
298	return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
299	Call_F32: RTLIB::FMAX_F32,
300	Call_F64: RTLIB::FMAX_F64,
301	Call_F80: RTLIB::FMAX_F80,
302	Call_F128: RTLIB::FMAX_F128,
303	Call_PPCF128: RTLIB::FMAX_PPCF128));
304	}
305
306	SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
307	return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
308	Call_F32: RTLIB::ADD_F32,
309	Call_F64: RTLIB::ADD_F64,
310	Call_F80: RTLIB::ADD_F80,
311	Call_F128: RTLIB::ADD_F128,
312	Call_PPCF128: RTLIB::ADD_PPCF128));
313	}
314
315	SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
316	return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
317	Call_F32: RTLIB::CBRT_F32,
318	Call_F64: RTLIB::CBRT_F64,
319	Call_F80: RTLIB::CBRT_F80,
320	Call_F128: RTLIB::CBRT_F128,
321	Call_PPCF128: RTLIB::CBRT_PPCF128));
322	}
323
324	SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
325	return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
326	Call_F32: RTLIB::CEIL_F32,
327	Call_F64: RTLIB::CEIL_F64,
328	Call_F80: RTLIB::CEIL_F80,
329	Call_F128: RTLIB::CEIL_F128,
330	Call_PPCF128: RTLIB::CEIL_PPCF128));
331	}
332
333	SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
334	SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 0));
335	SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
336	SDLoc dl(N);
337
338	EVT LVT = LHS.getValueType();
339	EVT RVT = RHS.getValueType();
340
341	unsigned LSize = LVT.getSizeInBits();
342	unsigned RSize = RVT.getSizeInBits();
343
344	// First get the sign bit of second operand.
345	SDValue SignBit = DAG.getNode(
346	Opcode: ISD::SHL, DL: dl, VT: RVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: RVT),
347	N2: DAG.getConstant(Val: RSize - 1, DL: dl,
348	VT: TLI.getShiftAmountTy(LHSTy: RVT, DL: DAG.getDataLayout())));
349	SignBit = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: RVT, N1: RHS, N2: SignBit);
350
351	// Shift right or sign-extend it if the two operands have different types.
352	int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
353	if (SizeDiff > 0) {
354	SignBit =
355	DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: SignBit,
356	N2: DAG.getConstant(Val: SizeDiff, DL: dl,
357	VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
358	DL: DAG.getDataLayout())));
359	SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LVT, Operand: SignBit);
360	} else if (SizeDiff < 0) {
361	SignBit = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: SignBit);
362	SignBit =
363	DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: LVT, N1: SignBit,
364	N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
365	VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
366	DL: DAG.getDataLayout())));
367	}
368
369	// Clear the sign bit of the first operand.
370	SDValue Mask = DAG.getNode(
371	Opcode: ISD::SHL, DL: dl, VT: LVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: LVT),
372	N2: DAG.getConstant(Val: LSize - 1, DL: dl,
373	VT: TLI.getShiftAmountTy(LHSTy: LVT, DL: DAG.getDataLayout())));
374	Mask = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: LVT, N1: Mask, N2: DAG.getConstant(Val: 1, DL: dl, VT: LVT));
375	LHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: LVT, N1: LHS, N2: Mask);
376
377	// Or the value with the sign bit.
378	return DAG.getNode(Opcode: ISD::OR, DL: dl, VT: LVT, N1: LHS, N2: SignBit);
379	}
380
381	SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
382	return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
383	Call_F32: RTLIB::COS_F32,
384	Call_F64: RTLIB::COS_F64,
385	Call_F80: RTLIB::COS_F80,
386	Call_F128: RTLIB::COS_F128,
387	Call_PPCF128: RTLIB::COS_PPCF128));
388	}
389
390	SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
391	return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
392	Call_F32: RTLIB::DIV_F32,
393	Call_F64: RTLIB::DIV_F64,
394	Call_F80: RTLIB::DIV_F80,
395	Call_F128: RTLIB::DIV_F128,
396	Call_PPCF128: RTLIB::DIV_PPCF128));
397	}
398
399	SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
400	return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
401	Call_F32: RTLIB::EXP_F32,
402	Call_F64: RTLIB::EXP_F64,
403	Call_F80: RTLIB::EXP_F80,
404	Call_F128: RTLIB::EXP_F128,
405	Call_PPCF128: RTLIB::EXP_PPCF128));
406	}
407
408	SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
409	return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
410	Call_F32: RTLIB::EXP2_F32,
411	Call_F64: RTLIB::EXP2_F64,
412	Call_F80: RTLIB::EXP2_F80,
413	Call_F128: RTLIB::EXP2_F128,
414	Call_PPCF128: RTLIB::EXP2_PPCF128));
415	}
416
417	SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) {
418	return SoftenFloatRes_Unary(
419	N,
420	LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::EXP10_F32, Call_F64: RTLIB::EXP10_F64,
421	Call_F80: RTLIB::EXP10_F80, Call_F128: RTLIB::EXP10_F128, Call_PPCF128: RTLIB::EXP10_PPCF128));
422	}
423
424	SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
425	return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
426	Call_F32: RTLIB::FLOOR_F32,
427	Call_F64: RTLIB::FLOOR_F64,
428	Call_F80: RTLIB::FLOOR_F80,
429	Call_F128: RTLIB::FLOOR_F128,
430	Call_PPCF128: RTLIB::FLOOR_PPCF128));
431	}
432
433	SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
434	return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
435	Call_F32: RTLIB::LOG_F32,
436	Call_F64: RTLIB::LOG_F64,
437	Call_F80: RTLIB::LOG_F80,
438	Call_F128: RTLIB::LOG_F128,
439	Call_PPCF128: RTLIB::LOG_PPCF128));
440	}
441
442	SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
443	return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
444	Call_F32: RTLIB::LOG2_F32,
445	Call_F64: RTLIB::LOG2_F64,
446	Call_F80: RTLIB::LOG2_F80,
447	Call_F128: RTLIB::LOG2_F128,
448	Call_PPCF128: RTLIB::LOG2_PPCF128));
449	}
450
451	SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
452	return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
453	Call_F32: RTLIB::LOG10_F32,
454	Call_F64: RTLIB::LOG10_F64,
455	Call_F80: RTLIB::LOG10_F80,
456	Call_F128: RTLIB::LOG10_F128,
457	Call_PPCF128: RTLIB::LOG10_PPCF128));
458	}
459
460	SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
461	bool IsStrict = N->isStrictFPOpcode();
462	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
463	unsigned Offset = IsStrict ? 1 : 0;
464	SDValue Ops[3] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
465	GetSoftenedFloat(Op: N->getOperand(Num: 1 + Offset)),
466	GetSoftenedFloat(Op: N->getOperand(Num: 2 + Offset)) };
467	SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
468	TargetLowering::MakeLibCallOptions CallOptions;
469	EVT OpsVT[3] = { N->getOperand(Num: 0 + Offset).getValueType(),
470	N->getOperand(Num: 1 + Offset).getValueType(),
471	N->getOperand(Num: 2 + Offset).getValueType() };
472	CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0), Value: true);
473	std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
474	LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
475	Call_F32: RTLIB::FMA_F32,
476	Call_F64: RTLIB::FMA_F64,
477	Call_F80: RTLIB::FMA_F80,
478	Call_F128: RTLIB::FMA_F128,
479	Call_PPCF128: RTLIB::FMA_PPCF128),
480	RetVT: NVT, Ops, CallOptions, dl: SDLoc(N), Chain);
481	if (IsStrict)
482	ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
483	return Tmp.first;
484	}
485
486	SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
487	return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
488	Call_F32: RTLIB::MUL_F32,
489	Call_F64: RTLIB::MUL_F64,
490	Call_F80: RTLIB::MUL_F80,
491	Call_F128: RTLIB::MUL_F128,
492	Call_PPCF128: RTLIB::MUL_PPCF128));
493	}
494
495	SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
496	return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
497	Call_F32: RTLIB::NEARBYINT_F32,
498	Call_F64: RTLIB::NEARBYINT_F64,
499	Call_F80: RTLIB::NEARBYINT_F80,
500	Call_F128: RTLIB::NEARBYINT_F128,
501	Call_PPCF128: RTLIB::NEARBYINT_PPCF128));
502	}
503
504	SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
505	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
506	SDLoc dl(N);
507
508	// Expand Y = FNEG(X) -> Y = X ^ sign mask
509	APInt SignMask = APInt::getSignMask(BitWidth: NVT.getSizeInBits());
510	return DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: NVT, N1: GetSoftenedFloat(Op: N->getOperand(Num: 0)),
511	N2: DAG.getConstant(Val: SignMask, DL: dl, VT: NVT));
512	}
513
514	SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
515	bool IsStrict = N->isStrictFPOpcode();
516	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
517	SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
518
519	SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
520
521	if (getTypeAction(VT: Op.getValueType()) == TargetLowering::TypePromoteFloat) {
522	Op = GetPromotedFloat(Op);
523	// If the promotion did the FP_EXTEND to the destination type for us,
524	// there's nothing left to do here.
525	if (Op.getValueType() == N->getValueType(ResNo: 0)) {
526	if (IsStrict)
527	ReplaceValueWith(From: SDValue(N, 1), To: Chain);
528	return BitConvertToInteger(Op);
529	}
530	}
531
532	// There's only a libcall for f16 -> f32 and shifting is only valid for bf16
533	// -> f32, so proceed in two stages. Also, it's entirely possible for both
534	// f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather
535	// than FP16_TO_FP.
536	if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) &&
537	N->getValueType(ResNo: 0) != MVT::f32) {
538	if (IsStrict) {
539	Op = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N),
540	{ MVT::f32, MVT::Other }, { Chain, Op });
541	Chain = Op.getValue(R: 1);
542	} else {
543	Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op);
544	}
545	}
546
547	if (Op.getValueType() == MVT::bf16) {
548	// FIXME: Need ReplaceValueWith on chain in strict case
549	return SoftenFloatRes_BF16_TO_FP(N);
550	}
551
552	RTLIB::Libcall LC = RTLIB::getFPEXT(OpVT: Op.getValueType(), RetVT: N->getValueType(ResNo: 0));
553	assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
554	TargetLowering::MakeLibCallOptions CallOptions;
555	EVT OpVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
556	CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0), Value: true);
557	std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
558	CallOptions, dl: SDLoc(N),
559	Chain);
560	if (IsStrict)
561	ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
562	return Tmp.first;
563	}
564
565	// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
566	// nodes?
567	SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
568	EVT MidVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), MVT::VT: f32);
569	SDValue Op = N->getOperand(Num: 0);
570	TargetLowering::MakeLibCallOptions CallOptions;
571	EVT OpsVT[1] = { N->getOperand(Num: 0).getValueType() };
572	CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0), Value: true);
573	SDValue Res32 = TLI.makeLibCall(DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MidVT, Ops: Op,
574	CallOptions, dl: SDLoc(N)).first;
575	if (N->getValueType(ResNo: 0) == MVT::f32)
576	return Res32;
577
578	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
579	RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::OpVT: f32, RetVT: N->getValueType(ResNo: 0));
580	assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
581	return TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Res32, CallOptions, dl: SDLoc(N)).first;
582	}
583
584	// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
585	// nodes?
586	SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) {
587	assert(N->getValueType(0) == MVT::f32 &&
588	"Can only soften BF16_TO_FP with f32 result");
589	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), MVT::VT: f32);
590	SDValue Op = N->getOperand(Num: 0);
591	SDLoc DL(N);
592	Op = DAG.getNode(ISD::ANY_EXTEND, DL, NVT,
593	DAG.getNode(ISD::BITCAST, DL, MVT::i16, Op));
594	SDValue Res = DAG.getNode(Opcode: ISD::SHL, DL, VT: NVT, N1: Op,
595	N2: DAG.getShiftAmountConstant(Val: 16, VT: NVT, DL));
596	return Res;
597	}
598
599	SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
600	bool IsStrict = N->isStrictFPOpcode();
601	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
602	SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
603	SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
604	RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op.getValueType(), RetVT: N->getValueType(ResNo: 0));
605	assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
606	TargetLowering::MakeLibCallOptions CallOptions;
607	EVT OpVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
608	CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0), Value: true);
609	std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
610	CallOptions, dl: SDLoc(N),
611	Chain);
612	if (IsStrict)
613	ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
614	return Tmp.first;
615	}
616
617	SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
618	return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
619	Call_F32: RTLIB::POW_F32,
620	Call_F64: RTLIB::POW_F64,
621	Call_F80: RTLIB::POW_F80,
622	Call_F128: RTLIB::POW_F128,
623	Call_PPCF128: RTLIB::POW_PPCF128));
624	}
625
626	SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
627	bool IsStrict = N->isStrictFPOpcode();
628	unsigned Offset = IsStrict ? 1 : 0;
629	assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 ||
630	N->getOperand(1 + Offset).getValueType() == MVT::i32) &&
631	"Unsupported power type!");
632	bool IsPowI =
633	N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
634
635	RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(RetVT: N->getValueType(ResNo: 0))
636	: RTLIB::getLDEXP(RetVT: N->getValueType(ResNo: 0));
637	assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
638	if (!TLI.getLibcallName(Call: LC)) {
639	// Some targets don't have a powi libcall; use pow instead.
640	// FIXME: Implement this if some target needs it.
641	DAG.getContext()->emitError(ErrorStr: "Don't know how to soften fpowi to fpow");
642	return DAG.getUNDEF(VT: N->getValueType(ResNo: 0));
643	}
644
645	if (DAG.getLibInfo().getIntSize() !=
646	N->getOperand(Num: 1 + Offset).getValueType().getSizeInBits()) {
647	// If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
648	// would use the wrong type for the argument.
649	DAG.getContext()->emitError(ErrorStr: "POWI exponent does not match sizeof(int)");
650	return DAG.getUNDEF(VT: N->getValueType(ResNo: 0));
651	}
652
653	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
654	SDValue Ops[2] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
655	N->getOperand(Num: 1 + Offset) };
656	SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
657	TargetLowering::MakeLibCallOptions CallOptions;
658	EVT OpsVT[2] = { N->getOperand(Num: 0 + Offset).getValueType(),
659	N->getOperand(Num: 1 + Offset).getValueType() };
660	CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0), Value: true);
661	std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops,
662	CallOptions, dl: SDLoc(N),
663	Chain);
664	if (IsStrict)
665	ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
666	return Tmp.first;
667	}
668
669	SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
670	assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp");
671	EVT VT0 = N->getValueType(ResNo: 0);
672	EVT VT1 = N->getValueType(ResNo: 1);
673	RTLIB::Libcall LC = RTLIB::getFREXP(RetVT: VT0);
674
675	if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) {
676	// If the exponent does not match with sizeof(int) a libcall would use the
677	// wrong type for the argument.
678	// TODO: Should be able to handle mismatches.
679	DAG.getContext()->emitError(ErrorStr: "ffrexp exponent does not match sizeof(int)");
680	return DAG.getUNDEF(VT: N->getValueType(ResNo: 0));
681	}
682
683	EVT NVT0 = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: VT0);
684	SDValue StackSlot = DAG.CreateStackTemporary(VT: VT1);
685
686	SDLoc DL(N);
687
688	TargetLowering::MakeLibCallOptions CallOptions;
689	SDValue Ops[2] = {GetSoftenedFloat(Op: N->getOperand(Num: 0)), StackSlot};
690	EVT OpsVT[2] = {VT0, StackSlot.getValueType()};
691
692	// TODO: setTypeListBeforeSoften can't properly express multiple return types,
693	// but we only really need to handle the 0th one for softening anyway.
694	CallOptions.setTypeListBeforeSoften(OpsVT: {OpsVT}, RetVT: VT0, Value: true);
695
696	auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, RetVT: NVT0, Ops, CallOptions, dl: DL,
697	/*Chain=*/SDValue());
698	int FrameIdx = cast<FrameIndexSDNode>(Val&: StackSlot)->getIndex();
699	auto PtrInfo =
700	MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
701
702	SDValue LoadExp = DAG.getLoad(VT: VT1, dl: DL, Chain, Ptr: StackSlot, PtrInfo);
703
704	ReplaceValueWith(From: SDValue(N, 1), To: LoadExp);
705	return ReturnVal;
706	}
707
708	SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
709	return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
710	Call_F32: RTLIB::REM_F32,
711	Call_F64: RTLIB::REM_F64,
712	Call_F80: RTLIB::REM_F80,
713	Call_F128: RTLIB::REM_F128,
714	Call_PPCF128: RTLIB::REM_PPCF128));
715	}
716
717	SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
718	return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
719	Call_F32: RTLIB::RINT_F32,
720	Call_F64: RTLIB::RINT_F64,
721	Call_F80: RTLIB::RINT_F80,
722	Call_F128: RTLIB::RINT_F128,
723	Call_PPCF128: RTLIB::RINT_PPCF128));
724	}
725
726	SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
727	return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
728	Call_F32: RTLIB::ROUND_F32,
729	Call_F64: RTLIB::ROUND_F64,
730	Call_F80: RTLIB::ROUND_F80,
731	Call_F128: RTLIB::ROUND_F128,
732	Call_PPCF128: RTLIB::ROUND_PPCF128));
733	}
734
735	SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
736	return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
737	Call_F32: RTLIB::ROUNDEVEN_F32,
738	Call_F64: RTLIB::ROUNDEVEN_F64,
739	Call_F80: RTLIB::ROUNDEVEN_F80,
740	Call_F128: RTLIB::ROUNDEVEN_F128,
741	Call_PPCF128: RTLIB::ROUNDEVEN_PPCF128));
742	}
743
744	SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
745	return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
746	Call_F32: RTLIB::SIN_F32,
747	Call_F64: RTLIB::SIN_F64,
748	Call_F80: RTLIB::SIN_F80,
749	Call_F128: RTLIB::SIN_F128,
750	Call_PPCF128: RTLIB::SIN_PPCF128));
751	}
752
753	SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
754	return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
755	Call_F32: RTLIB::SQRT_F32,
756	Call_F64: RTLIB::SQRT_F64,
757	Call_F80: RTLIB::SQRT_F80,
758	Call_F128: RTLIB::SQRT_F128,
759	Call_PPCF128: RTLIB::SQRT_PPCF128));
760	}
761
762	SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
763	return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
764	Call_F32: RTLIB::SUB_F32,
765	Call_F64: RTLIB::SUB_F64,
766	Call_F80: RTLIB::SUB_F80,
767	Call_F128: RTLIB::SUB_F128,
768	Call_PPCF128: RTLIB::SUB_PPCF128));
769	}
770
771	SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
772	return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
773	Call_F32: RTLIB::TRUNC_F32,
774	Call_F64: RTLIB::TRUNC_F64,
775	Call_F80: RTLIB::TRUNC_F80,
776	Call_F128: RTLIB::TRUNC_F128,
777	Call_PPCF128: RTLIB::TRUNC_PPCF128));
778	}
779
780	SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
781	LoadSDNode *L = cast<LoadSDNode>(Val: N);
782	EVT VT = N->getValueType(ResNo: 0);
783	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
784	SDLoc dl(N);
785
786	auto MMOFlags =
787	L->getMemOperand()->getFlags() &
788	~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
789	SDValue NewL;
790	if (L->getExtensionType() == ISD::NON_EXTLOAD) {
791	NewL = DAG.getLoad(AM: L->getAddressingMode(), ExtType: L->getExtensionType(), VT: NVT, dl,
792	Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(),
793	PtrInfo: L->getPointerInfo(), MemVT: NVT, Alignment: L->getOriginalAlign(),
794	MMOFlags, AAInfo: L->getAAInfo());
795	// Legalized the chain result - switch anything that used the old chain to
796	// use the new one.
797	ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
798	return NewL;
799	}
800
801	// Do a non-extending load followed by FP_EXTEND.
802	NewL = DAG.getLoad(AM: L->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: L->getMemoryVT(),
803	dl, Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(),
804	PtrInfo: L->getPointerInfo(), MemVT: L->getMemoryVT(),
805	Alignment: L->getOriginalAlign(), MMOFlags, AAInfo: L->getAAInfo());
806	// Legalized the chain result - switch anything that used the old chain to
807	// use the new one.
808	ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
809	auto ExtendNode = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT, Operand: NewL);
810	return BitConvertToInteger(Op: ExtendNode);
811	}
812
813	SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
814	SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 1));
815	SDValue RHS = GetSoftenedFloat(Op: N->getOperand(Num: 2));
816	return DAG.getSelect(DL: SDLoc(N),
817	VT: LHS.getValueType(), Cond: N->getOperand(Num: 0), LHS, RHS);
818	}
819
820	SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
821	SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 2));
822	SDValue RHS = GetSoftenedFloat(Op: N->getOperand(Num: 3));
823	return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
824	VT: LHS.getValueType(), N1: N->getOperand(Num: 0),
825	N2: N->getOperand(Num: 1), N3: LHS, N4: RHS, N5: N->getOperand(Num: 4));
826	}
827
828	SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
829	return DAG.getUNDEF(VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
830	VT: N->getValueType(ResNo: 0)));
831	}
832
833	SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
834	SDValue Chain = N->getOperand(Num: 0); // Get the chain.
835	SDValue Ptr = N->getOperand(Num: 1); // Get the pointer.
836	EVT VT = N->getValueType(ResNo: 0);
837	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
838	SDLoc dl(N);
839
840	SDValue NewVAARG;
841	NewVAARG = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV: N->getOperand(Num: 2),
842	Align: N->getConstantOperandVal(Num: 3));
843
844	// Legalized the chain result - switch anything that used the old chain to
845	// use the new one.
846	if (N != NewVAARG.getValue(R: 1).getNode())
847	ReplaceValueWith(From: SDValue(N, 1), To: NewVAARG.getValue(R: 1));
848	return NewVAARG;
849	}
850
851	SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
852	bool IsStrict = N->isStrictFPOpcode();
853	bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
854	N->getOpcode() == ISD::STRICT_SINT_TO_FP;
855	EVT SVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
856	EVT RVT = N->getValueType(ResNo: 0);
857	EVT NVT = EVT();
858	SDLoc dl(N);
859
860	// If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
861	// a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
862	// match. Look for an appropriate libcall.
863	RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
864	for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
865	t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
866	NVT = (MVT::SimpleValueType)t;
867	// The source needs to big enough to hold the operand.
868	if (NVT.bitsGE(VT: SVT))
869	LC = Signed ? RTLIB::getSINTTOFP(OpVT: NVT, RetVT: RVT):RTLIB::getUINTTOFP (OpVT: NVT, RetVT: RVT);
870	}
871	assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
872
873	SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
874	// Sign/zero extend the argument if the libcall takes a larger type.
875	SDValue Op = DAG.getNode(Opcode: Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
876	VT: NVT, Operand: N->getOperand(Num: IsStrict ? 1 : 0));
877	TargetLowering::MakeLibCallOptions CallOptions;
878	CallOptions.setSExt(Signed);
879	CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT, Value: true);
880	std::pair<SDValue, SDValue> Tmp =
881	TLI.makeLibCall(DAG, LC, RetVT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: RVT),
882	Ops: Op, CallOptions, dl, Chain);
883
884	if (IsStrict)
885	ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
886	return Tmp.first;
887	}
888
889	SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
890	// Expand and soften recursively.
891	ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduce(Node: N, DAG));
892	return SDValue();
893	}
894
895	SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
896	ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduceSeq(Node: N, DAG));
897	return SDValue();
898	}
899
900	//===----------------------------------------------------------------------===//
901	// Convert Float Operand to Integer
902	//===----------------------------------------------------------------------===//
903
904	bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
905	LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG));
906	SDValue Res = SDValue();
907
908	switch (N->getOpcode()) {
909	default:
910	#ifndef NDEBUG
911	dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
912	N->dump(G: &DAG); dbgs() << "\n";
913	#endif
914	report_fatal_error(reason: "Do not know how to soften this operator's operand!");
915
916	case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
917	case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
918	case ISD::STRICT_FP_TO_FP16:
919	case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
920	case ISD::FP_TO_BF16:
921	case ISD::STRICT_FP_ROUND:
922	case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
923	case ISD::STRICT_FP_TO_SINT:
924	case ISD::STRICT_FP_TO_UINT:
925	case ISD::FP_TO_SINT:
926	case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
927	case ISD::FP_TO_SINT_SAT:
928	case ISD::FP_TO_UINT_SAT:
929	Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
930	case ISD::STRICT_LROUND:
931	case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
932	case ISD::STRICT_LLROUND:
933	case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
934	case ISD::STRICT_LRINT:
935	case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
936	case ISD::STRICT_LLRINT:
937	case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
938	case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
939	case ISD::STRICT_FSETCC:
940	case ISD::STRICT_FSETCCS:
941	case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
942	case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
943	case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
944	}
945
946	// If the result is null, the sub-method took care of registering results etc.
947	if (!Res.getNode()) return false;
948
949	// If the result is N, the sub-method updated N in place. Tell the legalizer
950	// core about this to re-analyze.
951	if (Res.getNode() == N)
952	return true;
953
954	assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
955	"Invalid operand softening");
956
957	ReplaceValueWith(From: SDValue(N, 0), To: Res);
958	return false;
959	}
960
961	SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
962	SDValue Op0 = GetSoftenedFloat(Op: N->getOperand(Num: 0));
963
964	return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op0);
965	}
966
967	SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
968	// We actually deal with the partially-softened FP_TO_FP16 node too, which
969	// returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
970	assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
971	N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
972	N->getOpcode() == ISD::FP_TO_BF16 ||
973	N->getOpcode() == ISD::STRICT_FP_ROUND);
974
975	bool IsStrict = N->isStrictFPOpcode();
976	SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
977	EVT SVT = Op.getValueType();
978	EVT RVT = N->getValueType(ResNo: 0);
979	EVT FloatRVT = RVT;
980	if (N->getOpcode() == ISD::FP_TO_FP16 ||
981	N->getOpcode() == ISD::STRICT_FP_TO_FP16)
982	FloatRVT = MVT::f16;
983	else if (N->getOpcode() == ISD::FP_TO_BF16)
984	FloatRVT = MVT::bf16;
985
986	RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: SVT, RetVT: FloatRVT);
987	assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
988
989	SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
990	Op = GetSoftenedFloat(Op);
991	TargetLowering::MakeLibCallOptions CallOptions;
992	CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT, Value: true);
993	std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: RVT, Ops: Op,
994	CallOptions, dl: SDLoc(N),
995	Chain);
996	if (IsStrict) {
997	ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
998	ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
999	return SDValue();
1000	}
1001	return Tmp.first;
1002	}
1003
1004	SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
1005	SDValue NewLHS = N->getOperand(Num: 2), NewRHS = N->getOperand(Num: 3);
1006	ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get();
1007
1008	EVT VT = NewLHS.getValueType();
1009	NewLHS = GetSoftenedFloat(Op: NewLHS);
1010	NewRHS = GetSoftenedFloat(Op: NewRHS);
1011	TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N),
1012	OldLHS: N->getOperand(Num: 2), OldRHS: N->getOperand(Num: 3));
1013
1014	// If softenSetCCOperands returned a scalar, we need to compare the result
1015	// against zero to select between true and false values.
1016	if (!NewRHS.getNode()) {
1017	NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
1018	CCCode = ISD::SETNE;
1019	}
1020
1021	// Update N to have the operands specified.
1022	return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
1023	Op2: DAG.getCondCode(Cond: CCCode), Op3: NewLHS, Op4: NewRHS,
1024	Op5: N->getOperand(Num: 4)),
1025	0);
1026	}
1027
1028	// Even if the result type is legal, no libcall may exactly match. (e.g. We
1029	// don't have FP-i8 conversions) This helper method looks for an appropriate
1030	// promoted libcall.
1031	static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
1032	bool Signed) {
1033	RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1034	for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
1035	IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
1036	++IntVT) {
1037	Promoted = (MVT::SimpleValueType)IntVT;
1038	// The type needs to big enough to hold the result.
1039	if (Promoted.bitsGE(VT: RetVT))
1040	LC = Signed ? RTLIB::getFPTOSINT(OpVT: SrcVT, RetVT: Promoted)
1041	: RTLIB::getFPTOUINT(OpVT: SrcVT, RetVT: Promoted);
1042	}
1043	return LC;
1044	}
1045
1046	SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
1047	bool IsStrict = N->isStrictFPOpcode();
1048	bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1049	N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1050
1051	SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
1052	EVT SVT = Op.getValueType();
1053	EVT RVT = N->getValueType(ResNo: 0);
1054	EVT NVT = EVT();
1055	SDLoc dl(N);
1056
1057	// If the result is not legal, eg: fp -> i1, then it needs to be promoted to
1058	// a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
1059	// match, eg. we don't have fp -> i8 conversions.
1060	// Look for an appropriate libcall.
1061	RTLIB::Libcall LC = findFPToIntLibcall(SrcVT: SVT, RetVT: RVT, Promoted&: NVT, Signed);
1062	assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1063	"Unsupported FP_TO_XINT!");
1064
1065	Op = GetSoftenedFloat(Op);
1066	SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1067	TargetLowering::MakeLibCallOptions CallOptions;
1068	CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT, Value: true);
1069	std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
1070	CallOptions, dl, Chain);
1071
1072	// Truncate the result if the libcall returns a larger type.
1073	SDValue Res = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RVT, Operand: Tmp.first);
1074
1075	if (!IsStrict)
1076	return Res;
1077
1078	ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1079	ReplaceValueWith(From: SDValue(N, 0), To: Res);
1080	return SDValue();
1081	}
1082
1083	SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
1084	SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
1085	return Res;
1086	}
1087
1088	SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
1089	SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1);
1090	ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
1091
1092	EVT VT = NewLHS.getValueType();
1093	NewLHS = GetSoftenedFloat(Op: NewLHS);
1094	NewRHS = GetSoftenedFloat(Op: NewRHS);
1095	TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N),
1096	OldLHS: N->getOperand(Num: 0), OldRHS: N->getOperand(Num: 1));
1097
1098	// If softenSetCCOperands returned a scalar, we need to compare the result
1099	// against zero to select between true and false values.
1100	if (!NewRHS.getNode()) {
1101	NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
1102	CCCode = ISD::SETNE;
1103	}
1104
1105	// Update N to have the operands specified.
1106	return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
1107	Op3: N->getOperand(Num: 2), Op4: N->getOperand(Num: 3),
1108	Op5: DAG.getCondCode(Cond: CCCode)),
1109	0);
1110	}
1111
1112	SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1113	bool IsStrict = N->isStrictFPOpcode();
1114	SDValue Op0 = N->getOperand(Num: IsStrict ? 1 : 0);
1115	SDValue Op1 = N->getOperand(Num: IsStrict ? 2 : 1);
1116	SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1117	ISD::CondCode CCCode =
1118	cast<CondCodeSDNode>(Val: N->getOperand(Num: IsStrict ? 3 : 2))->get();
1119
1120	EVT VT = Op0.getValueType();
1121	SDValue NewLHS = GetSoftenedFloat(Op: Op0);
1122	SDValue NewRHS = GetSoftenedFloat(Op: Op1);
1123	TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N), OldLHS: Op0, OldRHS: Op1,
1124	Chain, IsSignaling: N->getOpcode() == ISD::STRICT_FSETCCS);
1125
1126	// Update N to have the operands specified.
1127	if (NewRHS.getNode()) {
1128	if (IsStrict)
1129	NewLHS = DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: NewLHS,
1130	N2: NewRHS, N3: DAG.getCondCode(Cond: CCCode));
1131	else
1132	return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
1133	Op3: DAG.getCondCode(Cond: CCCode)), 0);
1134	}
1135
1136	// Otherwise, softenSetCCOperands returned a scalar, use it.
1137	assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1138	"Unexpected setcc expansion!");
1139
1140	if (IsStrict) {
1141	ReplaceValueWith(From: SDValue(N, 0), To: NewLHS);
1142	ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1143	return SDValue();
1144	}
1145	return NewLHS;
1146	}
1147
1148	SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1149	assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1150	assert(OpNo == 1 && "Can only soften the stored value!");
1151	StoreSDNode *ST = cast<StoreSDNode>(Val: N);
1152	SDValue Val = ST->getValue();
1153	SDLoc dl(N);
1154
1155	if (ST->isTruncatingStore())
1156	// Do an FP_ROUND followed by a non-truncating store.
1157	Val = BitConvertToInteger(
1158	Op: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: ST->getMemoryVT(), N1: Val,
1159	N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
1160	else
1161	Val = GetSoftenedFloat(Op: Val);
1162
1163	return DAG.getStore(Chain: ST->getChain(), dl, Val, Ptr: ST->getBasePtr(),
1164	MMO: ST->getMemOperand());
1165	}
1166
1167	SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1168	SDValue LHS = N->getOperand(Num: 0);
1169	SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
1170	SDLoc dl(N);
1171
1172	EVT LVT = LHS.getValueType();
1173	EVT ILVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LVT.getSizeInBits());
1174	EVT RVT = RHS.getValueType();
1175
1176	unsigned LSize = LVT.getSizeInBits();
1177	unsigned RSize = RVT.getSizeInBits();
1178
1179	// Shift right or sign-extend it if the two operands have different types.
1180	int SizeDiff = RSize - LSize;
1181	if (SizeDiff > 0) {
1182	RHS =
1183	DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: RHS,
1184	N2: DAG.getConstant(Val: SizeDiff, DL: dl,
1185	VT: TLI.getShiftAmountTy(LHSTy: RHS.getValueType(),
1186	DL: DAG.getDataLayout())));
1187	RHS = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ILVT, Operand: RHS);
1188	} else if (SizeDiff < 0) {
1189	RHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: RHS);
1190	RHS =
1191	DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: ILVT, N1: RHS,
1192	N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
1193	VT: TLI.getShiftAmountTy(LHSTy: RHS.getValueType(),
1194	DL: DAG.getDataLayout())));
1195	}
1196
1197	RHS = DAG.getBitcast(VT: LVT, V: RHS);
1198	return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: dl, VT: LVT, N1: LHS, N2: RHS);
1199	}
1200
1201	SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1202	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1203	bool IsStrict = N->isStrictFPOpcode();
1204	unsigned Offset = IsStrict ? 1 : 0;
1205	SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset));
1206	SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1207	TargetLowering::MakeLibCallOptions CallOptions;
1208	EVT OpVT = N->getOperand(Num: 0 + Offset).getValueType();
1209	CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0), Value: true);
1210	std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
1211	CallOptions, dl: SDLoc(N),
1212	Chain);
1213	if (IsStrict) {
1214	ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1215	ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
1216	return SDValue();
1217	}
1218
1219	return Tmp.first;
1220	}
1221
1222	SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1223	EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1224	return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1225	Call_F32: RTLIB::LROUND_F32,
1226	Call_F64: RTLIB::LROUND_F64,
1227	Call_F80: RTLIB::LROUND_F80,
1228	Call_F128: RTLIB::LROUND_F128,
1229	Call_PPCF128: RTLIB::LROUND_PPCF128));
1230	}
1231
1232	SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1233	EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1234	return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1235	Call_F32: RTLIB::LLROUND_F32,
1236	Call_F64: RTLIB::LLROUND_F64,
1237	Call_F80: RTLIB::LLROUND_F80,
1238	Call_F128: RTLIB::LLROUND_F128,
1239	Call_PPCF128: RTLIB::LLROUND_PPCF128));
1240	}
1241
1242	SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1243	EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1244	return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1245	Call_F32: RTLIB::LRINT_F32,
1246	Call_F64: RTLIB::LRINT_F64,
1247	Call_F80: RTLIB::LRINT_F80,
1248	Call_F128: RTLIB::LRINT_F128,
1249	Call_PPCF128: RTLIB::LRINT_PPCF128));
1250	}
1251
1252	SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1253	EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1254	return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1255	Call_F32: RTLIB::LLRINT_F32,
1256	Call_F64: RTLIB::LLRINT_F64,
1257	Call_F80: RTLIB::LLRINT_F80,
1258	Call_F128: RTLIB::LLRINT_F128,
1259	Call_PPCF128: RTLIB::LLRINT_PPCF128));
1260	}
1261
1262	//===----------------------------------------------------------------------===//
1263	// Float Result Expansion
1264	//===----------------------------------------------------------------------===//
1265
1266	/// ExpandFloatResult - This method is called when the specified result of the
1267	/// specified node is found to need expansion. At this point, the node may also
1268	/// have invalid operands or may have other results that need promotion, we just
1269	/// know that (at least) one result needs expansion.
1270	void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1271	LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG));
1272	SDValue Lo, Hi;
1273	Lo = Hi = SDValue();
1274
1275	// See if the target wants to custom expand this node.
1276	if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
1277	return;
1278
1279	switch (N->getOpcode()) {
1280	default:
1281	#ifndef NDEBUG
1282	dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1283	N->dump(G: &DAG); dbgs() << "\n";
1284	#endif
1285	report_fatal_error(reason: "Do not know how to expand the result of this "
1286	"operator!");
1287
1288	case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1289	case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
1290	case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1291
1292	case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1293	case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1294	case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1295	case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1296	case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1297	case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1298
1299	case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1300	case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1301	case ISD::STRICT_FMINNUM:
1302	case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1303	case ISD::STRICT_FMAXNUM:
1304	case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1305	case ISD::STRICT_FADD:
1306	case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1307	case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1308	case ISD::STRICT_FCEIL:
1309	case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1310	case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1311	case ISD::STRICT_FCOS:
1312	case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1313	case ISD::STRICT_FDIV:
1314	case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1315	case ISD::STRICT_FEXP:
1316	case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1317	case ISD::STRICT_FEXP2:
1318	case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1319	case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break;
1320	case ISD::STRICT_FFLOOR:
1321	case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1322	case ISD::STRICT_FLOG:
1323	case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1324	case ISD::STRICT_FLOG2:
1325	case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1326	case ISD::STRICT_FLOG10:
1327	case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1328	case ISD::STRICT_FMA:
1329	case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1330	case ISD::STRICT_FMUL:
1331	case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1332	case ISD::STRICT_FNEARBYINT:
1333	case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1334	case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1335	case ISD::STRICT_FP_EXTEND:
1336	case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1337	case ISD::STRICT_FPOW:
1338	case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1339	case ISD::STRICT_FPOWI:
1340	case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1341	case ISD::FLDEXP:
1342	case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break;
1343	case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1344	case ISD::STRICT_FRINT:
1345	case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1346	case ISD::STRICT_FROUND:
1347	case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1348	case ISD::STRICT_FROUNDEVEN:
1349	case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1350	case ISD::STRICT_FSIN:
1351	case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1352	case ISD::STRICT_FSQRT:
1353	case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1354	case ISD::STRICT_FSUB:
1355	case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1356	case ISD::STRICT_FTRUNC:
1357	case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1358	case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1359	case ISD::STRICT_SINT_TO_FP:
1360	case ISD::STRICT_UINT_TO_FP:
1361	case ISD::SINT_TO_FP:
1362	case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1363	case ISD::STRICT_FREM:
1364	case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1365	}
1366
1367	// If Lo/Hi is null, the sub-method took care of registering results etc.
1368	if (Lo.getNode())
1369	SetExpandedFloat(Op: SDValue(N, ResNo), Lo, Hi);
1370	}
1371
1372	void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1373	SDValue &Hi) {
1374	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1375	assert(NVT.getSizeInBits() == 64 &&
1376	"Do not know how to expand this float constant!");
1377	APInt C = cast<ConstantFPSDNode>(Val: N)->getValueAPF().bitcastToAPInt();
1378	SDLoc dl(N);
1379	Lo = DAG.getConstantFP(Val: APFloat(DAG.EVTToAPFloatSemantics(VT: NVT),
1380	APInt(64, C.getRawData()[1])),
1381	DL: dl, VT: NVT);
1382	Hi = DAG.getConstantFP(Val: APFloat(DAG.EVTToAPFloatSemantics(VT: NVT),
1383	APInt(64, C.getRawData()[0])),
1384	DL: dl, VT: NVT);
1385	}
1386
1387	void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1388	SDValue &Lo, SDValue &Hi) {
1389	bool IsStrict = N->isStrictFPOpcode();
1390	unsigned Offset = IsStrict ? 1 : 0;
1391	SDValue Op = N->getOperand(Num: 0 + Offset);
1392	SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1393	TargetLowering::MakeLibCallOptions CallOptions;
1394	std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: N->getValueType(ResNo: 0),
1395	Ops: Op, CallOptions, dl: SDLoc(N),
1396	Chain);
1397	if (IsStrict)
1398	ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1399	GetPairElements(Pair: Tmp.first, Lo, Hi);
1400	}
1401
1402	void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1403	SDValue &Lo, SDValue &Hi) {
1404	bool IsStrict = N->isStrictFPOpcode();
1405	unsigned Offset = IsStrict ? 1 : 0;
1406	SDValue Ops[] = { N->getOperand(Num: 0 + Offset), N->getOperand(Num: 1 + Offset) };
1407	SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1408	TargetLowering::MakeLibCallOptions CallOptions;
1409	std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: N->getValueType(ResNo: 0),
1410	Ops, CallOptions, dl: SDLoc(N),
1411	Chain);
1412	if (IsStrict)
1413	ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1414	GetPairElements(Pair: Tmp.first, Lo, Hi);
1415	}
1416
1417	void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1418	SDValue &Hi) {
1419	assert(N->getValueType(0) == MVT::ppcf128 &&
1420	"Logic only correct for ppcf128!");
1421	SDLoc dl(N);
1422	SDValue Tmp;
1423	GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi&: Tmp);
1424	Hi = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT: Tmp.getValueType(), Operand: Tmp);
1425	// Lo = Hi==fabs(Hi) ? Lo : -Lo;
1426	Lo = DAG.getSelectCC(DL: dl, LHS: Tmp, RHS: Hi, True: Lo,
1427	False: DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Lo.getValueType(), Operand: Lo),
1428	Cond: ISD::SETEQ);
1429	}
1430
1431	void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1432	SDValue &Hi) {
1433	ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1434	Call_F32: RTLIB::FMIN_F32, Call_F64: RTLIB::FMIN_F64,
1435	Call_F80: RTLIB::FMIN_F80, Call_F128: RTLIB::FMIN_F128,
1436	Call_PPCF128: RTLIB::FMIN_PPCF128), Lo, Hi);
1437	}
1438
1439	void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1440	SDValue &Hi) {
1441	ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1442	Call_F32: RTLIB::FMAX_F32, Call_F64: RTLIB::FMAX_F64,
1443	Call_F80: RTLIB::FMAX_F80, Call_F128: RTLIB::FMAX_F128,
1444	Call_PPCF128: RTLIB::FMAX_PPCF128), Lo, Hi);
1445	}
1446
1447	void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1448	SDValue &Hi) {
1449	ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1450	Call_F32: RTLIB::ADD_F32, Call_F64: RTLIB::ADD_F64,
1451	Call_F80: RTLIB::ADD_F80, Call_F128: RTLIB::ADD_F128,
1452	Call_PPCF128: RTLIB::ADD_PPCF128), Lo, Hi);
1453	}
1454
1455	void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1456	SDValue &Hi) {
1457	ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::CBRT_F32,
1458	Call_F64: RTLIB::CBRT_F64, Call_F80: RTLIB::CBRT_F80,
1459	Call_F128: RTLIB::CBRT_F128,
1460	Call_PPCF128: RTLIB::CBRT_PPCF128), Lo, Hi);
1461	}
1462
1463	void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1464	SDValue &Lo, SDValue &Hi) {
1465	ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1466	Call_F32: RTLIB::CEIL_F32, Call_F64: RTLIB::CEIL_F64,
1467	Call_F80: RTLIB::CEIL_F80, Call_F128: RTLIB::CEIL_F128,
1468	Call_PPCF128: RTLIB::CEIL_PPCF128), Lo, Hi);
1469	}
1470
1471	void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1472	SDValue &Lo, SDValue &Hi) {
1473	ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1474	Call_F32: RTLIB::COPYSIGN_F32,
1475	Call_F64: RTLIB::COPYSIGN_F64,
1476	Call_F80: RTLIB::COPYSIGN_F80,
1477	Call_F128: RTLIB::COPYSIGN_F128,
1478	Call_PPCF128: RTLIB::COPYSIGN_PPCF128), Lo, Hi);
1479	}
1480
1481	void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1482	SDValue &Lo, SDValue &Hi) {
1483	ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1484	Call_F32: RTLIB::COS_F32, Call_F64: RTLIB::COS_F64,
1485	Call_F80: RTLIB::COS_F80, Call_F128: RTLIB::COS_F128,
1486	Call_PPCF128: RTLIB::COS_PPCF128), Lo, Hi);
1487	}
1488
1489	void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1490	SDValue &Hi) {
1491	ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1492	Call_F32: RTLIB::DIV_F32,
1493	Call_F64: RTLIB::DIV_F64,
1494	Call_F80: RTLIB::DIV_F80,
1495	Call_F128: RTLIB::DIV_F128,
1496	Call_PPCF128: RTLIB::DIV_PPCF128), Lo, Hi);
1497	}
1498
1499	void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1500	SDValue &Lo, SDValue &Hi) {
1501	ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1502	Call_F32: RTLIB::EXP_F32, Call_F64: RTLIB::EXP_F64,
1503	Call_F80: RTLIB::EXP_F80, Call_F128: RTLIB::EXP_F128,
1504	Call_PPCF128: RTLIB::EXP_PPCF128), Lo, Hi);
1505	}
1506
1507	void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1508	SDValue &Lo, SDValue &Hi) {
1509	ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1510	Call_F32: RTLIB::EXP2_F32, Call_F64: RTLIB::EXP2_F64,
1511	Call_F80: RTLIB::EXP2_F80, Call_F128: RTLIB::EXP2_F128,
1512	Call_PPCF128: RTLIB::EXP2_PPCF128), Lo, Hi);
1513	}
1514
1515	void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo,
1516	SDValue &Hi) {
1517	ExpandFloatRes_Unary(N,
1518	LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::EXP10_F32,
1519	Call_F64: RTLIB::EXP10_F64, Call_F80: RTLIB::EXP10_F80,
1520	Call_F128: RTLIB::EXP10_F128, Call_PPCF128: RTLIB::EXP10_PPCF128),
1521	Lo, Hi);
1522	}
1523
1524	void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1525	SDValue &Lo, SDValue &Hi) {
1526	ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1527	Call_F32: RTLIB::FLOOR_F32, Call_F64: RTLIB::FLOOR_F64,
1528	Call_F80: RTLIB::FLOOR_F80, Call_F128: RTLIB::FLOOR_F128,
1529	Call_PPCF128: RTLIB::FLOOR_PPCF128), Lo, Hi);
1530	}
1531
1532	void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1533	SDValue &Lo, SDValue &Hi) {
1534	ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1535	Call_F32: RTLIB::LOG_F32, Call_F64: RTLIB::LOG_F64,
1536	Call_F80: RTLIB::LOG_F80, Call_F128: RTLIB::LOG_F128,
1537	Call_PPCF128: RTLIB::LOG_PPCF128), Lo, Hi);
1538	}
1539
1540	void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1541	SDValue &Lo, SDValue &Hi) {
1542	ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1543	Call_F32: RTLIB::LOG2_F32, Call_F64: RTLIB::LOG2_F64,
1544	Call_F80: RTLIB::LOG2_F80, Call_F128: RTLIB::LOG2_F128,
1545	Call_PPCF128: RTLIB::LOG2_PPCF128), Lo, Hi);
1546	}
1547
1548	void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1549	SDValue &Lo, SDValue &Hi) {
1550	ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1551	Call_F32: RTLIB::LOG10_F32, Call_F64: RTLIB::LOG10_F64,
1552	Call_F80: RTLIB::LOG10_F80, Call_F128: RTLIB::LOG10_F128,
1553	Call_PPCF128: RTLIB::LOG10_PPCF128), Lo, Hi);
1554	}
1555
1556	void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1557	SDValue &Hi) {
1558	bool IsStrict = N->isStrictFPOpcode();
1559	unsigned Offset = IsStrict ? 1 : 0;
1560	SDValue Ops[3] = { N->getOperand(Num: 0 + Offset), N->getOperand(Num: 1 + Offset),
1561	N->getOperand(Num: 2 + Offset) };
1562	SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1563	TargetLowering::MakeLibCallOptions CallOptions;
1564	std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1565	Call_F32: RTLIB::FMA_F32,
1566	Call_F64: RTLIB::FMA_F64,
1567	Call_F80: RTLIB::FMA_F80,
1568	Call_F128: RTLIB::FMA_F128,
1569	Call_PPCF128: RTLIB::FMA_PPCF128),
1570	RetVT: N->getValueType(ResNo: 0), Ops, CallOptions,
1571	dl: SDLoc(N), Chain);
1572	if (IsStrict)
1573	ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1574	GetPairElements(Pair: Tmp.first, Lo, Hi);
1575	}
1576
1577	void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1578	SDValue &Hi) {
1579	ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1580	Call_F32: RTLIB::MUL_F32,
1581	Call_F64: RTLIB::MUL_F64,
1582	Call_F80: RTLIB::MUL_F80,
1583	Call_F128: RTLIB::MUL_F128,
1584	Call_PPCF128: RTLIB::MUL_PPCF128), Lo, Hi);
1585	}
1586
1587	void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1588	SDValue &Lo, SDValue &Hi) {
1589	ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1590	Call_F32: RTLIB::NEARBYINT_F32,
1591	Call_F64: RTLIB::NEARBYINT_F64,
1592	Call_F80: RTLIB::NEARBYINT_F80,
1593	Call_F128: RTLIB::NEARBYINT_F128,
1594	Call_PPCF128: RTLIB::NEARBYINT_PPCF128), Lo, Hi);
1595	}
1596
1597	void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1598	SDValue &Hi) {
1599	SDLoc dl(N);
1600	GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi);
1601	Lo = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Lo.getValueType(), Operand: Lo);
1602	Hi = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Hi.getValueType(), Operand: Hi);
1603	}
1604
1605	void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1606	SDValue &Hi) {
1607	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1608	SDLoc dl(N);
1609	bool IsStrict = N->isStrictFPOpcode();
1610
1611	SDValue Chain;
1612	if (IsStrict) {
1613	// If the expanded type is the same as the input type, just bypass the node.
1614	if (NVT == N->getOperand(Num: 1).getValueType()) {
1615	Hi = N->getOperand(Num: 1);
1616	Chain = N->getOperand(Num: 0);
1617	} else {
1618	// Other we need to extend.
1619	Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other },
1620	{ N->getOperand(0), N->getOperand(1) });
1621	Chain = Hi.getValue(R: 1);
1622	}
1623	} else {
1624	Hi = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: N->getOperand(Num: 0));
1625	}
1626
1627	Lo = DAG.getConstantFP(Val: APFloat(DAG.EVTToAPFloatSemantics(VT: NVT),
1628	APInt(NVT.getSizeInBits(), 0)), DL: dl, VT: NVT);
1629
1630	if (IsStrict)
1631	ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1632	}
1633
1634	void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
1635	SDValue &Lo, SDValue &Hi) {
1636	ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1637	Call_F32: RTLIB::POW_F32, Call_F64: RTLIB::POW_F64,
1638	Call_F80: RTLIB::POW_F80, Call_F128: RTLIB::POW_F128,
1639	Call_PPCF128: RTLIB::POW_PPCF128), Lo, Hi);
1640	}
1641
1642	void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
1643	SDValue &Lo, SDValue &Hi) {
1644	ExpandFloatRes_Binary(N, LC: RTLIB::getPOWI(RetVT: N->getValueType(ResNo: 0)), Lo, Hi);
1645	}
1646
1647	void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo,
1648	SDValue &Hi) {
1649	ExpandFloatRes_Binary(N, LC: RTLIB::getLDEXP(RetVT: N->getValueType(ResNo: 0)), Lo, Hi);
1650	}
1651
1652	void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
1653	SDValue &Lo, SDValue &Hi) {
1654	assert(N->getValueType(0) == MVT::ppcf128 &&
1655	"Logic only correct for ppcf128!");
1656
1657	SDLoc dl(N);
1658	GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi);
1659	Lo = DAG.getNode(Opcode: ISD::FREEZE, DL: dl, VT: Lo.getValueType(), Operand: Lo);
1660	Hi = DAG.getNode(Opcode: ISD::FREEZE, DL: dl, VT: Hi.getValueType(), Operand: Hi);
1661	}
1662
1663	void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
1664	SDValue &Lo, SDValue &Hi) {
1665	ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1666	Call_F32: RTLIB::REM_F32, Call_F64: RTLIB::REM_F64,
1667	Call_F80: RTLIB::REM_F80, Call_F128: RTLIB::REM_F128,
1668	Call_PPCF128: RTLIB::REM_PPCF128), Lo, Hi);
1669	}
1670
1671	void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
1672	SDValue &Lo, SDValue &Hi) {
1673	ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1674	Call_F32: RTLIB::RINT_F32, Call_F64: RTLIB::RINT_F64,
1675	Call_F80: RTLIB::RINT_F80, Call_F128: RTLIB::RINT_F128,
1676	Call_PPCF128: RTLIB::RINT_PPCF128), Lo, Hi);
1677	}
1678
1679	void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
1680	SDValue &Lo, SDValue &Hi) {
1681	ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1682	Call_F32: RTLIB::ROUND_F32,
1683	Call_F64: RTLIB::ROUND_F64,
1684	Call_F80: RTLIB::ROUND_F80,
1685	Call_F128: RTLIB::ROUND_F128,
1686	Call_PPCF128: RTLIB::ROUND_PPCF128), Lo, Hi);
1687	}
1688
1689	void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
1690	SDValue &Lo, SDValue &Hi) {
1691	ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1692	Call_F32: RTLIB::ROUNDEVEN_F32,
1693	Call_F64: RTLIB::ROUNDEVEN_F64,
1694	Call_F80: RTLIB::ROUNDEVEN_F80,
1695	Call_F128: RTLIB::ROUNDEVEN_F128,
1696	Call_PPCF128: RTLIB::ROUNDEVEN_PPCF128), Lo, Hi);
1697	}
1698
1699	void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
1700	SDValue &Lo, SDValue &Hi) {
1701	ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1702	Call_F32: RTLIB::SIN_F32, Call_F64: RTLIB::SIN_F64,
1703	Call_F80: RTLIB::SIN_F80, Call_F128: RTLIB::SIN_F128,
1704	Call_PPCF128: RTLIB::SIN_PPCF128), Lo, Hi);
1705	}
1706
1707	void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
1708	SDValue &Lo, SDValue &Hi) {
1709	ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1710	Call_F32: RTLIB::SQRT_F32, Call_F64: RTLIB::SQRT_F64,
1711	Call_F80: RTLIB::SQRT_F80, Call_F128: RTLIB::SQRT_F128,
1712	Call_PPCF128: RTLIB::SQRT_PPCF128), Lo, Hi);
1713	}
1714
1715	void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
1716	SDValue &Hi) {
1717	ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1718	Call_F32: RTLIB::SUB_F32,
1719	Call_F64: RTLIB::SUB_F64,
1720	Call_F80: RTLIB::SUB_F80,
1721	Call_F128: RTLIB::SUB_F128,
1722	Call_PPCF128: RTLIB::SUB_PPCF128), Lo, Hi);
1723	}
1724
1725	void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
1726	SDValue &Lo, SDValue &Hi) {
1727	ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1728	Call_F32: RTLIB::TRUNC_F32, Call_F64: RTLIB::TRUNC_F64,
1729	Call_F80: RTLIB::TRUNC_F80, Call_F128: RTLIB::TRUNC_F128,
1730	Call_PPCF128: RTLIB::TRUNC_PPCF128), Lo, Hi);
1731	}
1732
1733	void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
1734	SDValue &Hi) {
1735	if (ISD::isNormalLoad(N)) {
1736	ExpandRes_NormalLoad(N, Lo, Hi);
1737	return;
1738	}
1739
1740	assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
1741	LoadSDNode *LD = cast<LoadSDNode>(Val: N);
1742	SDValue Chain = LD->getChain();
1743	SDValue Ptr = LD->getBasePtr();
1744	SDLoc dl(N);
1745
1746	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: LD->getValueType(ResNo: 0));
1747	assert(NVT.isByteSized() && "Expanded type not byte sized!");
1748	assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
1749
1750	Hi = DAG.getExtLoad(ExtType: LD->getExtensionType(), dl, VT: NVT, Chain, Ptr,
1751	MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
1752
1753	// Remember the chain.
1754	Chain = Hi.getValue(R: 1);
1755
1756	// The low part is zero.
1757	Lo = DAG.getConstantFP(Val: APFloat(DAG.EVTToAPFloatSemantics(VT: NVT),
1758	APInt(NVT.getSizeInBits(), 0)), DL: dl, VT: NVT);
1759
1760	// Modified the chain - switch anything that used the old chain to use the
1761	// new one.
1762	ReplaceValueWith(From: SDValue(LD, 1), To: Chain);
1763	}
1764
1765	void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
1766	SDValue &Hi) {
1767	assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
1768	EVT VT = N->getValueType(ResNo: 0);
1769	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
1770	bool Strict = N->isStrictFPOpcode();
1771	SDValue Src = N->getOperand(Num: Strict ? 1 : 0);
1772	EVT SrcVT = Src.getValueType();
1773	bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
1774	N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1775	SDLoc dl(N);
1776	SDValue Chain = Strict ? N->getOperand(Num: 0) : DAG.getEntryNode();
1777
1778	// TODO: Any other flags to propagate?
1779	SDNodeFlags Flags;
1780	Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
1781
1782	// First do an SINT_TO_FP, whether the original was signed or unsigned.
1783	// When promoting partial word types to i32 we must honor the signedness,
1784	// though.
1785	if (SrcVT.bitsLE(MVT::VT: i32)) {
1786	// The integer can be represented exactly in an f64.
1787	Lo = DAG.getConstantFP(Val: APFloat(DAG.EVTToAPFloatSemantics(VT: NVT),
1788	APInt(NVT.getSizeInBits(), 0)), DL: dl, VT: NVT);
1789	if (Strict) {
1790	Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
1791	{Chain, Src}, Flags);
1792	Chain = Hi.getValue(R: 1);
1793	} else
1794	Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Src);
1795	} else {
1796	RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1797	if (SrcVT.bitsLE(MVT::VT: i64)) {
1798	Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
1799	MVT::i64, Src);
1800	LC = RTLIB::SINTTOFP_I64_PPCF128;
1801	} else if (SrcVT.bitsLE(MVT::VT: i128)) {
1802	Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src);
1803	LC = RTLIB::SINTTOFP_I128_PPCF128;
1804	}
1805	assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1806
1807	TargetLowering::MakeLibCallOptions CallOptions;
1808	CallOptions.setSExt(true);
1809	std::pair<SDValue, SDValue> Tmp =
1810	TLI.makeLibCall(DAG, LC, RetVT: VT, Ops: Src, CallOptions, dl, Chain);
1811	if (Strict)
1812	Chain = Tmp.second;
1813	GetPairElements(Pair: Tmp.first, Lo, Hi);
1814	}
1815
1816	// No need to complement for unsigned 32-bit integers
1817	if (isSigned || SrcVT.bitsLE(MVT::VT: i32)) {
1818	if (Strict)
1819	ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1820
1821	return;
1822	}
1823
1824	// Unsigned - fix up the SINT_TO_FP value just calculated.
1825	// FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
1826	// keep semantics correctness if the integer is not exactly representable
1827	// here. See ExpandLegalINT_TO_FP.
1828	Hi = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT, N1: Lo, N2: Hi);
1829	SrcVT = Src.getValueType();
1830
1831	// x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
1832	static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
1833	static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
1834	static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
1835	ArrayRef<uint64_t> Parts;
1836
1837	switch (SrcVT.getSimpleVT().SimpleTy) {
1838	default:
1839	llvm_unreachable("Unsupported UINT_TO_FP!");
1840	case MVT::i32:
1841	Parts = TwoE32;
1842	break;
1843	case MVT::i64:
1844	Parts = TwoE64;
1845	break;
1846	case MVT::i128:
1847	Parts = TwoE128;
1848	break;
1849	}
1850
1851	// TODO: Are there other fast-math-flags to propagate to this FADD?
1852	SDValue NewLo = DAG.getConstantFP(
1853	APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128);
1854	if (Strict) {
1855	Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other),
1856	{Chain, Hi, NewLo}, Flags);
1857	Chain = Lo.getValue(R: 1);
1858	ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1859	} else
1860	Lo = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Hi, N2: NewLo);
1861	Lo = DAG.getSelectCC(DL: dl, LHS: Src, RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT),
1862	True: Lo, False: Hi, Cond: ISD::SETLT);
1863	GetPairElements(Pair: Lo, Lo, Hi);
1864	}
1865
1866
1867	//===----------------------------------------------------------------------===//
1868	// Float Operand Expansion
1869	//===----------------------------------------------------------------------===//
1870
1871	/// ExpandFloatOperand - This method is called when the specified operand of the
1872	/// specified node is found to need expansion. At this point, all of the result
1873	/// types of the node are known to be legal, but other operands of the node may
1874	/// need promotion or expansion as well as the specified one.
1875	bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
1876	LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG));
1877	SDValue Res = SDValue();
1878
1879	// See if the target wants to custom expand this node.
1880	if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
1881	return false;
1882
1883	switch (N->getOpcode()) {
1884	default:
1885	#ifndef NDEBUG
1886	dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
1887	N->dump(G: &DAG); dbgs() << "\n";
1888	#endif
1889	report_fatal_error(reason: "Do not know how to expand this operator's operand!");
1890
1891	case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
1892	case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
1893	case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
1894
1895	case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
1896	case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
1897	case ISD::STRICT_FP_ROUND:
1898	case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
1899	case ISD::STRICT_FP_TO_SINT:
1900	case ISD::STRICT_FP_TO_UINT:
1901	case ISD::FP_TO_SINT:
1902	case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
1903	case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
1904	case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
1905	case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
1906	case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
1907	case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
1908	case ISD::STRICT_FSETCC:
1909	case ISD::STRICT_FSETCCS:
1910	case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
1911	case ISD::STORE: Res = ExpandFloatOp_STORE(N: cast<StoreSDNode>(Val: N),
1912	OpNo); break;
1913	}
1914
1915	// If the result is null, the sub-method took care of registering results etc.
1916	if (!Res.getNode()) return false;
1917
1918	// If the result is N, the sub-method updated N in place. Tell the legalizer
1919	// core about this.
1920	if (Res.getNode() == N)
1921	return true;
1922
1923	assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1924	"Invalid operand expansion");
1925
1926	ReplaceValueWith(From: SDValue(N, 0), To: Res);
1927	return false;
1928	}
1929
1930	/// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
1931	/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
1932	void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
1933	SDValue &NewRHS,
1934	ISD::CondCode &CCCode,
1935	const SDLoc &dl, SDValue &Chain,
1936	bool IsSignaling) {
1937	SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1938	GetExpandedFloat(Op: NewLHS, Lo&: LHSLo, Hi&: LHSHi);
1939	GetExpandedFloat(Op: NewRHS, Lo&: RHSLo, Hi&: RHSHi);
1940
1941	assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
1942
1943	// FIXME: This generated code sucks. We want to generate
1944	// FCMPU crN, hi1, hi2
1945	// BNE crN, L:
1946	// FCMPU crN, lo1, lo2
1947	// The following can be improved, but not that much.
1948	SDValue Tmp1, Tmp2, Tmp3, OutputChain;
1949	Tmp1 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi,
1950	RHS: RHSHi, Cond: ISD::SETOEQ, Chain, IsSignaling);
1951	OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(R: 1) : SDValue();
1952	Tmp2 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSLo.getValueType()), LHS: LHSLo,
1953	RHS: RHSLo, Cond: CCCode, Chain: OutputChain, IsSignaling);
1954	OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(R: 1) : SDValue();
1955	Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp2);
1956	Tmp1 =
1957	DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi, RHS: RHSHi,
1958	Cond: ISD::SETUNE, Chain: OutputChain, IsSignaling);
1959	OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(R: 1) : SDValue();
1960	Tmp2 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi,
1961	RHS: RHSHi, Cond: CCCode, Chain: OutputChain, IsSignaling);
1962	OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(R: 1) : SDValue();
1963	Tmp1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp2);
1964	NewLHS = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp3);
1965	NewRHS = SDValue(); // LHS is the result, not a compare.
1966	Chain = OutputChain;
1967	}
1968
1969	SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
1970	SDValue NewLHS = N->getOperand(Num: 2), NewRHS = N->getOperand(Num: 3);
1971	ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get();
1972	SDValue Chain;
1973	FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain);
1974
1975	// If ExpandSetCCOperands returned a scalar, we need to compare the result
1976	// against zero to select between true and false values.
1977	if (!NewRHS.getNode()) {
1978	NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
1979	CCCode = ISD::SETNE;
1980	}
1981
1982	// Update N to have the operands specified.
1983	return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
1984	Op2: DAG.getCondCode(Cond: CCCode), Op3: NewLHS, Op4: NewRHS,
1985	Op5: N->getOperand(Num: 4)), 0);
1986	}
1987
1988	SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
1989	assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
1990	"Logic only correct for ppcf128!");
1991	SDValue Lo, Hi;
1992	GetExpandedFloat(Op: N->getOperand(Num: 1), Lo, Hi);
1993	// The ppcf128 value is providing only the sign; take it from the
1994	// higher-order double (which must have the larger magnitude).
1995	return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SDLoc(N),
1996	VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0), N2: Hi);
1997	}
1998
1999	SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
2000	bool IsStrict = N->isStrictFPOpcode();
2001	assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
2002	"Logic only correct for ppcf128!");
2003	SDValue Lo, Hi;
2004	GetExpandedFloat(Op: N->getOperand(Num: IsStrict ? 1 : 0), Lo, Hi);
2005
2006	if (!IsStrict)
2007	// Round it the rest of the way (e.g. to f32) if needed.
2008	return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N),
2009	VT: N->getValueType(ResNo: 0), N1: Hi, N2: N->getOperand(Num: 1));
2010
2011	// Eliminate the node if the input float type is the same as the output float
2012	// type.
2013	if (Hi.getValueType() == N->getValueType(ResNo: 0)) {
2014	// Connect the output chain to the input chain, unlinking the node.
2015	ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
2016	ReplaceValueWith(From: SDValue(N, 0), To: Hi);
2017	return SDValue();
2018	}
2019
2020	SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
2021	{N->getValueType(0), MVT::Other},
2022	{N->getOperand(0), Hi, N->getOperand(2)});
2023	ReplaceValueWith(From: SDValue(N, 1), To: Expansion.getValue(R: 1));
2024	ReplaceValueWith(From: SDValue(N, 0), To: Expansion);
2025	return SDValue();
2026	}
2027
2028	SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
2029	EVT RVT = N->getValueType(ResNo: 0);
2030	SDLoc dl(N);
2031
2032	bool IsStrict = N->isStrictFPOpcode();
2033	bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
2034	N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2035	SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
2036	SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
2037
2038	EVT NVT;
2039	RTLIB::Libcall LC = findFPToIntLibcall(SrcVT: Op.getValueType(), RetVT: RVT, Promoted&: NVT, Signed);
2040	assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
2041	"Unsupported FP_TO_XINT!");
2042	TargetLowering::MakeLibCallOptions CallOptions;
2043	std::pair<SDValue, SDValue> Tmp =
2044	TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op, CallOptions, dl, Chain);
2045	if (!IsStrict)
2046	return Tmp.first;
2047
2048	ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
2049	ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
2050	return SDValue();
2051	}
2052
2053	SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
2054	SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1);
2055	ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
2056	SDValue Chain;
2057	FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain);
2058
2059	// If ExpandSetCCOperands returned a scalar, we need to compare the result
2060	// against zero to select between true and false values.
2061	if (!NewRHS.getNode()) {
2062	NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
2063	CCCode = ISD::SETNE;
2064	}
2065
2066	// Update N to have the operands specified.
2067	return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
2068	Op3: N->getOperand(Num: 2), Op4: N->getOperand(Num: 3),
2069	Op5: DAG.getCondCode(Cond: CCCode)), 0);
2070	}
2071
2072	SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
2073	bool IsStrict = N->isStrictFPOpcode();
2074	SDValue NewLHS = N->getOperand(Num: IsStrict ? 1 : 0);
2075	SDValue NewRHS = N->getOperand(Num: IsStrict ? 2 : 1);
2076	SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
2077	ISD::CondCode CCCode =
2078	cast<CondCodeSDNode>(Val: N->getOperand(Num: IsStrict ? 3 : 2))->get();
2079	FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain,
2080	IsSignaling: N->getOpcode() == ISD::STRICT_FSETCCS);
2081
2082	// FloatExpandSetCCOperands always returned a scalar.
2083	assert(!NewRHS.getNode() && "Expect to return scalar");
2084	assert(NewLHS.getValueType() == N->getValueType(0) &&
2085	"Unexpected setcc expansion!");
2086	if (Chain) {
2087	ReplaceValueWith(From: SDValue(N, 0), To: NewLHS);
2088	ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2089	return SDValue();
2090	}
2091	return NewLHS;
2092	}
2093
2094	SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
2095	if (ISD::isNormalStore(N))
2096	return ExpandOp_NormalStore(N, OpNo);
2097
2098	assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2099	assert(OpNo == 1 && "Can only expand the stored value so far");
2100	StoreSDNode *ST = cast<StoreSDNode>(Val: N);
2101
2102	SDValue Chain = ST->getChain();
2103	SDValue Ptr = ST->getBasePtr();
2104
2105	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
2106	VT: ST->getValue().getValueType());
2107	assert(NVT.isByteSized() && "Expanded type not byte sized!");
2108	assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2109	(void)NVT;
2110
2111	SDValue Lo, Hi;
2112	GetExpandedOp(Op: ST->getValue(), Lo, Hi);
2113
2114	return DAG.getTruncStore(Chain, dl: SDLoc(N), Val: Hi, Ptr,
2115	SVT: ST->getMemoryVT(), MMO: ST->getMemOperand());
2116	}
2117
2118	SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2119	EVT RVT = N->getValueType(ResNo: 0);
2120	EVT RetVT = N->getOperand(Num: 0).getValueType();
2121	TargetLowering::MakeLibCallOptions CallOptions;
2122	return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT,
2123	Call_F32: RTLIB::LROUND_F32,
2124	Call_F64: RTLIB::LROUND_F64,
2125	Call_F80: RTLIB::LROUND_F80,
2126	Call_F128: RTLIB::LROUND_F128,
2127	Call_PPCF128: RTLIB::LROUND_PPCF128),
2128	RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first;
2129	}
2130
2131	SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2132	EVT RVT = N->getValueType(ResNo: 0);
2133	EVT RetVT = N->getOperand(Num: 0).getValueType();
2134	TargetLowering::MakeLibCallOptions CallOptions;
2135	return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT,
2136	Call_F32: RTLIB::LLROUND_F32,
2137	Call_F64: RTLIB::LLROUND_F64,
2138	Call_F80: RTLIB::LLROUND_F80,
2139	Call_F128: RTLIB::LLROUND_F128,
2140	Call_PPCF128: RTLIB::LLROUND_PPCF128),
2141	RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first;
2142	}
2143
2144	SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2145	EVT RVT = N->getValueType(ResNo: 0);
2146	EVT RetVT = N->getOperand(Num: 0).getValueType();
2147	TargetLowering::MakeLibCallOptions CallOptions;
2148	return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT,
2149	Call_F32: RTLIB::LRINT_F32,
2150	Call_F64: RTLIB::LRINT_F64,
2151	Call_F80: RTLIB::LRINT_F80,
2152	Call_F128: RTLIB::LRINT_F128,
2153	Call_PPCF128: RTLIB::LRINT_PPCF128),
2154	RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first;
2155	}
2156
2157	SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2158	EVT RVT = N->getValueType(ResNo: 0);
2159	EVT RetVT = N->getOperand(Num: 0).getValueType();
2160	TargetLowering::MakeLibCallOptions CallOptions;
2161	return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT,
2162	Call_F32: RTLIB::LLRINT_F32,
2163	Call_F64: RTLIB::LLRINT_F64,
2164	Call_F80: RTLIB::LLRINT_F80,
2165	Call_F128: RTLIB::LLRINT_F128,
2166	Call_PPCF128: RTLIB::LLRINT_PPCF128),
2167	RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first;
2168	}
2169
2170	//===----------------------------------------------------------------------===//
2171	// Float Operand Promotion
2172	//===----------------------------------------------------------------------===//
2173	//
2174
2175	static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) {
2176	if (OpVT == MVT::f16) {
2177	return ISD::FP16_TO_FP;
2178	} else if (RetVT == MVT::f16) {
2179	return ISD::FP_TO_FP16;
2180	} else if (OpVT == MVT::bf16) {
2181	return ISD::BF16_TO_FP;
2182	} else if (RetVT == MVT::bf16) {
2183	return ISD::FP_TO_BF16;
2184	}
2185
2186	report_fatal_error(reason: "Attempt at an invalid promotion-related conversion");
2187	}
2188
2189	static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT) {
2190	if (OpVT == MVT::f16)
2191	return ISD::STRICT_FP16_TO_FP;
2192
2193	if (RetVT == MVT::f16)
2194	return ISD::STRICT_FP_TO_FP16;
2195
2196	if (OpVT == MVT::bf16) {
2197	// TODO: return ISD::STRICT_BF16_TO_FP;
2198	}
2199
2200	if (RetVT == MVT::bf16) {
2201	// TODO: return ISD::STRICT_FP_TO_BF16;
2202	}
2203
2204	report_fatal_error(reason: "Attempt at an invalid promotion-related conversion");
2205	}
2206
2207	bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
2208	LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG));
2209	SDValue R = SDValue();
2210
2211	if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false)) {
2212	LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2213	return false;
2214	}
2215
2216	// Nodes that use a promotion-requiring floating point operand, but doesn't
2217	// produce a promotion-requiring floating point result, need to be legalized
2218	// to use the promoted float operand. Nodes that produce at least one
2219	// promotion-requiring floating point result have their operands legalized as
2220	// a part of PromoteFloatResult.
2221	// clang-format off
2222	switch (N->getOpcode()) {
2223	default:
2224	#ifndef NDEBUG
2225	dbgs() << "PromoteFloatOperand Op #" << OpNo << ": ";
2226	N->dump(G: &DAG); dbgs() << "\n";
2227	#endif
2228	report_fatal_error(reason: "Do not know how to promote this operator's operand!");
2229
2230	case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break;
2231	case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break;
2232	case ISD::FP_TO_SINT:
2233	case ISD::FP_TO_UINT:
2234	case ISD::LRINT:
2235	case ISD::LLRINT: R = PromoteFloatOp_UnaryOp(N, OpNo); break;
2236	case ISD::FP_TO_SINT_SAT:
2237	case ISD::FP_TO_UINT_SAT:
2238	R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break;
2239	case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break;
2240	case ISD::STRICT_FP_EXTEND:
2241	R = PromoteFloatOp_STRICT_FP_EXTEND(N, OpNo);
2242	break;
2243	case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
2244	case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break;
2245	case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break;
2246	}
2247	// clang-format on
2248
2249	if (R.getNode())
2250	ReplaceValueWith(From: SDValue(N, 0), To: R);
2251	return false;
2252	}
2253
2254	SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) {
2255	SDValue Op = N->getOperand(Num: 0);
2256	EVT OpVT = Op->getValueType(ResNo: 0);
2257
2258	SDValue Promoted = GetPromotedFloat(Op: N->getOperand(Num: 0));
2259	EVT PromotedVT = Promoted->getValueType(ResNo: 0);
2260
2261	// Convert the promoted float value to the desired IVT.
2262	EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: OpVT.getSizeInBits());
2263	SDValue Convert = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: PromotedVT, RetVT: OpVT), DL: SDLoc(N),
2264	VT: IVT, Operand: Promoted);
2265	// The final result type might not be an scalar so we need a bitcast. The
2266	// bitcast will be further legalized if needed.
2267	return DAG.getBitcast(VT: N->getValueType(ResNo: 0), V: Convert);
2268	}
2269
2270	// Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by
2271	// PromoteFloatRes_FCOPYSIGN.
2272	SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
2273	assert (OpNo == 1 && "Only Operand 1 must need promotion here");
2274	SDValue Op1 = GetPromotedFloat(Op: N->getOperand(Num: 1));
2275
2276	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
2277	N1: N->getOperand(Num: 0), N2: Op1);
2278	}
2279
2280	// Convert the promoted float value to the desired integer type
2281	SDValue DAGTypeLegalizer::PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo) {
2282	SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 0));
2283	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
2284	}
2285
2286	SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N,
2287	unsigned OpNo) {
2288	SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 0));
2289	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Op,
2290	N2: N->getOperand(Num: 1));
2291	}
2292
2293	SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) {
2294	SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 0));
2295	EVT VT = N->getValueType(ResNo: 0);
2296
2297	// Desired VT is same as promoted type. Use promoted float directly.
2298	if (VT == Op->getValueType(ResNo: 0))
2299	return Op;
2300
2301	// Else, extend the promoted float value to the desired VT.
2302	return DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), VT, Operand: Op);
2303	}
2304
2305	SDValue DAGTypeLegalizer::PromoteFloatOp_STRICT_FP_EXTEND(SDNode *N,
2306	unsigned OpNo) {
2307	assert(OpNo == 1 && "Promoting unpromotable operand");
2308
2309	SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 1));
2310	EVT VT = N->getValueType(ResNo: 0);
2311
2312	// Desired VT is same as promoted type. Use promoted float directly.
2313	if (VT == Op->getValueType(ResNo: 0)) {
2314	ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
2315	return Op;
2316	}
2317
2318	// Else, extend the promoted float value to the desired VT.
2319	SDValue Res = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: SDLoc(N), VTList: N->getVTList(),
2320	N1: N->getOperand(Num: 0), N2: Op);
2321	ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
2322	return Res;
2323	}
2324
2325	// Promote the float operands used for comparison. The true- and false-
2326	// operands have the same type as the result and are promoted, if needed, by
2327	// PromoteFloatRes_SELECT_CC
2328	SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2329	SDValue LHS = GetPromotedFloat(Op: N->getOperand(Num: 0));
2330	SDValue RHS = GetPromotedFloat(Op: N->getOperand(Num: 1));
2331
2332	return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
2333	N1: LHS, N2: RHS, N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3),
2334	N5: N->getOperand(Num: 4));
2335	}
2336
2337	// Construct a SETCC that compares the promoted values and sets the conditional
2338	// code.
2339	SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) {
2340	EVT VT = N->getValueType(ResNo: 0);
2341	SDValue Op0 = GetPromotedFloat(Op: N->getOperand(Num: 0));
2342	SDValue Op1 = GetPromotedFloat(Op: N->getOperand(Num: 1));
2343	ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
2344
2345	return DAG.getSetCC(DL: SDLoc(N), VT, LHS: Op0, RHS: Op1, Cond: CCCode);
2346
2347	}
2348
2349	// Lower the promoted Float down to the integer value of same size and construct
2350	// a STORE of the integer value.
2351	SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
2352	StoreSDNode *ST = cast<StoreSDNode>(Val: N);
2353	SDValue Val = ST->getValue();
2354	SDLoc DL(N);
2355
2356	SDValue Promoted = GetPromotedFloat(Op: Val);
2357	EVT VT = ST->getOperand(Num: 1).getValueType();
2358	EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
2359
2360	SDValue NewVal;
2361	NewVal = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: Promoted.getValueType(), RetVT: VT), DL,
2362	VT: IVT, Operand: Promoted);
2363
2364	return DAG.getStore(Chain: ST->getChain(), dl: DL, Val: NewVal, Ptr: ST->getBasePtr(),
2365	MMO: ST->getMemOperand());
2366	}
2367
2368	//===----------------------------------------------------------------------===//
2369	// Float Result Promotion
2370	//===----------------------------------------------------------------------===//
2371
2372	void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
2373	LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG));
2374	SDValue R = SDValue();
2375
2376	// See if the target wants to custom expand this node.
2377	if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true)) {
2378	LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2379	return;
2380	}
2381
2382	switch (N->getOpcode()) {
2383	// These opcodes cannot appear if promotion of FP16 is done in the backend
2384	// instead of Clang
2385	case ISD::FP16_TO_FP:
2386	case ISD::FP_TO_FP16:
2387	default:
2388	#ifndef NDEBUG
2389	dbgs() << "PromoteFloatResult #" << ResNo << ": ";
2390	N->dump(G: &DAG); dbgs() << "\n";
2391	#endif
2392	report_fatal_error(reason: "Do not know how to promote this operator's result!");
2393
2394	case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break;
2395	case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break;
2396	case ISD::EXTRACT_VECTOR_ELT:
2397	R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break;
2398	case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break;
2399
2400	// Unary FP Operations
2401	case ISD::FABS:
2402	case ISD::FCBRT:
2403	case ISD::FCEIL:
2404	case ISD::FCOS:
2405	case ISD::FEXP:
2406	case ISD::FEXP2:
2407	case ISD::FEXP10:
2408	case ISD::FFLOOR:
2409	case ISD::FLOG:
2410	case ISD::FLOG2:
2411	case ISD::FLOG10:
2412	case ISD::FNEARBYINT:
2413	case ISD::FNEG:
2414	case ISD::FRINT:
2415	case ISD::FROUND:
2416	case ISD::FROUNDEVEN:
2417	case ISD::FSIN:
2418	case ISD::FSQRT:
2419	case ISD::FTRUNC:
2420	case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break;
2421
2422	// Binary FP Operations
2423	case ISD::FADD:
2424	case ISD::FDIV:
2425	case ISD::FMAXIMUM:
2426	case ISD::FMINIMUM:
2427	case ISD::FMAXNUM:
2428	case ISD::FMINNUM:
2429	case ISD::FMUL:
2430	case ISD::FPOW:
2431	case ISD::FREM:
2432	case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break;
2433
2434	case ISD::FMA: // FMA is same as FMAD
2435	case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break;
2436
2437	case ISD::FPOWI:
2438	case ISD::FLDEXP: R = PromoteFloatRes_ExpOp(N); break;
2439	case ISD::FFREXP: R = PromoteFloatRes_FFREXP(N); break;
2440
2441	case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
2442	case ISD::STRICT_FP_ROUND:
2443	R = PromoteFloatRes_STRICT_FP_ROUND(N);
2444	break;
2445	case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break;
2446	case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break;
2447	case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break;
2448
2449	case ISD::SINT_TO_FP:
2450	case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
2451	case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break;
2452	case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2453	case ISD::VECREDUCE_FADD:
2454	case ISD::VECREDUCE_FMUL:
2455	case ISD::VECREDUCE_FMIN:
2456	case ISD::VECREDUCE_FMAX:
2457	case ISD::VECREDUCE_FMAXIMUM:
2458	case ISD::VECREDUCE_FMINIMUM:
2459	R = PromoteFloatRes_VECREDUCE(N);
2460	break;
2461	case ISD::VECREDUCE_SEQ_FADD:
2462	case ISD::VECREDUCE_SEQ_FMUL:
2463	R = PromoteFloatRes_VECREDUCE_SEQ(N);
2464	break;
2465	}
2466
2467	if (R.getNode())
2468	SetPromotedFloat(Op: SDValue(N, ResNo), Result: R);
2469	}
2470
2471	// Bitcast from i16 to f16: convert the i16 to a f32 value instead.
2472	// At this point, it is not possible to determine if the bitcast value is
2473	// eventually stored to memory or promoted to f32 or promoted to a floating
2474	// point at a higher precision. Some of these cases are handled by FP_EXTEND,
2475	// STORE promotion handlers.
2476	SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) {
2477	EVT VT = N->getValueType(ResNo: 0);
2478	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2479	// Input type isn't guaranteed to be a scalar int so bitcast if not. The
2480	// bitcast will be legalized further if necessary.
2481	EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(),
2482	BitWidth: N->getOperand(Num: 0).getValueType().getSizeInBits());
2483	SDValue Cast = DAG.getBitcast(VT: IVT, V: N->getOperand(Num: 0));
2484	return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NVT), DL: SDLoc(N), VT: NVT, Operand: Cast);
2485	}
2486
2487	SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) {
2488	ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(Val: N);
2489	EVT VT = N->getValueType(ResNo: 0);
2490	SDLoc DL(N);
2491
2492	// Get the (bit-cast) APInt of the APFloat and build an integer constant
2493	EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
2494	SDValue C = DAG.getConstant(Val: CFPNode->getValueAPF().bitcastToAPInt(), DL,
2495	VT: IVT);
2496
2497	// Convert the Constant to the desired FP type
2498	// FIXME We might be able to do the conversion during compilation and get rid
2499	// of it from the object code
2500	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2501	return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NVT), DL, VT: NVT, Operand: C);
2502	}
2503
2504	// If the Index operand is a constant, try to redirect the extract operation to
2505	// the correct legalized vector. If not, bit-convert the input vector to
2506	// equivalent integer vector. Extract the element as an (bit-cast) integer
2507	// value and convert it to the promoted type.
2508	SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2509	SDLoc DL(N);
2510
2511	// If the index is constant, try to extract the value from the legalized
2512	// vector type.
2513	if (isa<ConstantSDNode>(Val: N->getOperand(Num: 1))) {
2514	SDValue Vec = N->getOperand(Num: 0);
2515	SDValue Idx = N->getOperand(Num: 1);
2516	EVT VecVT = Vec->getValueType(ResNo: 0);
2517	EVT EltVT = VecVT.getVectorElementType();
2518
2519	uint64_t IdxVal = Idx->getAsZExtVal();
2520
2521	switch (getTypeAction(VT: VecVT)) {
2522	default: break;
2523	case TargetLowering::TypeScalarizeVector: {
2524	SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
2525	ReplaceValueWith(From: SDValue(N, 0), To: Res);
2526	return SDValue();
2527	}
2528	case TargetLowering::TypeWidenVector: {
2529	Vec = GetWidenedVector(Op: Vec);
2530	SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL, VT: EltVT, N1: Vec, N2: Idx);
2531	ReplaceValueWith(From: SDValue(N, 0), To: Res);
2532	return SDValue();
2533	}
2534	case TargetLowering::TypeSplitVector: {
2535	SDValue Lo, Hi;
2536	GetSplitVector(Op: Vec, Lo, Hi);
2537
2538	uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2539	SDValue Res;
2540	if (IdxVal < LoElts)
2541	Res = DAG.getNode(Opcode: N->getOpcode(), DL, VT: EltVT, N1: Lo, N2: Idx);
2542	else
2543	Res = DAG.getNode(Opcode: N->getOpcode(), DL, VT: EltVT, N1: Hi,
2544	N2: DAG.getConstant(Val: IdxVal - LoElts, DL,
2545	VT: Idx.getValueType()));
2546	ReplaceValueWith(From: SDValue(N, 0), To: Res);
2547	return SDValue();
2548	}
2549
2550	}
2551	}
2552
2553	// Bit-convert the input vector to the equivalent integer vector
2554	SDValue NewOp = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0));
2555	EVT IVT = NewOp.getValueType().getVectorElementType();
2556
2557	// Extract the element as an (bit-cast) integer value
2558	SDValue NewVal = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: IVT,
2559	N1: NewOp, N2: N->getOperand(Num: 1));
2560
2561	// Convert the element to the desired FP type
2562	EVT VT = N->getValueType(ResNo: 0);
2563	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2564	return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NVT), DL: SDLoc(N), VT: NVT, Operand: NewVal);
2565	}
2566
2567	// FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result
2568	// needs promotion, so does the argument X. Note that Y, if needed, will be
2569	// handled during operand promotion.
2570	SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) {
2571	EVT VT = N->getValueType(ResNo: 0);
2572	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2573	SDValue Op0 = GetPromotedFloat(Op: N->getOperand(Num: 0));
2574
2575	SDValue Op1 = N->getOperand(Num: 1);
2576
2577	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NVT, N1: Op0, N2: Op1);
2578	}
2579
2580	// Unary operation where the result and the operand have PromoteFloat type
2581	// action. Construct a new SDNode with the promoted float value of the old
2582	// operand.
2583	SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) {
2584	EVT VT = N->getValueType(ResNo: 0);
2585	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2586	SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 0));
2587
2588	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NVT, Operand: Op);
2589	}
2590
2591	// Binary operations where the result and both operands have PromoteFloat type
2592	// action. Construct a new SDNode with the promoted float values of the old
2593	// operands.
2594	SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
2595	EVT VT = N->getValueType(ResNo: 0);
2596	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2597	SDValue Op0 = GetPromotedFloat(Op: N->getOperand(Num: 0));
2598	SDValue Op1 = GetPromotedFloat(Op: N->getOperand(Num: 1));
2599	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NVT, N1: Op0, N2: Op1, Flags: N->getFlags());
2600	}
2601
2602	SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
2603	EVT VT = N->getValueType(ResNo: 0);
2604	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2605	SDValue Op0 = GetPromotedFloat(Op: N->getOperand(Num: 0));
2606	SDValue Op1 = GetPromotedFloat(Op: N->getOperand(Num: 1));
2607	SDValue Op2 = GetPromotedFloat(Op: N->getOperand(Num: 2));
2608
2609	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NVT, N1: Op0, N2: Op1, N3: Op2);
2610	}
2611
2612	// Promote the Float (first) operand and retain the Integer (second) operand
2613	SDValue DAGTypeLegalizer::PromoteFloatRes_ExpOp(SDNode *N) {
2614	EVT VT = N->getValueType(ResNo: 0);
2615	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2616	SDValue Op0 = GetPromotedFloat(Op: N->getOperand(Num: 0));
2617	SDValue Op1 = N->getOperand(Num: 1);
2618
2619	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NVT, N1: Op0, N2: Op1);
2620	}
2621
2622	SDValue DAGTypeLegalizer::PromoteFloatRes_FFREXP(SDNode *N) {
2623	EVT VT = N->getValueType(ResNo: 0);
2624	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2625	SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 0));
2626	SDValue Res =
2627	DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), ResultTys: {NVT, N->getValueType(ResNo: 1)}, Ops: Op);
2628
2629	ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
2630	return Res;
2631	}
2632
2633	// Explicit operation to reduce precision. Reduce the value to half precision
2634	// and promote it back to the legal type.
2635	SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
2636	SDLoc DL(N);
2637
2638	SDValue Op = N->getOperand(Num: 0);
2639	EVT VT = N->getValueType(ResNo: 0);
2640	EVT OpVT = Op->getValueType(ResNo: 0);
2641	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
2642	EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
2643
2644	// Round promoted float to desired precision
2645	SDValue Round = DAG.getNode(Opcode: GetPromotionOpcode(OpVT, RetVT: VT), DL, VT: IVT, Operand: Op);
2646	// Promote it back to the legal output type
2647	return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NVT), DL, VT: NVT, Operand: Round);
2648	}
2649
2650	// Explicit operation to reduce precision. Reduce the value to half precision
2651	// and promote it back to the legal type.
2652	SDValue DAGTypeLegalizer::PromoteFloatRes_STRICT_FP_ROUND(SDNode *N) {
2653	SDLoc DL(N);
2654
2655	SDValue Chain = N->getOperand(Num: 0);
2656	SDValue Op = N->getOperand(Num: 1);
2657	EVT VT = N->getValueType(ResNo: 0);
2658	EVT OpVT = Op->getValueType(ResNo: 0);
2659	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
2660	EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
2661
2662	// Round promoted float to desired precision
2663	SDValue Round = DAG.getNode(GetPromotionOpcodeStrict(OpVT, VT), DL,
2664	DAG.getVTList(IVT, MVT::Other), Chain, Op);
2665	// Promote it back to the legal output type
2666	SDValue Res =
2667	DAG.getNode(GetPromotionOpcodeStrict(VT, NVT), DL,
2668	DAG.getVTList(NVT, MVT::Other), Round.getValue(1), Round);
2669	ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
2670	return Res;
2671	}
2672
2673	SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) {
2674	LoadSDNode *L = cast<LoadSDNode>(Val: N);
2675	EVT VT = N->getValueType(ResNo: 0);
2676
2677	// Load the value as an integer value with the same number of bits.
2678	EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
2679	SDValue newL = DAG.getLoad(
2680	AM: L->getAddressingMode(), ExtType: L->getExtensionType(), VT: IVT, dl: SDLoc(N),
2681	Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(), PtrInfo: L->getPointerInfo(), MemVT: IVT,
2682	Alignment: L->getOriginalAlign(), MMOFlags: L->getMemOperand()->getFlags(), AAInfo: L->getAAInfo());
2683	// Legalize the chain result by replacing uses of the old value chain with the
2684	// new one
2685	ReplaceValueWith(From: SDValue(N, 1), To: newL.getValue(R: 1));
2686
2687	// Convert the integer value to the desired FP type
2688	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2689	return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NVT), DL: SDLoc(N), VT: NVT, Operand: newL);
2690	}
2691
2692	// Construct a new SELECT node with the promoted true- and false- values.
2693	SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) {
2694	SDValue TrueVal = GetPromotedFloat(Op: N->getOperand(Num: 1));
2695	SDValue FalseVal = GetPromotedFloat(Op: N->getOperand(Num: 2));
2696
2697	return DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT: TrueVal->getValueType(ResNo: 0),
2698	N1: N->getOperand(Num: 0), N2: TrueVal, N3: FalseVal);
2699	}
2700
2701	// Construct a new SELECT_CC node with the promoted true- and false- values.
2702	// The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC.
2703	SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
2704	SDValue TrueVal = GetPromotedFloat(Op: N->getOperand(Num: 2));
2705	SDValue FalseVal = GetPromotedFloat(Op: N->getOperand(Num: 3));
2706
2707	return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
2708	VT: TrueVal.getNode()->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
2709	N2: N->getOperand(Num: 1), N3: TrueVal, N4: FalseVal, N5: N->getOperand(Num: 4));
2710	}
2711
2712	// Construct a SDNode that transforms the SINT or UINT operand to the promoted
2713	// float type.
2714	SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) {
2715	SDLoc DL(N);
2716	EVT VT = N->getValueType(ResNo: 0);
2717	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2718	SDValue NV = DAG.getNode(Opcode: N->getOpcode(), DL, VT: NVT, Operand: N->getOperand(Num: 0));
2719	// Round the value to the desired precision (that of the source type).
2720	return DAG.getNode(
2721	Opcode: ISD::FP_EXTEND, DL, VT: NVT,
2722	Operand: DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: NV,
2723	N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true)));
2724	}
2725
2726	SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
2727	return DAG.getUNDEF(VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
2728	VT: N->getValueType(ResNo: 0)));
2729	}
2730
2731	SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) {
2732	// Expand and promote recursively.
2733	// TODO: This is non-optimal, but dealing with the concurrently happening
2734	// vector-legalization is non-trivial. We could do something similar to
2735	// PromoteFloatRes_EXTRACT_VECTOR_ELT here.
2736	ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduce(Node: N, DAG));
2737	return SDValue();
2738	}
2739
2740	SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) {
2741	ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduceSeq(Node: N, DAG));
2742	return SDValue();
2743	}
2744
2745	SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2746	EVT VT = N->getValueType(ResNo: 0);
2747
2748	AtomicSDNode *AM = cast<AtomicSDNode>(Val: N);
2749	SDLoc SL(N);
2750
2751	SDValue CastVal = BitConvertToInteger(Op: AM->getVal());
2752	EVT CastVT = CastVal.getValueType();
2753
2754	SDValue NewAtomic
2755	= DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT,
2756	DAG.getVTList(CastVT, MVT::Other),
2757	{ AM->getChain(), AM->getBasePtr(), CastVal },
2758	AM->getMemOperand());
2759
2760	SDValue Result = NewAtomic;
2761
2762	if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) {
2763	EVT NFPVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2764	Result = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NFPVT), DL: SL, VT: NFPVT,
2765	Operand: NewAtomic);
2766	}
2767
2768	// Legalize the chain result by replacing uses of the old value chain with the
2769	// new one
2770	ReplaceValueWith(From: SDValue(N, 1), To: NewAtomic.getValue(R: 1));
2771
2772	return Result;
2773
2774	}
2775
2776	//===----------------------------------------------------------------------===//
2777	// Half Result Soft Promotion
2778	//===----------------------------------------------------------------------===//
2779
2780	void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
2781	LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
2782	N->dump(&DAG));
2783	SDValue R = SDValue();
2784
2785	// See if the target wants to custom expand this node.
2786	if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true)) {
2787	LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2788	return;
2789	}
2790
2791	switch (N->getOpcode()) {
2792	default:
2793	#ifndef NDEBUG
2794	dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
2795	N->dump(G: &DAG); dbgs() << "\n";
2796	#endif
2797	report_fatal_error(reason: "Do not know how to soft promote this operator's "
2798	"result!");
2799
2800	case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
2801	case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
2802	case ISD::EXTRACT_VECTOR_ELT:
2803	R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
2804	case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
2805	case ISD::STRICT_FP_ROUND:
2806	case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
2807
2808	// Unary FP Operations
2809	case ISD::FABS:
2810	case ISD::FCBRT:
2811	case ISD::FCEIL:
2812	case ISD::FCOS:
2813	case ISD::FEXP:
2814	case ISD::FEXP2:
2815	case ISD::FEXP10:
2816	case ISD::FFLOOR:
2817	case ISD::FLOG:
2818	case ISD::FLOG2:
2819	case ISD::FLOG10:
2820	case ISD::FNEARBYINT:
2821	case ISD::FNEG:
2822	case ISD::FREEZE:
2823	case ISD::FRINT:
2824	case ISD::FROUND:
2825	case ISD::FROUNDEVEN:
2826	case ISD::FSIN:
2827	case ISD::FSQRT:
2828	case ISD::FTRUNC:
2829	case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
2830
2831	// Binary FP Operations
2832	case ISD::FADD:
2833	case ISD::FDIV:
2834	case ISD::FMAXIMUM:
2835	case ISD::FMINIMUM:
2836	case ISD::FMAXNUM:
2837	case ISD::FMINNUM:
2838	case ISD::FMUL:
2839	case ISD::FPOW:
2840	case ISD::FREM:
2841	case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
2842
2843	case ISD::FMA: // FMA is same as FMAD
2844	case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
2845
2846	case ISD::FPOWI:
2847	case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break;
2848
2849	case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break;
2850
2851	case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
2852	case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
2853	case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
2854	case ISD::SINT_TO_FP:
2855	case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
2856	case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
2857	case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2858	case ISD::VECREDUCE_FADD:
2859	case ISD::VECREDUCE_FMUL:
2860	case ISD::VECREDUCE_FMIN:
2861	case ISD::VECREDUCE_FMAX:
2862	case ISD::VECREDUCE_FMAXIMUM:
2863	case ISD::VECREDUCE_FMINIMUM:
2864	R = SoftPromoteHalfRes_VECREDUCE(N);
2865	break;
2866	case ISD::VECREDUCE_SEQ_FADD:
2867	case ISD::VECREDUCE_SEQ_FMUL:
2868	R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
2869	break;
2870	}
2871
2872	if (R.getNode())
2873	SetSoftPromotedHalf(Op: SDValue(N, ResNo), Result: R);
2874	}
2875
2876	SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
2877	return BitConvertToInteger(Op: N->getOperand(Num: 0));
2878	}
2879
2880	SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
2881	ConstantFPSDNode *CN = cast<ConstantFPSDNode>(Val: N);
2882
2883	// Get the (bit-cast) APInt of the APFloat and build an integer constant
2884	return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
2885	MVT::i16);
2886	}
2887
2888	SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2889	SDValue NewOp = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0));
2890	return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
2891	VT: NewOp.getValueType().getVectorElementType(), N1: NewOp,
2892	N2: N->getOperand(Num: 1));
2893	}
2894
2895	SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
2896	SDValue LHS = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2897	SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
2898	SDLoc dl(N);
2899
2900	EVT LVT = LHS.getValueType();
2901	EVT RVT = RHS.getValueType();
2902
2903	unsigned LSize = LVT.getSizeInBits();
2904	unsigned RSize = RVT.getSizeInBits();
2905
2906	// First get the sign bit of second operand.
2907	SDValue SignBit = DAG.getNode(
2908	Opcode: ISD::SHL, DL: dl, VT: RVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: RVT),
2909	N2: DAG.getConstant(Val: RSize - 1, DL: dl,
2910	VT: TLI.getShiftAmountTy(LHSTy: RVT, DL: DAG.getDataLayout())));
2911	SignBit = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: RVT, N1: RHS, N2: SignBit);
2912
2913	// Shift right or sign-extend it if the two operands have different types.
2914	int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
2915	if (SizeDiff > 0) {
2916	SignBit =
2917	DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: SignBit,
2918	N2: DAG.getConstant(Val: SizeDiff, DL: dl,
2919	VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
2920	DL: DAG.getDataLayout())));
2921	SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LVT, Operand: SignBit);
2922	} else if (SizeDiff < 0) {
2923	SignBit = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: SignBit);
2924	SignBit =
2925	DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: LVT, N1: SignBit,
2926	N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
2927	VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
2928	DL: DAG.getDataLayout())));
2929	}
2930
2931	// Clear the sign bit of the first operand.
2932	SDValue Mask = DAG.getNode(
2933	Opcode: ISD::SHL, DL: dl, VT: LVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: LVT),
2934	N2: DAG.getConstant(Val: LSize - 1, DL: dl,
2935	VT: TLI.getShiftAmountTy(LHSTy: LVT, DL: DAG.getDataLayout())));
2936	Mask = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: LVT, N1: Mask, N2: DAG.getConstant(Val: 1, DL: dl, VT: LVT));
2937	LHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: LVT, N1: LHS, N2: Mask);
2938
2939	// Or the value with the sign bit.
2940	return DAG.getNode(Opcode: ISD::OR, DL: dl, VT: LVT, N1: LHS, N2: SignBit);
2941	}
2942
2943	SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
2944	EVT OVT = N->getValueType(ResNo: 0);
2945	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2946	SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2947	SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
2948	SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
2949	SDLoc dl(N);
2950
2951	// Promote to the larger FP type.
2952	auto PromotionOpcode = GetPromotionOpcode(OpVT: OVT, RetVT: NVT);
2953	Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
2954	Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
2955	Op2 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op2);
2956
2957	SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1, N3: Op2);
2958
2959	// Convert back to FP16 as an integer.
2960	return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2961	}
2962
2963	SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) {
2964	EVT OVT = N->getValueType(ResNo: 0);
2965	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2966	SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2967	SDValue Op1 = N->getOperand(Num: 1);
2968	SDLoc dl(N);
2969
2970	// Promote to the larger FP type.
2971	Op0 = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op0);
2972
2973	SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1);
2974
2975	// Convert back to FP16 as an integer.
2976	return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2977	}
2978
2979	SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) {
2980	EVT OVT = N->getValueType(ResNo: 0);
2981	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2982	SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2983	SDLoc dl(N);
2984
2985	// Promote to the larger FP type.
2986	Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op);
2987
2988	SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl,
2989	VTList: DAG.getVTList(VT1: NVT, VT2: N->getValueType(ResNo: 1)), N: Op);
2990
2991	ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
2992
2993	// Convert back to FP16 as an integer.
2994	return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2995	}
2996
2997	SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
2998	EVT RVT = N->getValueType(ResNo: 0);
2999	EVT SVT = N->getOperand(Num: 0).getValueType();
3000
3001	if (N->isStrictFPOpcode()) {
3002	assert(RVT == MVT::f16);
3003	SDValue Res =
3004	DAG.getNode(ISD::STRICT_FP_TO_FP16, SDLoc(N), {MVT::i16, MVT::Other},
3005	{N->getOperand(0), N->getOperand(1)});
3006	ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
3007	return Res;
3008	}
3009
3010	return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), MVT::i16,
3011	N->getOperand(0));
3012	}
3013
3014	SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
3015	LoadSDNode *L = cast<LoadSDNode>(Val: N);
3016
3017	// Load the value as an integer value with the same number of bits.
3018	assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
3019	SDValue NewL =
3020	DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16,
3021	SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(),
3022	L->getPointerInfo(), MVT::i16, L->getOriginalAlign(),
3023	L->getMemOperand()->getFlags(), L->getAAInfo());
3024	// Legalize the chain result by replacing uses of the old value chain with the
3025	// new one
3026	ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
3027	return NewL;
3028	}
3029
3030	SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
3031	SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
3032	SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
3033	return DAG.getSelect(DL: SDLoc(N), VT: Op1.getValueType(), Cond: N->getOperand(Num: 0), LHS: Op1,
3034	RHS: Op2);
3035	}
3036
3037	SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
3038	SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
3039	SDValue Op3 = GetSoftPromotedHalf(Op: N->getOperand(Num: 3));
3040	return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: Op2.getValueType(),
3041	N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1), N3: Op2, N4: Op3,
3042	N5: N->getOperand(Num: 4));
3043	}
3044
3045	SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
3046	EVT OVT = N->getValueType(ResNo: 0);
3047	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
3048	SDLoc dl(N);
3049
3050	SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: N->getOperand(Num: 0));
3051
3052	// Round the value to the softened type.
3053	return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3054	}
3055
3056	SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
3057	return DAG.getUNDEF(MVT::i16);
3058	}
3059
3060	SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
3061	EVT OVT = N->getValueType(ResNo: 0);
3062	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
3063	SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3064	SDLoc dl(N);
3065
3066	// Promote to the larger FP type.
3067	Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op);
3068
3069	SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Op);
3070
3071	// Convert back to FP16 as an integer.
3072	return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3073	}
3074
3075	SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
3076	EVT OVT = N->getValueType(ResNo: 0);
3077	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
3078	SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3079	SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
3080	SDLoc dl(N);
3081
3082	// Promote to the larger FP type.
3083	auto PromotionOpcode = GetPromotionOpcode(OpVT: OVT, RetVT: NVT);
3084	Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3085	Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3086
3087	SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1);
3088
3089	// Convert back to FP16 as an integer.
3090	return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3091	}
3092
3093	SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
3094	// Expand and soften recursively.
3095	ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduce(Node: N, DAG));
3096	return SDValue();
3097	}
3098
3099	SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
3100	// Expand and soften.
3101	ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduceSeq(Node: N, DAG));
3102	return SDValue();
3103	}
3104
3105	//===----------------------------------------------------------------------===//
3106	// Half Operand Soft Promotion
3107	//===----------------------------------------------------------------------===//
3108
3109	bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
3110	LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
3111	N->dump(&DAG));
3112	SDValue Res = SDValue();
3113
3114	if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false)) {
3115	LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
3116	return false;
3117	}
3118
3119	// Nodes that use a promotion-requiring floating point operand, but doesn't
3120	// produce a soft promotion-requiring floating point result, need to be
3121	// legalized to use the soft promoted float operand. Nodes that produce at
3122	// least one soft promotion-requiring floating point result have their
3123	// operands legalized as a part of PromoteFloatResult.
3124	switch (N->getOpcode()) {
3125	default:
3126	#ifndef NDEBUG
3127	dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
3128	N->dump(G: &DAG); dbgs() << "\n";
3129	#endif
3130	report_fatal_error(reason: "Do not know how to soft promote this operator's "
3131	"operand!");
3132
3133	case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
3134	case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break;
3135	case ISD::FP_TO_SINT:
3136	case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break;
3137	case ISD::FP_TO_SINT_SAT:
3138	case ISD::FP_TO_UINT_SAT:
3139	Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
3140	case ISD::STRICT_FP_EXTEND:
3141	case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
3142	case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
3143	case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
3144	case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
3145	case ISD::STACKMAP:
3146	Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
3147	break;
3148	case ISD::PATCHPOINT:
3149	Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
3150	break;
3151	}
3152
3153	if (!Res.getNode())
3154	return false;
3155
3156	assert(Res.getNode() != N && "Expected a new node!");
3157
3158	assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3159	"Invalid operand expansion");
3160
3161	ReplaceValueWith(From: SDValue(N, 0), To: Res);
3162	return false;
3163	}
3164
3165	SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
3166	SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3167
3168	return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op0);
3169	}
3170
3171	SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
3172	unsigned OpNo) {
3173	assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3174	SDValue Op1 = N->getOperand(Num: 1);
3175	EVT RVT = Op1.getValueType();
3176	SDLoc dl(N);
3177
3178	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op1.getValueType());
3179
3180	Op1 = GetSoftPromotedHalf(Op: Op1);
3181	Op1 = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: RVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op1);
3182
3183	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
3184	N2: Op1);
3185	}
3186
3187	SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
3188	EVT RVT = N->getValueType(ResNo: 0);
3189	bool IsStrict = N->isStrictFPOpcode();
3190	SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
3191	EVT SVT = Op.getValueType();
3192	Op = GetSoftPromotedHalf(Op: N->getOperand(Num: IsStrict ? 1 : 0));
3193
3194	if (IsStrict) {
3195	assert(SVT == MVT::f16);
3196	SDValue Res =
3197	DAG.getNode(ISD::STRICT_FP16_TO_FP, SDLoc(N),
3198	{N->getValueType(0), MVT::Other}, {N->getOperand(0), Op});
3199	ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
3200	ReplaceValueWith(From: SDValue(N, 0), To: Res);
3201	return SDValue();
3202	}
3203
3204	return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: SDLoc(N), VT: RVT, Operand: Op);
3205	}
3206
3207	SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) {
3208	EVT RVT = N->getValueType(ResNo: 0);
3209	SDValue Op = N->getOperand(Num: 0);
3210	EVT SVT = Op.getValueType();
3211	SDLoc dl(N);
3212
3213	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op.getValueType());
3214
3215	Op = GetSoftPromotedHalf(Op);
3216
3217	SDValue Res = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: dl, VT: NVT, Operand: Op);
3218
3219	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), Operand: Res);
3220	}
3221
3222	SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
3223	EVT RVT = N->getValueType(ResNo: 0);
3224	SDValue Op = N->getOperand(Num: 0);
3225	EVT SVT = Op.getValueType();
3226	SDLoc dl(N);
3227
3228	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op.getValueType());
3229
3230	Op = GetSoftPromotedHalf(Op);
3231
3232	SDValue Res = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: dl, VT: NVT, Operand: Op);
3233
3234	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), N1: Res,
3235	N2: N->getOperand(Num: 1));
3236	}
3237
3238	SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
3239	unsigned OpNo) {
3240	assert(OpNo == 0 && "Can only soften the comparison values");
3241	SDValue Op0 = N->getOperand(Num: 0);
3242	SDValue Op1 = N->getOperand(Num: 1);
3243	SDLoc dl(N);
3244
3245	EVT SVT = Op0.getValueType();
3246	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
3247
3248	Op0 = GetSoftPromotedHalf(Op: Op0);
3249	Op1 = GetSoftPromotedHalf(Op: Op1);
3250
3251	// Promote to the larger FP type.
3252	auto PromotionOpcode = GetPromotionOpcode(OpVT: SVT, RetVT: NVT);
3253	Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3254	Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3255
3256	return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Op0, N2: Op1,
3257	N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3), N5: N->getOperand(Num: 4));
3258	}
3259
3260	SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
3261	SDValue Op0 = N->getOperand(Num: 0);
3262	SDValue Op1 = N->getOperand(Num: 1);
3263	ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
3264	SDLoc dl(N);
3265
3266	EVT SVT = Op0.getValueType();
3267	EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op0.getValueType());
3268
3269	Op0 = GetSoftPromotedHalf(Op: Op0);
3270	Op1 = GetSoftPromotedHalf(Op: Op1);
3271
3272	// Promote to the larger FP type.
3273	auto PromotionOpcode = GetPromotionOpcode(OpVT: SVT, RetVT: NVT);
3274	Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3275	Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3276
3277	return DAG.getSetCC(DL: SDLoc(N), VT: N->getValueType(ResNo: 0), LHS: Op0, RHS: Op1, Cond: CCCode);
3278	}
3279
3280	SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3281	assert(OpNo == 1 && "Can only soften the stored value!");
3282	StoreSDNode *ST = cast<StoreSDNode>(Val: N);
3283	SDValue Val = ST->getValue();
3284	SDLoc dl(N);
3285
3286	assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3287	SDValue Promoted = GetSoftPromotedHalf(Op: Val);
3288	return DAG.getStore(Chain: ST->getChain(), dl, Val: Promoted, Ptr: ST->getBasePtr(),
3289	MMO: ST->getMemOperand());
3290	}
3291
3292	SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
3293	assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3294	SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3295	SDValue Op = N->getOperand(Num: OpNo);
3296	NewOps[OpNo] = GetSoftPromotedHalf(Op);
3297	SDValue NewNode =
3298	DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: N->getVTList(), Ops: NewOps);
3299
3300	for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3301	ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum));
3302
3303	return SDValue(); // Signal that we replaced the node ourselves.
3304	}
3305
3306	SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
3307	unsigned OpNo) {
3308	assert(OpNo >= 7);
3309	SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3310	SDValue Op = N->getOperand(Num: OpNo);
3311	NewOps[OpNo] = GetSoftPromotedHalf(Op);
3312	SDValue NewNode =
3313	DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: N->getVTList(), Ops: NewOps);
3314
3315	for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3316	ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum));
3317
3318	return SDValue(); // Signal that we replaced the node ourselves.
3319	}
3320