1	//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file performs vector type splitting and scalarization for LegalizeTypes.
10	// Scalarization is the act of changing a computation in an illegal one-element
11	// vector type to be a computation in its scalar element type. For example,
12	// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13	// as a base case when scalarizing vector arithmetic like <4 x f32>, which
14	// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15	// types.
16	// Splitting is the act of changing a computation in an invalid vector type to
17	// be a computation in two vectors of half the size. For example, implementing
18	// <128 x f32> operations in terms of two <64 x f32> operations.
19	//
20	//===----------------------------------------------------------------------===//
21
22	#include "LegalizeTypes.h"
23	#include "llvm/ADT/SmallBitVector.h"
24	#include "llvm/Analysis/MemoryLocation.h"
25	#include "llvm/Analysis/VectorUtils.h"
26	#include "llvm/IR/DataLayout.h"
27	#include "llvm/Support/ErrorHandling.h"
28	#include "llvm/Support/TypeSize.h"
29	#include "llvm/Support/raw_ostream.h"
30	#include <numeric>
31
32	using namespace llvm;
33
34	#define DEBUG_TYPE "legalize-types"
35
36	//===----------------------------------------------------------------------===//
37	// Result Vector Scalarization: <1 x ty> -> ty.
38	//===----------------------------------------------------------------------===//
39
40	void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
41	LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": ";
42	N->dump(&DAG));
43	SDValue R = SDValue();
44
45	switch (N->getOpcode()) {
46	default:
47	#ifndef NDEBUG
48	dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
49	N->dump(G: &DAG);
50	dbgs() << "\n";
51	#endif
52	report_fatal_error(reason: "Do not know how to scalarize the result of this "
53	"operator!\n");
54
55	case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
56	case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
57	case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
58	case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
59	case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
60	case ISD::FPOWI: R = ScalarizeVecRes_ExpOp(N); break;
61	case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
62	case ISD::LOAD: R = ScalarizeVecRes_LOAD(N: cast<LoadSDNode>(Val: N));break;
63	case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
64	case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
65	case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
66	case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
67	case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
68	case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
69	case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
70	case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
71	case ISD::IS_FPCLASS: R = ScalarizeVecRes_IS_FPCLASS(N); break;
72	case ISD::ANY_EXTEND_VECTOR_INREG:
73	case ISD::SIGN_EXTEND_VECTOR_INREG:
74	case ISD::ZERO_EXTEND_VECTOR_INREG:
75	R = ScalarizeVecRes_VecInregOp(N);
76	break;
77	case ISD::ABS:
78	case ISD::ANY_EXTEND:
79	case ISD::BITREVERSE:
80	case ISD::BSWAP:
81	case ISD::CTLZ:
82	case ISD::CTLZ_ZERO_UNDEF:
83	case ISD::CTPOP:
84	case ISD::CTTZ:
85	case ISD::CTTZ_ZERO_UNDEF:
86	case ISD::FABS:
87	case ISD::FCEIL:
88	case ISD::FCOS:
89	case ISD::FEXP:
90	case ISD::FEXP2:
91	case ISD::FEXP10:
92	case ISD::FFLOOR:
93	case ISD::FLOG:
94	case ISD::FLOG10:
95	case ISD::FLOG2:
96	case ISD::FNEARBYINT:
97	case ISD::FNEG:
98	case ISD::FREEZE:
99	case ISD::ARITH_FENCE:
100	case ISD::FP_EXTEND:
101	case ISD::FP_TO_SINT:
102	case ISD::FP_TO_UINT:
103	case ISD::FRINT:
104	case ISD::LRINT:
105	case ISD::LLRINT:
106	case ISD::FROUND:
107	case ISD::FROUNDEVEN:
108	case ISD::FSIN:
109	case ISD::FSQRT:
110	case ISD::FTRUNC:
111	case ISD::SIGN_EXTEND:
112	case ISD::SINT_TO_FP:
113	case ISD::TRUNCATE:
114	case ISD::UINT_TO_FP:
115	case ISD::ZERO_EXTEND:
116	case ISD::FCANONICALIZE:
117	R = ScalarizeVecRes_UnaryOp(N);
118	break;
119	case ISD::FFREXP:
120	R = ScalarizeVecRes_FFREXP(N, ResNo);
121	break;
122	case ISD::ADD:
123	case ISD::AND:
124	case ISD::FADD:
125	case ISD::FCOPYSIGN:
126	case ISD::FDIV:
127	case ISD::FMUL:
128	case ISD::FMINNUM:
129	case ISD::FMAXNUM:
130	case ISD::FMINNUM_IEEE:
131	case ISD::FMAXNUM_IEEE:
132	case ISD::FMINIMUM:
133	case ISD::FMAXIMUM:
134	case ISD::FLDEXP:
135	case ISD::SMIN:
136	case ISD::SMAX:
137	case ISD::UMIN:
138	case ISD::UMAX:
139
140	case ISD::SADDSAT:
141	case ISD::UADDSAT:
142	case ISD::SSUBSAT:
143	case ISD::USUBSAT:
144	case ISD::SSHLSAT:
145	case ISD::USHLSAT:
146
147	case ISD::FPOW:
148	case ISD::FREM:
149	case ISD::FSUB:
150	case ISD::MUL:
151	case ISD::MULHS:
152	case ISD::MULHU:
153	case ISD::OR:
154	case ISD::SDIV:
155	case ISD::SREM:
156	case ISD::SUB:
157	case ISD::UDIV:
158	case ISD::UREM:
159	case ISD::XOR:
160	case ISD::SHL:
161	case ISD::SRA:
162	case ISD::SRL:
163	case ISD::ROTL:
164	case ISD::ROTR:
165	R = ScalarizeVecRes_BinOp(N);
166	break;
167	case ISD::FMA:
168	case ISD::FSHL:
169	case ISD::FSHR:
170	R = ScalarizeVecRes_TernaryOp(N);
171	break;
172
173	#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
174	case ISD::STRICT_##DAGN:
175	#include "llvm/IR/ConstrainedOps.def"
176	R = ScalarizeVecRes_StrictFPOp(N);
177	break;
178
179	case ISD::FP_TO_UINT_SAT:
180	case ISD::FP_TO_SINT_SAT:
181	R = ScalarizeVecRes_FP_TO_XINT_SAT(N);
182	break;
183
184	case ISD::UADDO:
185	case ISD::SADDO:
186	case ISD::USUBO:
187	case ISD::SSUBO:
188	case ISD::UMULO:
189	case ISD::SMULO:
190	R = ScalarizeVecRes_OverflowOp(N, ResNo);
191	break;
192	case ISD::SMULFIX:
193	case ISD::SMULFIXSAT:
194	case ISD::UMULFIX:
195	case ISD::UMULFIXSAT:
196	case ISD::SDIVFIX:
197	case ISD::SDIVFIXSAT:
198	case ISD::UDIVFIX:
199	case ISD::UDIVFIXSAT:
200	R = ScalarizeVecRes_FIX(N);
201	break;
202	}
203
204	// If R is null, the sub-method took care of registering the result.
205	if (R.getNode())
206	SetScalarizedVector(Op: SDValue(N, ResNo), Result: R);
207	}
208
209	SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
210	SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
211	SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
212	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
213	VT: LHS.getValueType(), N1: LHS, N2: RHS, Flags: N->getFlags());
214	}
215
216	SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
217	SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0));
218	SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1));
219	SDValue Op2 = GetScalarizedVector(Op: N->getOperand(Num: 2));
220	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1,
221	N3: Op2, Flags: N->getFlags());
222	}
223
224	SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
225	SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0));
226	SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1));
227	SDValue Op2 = N->getOperand(Num: 2);
228	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1,
229	N3: Op2, Flags: N->getFlags());
230	}
231
232	SDValue DAGTypeLegalizer::ScalarizeVecRes_FFREXP(SDNode *N, unsigned ResNo) {
233	assert(N->getValueType(0).getVectorNumElements() == 1 &&
234	"Unexpected vector type!");
235	SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
236
237	EVT VT0 = N->getValueType(ResNo: 0);
238	EVT VT1 = N->getValueType(ResNo: 1);
239	SDLoc dl(N);
240
241	SDNode *ScalarNode =
242	DAG.getNode(Opcode: N->getOpcode(), DL: dl,
243	ResultTys: {VT0.getScalarType(), VT1.getScalarType()}, Ops: Elt)
244	.getNode();
245
246	// Replace the other vector result not being explicitly scalarized here.
247	unsigned OtherNo = 1 - ResNo;
248	EVT OtherVT = N->getValueType(ResNo: OtherNo);
249	if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) {
250	SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo));
251	} else {
252	SDValue OtherVal = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: OtherVT,
253	Operand: SDValue(ScalarNode, OtherNo));
254	ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
255	}
256
257	return SDValue(ScalarNode, ResNo);
258	}
259
260	SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
261	EVT VT = N->getValueType(ResNo: 0).getVectorElementType();
262	unsigned NumOpers = N->getNumOperands();
263	SDValue Chain = N->getOperand(Num: 0);
264	EVT ValueVTs[] = {VT, MVT::Other};
265	SDLoc dl(N);
266
267	SmallVector<SDValue, 4> Opers(NumOpers);
268
269	// The Chain is the first operand.
270	Opers[0] = Chain;
271
272	// Now process the remaining operands.
273	for (unsigned i = 1; i < NumOpers; ++i) {
274	SDValue Oper = N->getOperand(Num: i);
275	EVT OperVT = Oper.getValueType();
276
277	if (OperVT.isVector()) {
278	if (getTypeAction(VT: OperVT) == TargetLowering::TypeScalarizeVector)
279	Oper = GetScalarizedVector(Op: Oper);
280	else
281	Oper = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl,
282	VT: OperVT.getVectorElementType(), N1: Oper,
283	N2: DAG.getVectorIdxConstant(Val: 0, DL: dl));
284	}
285
286	Opers[i] = Oper;
287	}
288
289	SDValue Result = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(ValueVTs),
290	Ops: Opers, Flags: N->getFlags());
291
292	// Legalize the chain result - switch anything that used the old chain to
293	// use the new one.
294	ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
295	return Result;
296	}
297
298	SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
299	unsigned ResNo) {
300	SDLoc DL(N);
301	EVT ResVT = N->getValueType(ResNo: 0);
302	EVT OvVT = N->getValueType(ResNo: 1);
303
304	SDValue ScalarLHS, ScalarRHS;
305	if (getTypeAction(VT: ResVT) == TargetLowering::TypeScalarizeVector) {
306	ScalarLHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
307	ScalarRHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
308	} else {
309	SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
310	DAG.ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: ElemsLHS);
311	DAG.ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: ElemsRHS);
312	ScalarLHS = ElemsLHS[0];
313	ScalarRHS = ElemsRHS[0];
314	}
315
316	SDVTList ScalarVTs = DAG.getVTList(
317	VT1: ResVT.getVectorElementType(), VT2: OvVT.getVectorElementType());
318	SDNode *ScalarNode = DAG.getNode(
319	Opcode: N->getOpcode(), DL, VTList: ScalarVTs, N1: ScalarLHS, N2: ScalarRHS).getNode();
320	ScalarNode->setFlags(N->getFlags());
321
322	// Replace the other vector result not being explicitly scalarized here.
323	unsigned OtherNo = 1 - ResNo;
324	EVT OtherVT = N->getValueType(ResNo: OtherNo);
325	if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) {
326	SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo));
327	} else {
328	SDValue OtherVal = DAG.getNode(
329	Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: OtherVT, Operand: SDValue(ScalarNode, OtherNo));
330	ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
331	}
332
333	return SDValue(ScalarNode, ResNo);
334	}
335
336	SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
337	unsigned ResNo) {
338	SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
339	return GetScalarizedVector(Op);
340	}
341
342	SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
343	SDValue Op = N->getOperand(Num: 0);
344	if (Op.getValueType().isVector()
345	&& Op.getValueType().getVectorNumElements() == 1
346	&& !isSimpleLegalType(VT: Op.getValueType()))
347	Op = GetScalarizedVector(Op);
348	EVT NewVT = N->getValueType(ResNo: 0).getVectorElementType();
349	return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
350	VT: NewVT, Operand: Op);
351	}
352
353	SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
354	EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
355	SDValue InOp = N->getOperand(Num: 0);
356	// The BUILD_VECTOR operands may be of wider element types and
357	// we may need to truncate them back to the requested return type.
358	if (EltVT.isInteger())
359	return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp);
360	return InOp;
361	}
362
363	SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
364	return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
365	VT: N->getValueType(ResNo: 0).getVectorElementType(),
366	N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
367	}
368
369	SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
370	SDLoc DL(N);
371	SDValue Op = N->getOperand(Num: 0);
372	EVT OpVT = Op.getValueType();
373	// The result needs scalarizing, but it's not a given that the source does.
374	// See similar logic in ScalarizeVecRes_UnaryOp.
375	if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
376	Op = GetScalarizedVector(Op);
377	} else {
378	EVT VT = OpVT.getVectorElementType();
379	Op = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: Op,
380	N2: DAG.getVectorIdxConstant(Val: 0, DL));
381	}
382	return DAG.getNode(Opcode: ISD::FP_ROUND, DL,
383	VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
384	N2: N->getOperand(Num: 1));
385	}
386
387	SDValue DAGTypeLegalizer::ScalarizeVecRes_ExpOp(SDNode *N) {
388	SDValue Op = GetScalarizedVector(Op: N->getOperand(Num: 0));
389	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op.getValueType(), N1: Op,
390	N2: N->getOperand(Num: 1));
391	}
392
393	SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
394	// The value to insert may have a wider type than the vector element type,
395	// so be sure to truncate it to the element type if necessary.
396	SDValue Op = N->getOperand(Num: 1);
397	EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
398	if (Op.getValueType() != EltVT)
399	// FIXME: Can this happen for floating point types?
400	Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: Op);
401	return Op;
402	}
403
404	SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
405	assert(N->isUnindexed() && "Indexed vector load?");
406
407	SDValue Result = DAG.getLoad(
408	AM: ISD::UNINDEXED, ExtType: N->getExtensionType(),
409	VT: N->getValueType(ResNo: 0).getVectorElementType(), dl: SDLoc(N), Chain: N->getChain(),
410	Ptr: N->getBasePtr(), Offset: DAG.getUNDEF(VT: N->getBasePtr().getValueType()),
411	PtrInfo: N->getPointerInfo(), MemVT: N->getMemoryVT().getVectorElementType(),
412	Alignment: N->getOriginalAlign(), MMOFlags: N->getMemOperand()->getFlags(), AAInfo: N->getAAInfo());
413
414	// Legalize the chain result - switch anything that used the old chain to
415	// use the new one.
416	ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
417	return Result;
418	}
419
420	SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
421	// Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
422	EVT DestVT = N->getValueType(ResNo: 0).getVectorElementType();
423	SDValue Op = N->getOperand(Num: 0);
424	EVT OpVT = Op.getValueType();
425	SDLoc DL(N);
426	// The result needs scalarizing, but it's not a given that the source does.
427	// This is a workaround for targets where it's impossible to scalarize the
428	// result of a conversion, because the source type is legal.
429	// For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
430	// are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
431	// legal and was not scalarized.
432	// See the similar logic in ScalarizeVecRes_SETCC
433	if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
434	Op = GetScalarizedVector(Op);
435	} else {
436	EVT VT = OpVT.getVectorElementType();
437	Op = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: Op,
438	N2: DAG.getVectorIdxConstant(Val: 0, DL));
439	}
440	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: DestVT, Operand: Op, Flags: N->getFlags());
441	}
442
443	SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
444	EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
445	EVT ExtVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT().getVectorElementType();
446	SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
447	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: EltVT,
448	N1: LHS, N2: DAG.getValueType(ExtVT));
449	}
450
451	SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
452	SDLoc DL(N);
453	SDValue Op = N->getOperand(Num: 0);
454
455	EVT OpVT = Op.getValueType();
456	EVT OpEltVT = OpVT.getVectorElementType();
457	EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
458
459	if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
460	Op = GetScalarizedVector(Op);
461	} else {
462	Op = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: OpEltVT, N1: Op,
463	N2: DAG.getVectorIdxConstant(Val: 0, DL));
464	}
465
466	switch (N->getOpcode()) {
467	case ISD::ANY_EXTEND_VECTOR_INREG:
468	return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: EltVT, Operand: Op);
469	case ISD::SIGN_EXTEND_VECTOR_INREG:
470	return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: EltVT, Operand: Op);
471	case ISD::ZERO_EXTEND_VECTOR_INREG:
472	return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EltVT, Operand: Op);
473	}
474
475	llvm_unreachable("Illegal extend_vector_inreg opcode");
476	}
477
478	SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
479	// If the operand is wider than the vector element type then it is implicitly
480	// truncated. Make that explicit here.
481	EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
482	SDValue InOp = N->getOperand(Num: 0);
483	if (InOp.getValueType() != EltVT)
484	return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp);
485	return InOp;
486	}
487
488	SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
489	SDValue Cond = N->getOperand(Num: 0);
490	EVT OpVT = Cond.getValueType();
491	SDLoc DL(N);
492	// The vselect result and true/value operands needs scalarizing, but it's
493	// not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
494	// See the similar logic in ScalarizeVecRes_SETCC
495	if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
496	Cond = GetScalarizedVector(Op: Cond);
497	} else {
498	EVT VT = OpVT.getVectorElementType();
499	Cond = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: Cond,
500	N2: DAG.getVectorIdxConstant(Val: 0, DL));
501	}
502
503	SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
504	TargetLowering::BooleanContent ScalarBool =
505	TLI.getBooleanContents(isVec: false, isFloat: false);
506	TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(isVec: true, isFloat: false);
507
508	// If integer and float booleans have different contents then we can't
509	// reliably optimize in all cases. There is a full explanation for this in
510	// DAGCombiner::visitSELECT() where the same issue affects folding
511	// (select C, 0, 1) to (xor C, 1).
512	if (TLI.getBooleanContents(isVec: false, isFloat: false) !=
513	TLI.getBooleanContents(isVec: false, isFloat: true)) {
514	// At least try the common case where the boolean is generated by a
515	// comparison.
516	if (Cond->getOpcode() == ISD::SETCC) {
517	EVT OpVT = Cond->getOperand(Num: 0).getValueType();
518	ScalarBool = TLI.getBooleanContents(Type: OpVT.getScalarType());
519	VecBool = TLI.getBooleanContents(Type: OpVT);
520	} else
521	ScalarBool = TargetLowering::UndefinedBooleanContent;
522	}
523
524	EVT CondVT = Cond.getValueType();
525	if (ScalarBool != VecBool) {
526	switch (ScalarBool) {
527	case TargetLowering::UndefinedBooleanContent:
528	break;
529	case TargetLowering::ZeroOrOneBooleanContent:
530	assert(VecBool == TargetLowering::UndefinedBooleanContent ||
531	VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent);
532	// Vector read from all ones, scalar expects a single 1 so mask.
533	Cond = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: CondVT,
534	N1: Cond, N2: DAG.getConstant(Val: 1, DL: SDLoc(N), VT: CondVT));
535	break;
536	case TargetLowering::ZeroOrNegativeOneBooleanContent:
537	assert(VecBool == TargetLowering::UndefinedBooleanContent ||
538	VecBool == TargetLowering::ZeroOrOneBooleanContent);
539	// Vector reads from a one, scalar from all ones so sign extend.
540	Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
541	Cond, DAG.getValueType(MVT::i1));
542	break;
543	}
544	}
545
546	// Truncate the condition if needed
547	auto BoolVT = getSetCCResultType(VT: CondVT);
548	if (BoolVT.bitsLT(VT: CondVT))
549	Cond = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: BoolVT, Operand: Cond);
550
551	return DAG.getSelect(DL: SDLoc(N),
552	VT: LHS.getValueType(), Cond, LHS,
553	RHS: GetScalarizedVector(Op: N->getOperand(Num: 2)));
554	}
555
556	SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
557	SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
558	return DAG.getSelect(DL: SDLoc(N),
559	VT: LHS.getValueType(), Cond: N->getOperand(Num: 0), LHS,
560	RHS: GetScalarizedVector(Op: N->getOperand(Num: 2)));
561	}
562
563	SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
564	SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 2));
565	return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: LHS.getValueType(),
566	N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1),
567	N3: LHS, N4: GetScalarizedVector(Op: N->getOperand(Num: 3)),
568	N5: N->getOperand(Num: 4));
569	}
570
571	SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
572	return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType());
573	}
574
575	SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
576	// Figure out if the scalar is the LHS or RHS and return it.
577	SDValue Arg = N->getOperand(Num: 2).getOperand(i: 0);
578	if (Arg.isUndef())
579	return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType());
580	unsigned Op = !cast<ConstantSDNode>(Val&: Arg)->isZero();
581	return GetScalarizedVector(Op: N->getOperand(Num: Op));
582	}
583
584	SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) {
585	SDValue Src = N->getOperand(Num: 0);
586	EVT SrcVT = Src.getValueType();
587	SDLoc dl(N);
588
589	// Handle case where result is scalarized but operand is not
590	if (getTypeAction(VT: SrcVT) == TargetLowering::TypeScalarizeVector)
591	Src = GetScalarizedVector(Op: Src);
592	else
593	Src = DAG.getNode(
594	Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: SrcVT.getVectorElementType(), N1: Src,
595	N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())));
596
597	EVT DstVT = N->getValueType(ResNo: 0).getVectorElementType();
598	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVT, N1: Src, N2: N->getOperand(Num: 1));
599	}
600
601	SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
602	assert(N->getValueType(0).isVector() &&
603	N->getOperand(0).getValueType().isVector() &&
604	"Operand types must be vectors");
605	SDValue LHS = N->getOperand(Num: 0);
606	SDValue RHS = N->getOperand(Num: 1);
607	EVT OpVT = LHS.getValueType();
608	EVT NVT = N->getValueType(ResNo: 0).getVectorElementType();
609	SDLoc DL(N);
610
611	// The result needs scalarizing, but it's not a given that the source does.
612	if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
613	LHS = GetScalarizedVector(Op: LHS);
614	RHS = GetScalarizedVector(Op: RHS);
615	} else {
616	EVT VT = OpVT.getVectorElementType();
617	LHS = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: LHS,
618	N2: DAG.getVectorIdxConstant(Val: 0, DL));
619	RHS = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: RHS,
620	N2: DAG.getVectorIdxConstant(Val: 0, DL));
621	}
622
623	// Turn it into a scalar SETCC.
624	SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
625	N->getOperand(Num: 2));
626	// Vectors may have a different boolean contents to scalars. Promote the
627	// value appropriately.
628	ISD::NodeType ExtendCode =
629	TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
630	return DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
631	}
632
633	SDValue DAGTypeLegalizer::ScalarizeVecRes_IS_FPCLASS(SDNode *N) {
634	SDLoc DL(N);
635	SDValue Arg = N->getOperand(Num: 0);
636	SDValue Test = N->getOperand(Num: 1);
637	EVT ArgVT = Arg.getValueType();
638	EVT ResultVT = N->getValueType(ResNo: 0).getVectorElementType();
639
640	if (getTypeAction(VT: ArgVT) == TargetLowering::TypeScalarizeVector) {
641	Arg = GetScalarizedVector(Op: Arg);
642	} else {
643	EVT VT = ArgVT.getVectorElementType();
644	Arg = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: Arg,
645	N2: DAG.getVectorIdxConstant(Val: 0, DL));
646	}
647
648	SDValue Res =
649	DAG.getNode(ISD::IS_FPCLASS, DL, MVT::i1, {Arg, Test}, N->getFlags());
650	// Vectors may have a different boolean contents to scalars. Promote the
651	// value appropriately.
652	ISD::NodeType ExtendCode =
653	TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: ArgVT));
654	return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: Res);
655	}
656
657	//===----------------------------------------------------------------------===//
658	// Operand Vector Scalarization <1 x ty> -> ty.
659	//===----------------------------------------------------------------------===//
660
661	bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
662	LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": ";
663	N->dump(&DAG));
664	SDValue Res = SDValue();
665
666	switch (N->getOpcode()) {
667	default:
668	#ifndef NDEBUG
669	dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
670	N->dump(G: &DAG);
671	dbgs() << "\n";
672	#endif
673	report_fatal_error(reason: "Do not know how to scalarize this operator's "
674	"operand!\n");
675	case ISD::BITCAST:
676	Res = ScalarizeVecOp_BITCAST(N);
677	break;
678	case ISD::ANY_EXTEND:
679	case ISD::ZERO_EXTEND:
680	case ISD::SIGN_EXTEND:
681	case ISD::TRUNCATE:
682	case ISD::FP_TO_SINT:
683	case ISD::FP_TO_UINT:
684	case ISD::SINT_TO_FP:
685	case ISD::UINT_TO_FP:
686	case ISD::LRINT:
687	case ISD::LLRINT:
688	Res = ScalarizeVecOp_UnaryOp(N);
689	break;
690	case ISD::STRICT_SINT_TO_FP:
691	case ISD::STRICT_UINT_TO_FP:
692	case ISD::STRICT_FP_TO_SINT:
693	case ISD::STRICT_FP_TO_UINT:
694	Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
695	break;
696	case ISD::CONCAT_VECTORS:
697	Res = ScalarizeVecOp_CONCAT_VECTORS(N);
698	break;
699	case ISD::EXTRACT_VECTOR_ELT:
700	Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
701	break;
702	case ISD::VSELECT:
703	Res = ScalarizeVecOp_VSELECT(N);
704	break;
705	case ISD::SETCC:
706	Res = ScalarizeVecOp_VSETCC(N);
707	break;
708	case ISD::STORE:
709	Res = ScalarizeVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo);
710	break;
711	case ISD::STRICT_FP_ROUND:
712	Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
713	break;
714	case ISD::FP_ROUND:
715	Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
716	break;
717	case ISD::STRICT_FP_EXTEND:
718	Res = ScalarizeVecOp_STRICT_FP_EXTEND(N);
719	break;
720	case ISD::FP_EXTEND:
721	Res = ScalarizeVecOp_FP_EXTEND(N);
722	break;
723	case ISD::VECREDUCE_FADD:
724	case ISD::VECREDUCE_FMUL:
725	case ISD::VECREDUCE_ADD:
726	case ISD::VECREDUCE_MUL:
727	case ISD::VECREDUCE_AND:
728	case ISD::VECREDUCE_OR:
729	case ISD::VECREDUCE_XOR:
730	case ISD::VECREDUCE_SMAX:
731	case ISD::VECREDUCE_SMIN:
732	case ISD::VECREDUCE_UMAX:
733	case ISD::VECREDUCE_UMIN:
734	case ISD::VECREDUCE_FMAX:
735	case ISD::VECREDUCE_FMIN:
736	case ISD::VECREDUCE_FMAXIMUM:
737	case ISD::VECREDUCE_FMINIMUM:
738	Res = ScalarizeVecOp_VECREDUCE(N);
739	break;
740	case ISD::VECREDUCE_SEQ_FADD:
741	case ISD::VECREDUCE_SEQ_FMUL:
742	Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
743	break;
744	}
745
746	// If the result is null, the sub-method took care of registering results etc.
747	if (!Res.getNode()) return false;
748
749	// If the result is N, the sub-method updated N in place. Tell the legalizer
750	// core about this.
751	if (Res.getNode() == N)
752	return true;
753
754	assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
755	"Invalid operand expansion");
756
757	ReplaceValueWith(From: SDValue(N, 0), To: Res);
758	return false;
759	}
760
761	/// If the value to convert is a vector that needs to be scalarized, it must be
762	/// <1 x ty>. Convert the element instead.
763	SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
764	SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
765	return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
766	VT: N->getValueType(ResNo: 0), Operand: Elt);
767	}
768
769	/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
770	/// Do the operation on the element instead.
771	SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
772	assert(N->getValueType(0).getVectorNumElements() == 1 &&
773	"Unexpected vector type!");
774	SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
775	SDValue Op = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
776	VT: N->getValueType(ResNo: 0).getScalarType(), Operand: Elt);
777	// Revectorize the result so the types line up with what the uses of this
778	// expression expect.
779	return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
780	}
781
782	/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
783	/// Do the strict FP operation on the element instead.
784	SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
785	assert(N->getValueType(0).getVectorNumElements() == 1 &&
786	"Unexpected vector type!");
787	SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
788	SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
789	{ N->getValueType(0).getScalarType(), MVT::Other },
790	{ N->getOperand(0), Elt });
791	// Legalize the chain result - switch anything that used the old chain to
792	// use the new one.
793	ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
794	// Revectorize the result so the types line up with what the uses of this
795	// expression expect.
796	Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
797
798	// Do our own replacement and return SDValue() to tell the caller that we
799	// handled all replacements since caller can only handle a single result.
800	ReplaceValueWith(From: SDValue(N, 0), To: Res);
801	return SDValue();
802	}
803
804	/// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
805	SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
806	SmallVector<SDValue, 8> Ops(N->getNumOperands());
807	for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
808	Ops[i] = GetScalarizedVector(Op: N->getOperand(Num: i));
809	return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL: SDLoc(N), Ops);
810	}
811
812	/// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
813	/// so just return the element, ignoring the index.
814	SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
815	EVT VT = N->getValueType(ResNo: 0);
816	SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
817	if (Res.getValueType() != VT)
818	Res = VT.isFloatingPoint()
819	? DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), VT, Operand: Res)
820	: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT, Operand: Res);
821	return Res;
822	}
823
824	/// If the input condition is a vector that needs to be scalarized, it must be
825	/// <1 x i1>, so just convert to a normal ISD::SELECT
826	/// (still with vector output type since that was acceptable if we got here).
827	SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
828	SDValue ScalarCond = GetScalarizedVector(Op: N->getOperand(Num: 0));
829	EVT VT = N->getValueType(ResNo: 0);
830
831	return DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT, N1: ScalarCond, N2: N->getOperand(Num: 1),
832	N3: N->getOperand(Num: 2));
833	}
834
835	/// If the operand is a vector that needs to be scalarized then the
836	/// result must be v1i1, so just convert to a scalar SETCC and wrap
837	/// with a scalar_to_vector since the res type is legal if we got here
838	SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
839	assert(N->getValueType(0).isVector() &&
840	N->getOperand(0).getValueType().isVector() &&
841	"Operand types must be vectors");
842	assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
843
844	EVT VT = N->getValueType(ResNo: 0);
845	SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
846	SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
847
848	EVT OpVT = N->getOperand(Num: 0).getValueType();
849	EVT NVT = VT.getVectorElementType();
850	SDLoc DL(N);
851	// Turn it into a scalar SETCC.
852	SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
853	N->getOperand(Num: 2));
854
855	// Vectors may have a different boolean contents to scalars. Promote the
856	// value appropriately.
857	ISD::NodeType ExtendCode =
858	TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
859
860	Res = DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
861
862	return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Res);
863	}
864
865	/// If the value to store is a vector that needs to be scalarized, it must be
866	/// <1 x ty>. Just store the element.
867	SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
868	assert(N->isUnindexed() && "Indexed store of one-element vector?");
869	assert(OpNo == 1 && "Do not know how to scalarize this operand!");
870	SDLoc dl(N);
871
872	if (N->isTruncatingStore())
873	return DAG.getTruncStore(
874	Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)),
875	Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(),
876	SVT: N->getMemoryVT().getVectorElementType(), Alignment: N->getOriginalAlign(),
877	MMOFlags: N->getMemOperand()->getFlags(), AAInfo: N->getAAInfo());
878
879	return DAG.getStore(Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)),
880	Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(),
881	Alignment: N->getOriginalAlign(), MMOFlags: N->getMemOperand()->getFlags(),
882	AAInfo: N->getAAInfo());
883	}
884
885	/// If the value to round is a vector that needs to be scalarized, it must be
886	/// <1 x ty>. Convert the element instead.
887	SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
888	assert(OpNo == 0 && "Wrong operand for scalarization!");
889	SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
890	SDValue Res = DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N),
891	VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Elt,
892	N2: N->getOperand(Num: 1));
893	return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
894	}
895
896	SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
897	unsigned OpNo) {
898	assert(OpNo == 1 && "Wrong operand for scalarization!");
899	SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
900	SDValue Res = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
901	{ N->getValueType(0).getVectorElementType(),
902	MVT::Other },
903	{ N->getOperand(0), Elt, N->getOperand(2) });
904	// Legalize the chain result - switch anything that used the old chain to
905	// use the new one.
906	ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
907
908	Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
909
910	// Do our own replacement and return SDValue() to tell the caller that we
911	// handled all replacements since caller can only handle a single result.
912	ReplaceValueWith(From: SDValue(N, 0), To: Res);
913	return SDValue();
914	}
915
916	/// If the value to extend is a vector that needs to be scalarized, it must be
917	/// <1 x ty>. Convert the element instead.
918	SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) {
919	SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
920	SDValue Res = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N),
921	VT: N->getValueType(ResNo: 0).getVectorElementType(), Operand: Elt);
922	return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
923	}
924
925	/// If the value to extend is a vector that needs to be scalarized, it must be
926	/// <1 x ty>. Convert the element instead.
927	SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) {
928	SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
929	SDValue Res =
930	DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N),
931	{N->getValueType(0).getVectorElementType(), MVT::Other},
932	{N->getOperand(0), Elt});
933	// Legalize the chain result - switch anything that used the old chain to
934	// use the new one.
935	ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
936
937	Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
938
939	// Do our own replacement and return SDValue() to tell the caller that we
940	// handled all replacements since caller can only handle a single result.
941	ReplaceValueWith(From: SDValue(N, 0), To: Res);
942	return SDValue();
943	}
944
945	SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
946	SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
947	// Result type may be wider than element type.
948	if (Res.getValueType() != N->getValueType(ResNo: 0))
949	Res = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
950	return Res;
951	}
952
953	SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
954	SDValue AccOp = N->getOperand(Num: 0);
955	SDValue VecOp = N->getOperand(Num: 1);
956
957	unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode());
958
959	SDValue Op = GetScalarizedVector(Op: VecOp);
960	return DAG.getNode(Opcode: BaseOpc, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
961	N1: AccOp, N2: Op, Flags: N->getFlags());
962	}
963
964	//===----------------------------------------------------------------------===//
965	// Result Vector Splitting
966	//===----------------------------------------------------------------------===//
967
968	/// This method is called when the specified result of the specified node is
969	/// found to need vector splitting. At this point, the node may also have
970	/// invalid operands or may have other results that need legalization, we just
971	/// know that (at least) one result needs vector splitting.
972	void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
973	LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG));
974	SDValue Lo, Hi;
975
976	// See if the target wants to custom expand this node.
977	if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
978	return;
979
980	switch (N->getOpcode()) {
981	default:
982	#ifndef NDEBUG
983	dbgs() << "SplitVectorResult #" << ResNo << ": ";
984	N->dump(G: &DAG);
985	dbgs() << "\n";
986	#endif
987	report_fatal_error(reason: "Do not know how to split the result of this "
988	"operator!\n");
989
990	case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
991	case ISD::AssertZext: SplitVecRes_AssertZext(N, Lo, Hi); break;
992	case ISD::VSELECT:
993	case ISD::SELECT:
994	case ISD::VP_MERGE:
995	case ISD::VP_SELECT: SplitRes_Select(N, Lo, Hi); break;
996	case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
997	case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
998	case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
999	case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
1000	case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
1001	case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
1002	case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
1003	case ISD::FPOWI:
1004	case ISD::FLDEXP:
1005	case ISD::FCOPYSIGN: SplitVecRes_FPOp_MultiType(N, Lo, Hi); break;
1006	case ISD::IS_FPCLASS: SplitVecRes_IS_FPCLASS(N, Lo, Hi); break;
1007	case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
1008	case ISD::SPLAT_VECTOR:
1009	case ISD::SCALAR_TO_VECTOR:
1010	SplitVecRes_ScalarOp(N, Lo, Hi);
1011	break;
1012	case ISD::STEP_VECTOR:
1013	SplitVecRes_STEP_VECTOR(N, Lo, Hi);
1014	break;
1015	case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
1016	case ISD::LOAD:
1017	SplitVecRes_LOAD(LD: cast<LoadSDNode>(Val: N), Lo, Hi);
1018	break;
1019	case ISD::VP_LOAD:
1020	SplitVecRes_VP_LOAD(LD: cast<VPLoadSDNode>(Val: N), Lo, Hi);
1021	break;
1022	case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1023	SplitVecRes_VP_STRIDED_LOAD(SLD: cast<VPStridedLoadSDNode>(Val: N), Lo, Hi);
1024	break;
1025	case ISD::MLOAD:
1026	SplitVecRes_MLOAD(MLD: cast<MaskedLoadSDNode>(Val: N), Lo, Hi);
1027	break;
1028	case ISD::MGATHER:
1029	case ISD::VP_GATHER:
1030	SplitVecRes_Gather(VPGT: cast<MemSDNode>(Val: N), Lo, Hi, /*SplitSETCC*/ true);
1031	break;
1032	case ISD::SETCC:
1033	case ISD::VP_SETCC:
1034	SplitVecRes_SETCC(N, Lo, Hi);
1035	break;
1036	case ISD::VECTOR_REVERSE:
1037	SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
1038	break;
1039	case ISD::VECTOR_SHUFFLE:
1040	SplitVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N), Lo, Hi);
1041	break;
1042	case ISD::VECTOR_SPLICE:
1043	SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
1044	break;
1045	case ISD::VECTOR_DEINTERLEAVE:
1046	SplitVecRes_VECTOR_DEINTERLEAVE(N);
1047	return;
1048	case ISD::VECTOR_INTERLEAVE:
1049	SplitVecRes_VECTOR_INTERLEAVE(N);
1050	return;
1051	case ISD::VAARG:
1052	SplitVecRes_VAARG(N, Lo, Hi);
1053	break;
1054
1055	case ISD::ANY_EXTEND_VECTOR_INREG:
1056	case ISD::SIGN_EXTEND_VECTOR_INREG:
1057	case ISD::ZERO_EXTEND_VECTOR_INREG:
1058	SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1059	break;
1060
1061	case ISD::ABS:
1062	case ISD::VP_ABS:
1063	case ISD::BITREVERSE:
1064	case ISD::VP_BITREVERSE:
1065	case ISD::BSWAP:
1066	case ISD::VP_BSWAP:
1067	case ISD::CTLZ:
1068	case ISD::VP_CTLZ:
1069	case ISD::CTTZ:
1070	case ISD::VP_CTTZ:
1071	case ISD::CTLZ_ZERO_UNDEF:
1072	case ISD::VP_CTLZ_ZERO_UNDEF:
1073	case ISD::CTTZ_ZERO_UNDEF:
1074	case ISD::VP_CTTZ_ZERO_UNDEF:
1075	case ISD::CTPOP:
1076	case ISD::VP_CTPOP:
1077	case ISD::FABS: case ISD::VP_FABS:
1078	case ISD::FCEIL:
1079	case ISD::VP_FCEIL:
1080	case ISD::FCOS:
1081	case ISD::FEXP:
1082	case ISD::FEXP2:
1083	case ISD::FEXP10:
1084	case ISD::FFLOOR:
1085	case ISD::VP_FFLOOR:
1086	case ISD::FLOG:
1087	case ISD::FLOG10:
1088	case ISD::FLOG2:
1089	case ISD::FNEARBYINT:
1090	case ISD::VP_FNEARBYINT:
1091	case ISD::FNEG: case ISD::VP_FNEG:
1092	case ISD::FREEZE:
1093	case ISD::ARITH_FENCE:
1094	case ISD::FP_EXTEND:
1095	case ISD::VP_FP_EXTEND:
1096	case ISD::FP_ROUND:
1097	case ISD::VP_FP_ROUND:
1098	case ISD::FP_TO_SINT:
1099	case ISD::VP_FP_TO_SINT:
1100	case ISD::FP_TO_UINT:
1101	case ISD::VP_FP_TO_UINT:
1102	case ISD::FRINT:
1103	case ISD::VP_FRINT:
1104	case ISD::LRINT:
1105	case ISD::LLRINT:
1106	case ISD::FROUND:
1107	case ISD::VP_FROUND:
1108	case ISD::FROUNDEVEN:
1109	case ISD::VP_FROUNDEVEN:
1110	case ISD::FSIN:
1111	case ISD::FSQRT: case ISD::VP_SQRT:
1112	case ISD::FTRUNC:
1113	case ISD::VP_FROUNDTOZERO:
1114	case ISD::SINT_TO_FP:
1115	case ISD::VP_SINT_TO_FP:
1116	case ISD::TRUNCATE:
1117	case ISD::VP_TRUNCATE:
1118	case ISD::UINT_TO_FP:
1119	case ISD::VP_UINT_TO_FP:
1120	case ISD::FCANONICALIZE:
1121	SplitVecRes_UnaryOp(N, Lo, Hi);
1122	break;
1123	case ISD::FFREXP:
1124	SplitVecRes_FFREXP(N, ResNo, Lo, Hi);
1125	break;
1126
1127	case ISD::ANY_EXTEND:
1128	case ISD::SIGN_EXTEND:
1129	case ISD::ZERO_EXTEND:
1130	case ISD::VP_SIGN_EXTEND:
1131	case ISD::VP_ZERO_EXTEND:
1132	SplitVecRes_ExtendOp(N, Lo, Hi);
1133	break;
1134
1135	case ISD::ADD: case ISD::VP_ADD:
1136	case ISD::SUB: case ISD::VP_SUB:
1137	case ISD::MUL: case ISD::VP_MUL:
1138	case ISD::MULHS:
1139	case ISD::MULHU:
1140	case ISD::FADD: case ISD::VP_FADD:
1141	case ISD::FSUB: case ISD::VP_FSUB:
1142	case ISD::FMUL: case ISD::VP_FMUL:
1143	case ISD::FMINNUM: case ISD::VP_FMINNUM:
1144	case ISD::FMAXNUM: case ISD::VP_FMAXNUM:
1145	case ISD::FMINIMUM:
1146	case ISD::VP_FMINIMUM:
1147	case ISD::FMAXIMUM:
1148	case ISD::VP_FMAXIMUM:
1149	case ISD::SDIV: case ISD::VP_SDIV:
1150	case ISD::UDIV: case ISD::VP_UDIV:
1151	case ISD::FDIV: case ISD::VP_FDIV:
1152	case ISD::FPOW:
1153	case ISD::AND: case ISD::VP_AND:
1154	case ISD::OR: case ISD::VP_OR:
1155	case ISD::XOR: case ISD::VP_XOR:
1156	case ISD::SHL: case ISD::VP_SHL:
1157	case ISD::SRA: case ISD::VP_ASHR:
1158	case ISD::SRL: case ISD::VP_LSHR:
1159	case ISD::UREM: case ISD::VP_UREM:
1160	case ISD::SREM: case ISD::VP_SREM:
1161	case ISD::FREM: case ISD::VP_FREM:
1162	case ISD::SMIN: case ISD::VP_SMIN:
1163	case ISD::SMAX: case ISD::VP_SMAX:
1164	case ISD::UMIN: case ISD::VP_UMIN:
1165	case ISD::UMAX: case ISD::VP_UMAX:
1166	case ISD::SADDSAT:
1167	case ISD::UADDSAT:
1168	case ISD::SSUBSAT:
1169	case ISD::USUBSAT:
1170	case ISD::SSHLSAT:
1171	case ISD::USHLSAT:
1172	case ISD::ROTL:
1173	case ISD::ROTR:
1174	case ISD::VP_FCOPYSIGN:
1175	SplitVecRes_BinOp(N, Lo, Hi);
1176	break;
1177	case ISD::FMA: case ISD::VP_FMA:
1178	case ISD::FSHL:
1179	case ISD::VP_FSHL:
1180	case ISD::FSHR:
1181	case ISD::VP_FSHR:
1182	SplitVecRes_TernaryOp(N, Lo, Hi);
1183	break;
1184
1185	#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1186	case ISD::STRICT_##DAGN:
1187	#include "llvm/IR/ConstrainedOps.def"
1188	SplitVecRes_StrictFPOp(N, Lo, Hi);
1189	break;
1190
1191	case ISD::FP_TO_UINT_SAT:
1192	case ISD::FP_TO_SINT_SAT:
1193	SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi);
1194	break;
1195
1196	case ISD::UADDO:
1197	case ISD::SADDO:
1198	case ISD::USUBO:
1199	case ISD::SSUBO:
1200	case ISD::UMULO:
1201	case ISD::SMULO:
1202	SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
1203	break;
1204	case ISD::SMULFIX:
1205	case ISD::SMULFIXSAT:
1206	case ISD::UMULFIX:
1207	case ISD::UMULFIXSAT:
1208	case ISD::SDIVFIX:
1209	case ISD::SDIVFIXSAT:
1210	case ISD::UDIVFIX:
1211	case ISD::UDIVFIXSAT:
1212	SplitVecRes_FIX(N, Lo, Hi);
1213	break;
1214	case ISD::EXPERIMENTAL_VP_REVERSE:
1215	SplitVecRes_VP_REVERSE(N, Lo, Hi);
1216	break;
1217	}
1218
1219	// If Lo/Hi is null, the sub-method took care of registering results etc.
1220	if (Lo.getNode())
1221	SetSplitVector(Op: SDValue(N, ResNo), Lo, Hi);
1222	}
1223
1224	void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1225	MachinePointerInfo &MPI, SDValue &Ptr,
1226	uint64_t *ScaledOffset) {
1227	SDLoc DL(N);
1228	unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinValue() / 8;
1229
1230	if (MemVT.isScalableVector()) {
1231	SDNodeFlags Flags;
1232	SDValue BytesIncrement = DAG.getVScale(
1233	DL, VT: Ptr.getValueType(),
1234	MulImm: APInt(Ptr.getValueSizeInBits().getFixedValue(), IncrementSize));
1235	MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1236	Flags.setNoUnsignedWrap(true);
1237	if (ScaledOffset)
1238	*ScaledOffset += IncrementSize;
1239	Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: Ptr.getValueType(), N1: Ptr, N2: BytesIncrement,
1240	Flags);
1241	} else {
1242	MPI = N->getPointerInfo().getWithOffset(O: IncrementSize);
1243	// Increment the pointer to the other half.
1244	Ptr = DAG.getObjectPtrOffset(SL: DL, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
1245	}
1246	}
1247
1248	std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask) {
1249	return SplitMask(Mask, DL: SDLoc(Mask));
1250	}
1251
1252	std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask,
1253	const SDLoc &DL) {
1254	SDValue MaskLo, MaskHi;
1255	EVT MaskVT = Mask.getValueType();
1256	if (getTypeAction(VT: MaskVT) == TargetLowering::TypeSplitVector)
1257	GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
1258	else
1259	std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
1260	return std::make_pair(x&: MaskLo, y&: MaskHi);
1261	}
1262
1263	void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi) {
1264	SDValue LHSLo, LHSHi;
1265	GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1266	SDValue RHSLo, RHSHi;
1267	GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1268	SDLoc dl(N);
1269
1270	const SDNodeFlags Flags = N->getFlags();
1271	unsigned Opcode = N->getOpcode();
1272	if (N->getNumOperands() == 2) {
1273	Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, Flags);
1274	Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, Flags);
1275	return;
1276	}
1277
1278	assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1279	assert(N->isVPOpcode() && "Expected VP opcode");
1280
1281	SDValue MaskLo, MaskHi;
1282	std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2));
1283
1284	SDValue EVLLo, EVLHi;
1285	std::tie(args&: EVLLo, args&: EVLHi) =
1286	DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT: N->getValueType(ResNo: 0), DL: dl);
1287
1288	Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(),
1289	Ops: {LHSLo, RHSLo, MaskLo, EVLLo}, Flags);
1290	Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(),
1291	Ops: {LHSHi, RHSHi, MaskHi, EVLHi}, Flags);
1292	}
1293
1294	void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1295	SDValue &Hi) {
1296	SDValue Op0Lo, Op0Hi;
1297	GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Op0Lo, Hi&: Op0Hi);
1298	SDValue Op1Lo, Op1Hi;
1299	GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Op1Lo, Hi&: Op1Hi);
1300	SDValue Op2Lo, Op2Hi;
1301	GetSplitVector(Op: N->getOperand(Num: 2), Lo&: Op2Lo, Hi&: Op2Hi);
1302	SDLoc dl(N);
1303
1304	const SDNodeFlags Flags = N->getFlags();
1305	unsigned Opcode = N->getOpcode();
1306	if (N->getNumOperands() == 3) {
1307	Lo = DAG.getNode(Opcode, DL: dl, VT: Op0Lo.getValueType(), N1: Op0Lo, N2: Op1Lo, N3: Op2Lo, Flags);
1308	Hi = DAG.getNode(Opcode, DL: dl, VT: Op0Hi.getValueType(), N1: Op0Hi, N2: Op1Hi, N3: Op2Hi, Flags);
1309	return;
1310	}
1311
1312	assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
1313	assert(N->isVPOpcode() && "Expected VP opcode");
1314
1315	SDValue MaskLo, MaskHi;
1316	std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 3));
1317
1318	SDValue EVLLo, EVLHi;
1319	std::tie(args&: EVLLo, args&: EVLHi) =
1320	DAG.SplitEVL(N: N->getOperand(Num: 4), VecVT: N->getValueType(ResNo: 0), DL: dl);
1321
1322	Lo = DAG.getNode(Opcode, DL: dl, VT: Op0Lo.getValueType(),
1323	Ops: {Op0Lo, Op1Lo, Op2Lo, MaskLo, EVLLo}, Flags);
1324	Hi = DAG.getNode(Opcode, DL: dl, VT: Op0Hi.getValueType(),
1325	Ops: {Op0Hi, Op1Hi, Op2Hi, MaskHi, EVLHi}, Flags);
1326	}
1327
1328	void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1329	SDValue LHSLo, LHSHi;
1330	GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1331	SDValue RHSLo, RHSHi;
1332	GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1333	SDLoc dl(N);
1334	SDValue Op2 = N->getOperand(Num: 2);
1335
1336	unsigned Opcode = N->getOpcode();
1337	Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, N3: Op2,
1338	Flags: N->getFlags());
1339	Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, N3: Op2,
1340	Flags: N->getFlags());
1341	}
1342
1343	void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1344	SDValue &Hi) {
1345	// We know the result is a vector. The input may be either a vector or a
1346	// scalar value.
1347	EVT LoVT, HiVT;
1348	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1349	SDLoc dl(N);
1350
1351	SDValue InOp = N->getOperand(Num: 0);
1352	EVT InVT = InOp.getValueType();
1353
1354	// Handle some special cases efficiently.
1355	switch (getTypeAction(VT: InVT)) {
1356	case TargetLowering::TypeLegal:
1357	case TargetLowering::TypePromoteInteger:
1358	case TargetLowering::TypePromoteFloat:
1359	case TargetLowering::TypeSoftPromoteHalf:
1360	case TargetLowering::TypeSoftenFloat:
1361	case TargetLowering::TypeScalarizeVector:
1362	case TargetLowering::TypeWidenVector:
1363	break;
1364	case TargetLowering::TypeExpandInteger:
1365	case TargetLowering::TypeExpandFloat:
1366	// A scalar to vector conversion, where the scalar needs expansion.
1367	// If the vector is being split in two then we can just convert the
1368	// expanded pieces.
1369	if (LoVT == HiVT) {
1370	GetExpandedOp(Op: InOp, Lo, Hi);
1371	if (DAG.getDataLayout().isBigEndian())
1372	std::swap(a&: Lo, b&: Hi);
1373	Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1374	Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1375	return;
1376	}
1377	break;
1378	case TargetLowering::TypeSplitVector:
1379	// If the input is a vector that needs to be split, convert each split
1380	// piece of the input now.
1381	GetSplitVector(Op: InOp, Lo, Hi);
1382	Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1383	Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1384	return;
1385	case TargetLowering::TypeScalarizeScalableVector:
1386	report_fatal_error(reason: "Scalarization of scalable vectors is not supported.");
1387	}
1388
1389	// In the general case, convert the input to an integer and split it by hand.
1390	EVT LoIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoVT.getSizeInBits());
1391	EVT HiIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: HiVT.getSizeInBits());
1392	if (DAG.getDataLayout().isBigEndian())
1393	std::swap(a&: LoIntVT, b&: HiIntVT);
1394
1395	SplitInteger(Op: BitConvertToInteger(Op: InOp), LoVT: LoIntVT, HiVT: HiIntVT, Lo, Hi);
1396
1397	if (DAG.getDataLayout().isBigEndian())
1398	std::swap(a&: Lo, b&: Hi);
1399	Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1400	Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1401	}
1402
1403	void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1404	SDValue &Hi) {
1405	EVT LoVT, HiVT;
1406	SDLoc dl(N);
1407	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1408	unsigned LoNumElts = LoVT.getVectorNumElements();
1409	SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1410	Lo = DAG.getBuildVector(VT: LoVT, DL: dl, Ops: LoOps);
1411
1412	SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1413	Hi = DAG.getBuildVector(VT: HiVT, DL: dl, Ops: HiOps);
1414	}
1415
1416	void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1417	SDValue &Hi) {
1418	assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1419	SDLoc dl(N);
1420	unsigned NumSubvectors = N->getNumOperands() / 2;
1421	if (NumSubvectors == 1) {
1422	Lo = N->getOperand(Num: 0);
1423	Hi = N->getOperand(Num: 1);
1424	return;
1425	}
1426
1427	EVT LoVT, HiVT;
1428	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1429
1430	SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1431	Lo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: LoVT, Ops: LoOps);
1432
1433	SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1434	Hi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: HiVT, Ops: HiOps);
1435	}
1436
1437	void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1438	SDValue &Hi) {
1439	SDValue Vec = N->getOperand(Num: 0);
1440	SDValue Idx = N->getOperand(Num: 1);
1441	SDLoc dl(N);
1442
1443	EVT LoVT, HiVT;
1444	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1445
1446	Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: LoVT, N1: Vec, N2: Idx);
1447	uint64_t IdxVal = Idx->getAsZExtVal();
1448	Hi = DAG.getNode(
1449	Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: HiVT, N1: Vec,
1450	N2: DAG.getVectorIdxConstant(Val: IdxVal + LoVT.getVectorMinNumElements(), DL: dl));
1451	}
1452
1453	void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1454	SDValue &Hi) {
1455	SDValue Vec = N->getOperand(Num: 0);
1456	SDValue SubVec = N->getOperand(Num: 1);
1457	SDValue Idx = N->getOperand(Num: 2);
1458	SDLoc dl(N);
1459	GetSplitVector(Op: Vec, Lo, Hi);
1460
1461	EVT VecVT = Vec.getValueType();
1462	EVT LoVT = Lo.getValueType();
1463	EVT SubVecVT = SubVec.getValueType();
1464	unsigned VecElems = VecVT.getVectorMinNumElements();
1465	unsigned SubElems = SubVecVT.getVectorMinNumElements();
1466	unsigned LoElems = LoVT.getVectorMinNumElements();
1467
1468	// If we know the index is in the first half, and we know the subvector
1469	// doesn't cross the boundary between the halves, we can avoid spilling the
1470	// vector, and insert into the lower half of the split vector directly.
1471	unsigned IdxVal = Idx->getAsZExtVal();
1472	if (IdxVal + SubElems <= LoElems) {
1473	Lo = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: LoVT, N1: Lo, N2: SubVec, N3: Idx);
1474	return;
1475	}
1476	// Similarly if the subvector is fully in the high half, but mind that we
1477	// can't tell whether a fixed-length subvector is fully within the high half
1478	// of a scalable vector.
1479	if (VecVT.isScalableVector() == SubVecVT.isScalableVector() &&
1480	IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
1481	Hi = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: Hi.getValueType(), N1: Hi, N2: SubVec,
1482	N3: DAG.getVectorIdxConstant(Val: IdxVal - LoElems, DL: dl));
1483	return;
1484	}
1485
1486	// Spill the vector to the stack.
1487	// In cases where the vector is illegal it will be broken down into parts
1488	// and stored in parts - we should use the alignment for the smallest part.
1489	Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
1490	SDValue StackPtr =
1491	DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
1492	auto &MF = DAG.getMachineFunction();
1493	auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1494	auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
1495
1496	SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
1497	Alignment: SmallestAlign);
1498
1499	// Store the new subvector into the specified index.
1500	SDValue SubVecPtr =
1501	TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT, Index: Idx);
1502	Store = DAG.getStore(Chain: Store, dl, Val: SubVec, Ptr: SubVecPtr,
1503	PtrInfo: MachinePointerInfo::getUnknownStack(MF));
1504
1505	// Load the Lo part from the stack slot.
1506	Lo = DAG.getLoad(VT: Lo.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo,
1507	Alignment: SmallestAlign);
1508
1509	// Increment the pointer to the other part.
1510	auto *Load = cast<LoadSDNode>(Val&: Lo);
1511	MachinePointerInfo MPI = Load->getPointerInfo();
1512	IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr);
1513
1514	// Load the Hi part from the stack slot.
1515	Hi = DAG.getLoad(VT: Hi.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign);
1516	}
1517
1518	// Handle splitting an FP where the second operand does not match the first
1519	// type. The second operand may be a scalar, or a vector that has exactly as
1520	// many elements as the first
1521	void DAGTypeLegalizer::SplitVecRes_FPOp_MultiType(SDNode *N, SDValue &Lo,
1522	SDValue &Hi) {
1523	SDValue LHSLo, LHSHi;
1524	GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1525	SDLoc DL(N);
1526
1527	SDValue RHSLo, RHSHi;
1528	SDValue RHS = N->getOperand(Num: 1);
1529	EVT RHSVT = RHS.getValueType();
1530	if (RHSVT.isVector()) {
1531	if (getTypeAction(VT: RHSVT) == TargetLowering::TypeSplitVector)
1532	GetSplitVector(Op: RHS, Lo&: RHSLo, Hi&: RHSHi);
1533	else
1534	std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: RHS, DL: SDLoc(RHS));
1535
1536	Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo);
1537	Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi);
1538	} else {
1539	Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHS);
1540	Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHS);
1541	}
1542	}
1543
1544	void DAGTypeLegalizer::SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo,
1545	SDValue &Hi) {
1546	SDLoc DL(N);
1547	SDValue ArgLo, ArgHi;
1548	SDValue Test = N->getOperand(Num: 1);
1549	SDValue FpValue = N->getOperand(Num: 0);
1550	if (getTypeAction(VT: FpValue.getValueType()) == TargetLowering::TypeSplitVector)
1551	GetSplitVector(Op: FpValue, Lo&: ArgLo, Hi&: ArgHi);
1552	else
1553	std::tie(args&: ArgLo, args&: ArgHi) = DAG.SplitVector(N: FpValue, DL: SDLoc(FpValue));
1554	EVT LoVT, HiVT;
1555	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1556
1557	Lo = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: LoVT, N1: ArgLo, N2: Test, Flags: N->getFlags());
1558	Hi = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: HiVT, N1: ArgHi, N2: Test, Flags: N->getFlags());
1559	}
1560
1561	void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1562	SDValue &Hi) {
1563	SDValue LHSLo, LHSHi;
1564	GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1565	SDLoc dl(N);
1566
1567	EVT LoVT, HiVT;
1568	std::tie(args&: LoVT, args&: HiVT) =
1569	DAG.GetSplitDestVTs(VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT());
1570
1571	Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSLo.getValueType(), N1: LHSLo,
1572	N2: DAG.getValueType(LoVT));
1573	Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSHi.getValueType(), N1: LHSHi,
1574	N2: DAG.getValueType(HiVT));
1575	}
1576
1577	void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1578	SDValue &Hi) {
1579	unsigned Opcode = N->getOpcode();
1580	SDValue N0 = N->getOperand(Num: 0);
1581
1582	SDLoc dl(N);
1583	SDValue InLo, InHi;
1584
1585	if (getTypeAction(VT: N0.getValueType()) == TargetLowering::TypeSplitVector)
1586	GetSplitVector(Op: N0, Lo&: InLo, Hi&: InHi);
1587	else
1588	std::tie(args&: InLo, args&: InHi) = DAG.SplitVectorOperand(N, OpNo: 0);
1589
1590	EVT InLoVT = InLo.getValueType();
1591	unsigned InNumElements = InLoVT.getVectorNumElements();
1592
1593	EVT OutLoVT, OutHiVT;
1594	std::tie(args&: OutLoVT, args&: OutHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1595	unsigned OutNumElements = OutLoVT.getVectorNumElements();
1596	assert((2 * OutNumElements) <= InNumElements &&
1597	"Illegal extend vector in reg split");
1598
1599	// *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1600	// input vector (i.e. we only use InLo):
1601	// OutLo will extend the first OutNumElements from InLo.
1602	// OutHi will extend the next OutNumElements from InLo.
1603
1604	// Shuffle the elements from InLo for OutHi into the bottom elements to
1605	// create a 'fake' InHi.
1606	SmallVector<int, 8> SplitHi(InNumElements, -1);
1607	for (unsigned i = 0; i != OutNumElements; ++i)
1608	SplitHi[i] = i + OutNumElements;
1609	InHi = DAG.getVectorShuffle(VT: InLoVT, dl, N1: InLo, N2: DAG.getUNDEF(VT: InLoVT), Mask: SplitHi);
1610
1611	Lo = DAG.getNode(Opcode, DL: dl, VT: OutLoVT, Operand: InLo);
1612	Hi = DAG.getNode(Opcode, DL: dl, VT: OutHiVT, Operand: InHi);
1613	}
1614
1615	void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1616	SDValue &Hi) {
1617	unsigned NumOps = N->getNumOperands();
1618	SDValue Chain = N->getOperand(Num: 0);
1619	EVT LoVT, HiVT;
1620	SDLoc dl(N);
1621	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1622
1623	SmallVector<SDValue, 4> OpsLo(NumOps);
1624	SmallVector<SDValue, 4> OpsHi(NumOps);
1625
1626	// The Chain is the first operand.
1627	OpsLo[0] = Chain;
1628	OpsHi[0] = Chain;
1629
1630	// Now process the remaining operands.
1631	for (unsigned i = 1; i < NumOps; ++i) {
1632	SDValue Op = N->getOperand(Num: i);
1633	SDValue OpLo = Op;
1634	SDValue OpHi = Op;
1635
1636	EVT InVT = Op.getValueType();
1637	if (InVT.isVector()) {
1638	// If the input also splits, handle it directly for a
1639	// compile time speedup. Otherwise split it by hand.
1640	if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
1641	GetSplitVector(Op, Lo&: OpLo, Hi&: OpHi);
1642	else
1643	std::tie(args&: OpLo, args&: OpHi) = DAG.SplitVectorOperand(N, OpNo: i);
1644	}
1645
1646	OpsLo[i] = OpLo;
1647	OpsHi[i] = OpHi;
1648	}
1649
1650	EVT LoValueVTs[] = {LoVT, MVT::Other};
1651	EVT HiValueVTs[] = {HiVT, MVT::Other};
1652	Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(LoValueVTs), Ops: OpsLo,
1653	Flags: N->getFlags());
1654	Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(HiValueVTs), Ops: OpsHi,
1655	Flags: N->getFlags());
1656
1657	// Build a factor node to remember that this Op is independent of the
1658	// other one.
1659	Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1660	Lo.getValue(R: 1), Hi.getValue(R: 1));
1661
1662	// Legalize the chain result - switch anything that used the old chain to
1663	// use the new one.
1664	ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1665	}
1666
1667	SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
1668	SDValue Chain = N->getOperand(Num: 0);
1669	EVT VT = N->getValueType(ResNo: 0);
1670	unsigned NE = VT.getVectorNumElements();
1671	EVT EltVT = VT.getVectorElementType();
1672	SDLoc dl(N);
1673
1674	SmallVector<SDValue, 8> Scalars;
1675	SmallVector<SDValue, 4> Operands(N->getNumOperands());
1676
1677	// If ResNE is 0, fully unroll the vector op.
1678	if (ResNE == 0)
1679	ResNE = NE;
1680	else if (NE > ResNE)
1681	NE = ResNE;
1682
1683	//The results of each unrolled operation, including the chain.
1684	EVT ChainVTs[] = {EltVT, MVT::Other};
1685	SmallVector<SDValue, 8> Chains;
1686
1687	unsigned i;
1688	for (i = 0; i != NE; ++i) {
1689	Operands[0] = Chain;
1690	for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
1691	SDValue Operand = N->getOperand(Num: j);
1692	EVT OperandVT = Operand.getValueType();
1693	if (OperandVT.isVector()) {
1694	EVT OperandEltVT = OperandVT.getVectorElementType();
1695	Operands[j] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: OperandEltVT,
1696	N1: Operand, N2: DAG.getVectorIdxConstant(Val: i, DL: dl));
1697	} else {
1698	Operands[j] = Operand;
1699	}
1700	}
1701	SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands);
1702	Scalar.getNode()->setFlags(N->getFlags());
1703
1704	//Add in the scalar as well as its chain value to the
1705	//result vectors.
1706	Scalars.push_back(Elt: Scalar);
1707	Chains.push_back(Elt: Scalar.getValue(R: 1));
1708	}
1709
1710	for (; i < ResNE; ++i)
1711	Scalars.push_back(Elt: DAG.getUNDEF(VT: EltVT));
1712
1713	// Build a new factor node to connect the chain back together.
1714	Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1715	ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1716
1717	// Create a new BUILD_VECTOR node
1718	EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: ResNE);
1719	return DAG.getBuildVector(VT: VecVT, DL: dl, Ops: Scalars);
1720	}
1721
1722	void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
1723	SDValue &Lo, SDValue &Hi) {
1724	SDLoc dl(N);
1725	EVT ResVT = N->getValueType(ResNo: 0);
1726	EVT OvVT = N->getValueType(ResNo: 1);
1727	EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
1728	std::tie(args&: LoResVT, args&: HiResVT) = DAG.GetSplitDestVTs(VT: ResVT);
1729	std::tie(args&: LoOvVT, args&: HiOvVT) = DAG.GetSplitDestVTs(VT: OvVT);
1730
1731	SDValue LoLHS, HiLHS, LoRHS, HiRHS;
1732	if (getTypeAction(VT: ResVT) == TargetLowering::TypeSplitVector) {
1733	GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LoLHS, Hi&: HiLHS);
1734	GetSplitVector(Op: N->getOperand(Num: 1), Lo&: LoRHS, Hi&: HiRHS);
1735	} else {
1736	std::tie(args&: LoLHS, args&: HiLHS) = DAG.SplitVectorOperand(N, OpNo: 0);
1737	std::tie(args&: LoRHS, args&: HiRHS) = DAG.SplitVectorOperand(N, OpNo: 1);
1738	}
1739
1740	unsigned Opcode = N->getOpcode();
1741	SDVTList LoVTs = DAG.getVTList(VT1: LoResVT, VT2: LoOvVT);
1742	SDVTList HiVTs = DAG.getVTList(VT1: HiResVT, VT2: HiOvVT);
1743	SDNode *LoNode = DAG.getNode(Opcode, DL: dl, VTList: LoVTs, N1: LoLHS, N2: LoRHS).getNode();
1744	SDNode *HiNode = DAG.getNode(Opcode, DL: dl, VTList: HiVTs, N1: HiLHS, N2: HiRHS).getNode();
1745	LoNode->setFlags(N->getFlags());
1746	HiNode->setFlags(N->getFlags());
1747
1748	Lo = SDValue(LoNode, ResNo);
1749	Hi = SDValue(HiNode, ResNo);
1750
1751	// Replace the other vector result not being explicitly split here.
1752	unsigned OtherNo = 1 - ResNo;
1753	EVT OtherVT = N->getValueType(ResNo: OtherNo);
1754	if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) {
1755	SetSplitVector(Op: SDValue(N, OtherNo),
1756	Lo: SDValue(LoNode, OtherNo), Hi: SDValue(HiNode, OtherNo));
1757	} else {
1758	SDValue OtherVal = DAG.getNode(
1759	Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT,
1760	N1: SDValue(LoNode, OtherNo), N2: SDValue(HiNode, OtherNo));
1761	ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
1762	}
1763	}
1764
1765	void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1766	SDValue &Hi) {
1767	SDValue Vec = N->getOperand(Num: 0);
1768	SDValue Elt = N->getOperand(Num: 1);
1769	SDValue Idx = N->getOperand(Num: 2);
1770	SDLoc dl(N);
1771	GetSplitVector(Op: Vec, Lo, Hi);
1772
1773	if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Val&: Idx)) {
1774	unsigned IdxVal = CIdx->getZExtValue();
1775	unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
1776	if (IdxVal < LoNumElts) {
1777	Lo = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl,
1778	VT: Lo.getValueType(), N1: Lo, N2: Elt, N3: Idx);
1779	return;
1780	} else if (!Vec.getValueType().isScalableVector()) {
1781	Hi = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: Hi.getValueType(), N1: Hi, N2: Elt,
1782	N3: DAG.getVectorIdxConstant(Val: IdxVal - LoNumElts, DL: dl));
1783	return;
1784	}
1785	}
1786
1787	// See if the target wants to custom expand this node.
1788	if (CustomLowerNode(N, VT: N->getValueType(ResNo: 0), LegalizeResult: true))
1789	return;
1790
1791	// Make the vector elements byte-addressable if they aren't already.
1792	EVT VecVT = Vec.getValueType();
1793	EVT EltVT = VecVT.getVectorElementType();
1794	if (VecVT.getScalarSizeInBits() < 8) {
1795	EltVT = MVT::i8;
1796	VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
1797	EC: VecVT.getVectorElementCount());
1798	Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec);
1799	// Extend the element type to match if needed.
1800	if (EltVT.bitsGT(VT: Elt.getValueType()))
1801	Elt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: EltVT, Operand: Elt);
1802	}
1803
1804	// Spill the vector to the stack.
1805	// In cases where the vector is illegal it will be broken down into parts
1806	// and stored in parts - we should use the alignment for the smallest part.
1807	Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
1808	SDValue StackPtr =
1809	DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
1810	auto &MF = DAG.getMachineFunction();
1811	auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1812	auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
1813
1814	SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
1815	Alignment: SmallestAlign);
1816
1817	// Store the new element. This may be larger than the vector element type,
1818	// so use a truncating store.
1819	SDValue EltPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
1820	Store = DAG.getTruncStore(
1821	Chain: Store, dl, Val: Elt, Ptr: EltPtr, PtrInfo: MachinePointerInfo::getUnknownStack(MF), SVT: EltVT,
1822	Alignment: commonAlignment(A: SmallestAlign,
1823	Offset: EltVT.getFixedSizeInBits() / 8));
1824
1825	EVT LoVT, HiVT;
1826	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: VecVT);
1827
1828	// Load the Lo part from the stack slot.
1829	Lo = DAG.getLoad(VT: LoVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo, Alignment: SmallestAlign);
1830
1831	// Increment the pointer to the other part.
1832	auto Load = cast<LoadSDNode>(Val&: Lo);
1833	MachinePointerInfo MPI = Load->getPointerInfo();
1834	IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr);
1835
1836	Hi = DAG.getLoad(VT: HiVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign);
1837
1838	// If we adjusted the original type, we need to truncate the results.
1839	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1840	if (LoVT != Lo.getValueType())
1841	Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LoVT, Operand: Lo);
1842	if (HiVT != Hi.getValueType())
1843	Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiVT, Operand: Hi);
1844	}
1845
1846	void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
1847	SDValue &Hi) {
1848	EVT LoVT, HiVT;
1849	SDLoc dl(N);
1850	assert(N->getValueType(0).isScalableVector() &&
1851	"Only scalable vectors are supported for STEP_VECTOR");
1852	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1853	SDValue Step = N->getOperand(Num: 0);
1854
1855	Lo = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: LoVT, Operand: Step);
1856
1857	// Hi = Lo + (EltCnt * Step)
1858	EVT EltVT = Step.getValueType();
1859	APInt StepVal = Step->getAsAPIntVal();
1860	SDValue StartOfHi =
1861	DAG.getVScale(DL: dl, VT: EltVT, MulImm: StepVal * LoVT.getVectorMinNumElements());
1862	StartOfHi = DAG.getSExtOrTrunc(Op: StartOfHi, DL: dl, VT: HiVT.getVectorElementType());
1863	StartOfHi = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: HiVT, Operand: StartOfHi);
1864
1865	Hi = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: HiVT, Operand: Step);
1866	Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiVT, N1: Hi, N2: StartOfHi);
1867	}
1868
1869	void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
1870	SDValue &Hi) {
1871	EVT LoVT, HiVT;
1872	SDLoc dl(N);
1873	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1874	Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: N->getOperand(Num: 0));
1875	if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
1876	Hi = DAG.getUNDEF(VT: HiVT);
1877	} else {
1878	assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
1879	Hi = Lo;
1880	}
1881	}
1882
1883	void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1884	SDValue &Hi) {
1885	assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
1886	EVT LoVT, HiVT;
1887	SDLoc dl(LD);
1888	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
1889
1890	ISD::LoadExtType ExtType = LD->getExtensionType();
1891	SDValue Ch = LD->getChain();
1892	SDValue Ptr = LD->getBasePtr();
1893	SDValue Offset = DAG.getUNDEF(VT: Ptr.getValueType());
1894	EVT MemoryVT = LD->getMemoryVT();
1895	MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
1896	AAMDNodes AAInfo = LD->getAAInfo();
1897
1898	EVT LoMemVT, HiMemVT;
1899	std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
1900
1901	if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
1902	SDValue Value, NewChain;
1903	std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
1904	std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Value, DL: dl);
1905	ReplaceValueWith(From: SDValue(LD, 1), To: NewChain);
1906	return;
1907	}
1908
1909	Lo = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset,
1910	PtrInfo: LD->getPointerInfo(), MemVT: LoMemVT, Alignment: LD->getOriginalAlign(),
1911	MMOFlags, AAInfo);
1912
1913	MachinePointerInfo MPI;
1914	IncrementPointer(N: LD, MemVT: LoMemVT, MPI, Ptr);
1915
1916	Hi = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: HiVT, dl, Chain: Ch, Ptr, Offset, PtrInfo: MPI,
1917	MemVT: HiMemVT, Alignment: LD->getOriginalAlign(), MMOFlags, AAInfo);
1918
1919	// Build a factor node to remember that this load is independent of the
1920	// other one.
1921	Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(R: 1),
1922	Hi.getValue(R: 1));
1923
1924	// Legalize the chain result - switch anything that used the old chain to
1925	// use the new one.
1926	ReplaceValueWith(From: SDValue(LD, 1), To: Ch);
1927	}
1928
1929	void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
1930	SDValue &Hi) {
1931	assert(LD->isUnindexed() && "Indexed VP load during type legalization!");
1932	EVT LoVT, HiVT;
1933	SDLoc dl(LD);
1934	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
1935
1936	ISD::LoadExtType ExtType = LD->getExtensionType();
1937	SDValue Ch = LD->getChain();
1938	SDValue Ptr = LD->getBasePtr();
1939	SDValue Offset = LD->getOffset();
1940	assert(Offset.isUndef() && "Unexpected indexed variable-length load offset");
1941	Align Alignment = LD->getOriginalAlign();
1942	SDValue Mask = LD->getMask();
1943	SDValue EVL = LD->getVectorLength();
1944	EVT MemoryVT = LD->getMemoryVT();
1945
1946	EVT LoMemVT, HiMemVT;
1947	bool HiIsEmpty = false;
1948	std::tie(args&: LoMemVT, args&: HiMemVT) =
1949	DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
1950
1951	// Split Mask operand
1952	SDValue MaskLo, MaskHi;
1953	if (Mask.getOpcode() == ISD::SETCC) {
1954	SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
1955	} else {
1956	if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
1957	GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
1958	else
1959	std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
1960	}
1961
1962	// Split EVL operand
1963	SDValue EVLLo, EVLHi;
1964	std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: LD->getValueType(ResNo: 0), DL: dl);
1965
1966	MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
1967	PtrInfo: LD->getPointerInfo(), f: MachineMemOperand::MOLoad,
1968	s: MemoryLocation::UnknownSize, base_alignment: Alignment, AAInfo: LD->getAAInfo(), Ranges: LD->getRanges());
1969
1970	Lo =
1971	DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset,
1972	Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO, IsExpanding: LD->isExpandingLoad());
1973
1974	if (HiIsEmpty) {
1975	// The hi vp_load has zero storage size. We therefore simply set it to
1976	// the low vp_load and rely on subsequent removal from the chain.
1977	Hi = Lo;
1978	} else {
1979	// Generate hi vp_load.
1980	Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG,
1981	IsCompressedMemory: LD->isExpandingLoad());
1982
1983	MachinePointerInfo MPI;
1984	if (LoMemVT.isScalableVector())
1985	MPI = MachinePointerInfo(LD->getPointerInfo().getAddrSpace());
1986	else
1987	MPI = LD->getPointerInfo().getWithOffset(
1988	O: LoMemVT.getStoreSize().getFixedValue());
1989
1990	MMO = DAG.getMachineFunction().getMachineMemOperand(
1991	PtrInfo: MPI, f: MachineMemOperand::MOLoad, s: MemoryLocation::UnknownSize, base_alignment: Alignment,
1992	AAInfo: LD->getAAInfo(), Ranges: LD->getRanges());
1993
1994	Hi = DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: HiVT, dl, Chain: Ch, Ptr,
1995	Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO,
1996	IsExpanding: LD->isExpandingLoad());
1997	}
1998
1999	// Build a factor node to remember that this load is independent of the
2000	// other one.
2001	Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(R: 1),
2002	Hi.getValue(R: 1));
2003
2004	// Legalize the chain result - switch anything that used the old chain to
2005	// use the new one.
2006	ReplaceValueWith(From: SDValue(LD, 1), To: Ch);
2007	}
2008
2009	void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD,
2010	SDValue &Lo, SDValue &Hi) {
2011	assert(SLD->isUnindexed() &&
2012	"Indexed VP strided load during type legalization!");
2013	assert(SLD->getOffset().isUndef() &&
2014	"Unexpected indexed variable-length load offset");
2015
2016	SDLoc DL(SLD);
2017
2018	EVT LoVT, HiVT;
2019	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: SLD->getValueType(ResNo: 0));
2020
2021	EVT LoMemVT, HiMemVT;
2022	bool HiIsEmpty = false;
2023	std::tie(args&: LoMemVT, args&: HiMemVT) =
2024	DAG.GetDependentSplitDestVTs(VT: SLD->getMemoryVT(), EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2025
2026	SDValue Mask = SLD->getMask();
2027	SDValue LoMask, HiMask;
2028	if (Mask.getOpcode() == ISD::SETCC) {
2029	SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask);
2030	} else {
2031	if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2032	GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask);
2033	else
2034	std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
2035	}
2036
2037	SDValue LoEVL, HiEVL;
2038	std::tie(args&: LoEVL, args&: HiEVL) =
2039	DAG.SplitEVL(N: SLD->getVectorLength(), VecVT: SLD->getValueType(ResNo: 0), DL);
2040
2041	// Generate the low vp_strided_load
2042	Lo = DAG.getStridedLoadVP(
2043	AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(), VT: LoVT, DL,
2044	Chain: SLD->getChain(), Ptr: SLD->getBasePtr(), Offset: SLD->getOffset(), Stride: SLD->getStride(),
2045	Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: SLD->getMemOperand(), IsExpanding: SLD->isExpandingLoad());
2046
2047	if (HiIsEmpty) {
2048	// The high vp_strided_load has zero storage size. We therefore simply set
2049	// it to the low vp_strided_load and rely on subsequent removal from the
2050	// chain.
2051	Hi = Lo;
2052	} else {
2053	// Generate the high vp_strided_load.
2054	// To calculate the high base address, we need to sum to the low base
2055	// address stride number of bytes for each element already loaded by low,
2056	// that is: Ptr = Ptr + (LoEVL * Stride)
2057	EVT PtrVT = SLD->getBasePtr().getValueType();
2058	SDValue Increment =
2059	DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL,
2060	N2: DAG.getSExtOrTrunc(Op: SLD->getStride(), DL, VT: PtrVT));
2061	SDValue Ptr =
2062	DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: SLD->getBasePtr(), N2: Increment);
2063
2064	Align Alignment = SLD->getOriginalAlign();
2065	if (LoMemVT.isScalableVector())
2066	Alignment = commonAlignment(
2067	A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
2068
2069	MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2070	PtrInfo: MachinePointerInfo(SLD->getPointerInfo().getAddrSpace()),
2071	f: MachineMemOperand::MOLoad, s: MemoryLocation::UnknownSize, base_alignment: Alignment,
2072	AAInfo: SLD->getAAInfo(), Ranges: SLD->getRanges());
2073
2074	Hi = DAG.getStridedLoadVP(AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(),
2075	VT: HiVT, DL, Chain: SLD->getChain(), Ptr, Offset: SLD->getOffset(),
2076	Stride: SLD->getStride(), Mask: HiMask, EVL: HiEVL, MemVT: HiMemVT, MMO,
2077	IsExpanding: SLD->isExpandingLoad());
2078	}
2079
2080	// Build a factor node to remember that this load is independent of the
2081	// other one.
2082	SDValue Ch = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(R: 1),
2083	Hi.getValue(R: 1));
2084
2085	// Legalize the chain result - switch anything that used the old chain to
2086	// use the new one.
2087	ReplaceValueWith(From: SDValue(SLD, 1), To: Ch);
2088	}
2089
2090	void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
2091	SDValue &Lo, SDValue &Hi) {
2092	assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
2093	EVT LoVT, HiVT;
2094	SDLoc dl(MLD);
2095	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: MLD->getValueType(ResNo: 0));
2096
2097	SDValue Ch = MLD->getChain();
2098	SDValue Ptr = MLD->getBasePtr();
2099	SDValue Offset = MLD->getOffset();
2100	assert(Offset.isUndef() && "Unexpected indexed masked load offset");
2101	SDValue Mask = MLD->getMask();
2102	SDValue PassThru = MLD->getPassThru();
2103	Align Alignment = MLD->getOriginalAlign();
2104	ISD::LoadExtType ExtType = MLD->getExtensionType();
2105
2106	// Split Mask operand
2107	SDValue MaskLo, MaskHi;
2108	if (Mask.getOpcode() == ISD::SETCC) {
2109	SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2110	} else {
2111	if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2112	GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2113	else
2114	std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2115	}
2116
2117	EVT MemoryVT = MLD->getMemoryVT();
2118	EVT LoMemVT, HiMemVT;
2119	bool HiIsEmpty = false;
2120	std::tie(args&: LoMemVT, args&: HiMemVT) =
2121	DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2122
2123	SDValue PassThruLo, PassThruHi;
2124	if (getTypeAction(VT: PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2125	GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi);
2126	else
2127	std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl);
2128
2129	MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2130	PtrInfo: MLD->getPointerInfo(), f: MachineMemOperand::MOLoad,
2131	s: MemoryLocation::UnknownSize, base_alignment: Alignment, AAInfo: MLD->getAAInfo(),
2132	Ranges: MLD->getRanges());
2133
2134	Lo = DAG.getMaskedLoad(VT: LoVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskLo, Src0: PassThruLo, MemVT: LoMemVT,
2135	MMO, AM: MLD->getAddressingMode(), ExtType,
2136	IsExpanding: MLD->isExpandingLoad());
2137
2138	if (HiIsEmpty) {
2139	// The hi masked load has zero storage size. We therefore simply set it to
2140	// the low masked load and rely on subsequent removal from the chain.
2141	Hi = Lo;
2142	} else {
2143	// Generate hi masked load.
2144	Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG,
2145	IsCompressedMemory: MLD->isExpandingLoad());
2146
2147	MachinePointerInfo MPI;
2148	if (LoMemVT.isScalableVector())
2149	MPI = MachinePointerInfo(MLD->getPointerInfo().getAddrSpace());
2150	else
2151	MPI = MLD->getPointerInfo().getWithOffset(
2152	O: LoMemVT.getStoreSize().getFixedValue());
2153
2154	MMO = DAG.getMachineFunction().getMachineMemOperand(
2155	PtrInfo: MPI, f: MachineMemOperand::MOLoad, s: MemoryLocation::UnknownSize, base_alignment: Alignment,
2156	AAInfo: MLD->getAAInfo(), Ranges: MLD->getRanges());
2157
2158	Hi = DAG.getMaskedLoad(VT: HiVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskHi, Src0: PassThruHi,
2159	MemVT: HiMemVT, MMO, AM: MLD->getAddressingMode(), ExtType,
2160	IsExpanding: MLD->isExpandingLoad());
2161	}
2162
2163	// Build a factor node to remember that this load is independent of the
2164	// other one.
2165	Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(R: 1),
2166	Hi.getValue(R: 1));
2167
2168	// Legalize the chain result - switch anything that used the old chain to
2169	// use the new one.
2170	ReplaceValueWith(From: SDValue(MLD, 1), To: Ch);
2171
2172	}
2173
2174	void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
2175	SDValue &Hi, bool SplitSETCC) {
2176	EVT LoVT, HiVT;
2177	SDLoc dl(N);
2178	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2179
2180	SDValue Ch = N->getChain();
2181	SDValue Ptr = N->getBasePtr();
2182	struct Operands {
2183	SDValue Mask;
2184	SDValue Index;
2185	SDValue Scale;
2186	} Ops = [&]() -> Operands {
2187	if (auto *MSC = dyn_cast<MaskedGatherSDNode>(Val: N)) {
2188	return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale()};
2189	}
2190	auto *VPSC = cast<VPGatherSDNode>(Val: N);
2191	return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale()};
2192	}();
2193
2194	EVT MemoryVT = N->getMemoryVT();
2195	Align Alignment = N->getOriginalAlign();
2196
2197	// Split Mask operand
2198	SDValue MaskLo, MaskHi;
2199	if (SplitSETCC && Ops.Mask.getOpcode() == ISD::SETCC) {
2200	SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2201	} else {
2202	std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL: dl);
2203	}
2204
2205	EVT LoMemVT, HiMemVT;
2206	// Split MemoryVT
2207	std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
2208
2209	SDValue IndexHi, IndexLo;
2210	if (getTypeAction(VT: Ops.Index.getValueType()) ==
2211	TargetLowering::TypeSplitVector)
2212	GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi);
2213	else
2214	std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL: dl);
2215
2216	MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2217	PtrInfo: N->getPointerInfo(), f: MachineMemOperand::MOLoad,
2218	s: MemoryLocation::UnknownSize, base_alignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
2219
2220	if (auto *MGT = dyn_cast<MaskedGatherSDNode>(Val: N)) {
2221	SDValue PassThru = MGT->getPassThru();
2222	SDValue PassThruLo, PassThruHi;
2223	if (getTypeAction(VT: PassThru.getValueType()) ==
2224	TargetLowering::TypeSplitVector)
2225	GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi);
2226	else
2227	std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl);
2228
2229	ISD::LoadExtType ExtType = MGT->getExtensionType();
2230	ISD::MemIndexType IndexTy = MGT->getIndexType();
2231
2232	SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Ops.Scale};
2233	Lo = DAG.getMaskedGather(VTs: DAG.getVTList(LoVT, MVT::Other), MemVT: LoMemVT, dl,
2234	Ops: OpsLo, MMO, IndexType: IndexTy, ExtTy: ExtType);
2235
2236	SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Ops.Scale};
2237	Hi = DAG.getMaskedGather(VTs: DAG.getVTList(HiVT, MVT::Other), MemVT: HiMemVT, dl,
2238	Ops: OpsHi, MMO, IndexType: IndexTy, ExtTy: ExtType);
2239	} else {
2240	auto *VPGT = cast<VPGatherSDNode>(Val: N);
2241	SDValue EVLLo, EVLHi;
2242	std::tie(args&: EVLLo, args&: EVLHi) =
2243	DAG.SplitEVL(N: VPGT->getVectorLength(), VecVT: MemoryVT, DL: dl);
2244
2245	SDValue OpsLo[] = {Ch, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
2246	Lo = DAG.getGatherVP(VTs: DAG.getVTList(LoVT, MVT::Other), VT: LoMemVT, dl, Ops: OpsLo,
2247	MMO, IndexType: VPGT->getIndexType());
2248
2249	SDValue OpsHi[] = {Ch, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
2250	Hi = DAG.getGatherVP(VTs: DAG.getVTList(HiVT, MVT::Other), VT: HiMemVT, dl, Ops: OpsHi,
2251	MMO, IndexType: VPGT->getIndexType());
2252	}
2253
2254	// Build a factor node to remember that this load is independent of the
2255	// other one.
2256	Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(R: 1),
2257	Hi.getValue(R: 1));
2258
2259	// Legalize the chain result - switch anything that used the old chain to
2260	// use the new one.
2261	ReplaceValueWith(From: SDValue(N, 1), To: Ch);
2262	}
2263
2264	void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
2265	assert(N->getValueType(0).isVector() &&
2266	N->getOperand(0).getValueType().isVector() &&
2267	"Operand types must be vectors");
2268
2269	EVT LoVT, HiVT;
2270	SDLoc DL(N);
2271	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2272
2273	// If the input also splits, handle it directly. Otherwise split it by hand.
2274	SDValue LL, LH, RL, RH;
2275	if (getTypeAction(VT: N->getOperand(Num: 0).getValueType()) ==
2276	TargetLowering::TypeSplitVector)
2277	GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LL, Hi&: LH);
2278	else
2279	std::tie(args&: LL, args&: LH) = DAG.SplitVectorOperand(N, OpNo: 0);
2280
2281	if (getTypeAction(VT: N->getOperand(Num: 1).getValueType()) ==
2282	TargetLowering::TypeSplitVector)
2283	GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RL, Hi&: RH);
2284	else
2285	std::tie(args&: RL, args&: RH) = DAG.SplitVectorOperand(N, OpNo: 1);
2286
2287	if (N->getOpcode() == ISD::SETCC) {
2288	Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: LL, N2: RL, N3: N->getOperand(Num: 2));
2289	Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: LH, N2: RH, N3: N->getOperand(Num: 2));
2290	} else {
2291	assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
2292	SDValue MaskLo, MaskHi, EVLLo, EVLHi;
2293	std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 3));
2294	std::tie(args&: EVLLo, args&: EVLHi) =
2295	DAG.SplitEVL(N: N->getOperand(Num: 4), VecVT: N->getValueType(ResNo: 0), DL);
2296	Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: LL, N2: RL, N3: N->getOperand(Num: 2), N4: MaskLo,
2297	N5: EVLLo);
2298	Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: LH, N2: RH, N3: N->getOperand(Num: 2), N4: MaskHi,
2299	N5: EVLHi);
2300	}
2301	}
2302
2303	void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
2304	SDValue &Hi) {
2305	// Get the dest types - they may not match the input types, e.g. int_to_fp.
2306	EVT LoVT, HiVT;
2307	SDLoc dl(N);
2308	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2309
2310	// If the input also splits, handle it directly for a compile time speedup.
2311	// Otherwise split it by hand.
2312	EVT InVT = N->getOperand(Num: 0).getValueType();
2313	if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2314	GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
2315	else
2316	std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2317
2318	const SDNodeFlags Flags = N->getFlags();
2319	unsigned Opcode = N->getOpcode();
2320	if (N->getNumOperands() <= 2) {
2321	if (Opcode == ISD::FP_ROUND) {
2322	Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, N1: Lo, N2: N->getOperand(Num: 1), Flags);
2323	Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, N1: Hi, N2: N->getOperand(Num: 1), Flags);
2324	} else {
2325	Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, Operand: Lo, Flags);
2326	Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, Operand: Hi, Flags);
2327	}
2328	return;
2329	}
2330
2331	assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
2332	assert(N->isVPOpcode() && "Expected VP opcode");
2333
2334	SDValue MaskLo, MaskHi;
2335	std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
2336
2337	SDValue EVLLo, EVLHi;
2338	std::tie(args&: EVLLo, args&: EVLHi) =
2339	DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl);
2340
2341	Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, Ops: {Lo, MaskLo, EVLLo}, Flags);
2342	Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, Ops: {Hi, MaskHi, EVLHi}, Flags);
2343	}
2344
2345	void DAGTypeLegalizer::SplitVecRes_FFREXP(SDNode *N, unsigned ResNo,
2346	SDValue &Lo, SDValue &Hi) {
2347	SDLoc dl(N);
2348	auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2349	auto [LoVT1, HiVT1] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 1));
2350
2351	// If the input also splits, handle it directly for a compile time speedup.
2352	// Otherwise split it by hand.
2353	EVT InVT = N->getOperand(Num: 0).getValueType();
2354	if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2355	GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
2356	else
2357	std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2358
2359	Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {LoVT, LoVT1}, Ops: Lo);
2360	Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {HiVT, HiVT1}, Ops: Hi);
2361	Lo->setFlags(N->getFlags());
2362	Hi->setFlags(N->getFlags());
2363
2364	SDNode *HiNode = Hi.getNode();
2365	SDNode *LoNode = Lo.getNode();
2366
2367	// Replace the other vector result not being explicitly split here.
2368	unsigned OtherNo = 1 - ResNo;
2369	EVT OtherVT = N->getValueType(ResNo: OtherNo);
2370	if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) {
2371	SetSplitVector(Op: SDValue(N, OtherNo), Lo: SDValue(LoNode, OtherNo),
2372	Hi: SDValue(HiNode, OtherNo));
2373	} else {
2374	SDValue OtherVal =
2375	DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT, N1: SDValue(LoNode, OtherNo),
2376	N2: SDValue(HiNode, OtherNo));
2377	ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
2378	}
2379	}
2380
2381	void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
2382	SDValue &Hi) {
2383	SDLoc dl(N);
2384	EVT SrcVT = N->getOperand(Num: 0).getValueType();
2385	EVT DestVT = N->getValueType(ResNo: 0);
2386	EVT LoVT, HiVT;
2387	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: DestVT);
2388
2389	// We can do better than a generic split operation if the extend is doing
2390	// more than just doubling the width of the elements and the following are
2391	// true:
2392	// - The number of vector elements is even,
2393	// - the source type is legal,
2394	// - the type of a split source is illegal,
2395	// - the type of an extended (by doubling element size) source is legal, and
2396	// - the type of that extended source when split is legal.
2397	//
2398	// This won't necessarily completely legalize the operation, but it will
2399	// more effectively move in the right direction and prevent falling down
2400	// to scalarization in many cases due to the input vector being split too
2401	// far.
2402	if (SrcVT.getVectorElementCount().isKnownEven() &&
2403	SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
2404	LLVMContext &Ctx = *DAG.getContext();
2405	EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Context&: Ctx);
2406	EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Context&: Ctx);
2407
2408	EVT SplitLoVT, SplitHiVT;
2409	std::tie(args&: SplitLoVT, args&: SplitHiVT) = DAG.GetSplitDestVTs(VT: NewSrcVT);
2410	if (TLI.isTypeLegal(VT: SrcVT) && !TLI.isTypeLegal(VT: SplitSrcVT) &&
2411	TLI.isTypeLegal(VT: NewSrcVT) && TLI.isTypeLegal(VT: SplitLoVT)) {
2412	LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
2413	N->dump(&DAG); dbgs() << "\n");
2414	if (!N->isVPOpcode()) {
2415	// Extend the source vector by one step.
2416	SDValue NewSrc =
2417	DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewSrcVT, Operand: N->getOperand(Num: 0));
2418	// Get the low and high halves of the new, extended one step, vector.
2419	std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: NewSrc, DL: dl);
2420	// Extend those vector halves the rest of the way.
2421	Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: Lo);
2422	Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: HiVT, Operand: Hi);
2423	return;
2424	}
2425
2426	// Extend the source vector by one step.
2427	SDValue NewSrc =
2428	DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewSrcVT, N1: N->getOperand(Num: 0),
2429	N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
2430	// Get the low and high halves of the new, extended one step, vector.
2431	std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: NewSrc, DL: dl);
2432
2433	SDValue MaskLo, MaskHi;
2434	std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
2435
2436	SDValue EVLLo, EVLHi;
2437	std::tie(args&: EVLLo, args&: EVLHi) =
2438	DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl);
2439	// Extend those vector halves the rest of the way.
2440	Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Ops: {Lo, MaskLo, EVLLo});
2441	Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: HiVT, Ops: {Hi, MaskHi, EVLHi});
2442	return;
2443	}
2444	}
2445	// Fall back to the generic unary operator splitting otherwise.
2446	SplitVecRes_UnaryOp(N, Lo, Hi);
2447	}
2448
2449	void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
2450	SDValue &Lo, SDValue &Hi) {
2451	// The low and high parts of the original input give four input vectors.
2452	SDValue Inputs[4];
2453	SDLoc DL(N);
2454	GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Inputs[0], Hi&: Inputs[1]);
2455	GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Inputs[2], Hi&: Inputs[3]);
2456	EVT NewVT = Inputs[0].getValueType();
2457	unsigned NewElts = NewVT.getVectorNumElements();
2458
2459	auto &&IsConstant = [](const SDValue &N) {
2460	APInt SplatValue;
2461	return N.getResNo() == 0 &&
2462	(ISD::isConstantSplatVector(N: N.getNode(), SplatValue) ||
2463	ISD::isBuildVectorOfConstantSDNodes(N: N.getNode()));
2464	};
2465	auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &DL](SDValue &Input1,
2466	SDValue &Input2,
2467	ArrayRef<int> Mask) {
2468	assert(Input1->getOpcode() == ISD::BUILD_VECTOR &&
2469	Input2->getOpcode() == ISD::BUILD_VECTOR &&
2470	"Expected build vector node.");
2471	EVT EltVT = NewVT.getVectorElementType();
2472	SmallVector<SDValue> Ops(NewElts, DAG.getUNDEF(VT: EltVT));
2473	for (unsigned I = 0; I < NewElts; ++I) {
2474	if (Mask[I] == PoisonMaskElem)
2475	continue;
2476	unsigned Idx = Mask[I];
2477	if (Idx >= NewElts)
2478	Ops[I] = Input2.getOperand(i: Idx - NewElts);
2479	else
2480	Ops[I] = Input1.getOperand(i: Idx);
2481	// Make the type of all elements the same as the element type.
2482	if (Ops[I].getValueType().bitsGT(VT: EltVT))
2483	Ops[I] = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Ops[I]);
2484	}
2485	return DAG.getBuildVector(VT: NewVT, DL, Ops);
2486	};
2487
2488	// If Lo or Hi uses elements from at most two of the four input vectors, then
2489	// express it as a vector shuffle of those two inputs. Otherwise extract the
2490	// input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
2491	SmallVector<int> OrigMask(N->getMask());
2492	// Try to pack incoming shuffles/inputs.
2493	auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT, this, NewElts,
2494	&DL](SmallVectorImpl<int> &Mask) {
2495	// Check if all inputs are shuffles of the same operands or non-shuffles.
2496	MapVector<std::pair<SDValue, SDValue>, SmallVector<unsigned>> ShufflesIdxs;
2497	for (unsigned Idx = 0; Idx < std::size(Inputs); ++Idx) {
2498	SDValue Input = Inputs[Idx];
2499	auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Input.getNode());
2500	if (!Shuffle ||
2501	Input.getOperand(i: 0).getValueType() != Input.getValueType())
2502	continue;
2503	ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 0), y: Input.getOperand(i: 1))]
2504	.push_back(Elt: Idx);
2505	ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 1), y: Input.getOperand(i: 0))]
2506	.push_back(Elt: Idx);
2507	}
2508	for (auto &P : ShufflesIdxs) {
2509	if (P.second.size() < 2)
2510	continue;
2511	// Use shuffles operands instead of shuffles themselves.
2512	// 1. Adjust mask.
2513	for (int &Idx : Mask) {
2514	if (Idx == PoisonMaskElem)
2515	continue;
2516	unsigned SrcRegIdx = Idx / NewElts;
2517	if (Inputs[SrcRegIdx].isUndef()) {
2518	Idx = PoisonMaskElem;
2519	continue;
2520	}
2521	auto *Shuffle =
2522	dyn_cast<ShuffleVectorSDNode>(Val: Inputs[SrcRegIdx].getNode());
2523	if (!Shuffle || !is_contained(Range&: P.second, Element: SrcRegIdx))
2524	continue;
2525	int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
2526	if (MaskElt == PoisonMaskElem) {
2527	Idx = PoisonMaskElem;
2528	continue;
2529	}
2530	Idx = MaskElt % NewElts +
2531	P.second[Shuffle->getOperand(Num: MaskElt / NewElts) == P.first.first
2532	? 0
2533	: 1] *
2534	NewElts;
2535	}
2536	// 2. Update inputs.
2537	Inputs[P.second[0]] = P.first.first;
2538	Inputs[P.second[1]] = P.first.second;
2539	// Clear the pair data.
2540	P.second.clear();
2541	ShufflesIdxs[std::make_pair(x&: P.first.second, y&: P.first.first)].clear();
2542	}
2543	// Check if any concat_vectors can be simplified.
2544	SmallBitVector UsedSubVector(2 * std::size(Inputs));
2545	for (int &Idx : Mask) {
2546	if (Idx == PoisonMaskElem)
2547	continue;
2548	unsigned SrcRegIdx = Idx / NewElts;
2549	if (Inputs[SrcRegIdx].isUndef()) {
2550	Idx = PoisonMaskElem;
2551	continue;
2552	}
2553	TargetLowering::LegalizeTypeAction TypeAction =
2554	getTypeAction(VT: Inputs[SrcRegIdx].getValueType());
2555	if (Inputs[SrcRegIdx].getOpcode() == ISD::CONCAT_VECTORS &&
2556	Inputs[SrcRegIdx].getNumOperands() == 2 &&
2557	!Inputs[SrcRegIdx].getOperand(i: 1).isUndef() &&
2558	(TypeAction == TargetLowering::TypeLegal ||
2559	TypeAction == TargetLowering::TypeWidenVector))
2560	UsedSubVector.set(2 * SrcRegIdx + (Idx % NewElts) / (NewElts / 2));
2561	}
2562	if (UsedSubVector.count() > 1) {
2563	SmallVector<SmallVector<std::pair<unsigned, int>, 2>> Pairs;
2564	for (unsigned I = 0; I < std::size(Inputs); ++I) {
2565	if (UsedSubVector.test(Idx: 2 * I) == UsedSubVector.test(Idx: 2 * I + 1))
2566	continue;
2567	if (Pairs.empty() || Pairs.back().size() == 2)
2568	Pairs.emplace_back();
2569	if (UsedSubVector.test(Idx: 2 * I)) {
2570	Pairs.back().emplace_back(Args&: I, Args: 0);
2571	} else {
2572	assert(UsedSubVector.test(2 * I + 1) &&
2573	"Expected to be used one of the subvectors.");
2574	Pairs.back().emplace_back(Args&: I, Args: 1);
2575	}
2576	}
2577	if (!Pairs.empty() && Pairs.front().size() > 1) {
2578	// Adjust mask.
2579	for (int &Idx : Mask) {
2580	if (Idx == PoisonMaskElem)
2581	continue;
2582	unsigned SrcRegIdx = Idx / NewElts;
2583	auto *It = find_if(
2584	Range&: Pairs, P: [SrcRegIdx](ArrayRef<std::pair<unsigned, int>> Idxs) {
2585	return Idxs.front().first == SrcRegIdx ||
2586	Idxs.back().first == SrcRegIdx;
2587	});
2588	if (It == Pairs.end())
2589	continue;
2590	Idx = It->front().first * NewElts + (Idx % NewElts) % (NewElts / 2) +
2591	(SrcRegIdx == It->front().first ? 0 : (NewElts / 2));
2592	}
2593	// Adjust inputs.
2594	for (ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) {
2595	Inputs[Idxs.front().first] = DAG.getNode(
2596	Opcode: ISD::CONCAT_VECTORS, DL,
2597	VT: Inputs[Idxs.front().first].getValueType(),
2598	N1: Inputs[Idxs.front().first].getOperand(i: Idxs.front().second),
2599	N2: Inputs[Idxs.back().first].getOperand(i: Idxs.back().second));
2600	}
2601	}
2602	}
2603	bool Changed;
2604	do {
2605	// Try to remove extra shuffles (except broadcasts) and shuffles with the
2606	// reused operands.
2607	Changed = false;
2608	for (unsigned I = 0; I < std::size(Inputs); ++I) {
2609	auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Inputs[I].getNode());
2610	if (!Shuffle)
2611	continue;
2612	if (Shuffle->getOperand(Num: 0).getValueType() != NewVT)
2613	continue;
2614	int Op = -1;
2615	if (!Inputs[I].hasOneUse() && Shuffle->getOperand(Num: 1).isUndef() &&
2616	!Shuffle->isSplat()) {
2617	Op = 0;
2618	} else if (!Inputs[I].hasOneUse() &&
2619	!Shuffle->getOperand(Num: 1).isUndef()) {
2620	// Find the only used operand, if possible.
2621	for (int &Idx : Mask) {
2622	if (Idx == PoisonMaskElem)
2623	continue;
2624	unsigned SrcRegIdx = Idx / NewElts;
2625	if (SrcRegIdx != I)
2626	continue;
2627	int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
2628	if (MaskElt == PoisonMaskElem) {
2629	Idx = PoisonMaskElem;
2630	continue;
2631	}
2632	int OpIdx = MaskElt / NewElts;
2633	if (Op == -1) {
2634	Op = OpIdx;
2635	continue;
2636	}
2637	if (Op != OpIdx) {
2638	Op = -1;
2639	break;
2640	}
2641	}
2642	}
2643	if (Op < 0) {
2644	// Try to check if one of the shuffle operands is used already.
2645	for (int OpIdx = 0; OpIdx < 2; ++OpIdx) {
2646	if (Shuffle->getOperand(Num: OpIdx).isUndef())
2647	continue;
2648	auto *It = find(Range&: Inputs, Val: Shuffle->getOperand(Num: OpIdx));
2649	if (It == std::end(arr&: Inputs))
2650	continue;
2651	int FoundOp = std::distance(first: std::begin(arr&: Inputs), last: It);
2652	// Found that operand is used already.
2653	// 1. Fix the mask for the reused operand.
2654	for (int &Idx : Mask) {
2655	if (Idx == PoisonMaskElem)
2656	continue;
2657	unsigned SrcRegIdx = Idx / NewElts;
2658	if (SrcRegIdx != I)
2659	continue;
2660	int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
2661	if (MaskElt == PoisonMaskElem) {
2662	Idx = PoisonMaskElem;
2663	continue;
2664	}
2665	int MaskIdx = MaskElt / NewElts;
2666	if (OpIdx == MaskIdx)
2667	Idx = MaskElt % NewElts + FoundOp * NewElts;
2668	}
2669	// 2. Set Op to the unused OpIdx.
2670	Op = (OpIdx + 1) % 2;
2671	break;
2672	}
2673	}
2674	if (Op >= 0) {
2675	Changed = true;
2676	Inputs[I] = Shuffle->getOperand(Num: Op);
2677	// Adjust mask.
2678	for (int &Idx : Mask) {
2679	if (Idx == PoisonMaskElem)
2680	continue;
2681	unsigned SrcRegIdx = Idx / NewElts;
2682	if (SrcRegIdx != I)
2683	continue;
2684	int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
2685	int OpIdx = MaskElt / NewElts;
2686	if (OpIdx != Op)
2687	continue;
2688	Idx = MaskElt % NewElts + SrcRegIdx * NewElts;
2689	}
2690	}
2691	}
2692	} while (Changed);
2693	};
2694	TryPeekThroughShufflesInputs(OrigMask);
2695	// Proces unique inputs.
2696	auto &&MakeUniqueInputs = [&Inputs, &IsConstant,
2697	NewElts](SmallVectorImpl<int> &Mask) {
2698	SetVector<SDValue> UniqueInputs;
2699	SetVector<SDValue> UniqueConstantInputs;
2700	for (const auto &I : Inputs) {
2701	if (IsConstant(I))
2702	UniqueConstantInputs.insert(X: I);
2703	else if (!I.isUndef())
2704	UniqueInputs.insert(X: I);
2705	}
2706	// Adjust mask in case of reused inputs. Also, need to insert constant
2707	// inputs at first, otherwise it affects the final outcome.
2708	if (UniqueInputs.size() != std::size(Inputs)) {
2709	auto &&UniqueVec = UniqueInputs.takeVector();
2710	auto &&UniqueConstantVec = UniqueConstantInputs.takeVector();
2711	unsigned ConstNum = UniqueConstantVec.size();
2712	for (int &Idx : Mask) {
2713	if (Idx == PoisonMaskElem)
2714	continue;
2715	unsigned SrcRegIdx = Idx / NewElts;
2716	if (Inputs[SrcRegIdx].isUndef()) {
2717	Idx = PoisonMaskElem;
2718	continue;
2719	}
2720	const auto It = find(Range&: UniqueConstantVec, Val: Inputs[SrcRegIdx]);
2721	if (It != UniqueConstantVec.end()) {
2722	Idx = (Idx % NewElts) +
2723	NewElts * std::distance(first: UniqueConstantVec.begin(), last: It);
2724	assert(Idx >= 0 && "Expected defined mask idx.");
2725	continue;
2726	}
2727	const auto RegIt = find(Range&: UniqueVec, Val: Inputs[SrcRegIdx]);
2728	assert(RegIt != UniqueVec.end() && "Cannot find non-const value.");
2729	Idx = (Idx % NewElts) +
2730	NewElts * (std::distance(first: UniqueVec.begin(), last: RegIt) + ConstNum);
2731	assert(Idx >= 0 && "Expected defined mask idx.");
2732	}
2733	copy(Range&: UniqueConstantVec, Out: std::begin(arr&: Inputs));
2734	copy(Range&: UniqueVec, Out: std::next(x: std::begin(arr&: Inputs), n: ConstNum));
2735	}
2736	};
2737	MakeUniqueInputs(OrigMask);
2738	SDValue OrigInputs[4];
2739	copy(Range&: Inputs, Out: std::begin(arr&: OrigInputs));
2740	for (unsigned High = 0; High < 2; ++High) {
2741	SDValue &Output = High ? Hi : Lo;
2742
2743	// Build a shuffle mask for the output, discovering on the fly which
2744	// input vectors to use as shuffle operands.
2745	unsigned FirstMaskIdx = High * NewElts;
2746	SmallVector<int> Mask(NewElts * std::size(Inputs), PoisonMaskElem);
2747	copy(Range: ArrayRef(OrigMask).slice(N: FirstMaskIdx, M: NewElts), Out: Mask.begin());
2748	assert(!Output && "Expected default initialized initial value.");
2749	TryPeekThroughShufflesInputs(Mask);
2750	MakeUniqueInputs(Mask);
2751	SDValue TmpInputs[4];
2752	copy(Range&: Inputs, Out: std::begin(arr&: TmpInputs));
2753	// Track changes in the output registers.
2754	int UsedIdx = -1;
2755	bool SecondIteration = false;
2756	auto &&AccumulateResults = [&UsedIdx, &SecondIteration](unsigned Idx) {
2757	if (UsedIdx < 0) {
2758	UsedIdx = Idx;
2759	return false;
2760	}
2761	if (UsedIdx >= 0 && static_cast<unsigned>(UsedIdx) == Idx)
2762	SecondIteration = true;
2763	return SecondIteration;
2764	};
2765	processShuffleMasks(
2766	Mask, NumOfSrcRegs: std::size(Inputs), NumOfDestRegs: std::size(Inputs),
2767	/*NumOfUsedRegs=*/1,
2768	NoInputAction: [&Output, &DAG = DAG, NewVT]() { Output = DAG.getUNDEF(VT: NewVT); },
2769	SingleInputAction: [&Output, &DAG = DAG, NewVT, &DL, &Inputs,
2770	&BuildVector](ArrayRef<int> Mask, unsigned Idx, unsigned /*Unused*/) {
2771	if (Inputs[Idx]->getOpcode() == ISD::BUILD_VECTOR)
2772	Output = BuildVector(Inputs[Idx], Inputs[Idx], Mask);
2773	else
2774	Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx],
2775	N2: DAG.getUNDEF(VT: NewVT), Mask);
2776	Inputs[Idx] = Output;
2777	},
2778	ManyInputsAction: [&AccumulateResults, &Output, &DAG = DAG, NewVT, &DL, &Inputs,
2779	&TmpInputs,
2780	&BuildVector](ArrayRef<int> Mask, unsigned Idx1, unsigned Idx2) {
2781	if (AccumulateResults(Idx1)) {
2782	if (Inputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
2783	Inputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
2784	Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask);
2785	else
2786	Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx1],
2787	N2: Inputs[Idx2], Mask);
2788	} else {
2789	if (TmpInputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
2790	TmpInputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
2791	Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask);
2792	else
2793	Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: TmpInputs[Idx1],
2794	N2: TmpInputs[Idx2], Mask);
2795	}
2796	Inputs[Idx1] = Output;
2797	});
2798	copy(Range&: OrigInputs, Out: std::begin(arr&: Inputs));
2799	}
2800	}
2801
2802	void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2803	EVT OVT = N->getValueType(ResNo: 0);
2804	EVT NVT = OVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
2805	SDValue Chain = N->getOperand(Num: 0);
2806	SDValue Ptr = N->getOperand(Num: 1);
2807	SDValue SV = N->getOperand(Num: 2);
2808	SDLoc dl(N);
2809
2810	const Align Alignment =
2811	DAG.getDataLayout().getABITypeAlign(Ty: NVT.getTypeForEVT(Context&: *DAG.getContext()));
2812
2813	Lo = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV, Align: Alignment.value());
2814	Hi = DAG.getVAArg(VT: NVT, dl, Chain: Lo.getValue(R: 1), Ptr, SV, Align: Alignment.value());
2815	Chain = Hi.getValue(R: 1);
2816
2817	// Modified the chain - switch anything that used the old chain to use
2818	// the new one.
2819	ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2820	}
2821
2822	void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
2823	SDValue &Hi) {
2824	EVT DstVTLo, DstVTHi;
2825	std::tie(args&: DstVTLo, args&: DstVTHi) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2826	SDLoc dl(N);
2827
2828	SDValue SrcLo, SrcHi;
2829	EVT SrcVT = N->getOperand(Num: 0).getValueType();
2830	if (getTypeAction(VT: SrcVT) == TargetLowering::TypeSplitVector)
2831	GetSplitVector(Op: N->getOperand(Num: 0), Lo&: SrcLo, Hi&: SrcHi);
2832	else
2833	std::tie(args&: SrcLo, args&: SrcHi) = DAG.SplitVectorOperand(N, OpNo: 0);
2834
2835	Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTLo, N1: SrcLo, N2: N->getOperand(Num: 1));
2836	Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTHi, N1: SrcHi, N2: N->getOperand(Num: 1));
2837	}
2838
2839	void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo,
2840	SDValue &Hi) {
2841	SDValue InLo, InHi;
2842	GetSplitVector(Op: N->getOperand(Num: 0), Lo&: InLo, Hi&: InHi);
2843	SDLoc DL(N);
2844
2845	Lo = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InHi.getValueType(), Operand: InHi);
2846	Hi = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InLo.getValueType(), Operand: InLo);
2847	}
2848
2849	void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo,
2850	SDValue &Hi) {
2851	EVT VT = N->getValueType(ResNo: 0);
2852	SDLoc DL(N);
2853
2854	EVT LoVT, HiVT;
2855	std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT);
2856
2857	SDValue Expanded = TLI.expandVectorSplice(Node: N, DAG);
2858	Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LoVT, N1: Expanded,
2859	N2: DAG.getVectorIdxConstant(Val: 0, DL));
2860	Hi =
2861	DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: Expanded,
2862	N2: DAG.getVectorIdxConstant(Val: LoVT.getVectorMinNumElements(), DL));
2863	}
2864
2865	void DAGTypeLegalizer::SplitVecRes_VP_REVERSE(SDNode *N, SDValue &Lo,
2866	SDValue &Hi) {
2867	EVT VT = N->getValueType(ResNo: 0);
2868	SDValue Val = N->getOperand(Num: 0);
2869	SDValue Mask = N->getOperand(Num: 1);
2870	SDValue EVL = N->getOperand(Num: 2);
2871	SDLoc DL(N);
2872
2873	// Fallback to VP_STRIDED_STORE to stack followed by VP_LOAD.
2874	Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
2875
2876	EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
2877	EC: VT.getVectorElementCount());
2878	SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
2879	EVT PtrVT = StackPtr.getValueType();
2880	auto &MF = DAG.getMachineFunction();
2881	auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
2882	auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
2883
2884	MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
2885	PtrInfo, f: MachineMemOperand::MOStore, s: MemoryLocation::UnknownSize,
2886	base_alignment: Alignment);
2887	MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
2888	PtrInfo, f: MachineMemOperand::MOLoad, s: MemoryLocation::UnknownSize,
2889	base_alignment: Alignment);
2890
2891	unsigned EltWidth = VT.getScalarSizeInBits() / 8;
2892	SDValue NumElemMinus1 =
2893	DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: DAG.getZExtOrTrunc(Op: EVL, DL, VT: PtrVT),
2894	N2: DAG.getConstant(Val: 1, DL, VT: PtrVT));
2895	SDValue StartOffset = DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: NumElemMinus1,
2896	N2: DAG.getConstant(Val: EltWidth, DL, VT: PtrVT));
2897	SDValue StorePtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr, N2: StartOffset);
2898	SDValue Stride = DAG.getConstant(Val: -(int64_t)EltWidth, DL, VT: PtrVT);
2899
2900	SDValue TrueMask = DAG.getBoolConstant(V: true, DL, VT: Mask.getValueType(), OpVT: VT);
2901	SDValue Store = DAG.getStridedStoreVP(Chain: DAG.getEntryNode(), DL, Val, Ptr: StorePtr,
2902	Offset: DAG.getUNDEF(VT: PtrVT), Stride, Mask: TrueMask,
2903	EVL, MemVT, MMO: StoreMMO, AM: ISD::UNINDEXED);
2904
2905	SDValue Load = DAG.getLoadVP(VT, dl: DL, Chain: Store, Ptr: StackPtr, Mask, EVL, MMO: LoadMMO);
2906
2907	auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT);
2908	Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LoVT, N1: Load,
2909	N2: DAG.getVectorIdxConstant(Val: 0, DL));
2910	Hi =
2911	DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: Load,
2912	N2: DAG.getVectorIdxConstant(Val: LoVT.getVectorMinNumElements(), DL));
2913	}
2914
2915	void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
2916
2917	SDValue Op0Lo, Op0Hi, Op1Lo, Op1Hi;
2918	GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Op0Lo, Hi&: Op0Hi);
2919	GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Op1Lo, Hi&: Op1Hi);
2920	EVT VT = Op0Lo.getValueType();
2921	SDLoc DL(N);
2922	SDValue ResLo = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL,
2923	VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: Op0Lo, N2: Op0Hi);
2924	SDValue ResHi = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL,
2925	VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: Op1Lo, N2: Op1Hi);
2926
2927	SetSplitVector(Op: SDValue(N, 0), Lo: ResLo.getValue(R: 0), Hi: ResHi.getValue(R: 0));
2928	SetSplitVector(Op: SDValue(N, 1), Lo: ResLo.getValue(R: 1), Hi: ResHi.getValue(R: 1));
2929	}
2930
2931	void DAGTypeLegalizer::SplitVecRes_VECTOR_INTERLEAVE(SDNode *N) {
2932	SDValue Op0Lo, Op0Hi, Op1Lo, Op1Hi;
2933	GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Op0Lo, Hi&: Op0Hi);
2934	GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Op1Lo, Hi&: Op1Hi);
2935	EVT VT = Op0Lo.getValueType();
2936	SDLoc DL(N);
2937	SDValue Res[] = {DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL,
2938	VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: Op0Lo, N2: Op1Lo),
2939	DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL,
2940	VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: Op0Hi, N2: Op1Hi)};
2941
2942	SetSplitVector(Op: SDValue(N, 0), Lo: Res[0].getValue(R: 0), Hi: Res[0].getValue(R: 1));
2943	SetSplitVector(Op: SDValue(N, 1), Lo: Res[1].getValue(R: 0), Hi: Res[1].getValue(R: 1));
2944	}
2945
2946	//===----------------------------------------------------------------------===//
2947	// Operand Vector Splitting
2948	//===----------------------------------------------------------------------===//
2949
2950	/// This method is called when the specified operand of the specified node is
2951	/// found to need vector splitting. At this point, all of the result types of
2952	/// the node are known to be legal, but other operands of the node may need
2953	/// legalization as well as the specified one.
2954	bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
2955	LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG));
2956	SDValue Res = SDValue();
2957
2958	// See if the target wants to custom split this node.
2959	if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
2960	return false;
2961
2962	switch (N->getOpcode()) {
2963	default:
2964	#ifndef NDEBUG
2965	dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
2966	N->dump(G: &DAG);
2967	dbgs() << "\n";
2968	#endif
2969	report_fatal_error(reason: "Do not know how to split this operator's "
2970	"operand!\n");
2971
2972	case ISD::VP_SETCC:
2973	case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
2974	case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
2975	case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
2976	case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
2977	case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
2978	case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
2979	case ISD::VP_TRUNCATE:
2980	case ISD::TRUNCATE:
2981	Res = SplitVecOp_TruncateHelper(N);
2982	break;
2983	case ISD::STRICT_FP_ROUND:
2984	case ISD::VP_FP_ROUND:
2985	case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
2986	case ISD::FCOPYSIGN: Res = SplitVecOp_FPOpDifferentTypes(N); break;
2987	case ISD::STORE:
2988	Res = SplitVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo);
2989	break;
2990	case ISD::VP_STORE:
2991	Res = SplitVecOp_VP_STORE(N: cast<VPStoreSDNode>(Val: N), OpNo);
2992	break;
2993	case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
2994	Res = SplitVecOp_VP_STRIDED_STORE(N: cast<VPStridedStoreSDNode>(Val: N), OpNo);
2995	break;
2996	case ISD::MSTORE:
2997	Res = SplitVecOp_MSTORE(N: cast<MaskedStoreSDNode>(Val: N), OpNo);
2998	break;
2999	case ISD::MSCATTER:
3000	case ISD::VP_SCATTER:
3001	Res = SplitVecOp_Scatter(N: cast<MemSDNode>(Val: N), OpNo);
3002	break;
3003	case ISD::MGATHER:
3004	case ISD::VP_GATHER:
3005	Res = SplitVecOp_Gather(MGT: cast<MemSDNode>(Val: N), OpNo);
3006	break;
3007	case ISD::VSELECT:
3008	Res = SplitVecOp_VSELECT(N, OpNo);
3009	break;
3010	case ISD::STRICT_SINT_TO_FP:
3011	case ISD::STRICT_UINT_TO_FP:
3012	case ISD::SINT_TO_FP:
3013	case ISD::UINT_TO_FP:
3014	case ISD::VP_SINT_TO_FP:
3015	case ISD::VP_UINT_TO_FP:
3016	if (N->getValueType(ResNo: 0).bitsLT(
3017	VT: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType()))
3018	Res = SplitVecOp_TruncateHelper(N);
3019	else
3020	Res = SplitVecOp_UnaryOp(N);
3021	break;
3022	case ISD::FP_TO_SINT_SAT:
3023	case ISD::FP_TO_UINT_SAT:
3024	Res = SplitVecOp_FP_TO_XINT_SAT(N);
3025	break;
3026	case ISD::FP_TO_SINT:
3027	case ISD::FP_TO_UINT:
3028	case ISD::VP_FP_TO_SINT:
3029	case ISD::VP_FP_TO_UINT:
3030	case ISD::STRICT_FP_TO_SINT:
3031	case ISD::STRICT_FP_TO_UINT:
3032	case ISD::STRICT_FP_EXTEND:
3033	case ISD::FP_EXTEND:
3034	case ISD::SIGN_EXTEND:
3035	case ISD::ZERO_EXTEND:
3036	case ISD::ANY_EXTEND:
3037	case ISD::FTRUNC:
3038	case ISD::LRINT:
3039	case ISD::LLRINT:
3040	Res = SplitVecOp_UnaryOp(N);
3041	break;
3042	case ISD::FLDEXP:
3043	Res = SplitVecOp_FPOpDifferentTypes(N);
3044	break;
3045
3046	case ISD::ANY_EXTEND_VECTOR_INREG:
3047	case ISD::SIGN_EXTEND_VECTOR_INREG:
3048	case ISD::ZERO_EXTEND_VECTOR_INREG:
3049	Res = SplitVecOp_ExtVecInRegOp(N);
3050	break;
3051
3052	case ISD::VECREDUCE_FADD:
3053	case ISD::VECREDUCE_FMUL:
3054	case ISD::VECREDUCE_ADD:
3055	case ISD::VECREDUCE_MUL:
3056	case ISD::VECREDUCE_AND:
3057	case ISD::VECREDUCE_OR:
3058	case ISD::VECREDUCE_XOR:
3059	case ISD::VECREDUCE_SMAX:
3060	case ISD::VECREDUCE_SMIN:
3061	case ISD::VECREDUCE_UMAX:
3062	case ISD::VECREDUCE_UMIN:
3063	case ISD::VECREDUCE_FMAX:
3064	case ISD::VECREDUCE_FMIN:
3065	case ISD::VECREDUCE_FMAXIMUM:
3066	case ISD::VECREDUCE_FMINIMUM:
3067	Res = SplitVecOp_VECREDUCE(N, OpNo);
3068	break;
3069	case ISD::VECREDUCE_SEQ_FADD:
3070	case ISD::VECREDUCE_SEQ_FMUL:
3071	Res = SplitVecOp_VECREDUCE_SEQ(N);
3072	break;
3073	case ISD::VP_REDUCE_FADD:
3074	case ISD::VP_REDUCE_SEQ_FADD:
3075	case ISD::VP_REDUCE_FMUL:
3076	case ISD::VP_REDUCE_SEQ_FMUL:
3077	case ISD::VP_REDUCE_ADD:
3078	case ISD::VP_REDUCE_MUL:
3079	case ISD::VP_REDUCE_AND:
3080	case ISD::VP_REDUCE_OR:
3081	case ISD::VP_REDUCE_XOR:
3082	case ISD::VP_REDUCE_SMAX:
3083	case ISD::VP_REDUCE_SMIN:
3084	case ISD::VP_REDUCE_UMAX:
3085	case ISD::VP_REDUCE_UMIN:
3086	case ISD::VP_REDUCE_FMAX:
3087	case ISD::VP_REDUCE_FMIN:
3088	Res = SplitVecOp_VP_REDUCE(N, OpNo);
3089	break;
3090	}
3091
3092	// If the result is null, the sub-method took care of registering results etc.
3093	if (!Res.getNode()) return false;
3094
3095	// If the result is N, the sub-method updated N in place. Tell the legalizer
3096	// core about this.
3097	if (Res.getNode() == N)
3098	return true;
3099
3100	if (N->isStrictFPOpcode())
3101	assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
3102	"Invalid operand expansion");
3103	else
3104	assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3105	"Invalid operand expansion");
3106
3107	ReplaceValueWith(From: SDValue(N, 0), To: Res);
3108	return false;
3109	}
3110
3111	SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
3112	// The only possibility for an illegal operand is the mask, since result type
3113	// legalization would have handled this node already otherwise.
3114	assert(OpNo == 0 && "Illegal operand must be mask");
3115
3116	SDValue Mask = N->getOperand(Num: 0);
3117	SDValue Src0 = N->getOperand(Num: 1);
3118	SDValue Src1 = N->getOperand(Num: 2);
3119	EVT Src0VT = Src0.getValueType();
3120	SDLoc DL(N);
3121	assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
3122
3123	SDValue Lo, Hi;
3124	GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3125	assert(Lo.getValueType() == Hi.getValueType() &&
3126	"Lo and Hi have differing types");
3127
3128	EVT LoOpVT, HiOpVT;
3129	std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: Src0VT);
3130	assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
3131
3132	SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
3133	std::tie(args&: LoOp0, args&: HiOp0) = DAG.SplitVector(N: Src0, DL);
3134	std::tie(args&: LoOp1, args&: HiOp1) = DAG.SplitVector(N: Src1, DL);
3135	std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
3136
3137	SDValue LoSelect =
3138	DAG.getNode(Opcode: ISD::VSELECT, DL, VT: LoOpVT, N1: LoMask, N2: LoOp0, N3: LoOp1);
3139	SDValue HiSelect =
3140	DAG.getNode(Opcode: ISD::VSELECT, DL, VT: HiOpVT, N1: HiMask, N2: HiOp0, N3: HiOp1);
3141
3142	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: Src0VT, N1: LoSelect, N2: HiSelect);
3143	}
3144
3145	SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
3146	EVT ResVT = N->getValueType(ResNo: 0);
3147	SDValue Lo, Hi;
3148	SDLoc dl(N);
3149
3150	SDValue VecOp = N->getOperand(Num: OpNo);
3151	EVT VecVT = VecOp.getValueType();
3152	assert(VecVT.isVector() && "Can only split reduce vector operand");
3153	GetSplitVector(Op: VecOp, Lo, Hi);
3154	EVT LoOpVT, HiOpVT;
3155	std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT);
3156
3157	// Use the appropriate scalar instruction on the split subvectors before
3158	// reducing the now partially reduced smaller vector.
3159	unsigned CombineOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode());
3160	SDValue Partial = DAG.getNode(Opcode: CombineOpc, DL: dl, VT: LoOpVT, N1: Lo, N2: Hi, Flags: N->getFlags());
3161	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, Operand: Partial, Flags: N->getFlags());
3162	}
3163
3164	SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
3165	EVT ResVT = N->getValueType(ResNo: 0);
3166	SDValue Lo, Hi;
3167	SDLoc dl(N);
3168
3169	SDValue AccOp = N->getOperand(Num: 0);
3170	SDValue VecOp = N->getOperand(Num: 1);
3171	SDNodeFlags Flags = N->getFlags();
3172
3173	EVT VecVT = VecOp.getValueType();
3174	assert(VecVT.isVector() && "Can only split reduce vector operand");
3175	GetSplitVector(Op: VecOp, Lo, Hi);
3176	EVT LoOpVT, HiOpVT;
3177	std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT);
3178
3179	// Reduce low half.
3180	SDValue Partial = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: AccOp, N2: Lo, Flags);
3181
3182	// Reduce high half, using low half result as initial value.
3183	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: Partial, N2: Hi, Flags);
3184	}
3185
3186	SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
3187	assert(N->isVPOpcode() && "Expected VP opcode");
3188	assert(OpNo == 1 && "Can only split reduce vector operand");
3189
3190	unsigned Opc = N->getOpcode();
3191	EVT ResVT = N->getValueType(ResNo: 0);
3192	SDValue Lo, Hi;
3193	SDLoc dl(N);
3194
3195	SDValue VecOp = N->getOperand(Num: OpNo);
3196	EVT VecVT = VecOp.getValueType();
3197	assert(VecVT.isVector() && "Can only split reduce vector operand");
3198	GetSplitVector(Op: VecOp, Lo, Hi);
3199
3200	SDValue MaskLo, MaskHi;
3201	std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2));
3202
3203	SDValue EVLLo, EVLHi;
3204	std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT, DL: dl);
3205
3206	const SDNodeFlags Flags = N->getFlags();
3207
3208	SDValue ResLo =
3209	DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {N->getOperand(Num: 0), Lo, MaskLo, EVLLo}, Flags);
3210	return DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {ResLo, Hi, MaskHi, EVLHi}, Flags);
3211	}
3212
3213	SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
3214	// The result has a legal vector type, but the input needs splitting.
3215	EVT ResVT = N->getValueType(ResNo: 0);
3216	SDValue Lo, Hi;
3217	SDLoc dl(N);
3218	GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
3219	EVT InVT = Lo.getValueType();
3220
3221	EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
3222	EC: InVT.getVectorElementCount());
3223
3224	if (N->isStrictFPOpcode()) {
3225	Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
3226	{ N->getOperand(0), Lo });
3227	Hi = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
3228	{ N->getOperand(0), Hi });
3229
3230	// Build a factor node to remember that this operation is independent
3231	// of the other one.
3232	SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(R: 1),
3233	Hi.getValue(R: 1));
3234
3235	// Legalize the chain result - switch anything that used the old chain to
3236	// use the new one.
3237	ReplaceValueWith(From: SDValue(N, 1), To: Ch);
3238	} else if (N->getNumOperands() == 3) {
3239	assert(N->isVPOpcode() && "Expected VP opcode");
3240	SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3241	std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
3242	std::tie(args&: EVLLo, args&: EVLHi) =
3243	DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl);
3244	Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, N1: Lo, N2: MaskLo, N3: EVLLo);
3245	Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, N1: Hi, N2: MaskHi, N3: EVLHi);
3246	} else {
3247	Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Lo);
3248	Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Hi);
3249	}
3250
3251	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
3252	}
3253
3254	SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
3255	// For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
3256	// end up being split all the way down to individual components. Convert the
3257	// split pieces into integers and reassemble.
3258	SDValue Lo, Hi;
3259	GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3260	Lo = BitConvertToInteger(Op: Lo);
3261	Hi = BitConvertToInteger(Op: Hi);
3262
3263	if (DAG.getDataLayout().isBigEndian())
3264	std::swap(a&: Lo, b&: Hi);
3265
3266	return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
3267	Operand: JoinIntegers(Lo, Hi));
3268	}
3269
3270	SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
3271	unsigned OpNo) {
3272	assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
3273	// We know that the result type is legal.
3274	EVT ResVT = N->getValueType(ResNo: 0);
3275
3276	SDValue Vec = N->getOperand(Num: 0);
3277	SDValue SubVec = N->getOperand(Num: 1);
3278	SDValue Idx = N->getOperand(Num: 2);
3279	SDLoc dl(N);
3280
3281	SDValue Lo, Hi;
3282	GetSplitVector(Op: SubVec, Lo, Hi);
3283
3284	uint64_t IdxVal = Idx->getAsZExtVal();
3285	uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
3286
3287	SDValue FirstInsertion =
3288	DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: Vec, N2: Lo, N3: Idx);
3289	SDValue SecondInsertion =
3290	DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: FirstInsertion, N2: Hi,
3291	N3: DAG.getVectorIdxConstant(Val: IdxVal + LoElts, DL: dl));
3292
3293	return SecondInsertion;
3294	}
3295
3296	SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
3297	// We know that the extracted result type is legal.
3298	EVT SubVT = N->getValueType(ResNo: 0);
3299	SDValue Idx = N->getOperand(Num: 1);
3300	SDLoc dl(N);
3301	SDValue Lo, Hi;
3302
3303	GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3304
3305	uint64_t LoEltsMin = Lo.getValueType().getVectorMinNumElements();
3306	uint64_t IdxVal = Idx->getAsZExtVal();
3307
3308	if (IdxVal < LoEltsMin) {
3309	assert(IdxVal + SubVT.getVectorMinNumElements() <= LoEltsMin &&
3310	"Extracted subvector crosses vector split!");
3311	return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: SubVT, N1: Lo, N2: Idx);
3312	} else if (SubVT.isScalableVector() ==
3313	N->getOperand(Num: 0).getValueType().isScalableVector())
3314	return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: SubVT, N1: Hi,
3315	N2: DAG.getVectorIdxConstant(Val: IdxVal - LoEltsMin, DL: dl));
3316
3317	// After this point the DAG node only permits extracting fixed-width
3318	// subvectors from scalable vectors.
3319	assert(SubVT.isFixedLengthVector() &&
3320	"Extracting scalable subvector from fixed-width unsupported");
3321
3322	// If the element type is i1 and we're not promoting the result, then we may
3323	// end up loading the wrong data since the bits are packed tightly into
3324	// bytes. For example, if we extract a v4i1 (legal) from a nxv4i1 (legal)
3325	// type at index 4, then we will load a byte starting at index 0.
3326	if (SubVT.getScalarType() == MVT::i1)
3327	report_fatal_error(reason: "Don't know how to extract fixed-width predicate "
3328	"subvector from a scalable predicate vector");
3329
3330	// Spill the vector to the stack. We should use the alignment for
3331	// the smallest part.
3332	SDValue Vec = N->getOperand(Num: 0);
3333	EVT VecVT = Vec.getValueType();
3334	Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
3335	SDValue StackPtr =
3336	DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
3337	auto &MF = DAG.getMachineFunction();
3338	auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
3339	auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
3340
3341	SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
3342	Alignment: SmallestAlign);
3343
3344	// Extract the subvector by loading the correct part.
3345	StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT: SubVT, Index: Idx);
3346
3347	return DAG.getLoad(
3348	VT: SubVT, dl, Chain: Store, Ptr: StackPtr,
3349	PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
3350	}
3351
3352	SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3353	SDValue Vec = N->getOperand(Num: 0);
3354	SDValue Idx = N->getOperand(Num: 1);
3355	EVT VecVT = Vec.getValueType();
3356
3357	if (const ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Val&: Idx)) {
3358	uint64_t IdxVal = Index->getZExtValue();
3359
3360	SDValue Lo, Hi;
3361	GetSplitVector(Op: Vec, Lo, Hi);
3362
3363	uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
3364
3365	if (IdxVal < LoElts)
3366	return SDValue(DAG.UpdateNodeOperands(N, Op1: Lo, Op2: Idx), 0);
3367	else if (!Vec.getValueType().isScalableVector())
3368	return SDValue(DAG.UpdateNodeOperands(N, Op1: Hi,
3369	Op2: DAG.getConstant(Val: IdxVal - LoElts, DL: SDLoc(N),
3370	VT: Idx.getValueType())), 0);
3371	}
3372
3373	// See if the target wants to custom expand this node.
3374	if (CustomLowerNode(N, VT: N->getValueType(ResNo: 0), LegalizeResult: true))
3375	return SDValue();
3376
3377	// Make the vector elements byte-addressable if they aren't already.
3378	SDLoc dl(N);
3379	EVT EltVT = VecVT.getVectorElementType();
3380	if (VecVT.getScalarSizeInBits() < 8) {
3381	EltVT = MVT::i8;
3382	VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
3383	EC: VecVT.getVectorElementCount());
3384	Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec);
3385	}
3386
3387	// Store the vector to the stack.
3388	// In cases where the vector is illegal it will be broken down into parts
3389	// and stored in parts - we should use the alignment for the smallest part.
3390	Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
3391	SDValue StackPtr =
3392	DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
3393	auto &MF = DAG.getMachineFunction();
3394	auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
3395	auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
3396	SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
3397	Alignment: SmallestAlign);
3398
3399	// Load back the required element.
3400	StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
3401
3402	// FIXME: This is to handle i1 vectors with elements promoted to i8.
3403	// i1 vector handling needs general improvement.
3404	if (N->getValueType(ResNo: 0).bitsLT(VT: EltVT)) {
3405	SDValue Load = DAG.getLoad(VT: EltVT, dl, Chain: Store, Ptr: StackPtr,
3406	PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
3407	return DAG.getZExtOrTrunc(Op: Load, DL: dl, VT: N->getValueType(ResNo: 0));
3408	}
3409
3410	return DAG.getExtLoad(
3411	ExtType: ISD::EXTLOAD, dl, VT: N->getValueType(ResNo: 0), Chain: Store, Ptr: StackPtr,
3412	PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()), MemVT: EltVT,
3413	Alignment: commonAlignment(A: SmallestAlign, Offset: EltVT.getFixedSizeInBits() / 8));
3414	}
3415
3416	SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
3417	SDValue Lo, Hi;
3418
3419	// *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
3420	// splitting the result has the same effect as splitting the input operand.
3421	SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
3422
3423	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
3424	}
3425
3426	SDValue DAGTypeLegalizer::SplitVecOp_Gather(MemSDNode *N, unsigned OpNo) {
3427	(void)OpNo;
3428	SDValue Lo, Hi;
3429	SplitVecRes_Gather(N, Lo, Hi);
3430
3431	SDValue Res = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: N, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
3432	ReplaceValueWith(From: SDValue(N, 0), To: Res);
3433	return SDValue();
3434	}
3435
3436	SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
3437	assert(N->isUnindexed() && "Indexed vp_store of vector?");
3438	SDValue Ch = N->getChain();
3439	SDValue Ptr = N->getBasePtr();
3440	SDValue Offset = N->getOffset();
3441	assert(Offset.isUndef() && "Unexpected VP store offset");
3442	SDValue Mask = N->getMask();
3443	SDValue EVL = N->getVectorLength();
3444	SDValue Data = N->getValue();
3445	Align Alignment = N->getOriginalAlign();
3446	SDLoc DL(N);
3447
3448	SDValue DataLo, DataHi;
3449	if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
3450	// Split Data operand
3451	GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi);
3452	else
3453	std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL);
3454
3455	// Split Mask operand
3456	SDValue MaskLo, MaskHi;
3457	if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
3458	SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
3459	} else {
3460	if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
3461	GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
3462	else
3463	std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
3464	}
3465
3466	EVT MemoryVT = N->getMemoryVT();
3467	EVT LoMemVT, HiMemVT;
3468	bool HiIsEmpty = false;
3469	std::tie(args&: LoMemVT, args&: HiMemVT) =
3470	DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty);
3471
3472	// Split EVL
3473	SDValue EVLLo, EVLHi;
3474	std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: Data.getValueType(), DL);
3475
3476	SDValue Lo, Hi;
3477	MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3478	PtrInfo: N->getPointerInfo(), f: MachineMemOperand::MOStore,
3479	s: MemoryLocation::UnknownSize, base_alignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
3480
3481	Lo = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataLo, Ptr, Offset, Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO,
3482	AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
3483	IsCompressing: N->isCompressingStore());
3484
3485	// If the hi vp_store has zero storage size, only the lo vp_store is needed.
3486	if (HiIsEmpty)
3487	return Lo;
3488
3489	Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG,
3490	IsCompressedMemory: N->isCompressingStore());
3491
3492	MachinePointerInfo MPI;
3493	if (LoMemVT.isScalableVector()) {
3494	Alignment = commonAlignment(A: Alignment,
3495	Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
3496	MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
3497	} else
3498	MPI = N->getPointerInfo().getWithOffset(
3499	O: LoMemVT.getStoreSize().getFixedValue());
3500
3501	MMO = DAG.getMachineFunction().getMachineMemOperand(
3502	PtrInfo: MPI, f: MachineMemOperand::MOStore, s: MemoryLocation::UnknownSize, base_alignment: Alignment,
3503	AAInfo: N->getAAInfo(), Ranges: N->getRanges());
3504
3505	Hi = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataHi, Ptr, Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO,
3506	AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
3507	IsCompressing: N->isCompressingStore());
3508
3509	// Build a factor node to remember that this store is independent of the
3510	// other one.
3511	return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3512	}
3513
3514	SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N,
3515	unsigned OpNo) {
3516	assert(N->isUnindexed() && "Indexed vp_strided_store of a vector?");
3517	assert(N->getOffset().isUndef() && "Unexpected VP strided store offset");
3518
3519	SDLoc DL(N);
3520
3521	SDValue Data = N->getValue();
3522	SDValue LoData, HiData;
3523	if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
3524	GetSplitVector(Op: Data, Lo&: LoData, Hi&: HiData);
3525	else
3526	std::tie(args&: LoData, args&: HiData) = DAG.SplitVector(N: Data, DL);
3527
3528	EVT LoMemVT, HiMemVT;
3529	bool HiIsEmpty = false;
3530	std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetDependentSplitDestVTs(
3531	VT: N->getMemoryVT(), EnvVT: LoData.getValueType(), HiIsEmpty: &HiIsEmpty);
3532
3533	SDValue Mask = N->getMask();
3534	SDValue LoMask, HiMask;
3535	if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC)
3536	SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask);
3537	else if (getTypeAction(VT: Mask.getValueType()) ==
3538	TargetLowering::TypeSplitVector)
3539	GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask);
3540	else
3541	std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
3542
3543	SDValue LoEVL, HiEVL;
3544	std::tie(args&: LoEVL, args&: HiEVL) =
3545	DAG.SplitEVL(N: N->getVectorLength(), VecVT: Data.getValueType(), DL);
3546
3547	// Generate the low vp_strided_store
3548	SDValue Lo = DAG.getStridedStoreVP(
3549	Chain: N->getChain(), DL, Val: LoData, Ptr: N->getBasePtr(), Offset: N->getOffset(),
3550	Stride: N->getStride(), Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: N->getMemOperand(),
3551	AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(), IsCompressing: N->isCompressingStore());
3552
3553	// If the high vp_strided_store has zero storage size, only the low
3554	// vp_strided_store is needed.
3555	if (HiIsEmpty)
3556	return Lo;
3557
3558	// Generate the high vp_strided_store.
3559	// To calculate the high base address, we need to sum to the low base
3560	// address stride number of bytes for each element already stored by low,
3561	// that is: Ptr = Ptr + (LoEVL * Stride)
3562	EVT PtrVT = N->getBasePtr().getValueType();
3563	SDValue Increment =
3564	DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL,
3565	N2: DAG.getSExtOrTrunc(Op: N->getStride(), DL, VT: PtrVT));
3566	SDValue Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: N->getBasePtr(), N2: Increment);
3567
3568	Align Alignment = N->getOriginalAlign();
3569	if (LoMemVT.isScalableVector())
3570	Alignment = commonAlignment(A: Alignment,
3571	Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
3572
3573	MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3574	PtrInfo: MachinePointerInfo(N->getPointerInfo().getAddrSpace()),
3575	f: MachineMemOperand::MOStore, s: MemoryLocation::UnknownSize, base_alignment: Alignment,
3576	AAInfo: N->getAAInfo(), Ranges: N->getRanges());
3577
3578	SDValue Hi = DAG.getStridedStoreVP(
3579	Chain: N->getChain(), DL, Val: HiData, Ptr, Offset: N->getOffset(), Stride: N->getStride(), Mask: HiMask,
3580	EVL: HiEVL, MemVT: HiMemVT, MMO, AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
3581	IsCompressing: N->isCompressingStore());
3582
3583	// Build a factor node to remember that this store is independent of the
3584	// other one.
3585	return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3586	}
3587
3588	SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
3589	unsigned OpNo) {
3590	assert(N->isUnindexed() && "Indexed masked store of vector?");
3591	SDValue Ch = N->getChain();
3592	SDValue Ptr = N->getBasePtr();
3593	SDValue Offset = N->getOffset();
3594	assert(Offset.isUndef() && "Unexpected indexed masked store offset");
3595	SDValue Mask = N->getMask();
3596	SDValue Data = N->getValue();
3597	Align Alignment = N->getOriginalAlign();
3598	SDLoc DL(N);
3599
3600	SDValue DataLo, DataHi;
3601	if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
3602	// Split Data operand
3603	GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi);
3604	else
3605	std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL);
3606
3607	// Split Mask operand
3608	SDValue MaskLo, MaskHi;
3609	if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
3610	SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
3611	} else {
3612	if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
3613	GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
3614	else
3615	std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
3616	}
3617
3618	EVT MemoryVT = N->getMemoryVT();
3619	EVT LoMemVT, HiMemVT;
3620	bool HiIsEmpty = false;
3621	std::tie(args&: LoMemVT, args&: HiMemVT) =
3622	DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty);
3623
3624	SDValue Lo, Hi, Res;
3625	MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3626	PtrInfo: N->getPointerInfo(), f: MachineMemOperand::MOStore,
3627	s: MemoryLocation::UnknownSize, base_alignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
3628
3629	Lo = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataLo, Base: Ptr, Offset, Mask: MaskLo, MemVT: LoMemVT, MMO,
3630	AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
3631	IsCompressing: N->isCompressingStore());
3632
3633	if (HiIsEmpty) {
3634	// The hi masked store has zero storage size.
3635	// Only the lo masked store is needed.
3636	Res = Lo;
3637	} else {
3638
3639	Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG,
3640	IsCompressedMemory: N->isCompressingStore());
3641
3642	MachinePointerInfo MPI;
3643	if (LoMemVT.isScalableVector()) {
3644	Alignment = commonAlignment(
3645	A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
3646	MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
3647	} else
3648	MPI = N->getPointerInfo().getWithOffset(
3649	O: LoMemVT.getStoreSize().getFixedValue());
3650
3651	MMO = DAG.getMachineFunction().getMachineMemOperand(
3652	PtrInfo: MPI, f: MachineMemOperand::MOStore, s: MemoryLocation::UnknownSize, base_alignment: Alignment,
3653	AAInfo: N->getAAInfo(), Ranges: N->getRanges());
3654
3655	Hi = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataHi, Base: Ptr, Offset, Mask: MaskHi, MemVT: HiMemVT, MMO,
3656	AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
3657	IsCompressing: N->isCompressingStore());
3658
3659	// Build a factor node to remember that this store is independent of the
3660	// other one.
3661	Res = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3662	}
3663
3664	return Res;
3665	}
3666
3667	SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
3668	SDValue Ch = N->getChain();
3669	SDValue Ptr = N->getBasePtr();
3670	EVT MemoryVT = N->getMemoryVT();
3671	Align Alignment = N->getOriginalAlign();
3672	SDLoc DL(N);
3673	struct Operands {
3674	SDValue Mask;
3675	SDValue Index;
3676	SDValue Scale;
3677	SDValue Data;
3678	} Ops = [&]() -> Operands {
3679	if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) {
3680	return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale(),
3681	.Data: MSC->getValue()};
3682	}
3683	auto *VPSC = cast<VPScatterSDNode>(Val: N);
3684	return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale(),
3685	.Data: VPSC->getValue()};
3686	}();
3687	// Split all operands
3688
3689	EVT LoMemVT, HiMemVT;
3690	std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
3691
3692	SDValue DataLo, DataHi;
3693	if (getTypeAction(VT: Ops.Data.getValueType()) == TargetLowering::TypeSplitVector)
3694	// Split Data operand
3695	GetSplitVector(Op: Ops.Data, Lo&: DataLo, Hi&: DataHi);
3696	else
3697	std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Ops.Data, DL);
3698
3699	// Split Mask operand
3700	SDValue MaskLo, MaskHi;
3701	if (OpNo == 1 && Ops.Mask.getOpcode() == ISD::SETCC) {
3702	SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
3703	} else {
3704	std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL);
3705	}
3706
3707	SDValue IndexHi, IndexLo;
3708	if (getTypeAction(VT: Ops.Index.getValueType()) ==
3709	TargetLowering::TypeSplitVector)
3710	GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi);
3711	else
3712	std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL);
3713
3714	SDValue Lo;
3715	MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3716	PtrInfo: N->getPointerInfo(), f: MachineMemOperand::MOStore,
3717	s: MemoryLocation::UnknownSize, base_alignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
3718
3719	if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) {
3720	SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
3721	Lo =
3722	DAG.getMaskedScatter(VTs: DAG.getVTList(MVT::Other), MemVT: LoMemVT, dl: DL, Ops: OpsLo, MMO,
3723	IndexType: MSC->getIndexType(), IsTruncating: MSC->isTruncatingStore());
3724
3725	// The order of the Scatter operation after split is well defined. The "Hi"
3726	// part comes after the "Lo". So these two operations should be chained one
3727	// after another.
3728	SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Ops.Scale};
3729	return DAG.getMaskedScatter(VTs: DAG.getVTList(MVT::Other), MemVT: HiMemVT, dl: DL, Ops: OpsHi,
3730	MMO, IndexType: MSC->getIndexType(),
3731	IsTruncating: MSC->isTruncatingStore());
3732	}
3733	auto *VPSC = cast<VPScatterSDNode>(Val: N);
3734	SDValue EVLLo, EVLHi;
3735	std::tie(args&: EVLLo, args&: EVLHi) =
3736	DAG.SplitEVL(N: VPSC->getVectorLength(), VecVT: Ops.Data.getValueType(), DL);
3737
3738	SDValue OpsLo[] = {Ch, DataLo, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
3739	Lo = DAG.getScatterVP(VTs: DAG.getVTList(MVT::Other), VT: LoMemVT, dl: DL, Ops: OpsLo, MMO,
3740	IndexType: VPSC->getIndexType());
3741
3742	// The order of the Scatter operation after split is well defined. The "Hi"
3743	// part comes after the "Lo". So these two operations should be chained one
3744	// after another.
3745	SDValue OpsHi[] = {Lo, DataHi, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
3746	return DAG.getScatterVP(VTs: DAG.getVTList(MVT::Other), VT: HiMemVT, dl: DL, Ops: OpsHi, MMO,
3747	IndexType: VPSC->getIndexType());
3748	}
3749
3750	SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
3751	assert(N->isUnindexed() && "Indexed store of vector?");
3752	assert(OpNo == 1 && "Can only split the stored value");
3753	SDLoc DL(N);
3754
3755	bool isTruncating = N->isTruncatingStore();
3756	SDValue Ch = N->getChain();
3757	SDValue Ptr = N->getBasePtr();
3758	EVT MemoryVT = N->getMemoryVT();
3759	Align Alignment = N->getOriginalAlign();
3760	MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3761	AAMDNodes AAInfo = N->getAAInfo();
3762	SDValue Lo, Hi;
3763	GetSplitVector(Op: N->getOperand(Num: 1), Lo, Hi);
3764
3765	EVT LoMemVT, HiMemVT;
3766	std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
3767
3768	// Scalarize if the split halves are not byte-sized.
3769	if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
3770	return TLI.scalarizeVectorStore(ST: N, DAG);
3771
3772	if (isTruncating)
3773	Lo = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), SVT: LoMemVT,
3774	Alignment, MMOFlags, AAInfo);
3775	else
3776	Lo = DAG.getStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), Alignment, MMOFlags,
3777	AAInfo);
3778
3779	MachinePointerInfo MPI;
3780	IncrementPointer(N, MemVT: LoMemVT, MPI, Ptr);
3781
3782	if (isTruncating)
3783	Hi = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI,
3784	SVT: HiMemVT, Alignment, MMOFlags, AAInfo);
3785	else
3786	Hi = DAG.getStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI, Alignment, MMOFlags, AAInfo);
3787
3788	return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3789	}
3790
3791	SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
3792	SDLoc DL(N);
3793
3794	// The input operands all must have the same type, and we know the result
3795	// type is valid. Convert this to a buildvector which extracts all the
3796	// input elements.
3797	// TODO: If the input elements are power-two vectors, we could convert this to
3798	// a new CONCAT_VECTORS node with elements that are half-wide.
3799	SmallVector<SDValue, 32> Elts;
3800	EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
3801	for (const SDValue &Op : N->op_values()) {
3802	for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
3803	i != e; ++i) {
3804	Elts.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Op,
3805	N2: DAG.getVectorIdxConstant(Val: i, DL)));
3806	}
3807	}
3808
3809	return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL, Ops: Elts);
3810	}
3811
3812	SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
3813	// The result type is legal, but the input type is illegal. If splitting
3814	// ends up with the result type of each half still being legal, just
3815	// do that. If, however, that would result in an illegal result type,
3816	// we can try to get more clever with power-two vectors. Specifically,
3817	// split the input type, but also widen the result element size, then
3818	// concatenate the halves and truncate again. For example, consider a target
3819	// where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
3820	// vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
3821	// %inlo = v4i32 extract_subvector %in, 0
3822	// %inhi = v4i32 extract_subvector %in, 4
3823	// %lo16 = v4i16 trunc v4i32 %inlo
3824	// %hi16 = v4i16 trunc v4i32 %inhi
3825	// %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
3826	// %res = v8i8 trunc v8i16 %in16
3827	//
3828	// Without this transform, the original truncate would end up being
3829	// scalarized, which is pretty much always a last resort.
3830	unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
3831	SDValue InVec = N->getOperand(Num: OpNo);
3832	EVT InVT = InVec->getValueType(ResNo: 0);
3833	EVT OutVT = N->getValueType(ResNo: 0);
3834	ElementCount NumElements = OutVT.getVectorElementCount();
3835	bool IsFloat = OutVT.isFloatingPoint();
3836
3837	unsigned InElementSize = InVT.getScalarSizeInBits();
3838	unsigned OutElementSize = OutVT.getScalarSizeInBits();
3839
3840	// Determine the split output VT. If its legal we can just split dirctly.
3841	EVT LoOutVT, HiOutVT;
3842	std::tie(args&: LoOutVT, args&: HiOutVT) = DAG.GetSplitDestVTs(VT: OutVT);
3843	assert(LoOutVT == HiOutVT && "Unequal split?");
3844
3845	// If the input elements are only 1/2 the width of the result elements,
3846	// just use the normal splitting. Our trick only work if there's room
3847	// to split more than once.
3848	if (isTypeLegal(VT: LoOutVT) ||
3849	InElementSize <= OutElementSize * 2)
3850	return SplitVecOp_UnaryOp(N);
3851	SDLoc DL(N);
3852
3853	// Don't touch if this will be scalarized.
3854	EVT FinalVT = InVT;
3855	while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector)
3856	FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
3857
3858	if (getTypeAction(VT: FinalVT) == TargetLowering::TypeScalarizeVector)
3859	return SplitVecOp_UnaryOp(N);
3860
3861	// Get the split input vector.
3862	SDValue InLoVec, InHiVec;
3863	GetSplitVector(Op: InVec, Lo&: InLoVec, Hi&: InHiVec);
3864
3865	// Truncate them to 1/2 the element size.
3866	//
3867	// This assumes the number of elements is a power of two; any vector that
3868	// isn't should be widened, not split.
3869	EVT HalfElementVT = IsFloat ?
3870	EVT::getFloatingPointVT(BitWidth: InElementSize/2) :
3871	EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: InElementSize/2);
3872	EVT HalfVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT,
3873	EC: NumElements.divideCoefficientBy(RHS: 2));
3874
3875	SDValue HalfLo;
3876	SDValue HalfHi;
3877	SDValue Chain;
3878	if (N->isStrictFPOpcode()) {
3879	HalfLo = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
3880	{N->getOperand(0), InLoVec});
3881	HalfHi = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
3882	{N->getOperand(0), InHiVec});
3883	// Legalize the chain result - switch anything that used the old chain to
3884	// use the new one.
3885	Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, HalfLo.getValue(R: 1),
3886	HalfHi.getValue(R: 1));
3887	} else {
3888	HalfLo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InLoVec);
3889	HalfHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InHiVec);
3890	}
3891
3892	// Concatenate them to get the full intermediate truncation result.
3893	EVT InterVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT, EC: NumElements);
3894	SDValue InterVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InterVT, N1: HalfLo,
3895	N2: HalfHi);
3896	// Now finish up by truncating all the way down to the original result
3897	// type. This should normally be something that ends up being legal directly,
3898	// but in theory if a target has very wide vectors and an annoyingly
3899	// restricted set of legal types, this split can chain to build things up.
3900
3901	if (N->isStrictFPOpcode()) {
3902	SDValue Res = DAG.getNode(
3903	ISD::STRICT_FP_ROUND, DL, {OutVT, MVT::Other},
3904	{Chain, InterVec,
3905	DAG.getTargetConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()))});
3906	// Relink the chain
3907	ReplaceValueWith(From: SDValue(N, 1), To: SDValue(Res.getNode(), 1));
3908	return Res;
3909	}
3910
3911	return IsFloat
3912	? DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: OutVT, N1: InterVec,
3913	N2: DAG.getTargetConstant(
3914	Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout())))
3915	: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OutVT, Operand: InterVec);
3916	}
3917
3918	SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
3919	assert(N->getValueType(0).isVector() &&
3920	N->getOperand(0).getValueType().isVector() &&
3921	"Operand types must be vectors");
3922	// The result has a legal vector type, but the input needs splitting.
3923	SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
3924	SDLoc DL(N);
3925	GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Lo0, Hi&: Hi0);
3926	GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Lo1, Hi&: Hi1);
3927	auto PartEltCnt = Lo0.getValueType().getVectorElementCount();
3928
3929	LLVMContext &Context = *DAG.getContext();
3930	EVT PartResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt);
3931	EVT WideResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt*2);
3932
3933	if (N->getOpcode() == ISD::SETCC) {
3934	LoRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Lo0, N2: Lo1, N3: N->getOperand(Num: 2));
3935	HiRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Hi0, N2: Hi1, N3: N->getOperand(Num: 2));
3936	} else {
3937	assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
3938	SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3939	std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 3));
3940	std::tie(args&: EVLLo, args&: EVLHi) =
3941	DAG.SplitEVL(N: N->getOperand(Num: 4), VecVT: N->getValueType(ResNo: 0), DL);
3942	LoRes = DAG.getNode(Opcode: ISD::VP_SETCC, DL, VT: PartResVT, N1: Lo0, N2: Lo1,
3943	N3: N->getOperand(Num: 2), N4: MaskLo, N5: EVLLo);
3944	HiRes = DAG.getNode(Opcode: ISD::VP_SETCC, DL, VT: PartResVT, N1: Hi0, N2: Hi1,
3945	N3: N->getOperand(Num: 2), N4: MaskHi, N5: EVLHi);
3946	}
3947	SDValue Con = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: WideResVT, N1: LoRes, N2: HiRes);
3948
3949	EVT OpVT = N->getOperand(Num: 0).getValueType();
3950	ISD::NodeType ExtendCode =
3951	TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
3952	return DAG.getNode(Opcode: ExtendCode, DL, VT: N->getValueType(ResNo: 0), Operand: Con);
3953	}
3954
3955
3956	SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
3957	// The result has a legal vector type, but the input needs splitting.
3958	EVT ResVT = N->getValueType(ResNo: 0);
3959	SDValue Lo, Hi;
3960	SDLoc DL(N);
3961	GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
3962	EVT InVT = Lo.getValueType();
3963
3964	EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
3965	EC: InVT.getVectorElementCount());
3966
3967	if (N->isStrictFPOpcode()) {
3968	Lo = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
3969	{ N->getOperand(0), Lo, N->getOperand(2) });
3970	Hi = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
3971	{ N->getOperand(0), Hi, N->getOperand(2) });
3972	// Legalize the chain result - switch anything that used the old chain to
3973	// use the new one.
3974	SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
3975	Lo.getValue(1), Hi.getValue(1));
3976	ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
3977	} else if (N->getOpcode() == ISD::VP_FP_ROUND) {
3978	SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3979	std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
3980	std::tie(args&: EVLLo, args&: EVLHi) =
3981	DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL);
3982	Lo = DAG.getNode(Opcode: ISD::VP_FP_ROUND, DL, VT: OutVT, N1: Lo, N2: MaskLo, N3: EVLLo);
3983	Hi = DAG.getNode(Opcode: ISD::VP_FP_ROUND, DL, VT: OutVT, N1: Hi, N2: MaskHi, N3: EVLHi);
3984	} else {
3985	Lo = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: OutVT, N1: Lo, N2: N->getOperand(Num: 1));
3986	Hi = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: OutVT, N1: Hi, N2: N->getOperand(Num: 1));
3987	}
3988
3989	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ResVT, N1: Lo, N2: Hi);
3990	}
3991
3992	// Split a vector type in an FP binary operation where the second operand has a
3993	// different type from the first.
3994	//
3995	// The result (and the first input) has a legal vector type, but the second
3996	// input needs splitting.
3997	SDValue DAGTypeLegalizer::SplitVecOp_FPOpDifferentTypes(SDNode *N) {
3998	SDLoc DL(N);
3999
4000	EVT LHSLoVT, LHSHiVT;
4001	std::tie(args&: LHSLoVT, args&: LHSHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
4002
4003	if (!isTypeLegal(VT: LHSLoVT) || !isTypeLegal(VT: LHSHiVT))
4004	return DAG.UnrollVectorOp(N, ResNE: N->getValueType(ResNo: 0).getVectorNumElements());
4005
4006	SDValue LHSLo, LHSHi;
4007	std::tie(args&: LHSLo, args&: LHSHi) =
4008	DAG.SplitVector(N: N->getOperand(Num: 0), DL, LoVT: LHSLoVT, HiVT: LHSHiVT);
4009
4010	SDValue RHSLo, RHSHi;
4011	std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: N->getOperand(Num: 1), DL);
4012
4013	SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLoVT, N1: LHSLo, N2: RHSLo);
4014	SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHiVT, N1: LHSHi, N2: RHSHi);
4015
4016	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4017	}
4018
4019	SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
4020	EVT ResVT = N->getValueType(ResNo: 0);
4021	SDValue Lo, Hi;
4022	SDLoc dl(N);
4023	GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4024	EVT InVT = Lo.getValueType();
4025
4026	EVT NewResVT =
4027	EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
4028	EC: InVT.getVectorElementCount());
4029
4030	Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Lo, N2: N->getOperand(Num: 1));
4031	Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Hi, N2: N->getOperand(Num: 1));
4032
4033	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
4034	}
4035
4036	//===----------------------------------------------------------------------===//
4037	// Result Vector Widening
4038	//===----------------------------------------------------------------------===//
4039
4040	void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
4041	LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG));
4042
4043	// See if the target wants to custom widen this node.
4044	if (CustomWidenLowerNode(N, VT: N->getValueType(ResNo)))
4045	return;
4046
4047	SDValue Res = SDValue();
4048
4049	auto unrollExpandedOp = [&]() {
4050	// We're going to widen this vector op to a legal type by padding with undef
4051	// elements. If the wide vector op is eventually going to be expanded to
4052	// scalar libcalls, then unroll into scalar ops now to avoid unnecessary
4053	// libcalls on the undef elements.
4054	EVT VT = N->getValueType(ResNo: 0);
4055	EVT WideVecVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
4056	if (!TLI.isOperationLegalOrCustom(Op: N->getOpcode(), VT: WideVecVT) &&
4057	TLI.isOperationExpand(Op: N->getOpcode(), VT: VT.getScalarType())) {
4058	Res = DAG.UnrollVectorOp(N, ResNE: WideVecVT.getVectorNumElements());
4059	return true;
4060	}
4061	return false;
4062	};
4063
4064	switch (N->getOpcode()) {
4065	default:
4066	#ifndef NDEBUG
4067	dbgs() << "WidenVectorResult #" << ResNo << ": ";
4068	N->dump(G: &DAG);
4069	dbgs() << "\n";
4070	#endif
4071	report_fatal_error(reason: "Do not know how to widen the result of this operator!");
4072
4073	case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
4074	case ISD::AssertZext: Res = WidenVecRes_AssertZext(N); break;
4075	case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
4076	case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
4077	case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
4078	case ISD::INSERT_SUBVECTOR:
4079	Res = WidenVecRes_INSERT_SUBVECTOR(N);
4080	break;
4081	case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
4082	case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
4083	case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
4084	case ISD::STEP_VECTOR:
4085	case ISD::SPLAT_VECTOR:
4086	case ISD::SCALAR_TO_VECTOR:
4087	Res = WidenVecRes_ScalarOp(N);
4088	break;
4089	case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
4090	case ISD::VSELECT:
4091	case ISD::SELECT:
4092	case ISD::VP_SELECT:
4093	case ISD::VP_MERGE:
4094	Res = WidenVecRes_Select(N);
4095	break;
4096	case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
4097	case ISD::VP_SETCC:
4098	case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
4099	case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
4100	case ISD::VECTOR_SHUFFLE:
4101	Res = WidenVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N));
4102	break;
4103	case ISD::VP_LOAD:
4104	Res = WidenVecRes_VP_LOAD(N: cast<VPLoadSDNode>(Val: N));
4105	break;
4106	case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
4107	Res = WidenVecRes_VP_STRIDED_LOAD(N: cast<VPStridedLoadSDNode>(Val: N));
4108	break;
4109	case ISD::MLOAD:
4110	Res = WidenVecRes_MLOAD(N: cast<MaskedLoadSDNode>(Val: N));
4111	break;
4112	case ISD::MGATHER:
4113	Res = WidenVecRes_MGATHER(N: cast<MaskedGatherSDNode>(Val: N));
4114	break;
4115	case ISD::VP_GATHER:
4116	Res = WidenVecRes_VP_GATHER(N: cast<VPGatherSDNode>(Val: N));
4117	break;
4118	case ISD::VECTOR_REVERSE:
4119	Res = WidenVecRes_VECTOR_REVERSE(N);
4120	break;
4121
4122	case ISD::ADD: case ISD::VP_ADD:
4123	case ISD::AND: case ISD::VP_AND:
4124	case ISD::MUL: case ISD::VP_MUL:
4125	case ISD::MULHS:
4126	case ISD::MULHU:
4127	case ISD::OR: case ISD::VP_OR:
4128	case ISD::SUB: case ISD::VP_SUB:
4129	case ISD::XOR: case ISD::VP_XOR:
4130	case ISD::SHL: case ISD::VP_SHL:
4131	case ISD::SRA: case ISD::VP_ASHR:
4132	case ISD::SRL: case ISD::VP_LSHR:
4133	case ISD::FMINNUM: case ISD::VP_FMINNUM:
4134	case ISD::FMAXNUM: case ISD::VP_FMAXNUM:
4135	case ISD::FMINIMUM:
4136	case ISD::VP_FMINIMUM:
4137	case ISD::FMAXIMUM:
4138	case ISD::VP_FMAXIMUM:
4139	case ISD::SMIN: case ISD::VP_SMIN:
4140	case ISD::SMAX: case ISD::VP_SMAX:
4141	case ISD::UMIN: case ISD::VP_UMIN:
4142	case ISD::UMAX: case ISD::VP_UMAX:
4143	case ISD::UADDSAT:
4144	case ISD::SADDSAT:
4145	case ISD::USUBSAT:
4146	case ISD::SSUBSAT:
4147	case ISD::SSHLSAT:
4148	case ISD::USHLSAT:
4149	case ISD::ROTL:
4150	case ISD::ROTR:
4151	case ISD::AVGFLOORS:
4152	case ISD::AVGFLOORU:
4153	case ISD::AVGCEILS:
4154	case ISD::AVGCEILU:
4155	// Vector-predicated binary op widening. Note that -- unlike the
4156	// unpredicated versions -- we don't have to worry about trapping on
4157	// operations like UDIV, FADD, etc., as we pass on the original vector
4158	// length parameter. This means the widened elements containing garbage
4159	// aren't active.
4160	case ISD::VP_SDIV:
4161	case ISD::VP_UDIV:
4162	case ISD::VP_SREM:
4163	case ISD::VP_UREM:
4164	case ISD::VP_FADD:
4165	case ISD::VP_FSUB:
4166	case ISD::VP_FMUL:
4167	case ISD::VP_FDIV:
4168	case ISD::VP_FREM:
4169	case ISD::VP_FCOPYSIGN:
4170	Res = WidenVecRes_Binary(N);
4171	break;
4172
4173	case ISD::FPOW:
4174	case ISD::FREM:
4175	if (unrollExpandedOp())
4176	break;
4177	// If the target has custom/legal support for the scalar FP intrinsic ops
4178	// (they are probably not destined to become libcalls), then widen those
4179	// like any other binary ops.
4180	[[fallthrough]];
4181
4182	case ISD::FADD:
4183	case ISD::FMUL:
4184	case ISD::FSUB:
4185	case ISD::FDIV:
4186	case ISD::SDIV:
4187	case ISD::UDIV:
4188	case ISD::SREM:
4189	case ISD::UREM:
4190	Res = WidenVecRes_BinaryCanTrap(N);
4191	break;
4192
4193	case ISD::SMULFIX:
4194	case ISD::SMULFIXSAT:
4195	case ISD::UMULFIX:
4196	case ISD::UMULFIXSAT:
4197	// These are binary operations, but with an extra operand that shouldn't
4198	// be widened (the scale).
4199	Res = WidenVecRes_BinaryWithExtraScalarOp(N);
4200	break;
4201
4202	#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
4203	case ISD::STRICT_##DAGN:
4204	#include "llvm/IR/ConstrainedOps.def"
4205	Res = WidenVecRes_StrictFP(N);
4206	break;
4207
4208	case ISD::UADDO:
4209	case ISD::SADDO:
4210	case ISD::USUBO:
4211	case ISD::SSUBO:
4212	case ISD::UMULO:
4213	case ISD::SMULO:
4214	Res = WidenVecRes_OverflowOp(N, ResNo);
4215	break;
4216
4217	case ISD::FCOPYSIGN:
4218	Res = WidenVecRes_FCOPYSIGN(N);
4219	break;
4220
4221	case ISD::IS_FPCLASS:
4222	Res = WidenVecRes_IS_FPCLASS(N);
4223	break;
4224
4225	case ISD::FLDEXP:
4226	case ISD::FPOWI:
4227	if (!unrollExpandedOp())
4228	Res = WidenVecRes_ExpOp(N);
4229	break;
4230
4231	case ISD::ANY_EXTEND_VECTOR_INREG:
4232	case ISD::SIGN_EXTEND_VECTOR_INREG:
4233	case ISD::ZERO_EXTEND_VECTOR_INREG:
4234	Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
4235	break;
4236
4237	case ISD::ANY_EXTEND:
4238	case ISD::FP_EXTEND:
4239	case ISD::VP_FP_EXTEND:
4240	case ISD::FP_ROUND:
4241	case ISD::VP_FP_ROUND:
4242	case ISD::FP_TO_SINT:
4243	case ISD::VP_FP_TO_SINT:
4244	case ISD::FP_TO_UINT:
4245	case ISD::VP_FP_TO_UINT:
4246	case ISD::SIGN_EXTEND:
4247	case ISD::VP_SIGN_EXTEND:
4248	case ISD::SINT_TO_FP:
4249	case ISD::VP_SINT_TO_FP:
4250	case ISD::VP_TRUNCATE:
4251	case ISD::TRUNCATE:
4252	case ISD::UINT_TO_FP:
4253	case ISD::VP_UINT_TO_FP:
4254	case ISD::ZERO_EXTEND:
4255	case ISD::VP_ZERO_EXTEND:
4256	Res = WidenVecRes_Convert(N);
4257	break;
4258
4259	case ISD::FP_TO_SINT_SAT:
4260	case ISD::FP_TO_UINT_SAT:
4261	Res = WidenVecRes_FP_TO_XINT_SAT(N);
4262	break;
4263
4264	case ISD::LRINT:
4265	case ISD::LLRINT:
4266	Res = WidenVecRes_XRINT(N);
4267	break;
4268
4269	case ISD::FABS:
4270	case ISD::FCEIL:
4271	case ISD::FCOS:
4272	case ISD::FEXP:
4273	case ISD::FEXP2:
4274	case ISD::FEXP10:
4275	case ISD::FFLOOR:
4276	case ISD::FLOG:
4277	case ISD::FLOG10:
4278	case ISD::FLOG2:
4279	case ISD::FNEARBYINT:
4280	case ISD::FRINT:
4281	case ISD::FROUND:
4282	case ISD::FROUNDEVEN:
4283	case ISD::FSIN:
4284	case ISD::FSQRT:
4285	case ISD::FTRUNC:
4286	if (unrollExpandedOp())
4287	break;
4288	// If the target has custom/legal support for the scalar FP intrinsic ops
4289	// (they are probably not destined to become libcalls), then widen those
4290	// like any other unary ops.
4291	[[fallthrough]];
4292
4293	case ISD::ABS:
4294	case ISD::VP_ABS:
4295	case ISD::BITREVERSE:
4296	case ISD::VP_BITREVERSE:
4297	case ISD::BSWAP:
4298	case ISD::VP_BSWAP:
4299	case ISD::CTLZ:
4300	case ISD::VP_CTLZ:
4301	case ISD::CTLZ_ZERO_UNDEF:
4302	case ISD::VP_CTLZ_ZERO_UNDEF:
4303	case ISD::CTPOP:
4304	case ISD::VP_CTPOP:
4305	case ISD::CTTZ:
4306	case ISD::VP_CTTZ:
4307	case ISD::CTTZ_ZERO_UNDEF:
4308	case ISD::VP_CTTZ_ZERO_UNDEF:
4309	case ISD::FNEG: case ISD::VP_FNEG:
4310	case ISD::VP_FABS:
4311	case ISD::VP_SQRT:
4312	case ISD::VP_FCEIL:
4313	case ISD::VP_FFLOOR:
4314	case ISD::VP_FRINT:
4315	case ISD::VP_FNEARBYINT:
4316	case ISD::VP_FROUND:
4317	case ISD::VP_FROUNDEVEN:
4318	case ISD::VP_FROUNDTOZERO:
4319	case ISD::FREEZE:
4320	case ISD::ARITH_FENCE:
4321	case ISD::FCANONICALIZE:
4322	Res = WidenVecRes_Unary(N);
4323	break;
4324	case ISD::FMA: case ISD::VP_FMA:
4325	case ISD::FSHL:
4326	case ISD::VP_FSHL:
4327	case ISD::FSHR:
4328	case ISD::VP_FSHR:
4329	Res = WidenVecRes_Ternary(N);
4330	break;
4331	}
4332
4333	// If Res is null, the sub-method took care of registering the result.
4334	if (Res.getNode())
4335	SetWidenedVector(Op: SDValue(N, ResNo), Result: Res);
4336	}
4337
4338	SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
4339	// Ternary op widening.
4340	SDLoc dl(N);
4341	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
4342	SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
4343	SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
4344	SDValue InOp3 = GetWidenedVector(Op: N->getOperand(Num: 2));
4345	if (N->getNumOperands() == 3)
4346	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3);
4347
4348	assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
4349	assert(N->isVPOpcode() && "Expected VP opcode");
4350
4351	SDValue Mask =
4352	GetWidenedMask(Mask: N->getOperand(Num: 3), EC: WidenVT.getVectorElementCount());
4353	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT,
4354	Ops: {InOp1, InOp2, InOp3, Mask, N->getOperand(Num: 4)});
4355	}
4356
4357	SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
4358	// Binary op widening.
4359	SDLoc dl(N);
4360	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
4361	SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
4362	SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
4363	if (N->getNumOperands() == 2)
4364	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2,
4365	Flags: N->getFlags());
4366
4367	assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
4368	assert(N->isVPOpcode() && "Expected VP opcode");
4369
4370	SDValue Mask =
4371	GetWidenedMask(Mask: N->getOperand(Num: 2), EC: WidenVT.getVectorElementCount());
4372	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT,
4373	Ops: {InOp1, InOp2, Mask, N->getOperand(Num: 3)}, Flags: N->getFlags());
4374	}
4375
4376	SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
4377	// Binary op widening, but with an extra operand that shouldn't be widened.
4378	SDLoc dl(N);
4379	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
4380	SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
4381	SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
4382	SDValue InOp3 = N->getOperand(Num: 2);
4383	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3,
4384	Flags: N->getFlags());
4385	}
4386
4387	// Given a vector of operations that have been broken up to widen, see
4388	// if we can collect them together into the next widest legal VT. This
4389	// implementation is trap-safe.
4390	static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI,
4391	SmallVectorImpl<SDValue> &ConcatOps,
4392	unsigned ConcatEnd, EVT VT, EVT MaxVT,
4393	EVT WidenVT) {
4394	// Check to see if we have a single operation with the widen type.
4395	if (ConcatEnd == 1) {
4396	VT = ConcatOps[0].getValueType();
4397	if (VT == WidenVT)
4398	return ConcatOps[0];
4399	}
4400
4401	SDLoc dl(ConcatOps[0]);
4402	EVT WidenEltVT = WidenVT.getVectorElementType();
4403
4404	// while (Some element of ConcatOps is not of type MaxVT) {
4405	// From the end of ConcatOps, collect elements of the same type and put
4406	// them into an op of the next larger supported type
4407	// }
4408	while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
4409	int Idx = ConcatEnd - 1;
4410	VT = ConcatOps[Idx--].getValueType();
4411	while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
4412	Idx--;
4413
4414	int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
4415	EVT NextVT;
4416	do {
4417	NextSize *= 2;
4418	NextVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NextSize);
4419	} while (!TLI.isTypeLegal(VT: NextVT));
4420
4421	if (!VT.isVector()) {
4422	// Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
4423	SDValue VecOp = DAG.getUNDEF(VT: NextVT);
4424	unsigned NumToInsert = ConcatEnd - Idx - 1;
4425	for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
4426	VecOp = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: NextVT, N1: VecOp,
4427	N2: ConcatOps[OpIdx], N3: DAG.getVectorIdxConstant(Val: i, DL: dl));
4428	}
4429	ConcatOps[Idx+1] = VecOp;
4430	ConcatEnd = Idx + 2;
4431	} else {
4432	// Vector type, create a CONCAT_VECTORS of type NextVT
4433	SDValue undefVec = DAG.getUNDEF(VT);
4434	unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
4435	SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
4436	unsigned RealVals = ConcatEnd - Idx - 1;
4437	unsigned SubConcatEnd = 0;
4438	unsigned SubConcatIdx = Idx + 1;
4439	while (SubConcatEnd < RealVals)
4440	SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
4441	while (SubConcatEnd < OpsToConcat)
4442	SubConcatOps[SubConcatEnd++] = undefVec;
4443	ConcatOps[SubConcatIdx] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl,
4444	VT: NextVT, Ops: SubConcatOps);
4445	ConcatEnd = SubConcatIdx + 1;
4446	}
4447	}
4448
4449	// Check to see if we have a single operation with the widen type.
4450	if (ConcatEnd == 1) {
4451	VT = ConcatOps[0].getValueType();
4452	if (VT == WidenVT)
4453	return ConcatOps[0];
4454	}
4455
4456	// add undefs of size MaxVT until ConcatOps grows to length of WidenVT
4457	unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
4458	if (NumOps != ConcatEnd ) {
4459	SDValue UndefVal = DAG.getUNDEF(VT: MaxVT);
4460	for (unsigned j = ConcatEnd; j < NumOps; ++j)
4461	ConcatOps[j] = UndefVal;
4462	}
4463	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT,
4464	Ops: ArrayRef(ConcatOps.data(), NumOps));
4465	}
4466
4467	SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
4468	// Binary op widening for operations that can trap.
4469	unsigned Opcode = N->getOpcode();
4470	SDLoc dl(N);
4471	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
4472	EVT WidenEltVT = WidenVT.getVectorElementType();
4473	EVT VT = WidenVT;
4474	unsigned NumElts = VT.getVectorMinNumElements();
4475	const SDNodeFlags Flags = N->getFlags();
4476	while (!TLI.isTypeLegal(VT) && NumElts != 1) {
4477	NumElts = NumElts / 2;
4478	VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
4479	}
4480
4481	if (NumElts != 1 && !TLI.canOpTrap(Op: N->getOpcode(), VT)) {
4482	// Operation doesn't trap so just widen as normal.
4483	SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
4484	SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
4485	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, Flags);
4486	}
4487
4488	// FIXME: Improve support for scalable vectors.
4489	assert(!VT.isScalableVector() && "Scalable vectors not handled yet.");
4490
4491	// No legal vector version so unroll the vector operation and then widen.
4492	if (NumElts == 1)
4493	return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
4494
4495	// Since the operation can trap, apply operation on the original vector.
4496	EVT MaxVT = VT;
4497	SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
4498	SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
4499	unsigned CurNumElts = N->getValueType(ResNo: 0).getVectorNumElements();
4500
4501	SmallVector<SDValue, 16> ConcatOps(CurNumElts);
4502	unsigned ConcatEnd = 0; // Current ConcatOps index.
4503	int Idx = 0; // Current Idx into input vectors.
4504
4505	// NumElts := greatest legal vector size (at most WidenVT)
4506	// while (orig. vector has unhandled elements) {
4507	// take munches of size NumElts from the beginning and add to ConcatOps
4508	// NumElts := next smaller supported vector size or 1
4509	// }
4510	while (CurNumElts != 0) {
4511	while (CurNumElts >= NumElts) {
4512	SDValue EOp1 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT, N1: InOp1,
4513	N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4514	SDValue EOp2 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT, N1: InOp2,
4515	N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4516	ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, DL: dl, VT, N1: EOp1, N2: EOp2, Flags);
4517	Idx += NumElts;
4518	CurNumElts -= NumElts;
4519	}
4520	do {
4521	NumElts = NumElts / 2;
4522	VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
4523	} while (!TLI.isTypeLegal(VT) && NumElts != 1);
4524
4525	if (NumElts == 1) {
4526	for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
4527	SDValue EOp1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: WidenEltVT,
4528	N1: InOp1, N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4529	SDValue EOp2 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: WidenEltVT,
4530	N1: InOp2, N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4531	ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, DL: dl, VT: WidenEltVT,
4532	N1: EOp1, N2: EOp2, Flags);
4533	}
4534	CurNumElts = 0;
4535	}
4536	}
4537
4538	return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
4539	}
4540
4541	SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
4542	switch (N->getOpcode()) {
4543	case ISD::STRICT_FSETCC:
4544	case ISD::STRICT_FSETCCS:
4545	return WidenVecRes_STRICT_FSETCC(N);
4546	case ISD::STRICT_FP_EXTEND:
4547	case ISD::STRICT_FP_ROUND:
4548	case ISD::STRICT_FP_TO_SINT:
4549	case ISD::STRICT_FP_TO_UINT:
4550	case ISD::STRICT_SINT_TO_FP:
4551	case ISD::STRICT_UINT_TO_FP:
4552	return WidenVecRes_Convert_StrictFP(N);
4553	default:
4554	break;
4555	}
4556
4557	// StrictFP op widening for operations that can trap.
4558	unsigned NumOpers = N->getNumOperands();
4559	unsigned Opcode = N->getOpcode();
4560	SDLoc dl(N);
4561	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
4562	EVT WidenEltVT = WidenVT.getVectorElementType();
4563	EVT VT = WidenVT;
4564	unsigned NumElts = VT.getVectorNumElements();
4565	while (!TLI.isTypeLegal(VT) && NumElts != 1) {
4566	NumElts = NumElts / 2;
4567	VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
4568	}
4569
4570	// No legal vector version so unroll the vector operation and then widen.
4571	if (NumElts == 1)
4572	return UnrollVectorOp_StrictFP(N, ResNE: WidenVT.getVectorNumElements());
4573
4574	// Since the operation can trap, apply operation on the original vector.
4575	EVT MaxVT = VT;
4576	SmallVector<SDValue, 4> InOps;
4577	unsigned CurNumElts = N->getValueType(ResNo: 0).getVectorNumElements();
4578
4579	SmallVector<SDValue, 16> ConcatOps(CurNumElts);
4580	SmallVector<SDValue, 16> Chains;
4581	unsigned ConcatEnd = 0; // Current ConcatOps index.
4582	int Idx = 0; // Current Idx into input vectors.
4583
4584	// The Chain is the first operand.
4585	InOps.push_back(Elt: N->getOperand(Num: 0));
4586
4587	// Now process the remaining operands.
4588	for (unsigned i = 1; i < NumOpers; ++i) {
4589	SDValue Oper = N->getOperand(Num: i);
4590
4591	EVT OpVT = Oper.getValueType();
4592	if (OpVT.isVector()) {
4593	if (getTypeAction(VT: OpVT) == TargetLowering::TypeWidenVector)
4594	Oper = GetWidenedVector(Op: Oper);
4595	else {
4596	EVT WideOpVT =
4597	EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(),
4598	EC: WidenVT.getVectorElementCount());
4599	Oper = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WideOpVT,
4600	N1: DAG.getUNDEF(VT: WideOpVT), N2: Oper,
4601	N3: DAG.getVectorIdxConstant(Val: 0, DL: dl));
4602	}
4603	}
4604
4605	InOps.push_back(Elt: Oper);
4606	}
4607
4608	// NumElts := greatest legal vector size (at most WidenVT)
4609	// while (orig. vector has unhandled elements) {
4610	// take munches of size NumElts from the beginning and add to ConcatOps
4611	// NumElts := next smaller supported vector size or 1
4612	// }
4613	while (CurNumElts != 0) {
4614	while (CurNumElts >= NumElts) {
4615	SmallVector<SDValue, 4> EOps;
4616
4617	for (unsigned i = 0; i < NumOpers; ++i) {
4618	SDValue Op = InOps[i];
4619
4620	EVT OpVT = Op.getValueType();
4621	if (OpVT.isVector()) {
4622	EVT OpExtractVT =
4623	EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(),
4624	EC: VT.getVectorElementCount());
4625	Op = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: OpExtractVT, N1: Op,
4626	N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4627	}
4628
4629	EOps.push_back(Elt: Op);
4630	}
4631
4632	EVT OperVT[] = {VT, MVT::Other};
4633	SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps);
4634	ConcatOps[ConcatEnd++] = Oper;
4635	Chains.push_back(Elt: Oper.getValue(R: 1));
4636	Idx += NumElts;
4637	CurNumElts -= NumElts;
4638	}
4639	do {
4640	NumElts = NumElts / 2;
4641	VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
4642	} while (!TLI.isTypeLegal(VT) && NumElts != 1);
4643
4644	if (NumElts == 1) {
4645	for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
4646	SmallVector<SDValue, 4> EOps;
4647
4648	for (unsigned i = 0; i < NumOpers; ++i) {
4649	SDValue Op = InOps[i];
4650
4651	EVT OpVT = Op.getValueType();
4652	if (OpVT.isVector())
4653	Op = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl,
4654	VT: OpVT.getVectorElementType(), N1: Op,
4655	N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4656
4657	EOps.push_back(Elt: Op);
4658	}
4659
4660	EVT WidenVT[] = {WidenEltVT, MVT::Other};
4661	SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps);
4662	ConcatOps[ConcatEnd++] = Oper;
4663	Chains.push_back(Elt: Oper.getValue(R: 1));
4664	}
4665	CurNumElts = 0;
4666	}
4667	}
4668
4669	// Build a factor node to remember all the Ops that have been created.
4670	SDValue NewChain;
4671	if (Chains.size() == 1)
4672	NewChain = Chains[0];
4673	else
4674	NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
4675	ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
4676
4677	return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
4678	}
4679
4680	SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
4681	SDLoc DL(N);
4682	EVT ResVT = N->getValueType(ResNo: 0);
4683	EVT OvVT = N->getValueType(ResNo: 1);
4684	EVT WideResVT, WideOvVT;
4685	SDValue WideLHS, WideRHS;
4686
4687	// TODO: This might result in a widen/split loop.
4688	if (ResNo == 0) {
4689	WideResVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: ResVT);
4690	WideOvVT = EVT::getVectorVT(
4691	Context&: *DAG.getContext(), VT: OvVT.getVectorElementType(),
4692	NumElements: WideResVT.getVectorNumElements());
4693
4694	WideLHS = GetWidenedVector(Op: N->getOperand(Num: 0));
4695	WideRHS = GetWidenedVector(Op: N->getOperand(Num: 1));
4696	} else {
4697	WideOvVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OvVT);
4698	WideResVT = EVT::getVectorVT(
4699	Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
4700	NumElements: WideOvVT.getVectorNumElements());
4701
4702	SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL);
4703	WideLHS = DAG.getNode(
4704	Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: DAG.getUNDEF(VT: WideResVT),
4705	N2: N->getOperand(Num: 0), N3: Zero);
4706	WideRHS = DAG.getNode(
4707	Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: DAG.getUNDEF(VT: WideResVT),
4708	N2: N->getOperand(Num: 1), N3: Zero);
4709	}
4710
4711	SDVTList WideVTs = DAG.getVTList(VT1: WideResVT, VT2: WideOvVT);
4712	SDNode *WideNode = DAG.getNode(
4713	Opcode: N->getOpcode(), DL, VTList: WideVTs, N1: WideLHS, N2: WideRHS).getNode();
4714
4715	// Replace the other vector result not being explicitly widened here.
4716	unsigned OtherNo = 1 - ResNo;
4717	EVT OtherVT = N->getValueType(ResNo: OtherNo);
4718	if (getTypeAction(VT: OtherVT) == TargetLowering::TypeWidenVector) {
4719	SetWidenedVector(Op: SDValue(N, OtherNo), Result: SDValue(WideNode, OtherNo));
4720	} else {
4721	SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL);
4722	SDValue OtherVal = DAG.getNode(
4723	Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: OtherVT, N1: SDValue(WideNode, OtherNo), N2: Zero);
4724	ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
4725	}
4726
4727	return SDValue(WideNode, ResNo);
4728	}
4729
4730	SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
4731	LLVMContext &Ctx = *DAG.getContext();
4732	SDValue InOp = N->getOperand(Num: 0);
4733	SDLoc DL(N);
4734
4735	EVT WidenVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: N->getValueType(ResNo: 0));
4736	ElementCount WidenEC = WidenVT.getVectorElementCount();
4737
4738	EVT InVT = InOp.getValueType();
4739
4740	unsigned Opcode = N->getOpcode();
4741	const SDNodeFlags Flags = N->getFlags();
4742
4743	// Handle the case of ZERO_EXTEND where the promoted InVT element size does
4744	// not equal that of WidenVT.
4745	if (N->getOpcode() == ISD::ZERO_EXTEND &&
4746	getTypeAction(VT: InVT) == TargetLowering::TypePromoteInteger &&
4747	TLI.getTypeToTransformTo(Context&: Ctx, VT: InVT).getScalarSizeInBits() !=
4748	WidenVT.getScalarSizeInBits()) {
4749	InOp = ZExtPromotedInteger(Op: InOp);
4750	InVT = InOp.getValueType();
4751	if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits())
4752	Opcode = ISD::TRUNCATE;
4753	}
4754
4755	EVT InEltVT = InVT.getVectorElementType();
4756	EVT InWidenVT = EVT::getVectorVT(Context&: Ctx, VT: InEltVT, EC: WidenEC);
4757	ElementCount InVTEC = InVT.getVectorElementCount();
4758
4759	if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
4760	InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
4761	InVT = InOp.getValueType();
4762	InVTEC = InVT.getVectorElementCount();
4763	if (InVTEC == WidenEC) {
4764	if (N->getNumOperands() == 1)
4765	return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InOp);
4766	if (N->getNumOperands() == 3) {
4767	assert(N->isVPOpcode() && "Expected VP opcode");
4768	SDValue Mask =
4769	GetWidenedMask(Mask: N->getOperand(Num: 1), EC: WidenVT.getVectorElementCount());
4770	return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InOp, N2: Mask, N3: N->getOperand(Num: 2));
4771	}
4772	return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InOp, N2: N->getOperand(Num: 1), Flags);
4773	}
4774	if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
4775	// If both input and result vector types are of same width, extend
4776	// operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
4777	// accepts fewer elements in the result than in the input.
4778	if (Opcode == ISD::ANY_EXTEND)
4779	return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
4780	if (Opcode == ISD::SIGN_EXTEND)
4781	return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
4782	if (Opcode == ISD::ZERO_EXTEND)
4783	return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
4784	}
4785	}
4786
4787	if (TLI.isTypeLegal(VT: InWidenVT)) {
4788	// Because the result and the input are different vector types, widening
4789	// the result could create a legal type but widening the input might make
4790	// it an illegal type that might lead to repeatedly splitting the input
4791	// and then widening it. To avoid this, we widen the input only if
4792	// it results in a legal type.
4793	if (WidenEC.isKnownMultipleOf(RHS: InVTEC.getKnownMinValue())) {
4794	// Widen the input and call convert on the widened input vector.
4795	unsigned NumConcat =
4796	WidenEC.getKnownMinValue() / InVTEC.getKnownMinValue();
4797	SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(VT: InVT));
4798	Ops[0] = InOp;
4799	SDValue InVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InWidenVT, Ops);
4800	if (N->getNumOperands() == 1)
4801	return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InVec);
4802	return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InVec, N2: N->getOperand(Num: 1), Flags);
4803	}
4804
4805	if (InVTEC.isKnownMultipleOf(RHS: WidenEC.getKnownMinValue())) {
4806	SDValue InVal = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: InWidenVT, N1: InOp,
4807	N2: DAG.getVectorIdxConstant(Val: 0, DL));
4808	// Extract the input and convert the shorten input vector.
4809	if (N->getNumOperands() == 1)
4810	return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InVal);
4811	return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InVal, N2: N->getOperand(Num: 1), Flags);
4812	}
4813	}
4814
4815	// Otherwise unroll into some nasty scalar code and rebuild the vector.
4816	EVT EltVT = WidenVT.getVectorElementType();
4817	SmallVector<SDValue, 16> Ops(WidenEC.getFixedValue(), DAG.getUNDEF(VT: EltVT));
4818	// Use the original element count so we don't do more scalar opts than
4819	// necessary.
4820	unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements();
4821	for (unsigned i=0; i < MinElts; ++i) {
4822	SDValue Val = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: InEltVT, N1: InOp,
4823	N2: DAG.getVectorIdxConstant(Val: i, DL));
4824	if (N->getNumOperands() == 1)
4825	Ops[i] = DAG.getNode(Opcode, DL, VT: EltVT, Operand: Val);
4826	else
4827	Ops[i] = DAG.getNode(Opcode, DL, VT: EltVT, N1: Val, N2: N->getOperand(Num: 1), Flags);
4828	}
4829
4830	return DAG.getBuildVector(VT: WidenVT, DL, Ops);
4831	}
4832
4833	SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) {
4834	SDLoc dl(N);
4835	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
4836	ElementCount WidenNumElts = WidenVT.getVectorElementCount();
4837
4838	SDValue Src = N->getOperand(Num: 0);
4839	EVT SrcVT = Src.getValueType();
4840
4841	// Also widen the input.
4842	if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) {
4843	Src = GetWidenedVector(Op: Src);
4844	SrcVT = Src.getValueType();
4845	}
4846
4847	// Input and output not widened to the same size, give up.
4848	if (WidenNumElts != SrcVT.getVectorElementCount())
4849	return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue());
4850
4851	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: Src, N2: N->getOperand(Num: 1));
4852	}
4853
4854	SDValue DAGTypeLegalizer::WidenVecRes_XRINT(SDNode *N) {
4855	SDLoc dl(N);
4856	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
4857	ElementCount WidenNumElts = WidenVT.getVectorElementCount();
4858
4859	SDValue Src = N->getOperand(Num: 0);
4860	EVT SrcVT = Src.getValueType();
4861
4862	// Also widen the input.
4863	if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) {
4864	Src = GetWidenedVector(Op: Src);
4865	SrcVT = Src.getValueType();
4866	}
4867
4868	// Input and output not widened to the same size, give up.
4869	if (WidenNumElts != SrcVT.getVectorElementCount())
4870	return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue());
4871
4872	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, Operand: Src);
4873	}
4874
4875	SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
4876	SDValue InOp = N->getOperand(Num: 1);
4877	SDLoc DL(N);
4878	SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
4879
4880	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
4881	unsigned WidenNumElts = WidenVT.getVectorNumElements();
4882
4883	EVT InVT = InOp.getValueType();
4884	EVT InEltVT = InVT.getVectorElementType();
4885
4886	unsigned Opcode = N->getOpcode();
4887
4888	// FIXME: Optimizations need to be implemented here.
4889
4890	// Otherwise unroll into some nasty scalar code and rebuild the vector.
4891	EVT EltVT = WidenVT.getVectorElementType();
4892	std::array<EVT, 2> EltVTs = {{EltVT, MVT::Other}};
4893	SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(VT: EltVT));
4894	SmallVector<SDValue, 32> OpChains;
4895	// Use the original element count so we don't do more scalar opts than
4896	// necessary.
4897	unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements();
4898	for (unsigned i=0; i < MinElts; ++i) {
4899	NewOps[1] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: InEltVT, N1: InOp,
4900	N2: DAG.getVectorIdxConstant(Val: i, DL));
4901	Ops[i] = DAG.getNode(Opcode, DL, ResultTys: EltVTs, Ops: NewOps);
4902	OpChains.push_back(Elt: Ops[i].getValue(R: 1));
4903	}
4904	SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OpChains);
4905	ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
4906
4907	return DAG.getBuildVector(VT: WidenVT, DL, Ops);
4908	}
4909
4910	SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
4911	unsigned Opcode = N->getOpcode();
4912	SDValue InOp = N->getOperand(Num: 0);
4913	SDLoc DL(N);
4914
4915	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
4916	EVT WidenSVT = WidenVT.getVectorElementType();
4917	unsigned WidenNumElts = WidenVT.getVectorNumElements();
4918
4919	EVT InVT = InOp.getValueType();
4920	EVT InSVT = InVT.getVectorElementType();
4921	unsigned InVTNumElts = InVT.getVectorNumElements();
4922
4923	if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
4924	InOp = GetWidenedVector(Op: InOp);
4925	InVT = InOp.getValueType();
4926	if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
4927	switch (Opcode) {
4928	case ISD::ANY_EXTEND_VECTOR_INREG:
4929	case ISD::SIGN_EXTEND_VECTOR_INREG:
4930	case ISD::ZERO_EXTEND_VECTOR_INREG:
4931	return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InOp);
4932	}
4933	}
4934	}
4935
4936	// Unroll, extend the scalars and rebuild the vector.
4937	SmallVector<SDValue, 16> Ops;
4938	for (unsigned i = 0, e = std::min(a: InVTNumElts, b: WidenNumElts); i != e; ++i) {
4939	SDValue Val = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: InSVT, N1: InOp,
4940	N2: DAG.getVectorIdxConstant(Val: i, DL));
4941	switch (Opcode) {
4942	case ISD::ANY_EXTEND_VECTOR_INREG:
4943	Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: WidenSVT, Operand: Val);
4944	break;
4945	case ISD::SIGN_EXTEND_VECTOR_INREG:
4946	Val = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: WidenSVT, Operand: Val);
4947	break;
4948	case ISD::ZERO_EXTEND_VECTOR_INREG:
4949	Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenSVT, Operand: Val);
4950	break;
4951	default:
4952	llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
4953	}
4954	Ops.push_back(Elt: Val);
4955	}
4956
4957	while (Ops.size() != WidenNumElts)
4958	Ops.push_back(Elt: DAG.getUNDEF(VT: WidenSVT));
4959
4960	return DAG.getBuildVector(VT: WidenVT, DL, Ops);
4961	}
4962
4963	SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
4964	// If this is an FCOPYSIGN with same input types, we can treat it as a
4965	// normal (can trap) binary op.
4966	if (N->getOperand(Num: 0).getValueType() == N->getOperand(Num: 1).getValueType())
4967	return WidenVecRes_BinaryCanTrap(N);
4968
4969	// If the types are different, fall back to unrolling.
4970	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
4971	return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
4972	}
4973
4974	SDValue DAGTypeLegalizer::WidenVecRes_IS_FPCLASS(SDNode *N) {
4975	SDValue FpValue = N->getOperand(Num: 0);
4976	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
4977	if (getTypeAction(VT: FpValue.getValueType()) != TargetLowering::TypeWidenVector)
4978	return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
4979	SDValue Arg = GetWidenedVector(Op: FpValue);
4980	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Ops: {Arg, N->getOperand(Num: 1)},
4981	Flags: N->getFlags());
4982	}
4983
4984	SDValue DAGTypeLegalizer::WidenVecRes_ExpOp(SDNode *N) {
4985	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
4986	SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
4987	SDValue RHS = N->getOperand(Num: 1);
4988	SDValue ExpOp = RHS.getValueType().isVector() ? GetWidenedVector(Op: RHS) : RHS;
4989
4990	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, N1: InOp, N2: ExpOp);
4991	}
4992
4993	SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
4994	// Unary op widening.
4995	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
4996	SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
4997	if (N->getNumOperands() == 1)
4998	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: InOp, Flags: N->getFlags());
4999
5000	assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
5001	assert(N->isVPOpcode() && "Expected VP opcode");
5002
5003	SDValue Mask =
5004	GetWidenedMask(Mask: N->getOperand(Num: 1), EC: WidenVT.getVectorElementCount());
5005	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT,
5006	Ops: {InOp, Mask, N->getOperand(Num: 2)});
5007	}
5008
5009	SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
5010	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5011	EVT ExtVT = EVT::getVectorVT(Context&: *DAG.getContext(),
5012	VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT()
5013	.getVectorElementType(),
5014	NumElements: WidenVT.getVectorNumElements());
5015	SDValue WidenLHS = GetWidenedVector(Op: N->getOperand(Num: 0));
5016	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
5017	VT: WidenVT, N1: WidenLHS, N2: DAG.getValueType(ExtVT));
5018	}
5019
5020	SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
5021	SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
5022	return GetWidenedVector(Op: WidenVec);
5023	}
5024
5025	SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
5026	SDValue InOp = N->getOperand(Num: 0);
5027	EVT InVT = InOp.getValueType();
5028	EVT VT = N->getValueType(ResNo: 0);
5029	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
5030	SDLoc dl(N);
5031
5032	switch (getTypeAction(VT: InVT)) {
5033	case TargetLowering::TypeLegal:
5034	break;
5035	case TargetLowering::TypeScalarizeScalableVector:
5036	report_fatal_error(reason: "Scalarization of scalable vectors is not supported.");
5037	case TargetLowering::TypePromoteInteger: {
5038	// If the incoming type is a vector that is being promoted, then
5039	// we know that the elements are arranged differently and that we
5040	// must perform the conversion using a stack slot.
5041	if (InVT.isVector())
5042	break;
5043
5044	// If the InOp is promoted to the same size, convert it. Otherwise,
5045	// fall out of the switch and widen the promoted input.
5046	SDValue NInOp = GetPromotedInteger(Op: InOp);
5047	EVT NInVT = NInOp.getValueType();
5048	if (WidenVT.bitsEq(VT: NInVT)) {
5049	// For big endian targets we need to shift the input integer or the
5050	// interesting bits will end up at the wrong place.
5051	if (DAG.getDataLayout().isBigEndian()) {
5052	unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
5053	EVT ShiftAmtTy = TLI.getShiftAmountTy(LHSTy: NInVT, DL: DAG.getDataLayout());
5054	assert(ShiftAmt < WidenVT.getSizeInBits() && "Too large shift amount!");
5055	NInOp = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NInVT, N1: NInOp,
5056	N2: DAG.getConstant(Val: ShiftAmt, DL: dl, VT: ShiftAmtTy));
5057	}
5058	return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NInOp);
5059	}
5060	InOp = NInOp;
5061	InVT = NInVT;
5062	break;
5063	}
5064	case TargetLowering::TypeSoftenFloat:
5065	case TargetLowering::TypePromoteFloat:
5066	case TargetLowering::TypeSoftPromoteHalf:
5067	case TargetLowering::TypeExpandInteger:
5068	case TargetLowering::TypeExpandFloat:
5069	case TargetLowering::TypeScalarizeVector:
5070	case TargetLowering::TypeSplitVector:
5071	break;
5072	case TargetLowering::TypeWidenVector:
5073	// If the InOp is widened to the same size, convert it. Otherwise, fall
5074	// out of the switch and widen the widened input.
5075	InOp = GetWidenedVector(Op: InOp);
5076	InVT = InOp.getValueType();
5077	if (WidenVT.bitsEq(VT: InVT))
5078	// The input widens to the same size. Convert to the widen value.
5079	return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: InOp);
5080	break;
5081	}
5082
5083	unsigned WidenSize = WidenVT.getSizeInBits();
5084	unsigned InSize = InVT.getSizeInBits();
5085	unsigned InScalarSize = InVT.getScalarSizeInBits();
5086	// x86mmx is not an acceptable vector element type, so don't try.
5087	if (WidenSize % InScalarSize == 0 && InVT != MVT::x86mmx) {
5088	// Determine new input vector type. The new input vector type will use
5089	// the same element type (if its a vector) or use the input type as a
5090	// vector. It is the same size as the type to widen to.
5091	EVT NewInVT;
5092	unsigned NewNumParts = WidenSize / InSize;
5093	if (InVT.isVector()) {
5094	EVT InEltVT = InVT.getVectorElementType();
5095	NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: InEltVT,
5096	NumElements: WidenSize / InEltVT.getSizeInBits());
5097	} else {
5098	// For big endian systems, using the promoted input scalar type
5099	// to produce the scalar_to_vector would put the desired bits into
5100	// the least significant byte(s) of the wider element zero. This
5101	// will mean that the users of the result vector are using incorrect
5102	// bits. Use the original input type instead. Although either input
5103	// type can be used on little endian systems, for consistency we
5104	// use the original type there as well.
5105	EVT OrigInVT = N->getOperand(Num: 0).getValueType();
5106	NewNumParts = WidenSize / OrigInVT.getSizeInBits();
5107	NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: OrigInVT, NumElements: NewNumParts);
5108	}
5109
5110	if (TLI.isTypeLegal(VT: NewInVT)) {
5111	SDValue NewVec;
5112	if (InVT.isVector()) {
5113	// Because the result and the input are different vector types, widening
5114	// the result could create a legal type but widening the input might
5115	// make it an illegal type that might lead to repeatedly splitting the
5116	// input and then widening it. To avoid this, we widen the input only if
5117	// it results in a legal type.
5118	if (WidenSize % InSize == 0) {
5119	SmallVector<SDValue, 16> Ops(NewNumParts, DAG.getUNDEF(VT: InVT));
5120	Ops[0] = InOp;
5121
5122	NewVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewInVT, Ops);
5123	} else {
5124	SmallVector<SDValue, 16> Ops;
5125	DAG.ExtractVectorElements(Op: InOp, Args&: Ops);
5126	Ops.append(NumInputs: WidenSize / InScalarSize - Ops.size(),
5127	Elt: DAG.getUNDEF(VT: InVT.getVectorElementType()));
5128
5129	NewVec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: dl, VT: NewInVT, Ops);
5130	}
5131	} else {
5132	NewVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewInVT, Operand: InOp);
5133	}
5134	return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NewVec);
5135	}
5136	}
5137
5138	return CreateStackStoreLoad(Op: InOp, DestVT: WidenVT);
5139	}
5140
5141	SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
5142	SDLoc dl(N);
5143	// Build a vector with undefined for the new nodes.
5144	EVT VT = N->getValueType(ResNo: 0);
5145
5146	// Integer BUILD_VECTOR operands may be larger than the node's vector element
5147	// type. The UNDEFs need to have the same type as the existing operands.
5148	EVT EltVT = N->getOperand(Num: 0).getValueType();
5149	unsigned NumElts = VT.getVectorNumElements();
5150
5151	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
5152	unsigned WidenNumElts = WidenVT.getVectorNumElements();
5153
5154	SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
5155	assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
5156	NewOps.append(NumInputs: WidenNumElts - NumElts, Elt: DAG.getUNDEF(VT: EltVT));
5157
5158	return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: NewOps);
5159	}
5160
5161	SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
5162	EVT InVT = N->getOperand(Num: 0).getValueType();
5163	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5164	SDLoc dl(N);
5165	unsigned NumOperands = N->getNumOperands();
5166
5167	bool InputWidened = false; // Indicates we need to widen the input.
5168	if (getTypeAction(VT: InVT) != TargetLowering::TypeWidenVector) {
5169	unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
5170	unsigned NumInElts = InVT.getVectorMinNumElements();
5171	if (WidenNumElts % NumInElts == 0) {
5172	// Add undef vectors to widen to correct length.
5173	unsigned NumConcat = WidenNumElts / NumInElts;
5174	SDValue UndefVal = DAG.getUNDEF(VT: InVT);
5175	SmallVector<SDValue, 16> Ops(NumConcat);
5176	for (unsigned i=0; i < NumOperands; ++i)
5177	Ops[i] = N->getOperand(Num: i);
5178	for (unsigned i = NumOperands; i != NumConcat; ++i)
5179	Ops[i] = UndefVal;
5180	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops);
5181	}
5182	} else {
5183	InputWidened = true;
5184	if (WidenVT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) {
5185	// The inputs and the result are widen to the same value.
5186	unsigned i;
5187	for (i=1; i < NumOperands; ++i)
5188	if (!N->getOperand(Num: i).isUndef())
5189	break;
5190
5191	if (i == NumOperands)
5192	// Everything but the first operand is an UNDEF so just return the
5193	// widened first operand.
5194	return GetWidenedVector(Op: N->getOperand(Num: 0));
5195
5196	if (NumOperands == 2) {
5197	assert(!WidenVT.isScalableVector() &&
5198	"Cannot use vector shuffles to widen CONCAT_VECTOR result");
5199	unsigned WidenNumElts = WidenVT.getVectorNumElements();
5200	unsigned NumInElts = InVT.getVectorNumElements();
5201
5202	// Replace concat of two operands with a shuffle.
5203	SmallVector<int, 16> MaskOps(WidenNumElts, -1);
5204	for (unsigned i = 0; i < NumInElts; ++i) {
5205	MaskOps[i] = i;
5206	MaskOps[i + NumInElts] = i + WidenNumElts;
5207	}
5208	return DAG.getVectorShuffle(VT: WidenVT, dl,
5209	N1: GetWidenedVector(Op: N->getOperand(Num: 0)),
5210	N2: GetWidenedVector(Op: N->getOperand(Num: 1)),
5211	Mask: MaskOps);
5212	}
5213	}
5214	}
5215
5216	assert(!WidenVT.isScalableVector() &&
5217	"Cannot use build vectors to widen CONCAT_VECTOR result");
5218	unsigned WidenNumElts = WidenVT.getVectorNumElements();
5219	unsigned NumInElts = InVT.getVectorNumElements();
5220
5221	// Fall back to use extracts and build vector.
5222	EVT EltVT = WidenVT.getVectorElementType();
5223	SmallVector<SDValue, 16> Ops(WidenNumElts);
5224	unsigned Idx = 0;
5225	for (unsigned i=0; i < NumOperands; ++i) {
5226	SDValue InOp = N->getOperand(Num: i);
5227	if (InputWidened)
5228	InOp = GetWidenedVector(Op: InOp);
5229	for (unsigned j = 0; j < NumInElts; ++j)
5230	Ops[Idx++] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: InOp,
5231	N2: DAG.getVectorIdxConstant(Val: j, DL: dl));
5232	}
5233	SDValue UndefVal = DAG.getUNDEF(VT: EltVT);
5234	for (; Idx < WidenNumElts; ++Idx)
5235	Ops[Idx] = UndefVal;
5236	return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
5237	}
5238
5239	SDValue DAGTypeLegalizer::WidenVecRes_INSERT_SUBVECTOR(SDNode *N) {
5240	EVT VT = N->getValueType(ResNo: 0);
5241	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
5242	SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5243	SDValue InOp2 = N->getOperand(Num: 1);
5244	SDValue Idx = N->getOperand(Num: 2);
5245	SDLoc dl(N);
5246	return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Idx);
5247	}
5248
5249	SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
5250	EVT VT = N->getValueType(ResNo: 0);
5251	EVT EltVT = VT.getVectorElementType();
5252	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
5253	SDValue InOp = N->getOperand(Num: 0);
5254	SDValue Idx = N->getOperand(Num: 1);
5255	SDLoc dl(N);
5256
5257	auto InOpTypeAction = getTypeAction(VT: InOp.getValueType());
5258	if (InOpTypeAction == TargetLowering::TypeWidenVector)
5259	InOp = GetWidenedVector(Op: InOp);
5260
5261	EVT InVT = InOp.getValueType();
5262
5263	// Check if we can just return the input vector after widening.
5264	uint64_t IdxVal = Idx->getAsZExtVal();
5265	if (IdxVal == 0 && InVT == WidenVT)
5266	return InOp;
5267
5268	// Check if we can extract from the vector.
5269	unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
5270	unsigned InNumElts = InVT.getVectorMinNumElements();
5271	unsigned VTNumElts = VT.getVectorMinNumElements();
5272	assert(IdxVal % VTNumElts == 0 &&
5273	"Expected Idx to be a multiple of subvector minimum vector length");
5274	if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
5275	return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp, N2: Idx);
5276
5277	if (VT.isScalableVector()) {
5278	// Try to split the operation up into smaller extracts and concat the
5279	// results together, e.g.
5280	// nxv6i64 extract_subvector(nxv12i64, 6)
5281	// <->
5282	// nxv8i64 concat(
5283	// nxv2i64 extract_subvector(nxv16i64, 6)
5284	// nxv2i64 extract_subvector(nxv16i64, 8)
5285	// nxv2i64 extract_subvector(nxv16i64, 10)
5286	// undef)
5287	unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts);
5288	assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
5289	"down type's element count");
5290	EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
5291	EC: ElementCount::getScalable(MinVal: GCD));
5292	// Avoid recursion around e.g. nxv1i8.
5293	if (getTypeAction(VT: PartVT) != TargetLowering::TypeWidenVector) {
5294	SmallVector<SDValue> Parts;
5295	unsigned I = 0;
5296	for (; I < VTNumElts / GCD; ++I)
5297	Parts.push_back(
5298	Elt: DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: PartVT, N1: InOp,
5299	N2: DAG.getVectorIdxConstant(Val: IdxVal + I * GCD, DL: dl)));
5300	for (; I < WidenNumElts / GCD; ++I)
5301	Parts.push_back(Elt: DAG.getUNDEF(VT: PartVT));
5302
5303	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts);
5304	}
5305
5306	report_fatal_error(reason: "Don't know how to widen the result of "
5307	"EXTRACT_SUBVECTOR for scalable vectors");
5308	}
5309
5310	// We could try widening the input to the right length but for now, extract
5311	// the original elements, fill the rest with undefs and build a vector.
5312	SmallVector<SDValue, 16> Ops(WidenNumElts);
5313	unsigned i;
5314	for (i = 0; i < VTNumElts; ++i)
5315	Ops[i] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: InOp,
5316	N2: DAG.getVectorIdxConstant(Val: IdxVal + i, DL: dl));
5317
5318	SDValue UndefVal = DAG.getUNDEF(VT: EltVT);
5319	for (; i < WidenNumElts; ++i)
5320	Ops[i] = UndefVal;
5321	return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
5322	}
5323
5324	SDValue DAGTypeLegalizer::WidenVecRes_AssertZext(SDNode *N) {
5325	SDValue InOp = ModifyToType(
5326	InOp: N->getOperand(Num: 0),
5327	NVT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)), FillWithZeroes: true);
5328	return DAG.getNode(Opcode: ISD::AssertZext, DL: SDLoc(N), VT: InOp.getValueType(), N1: InOp,
5329	N2: N->getOperand(Num: 1));
5330	}
5331
5332	SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
5333	SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
5334	return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N),
5335	VT: InOp.getValueType(), N1: InOp,
5336	N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
5337	}
5338
5339	SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
5340	LoadSDNode *LD = cast<LoadSDNode>(Val: N);
5341	ISD::LoadExtType ExtType = LD->getExtensionType();
5342
5343	// A vector must always be stored in memory as-is, i.e. without any padding
5344	// between the elements, since various code depend on it, e.g. in the
5345	// handling of a bitcast of a vector type to int, which may be done with a
5346	// vector store followed by an integer load. A vector that does not have
5347	// elements that are byte-sized must therefore be stored as an integer
5348	// built out of the extracted vector elements.
5349	if (!LD->getMemoryVT().isByteSized()) {
5350	SDValue Value, NewChain;
5351	std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
5352	ReplaceValueWith(From: SDValue(LD, 0), To: Value);
5353	ReplaceValueWith(From: SDValue(LD, 1), To: NewChain);
5354	return SDValue();
5355	}
5356
5357	// Generate a vector-predicated load if it is custom/legal on the target. To
5358	// avoid possible recursion, only do this if the widened mask type is legal.
5359	// FIXME: Not all targets may support EVL in VP_LOAD. These will have been
5360	// removed from the IR by the ExpandVectorPredication pass but we're
5361	// reintroducing them here.
5362	EVT LdVT = LD->getMemoryVT();
5363	EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: LdVT);
5364	EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
5365	WideVT.getVectorElementCount());
5366	if (ExtType == ISD::NON_EXTLOAD &&
5367	TLI.isOperationLegalOrCustom(Op: ISD::VP_LOAD, VT: WideVT) &&
5368	TLI.isTypeLegal(VT: WideMaskVT)) {
5369	SDLoc DL(N);
5370	SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT);
5371	SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(),
5372	EC: LdVT.getVectorElementCount());
5373	const auto *MMO = LD->getMemOperand();
5374	SDValue NewLoad =
5375	DAG.getLoadVP(VT: WideVT, dl: DL, Chain: LD->getChain(), Ptr: LD->getBasePtr(), Mask, EVL,
5376	PtrInfo: MMO->getPointerInfo(), Alignment: MMO->getAlign(), MMOFlags: MMO->getFlags(),
5377	AAInfo: MMO->getAAInfo());
5378
5379	// Modified the chain - switch anything that used the old chain to use
5380	// the new one.
5381	ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
5382
5383	return NewLoad;
5384	}
5385
5386	SDValue Result;
5387	SmallVector<SDValue, 16> LdChain; // Chain for the series of load
5388	if (ExtType != ISD::NON_EXTLOAD)
5389	Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
5390	else
5391	Result = GenWidenVectorLoads(LdChain, LD);
5392
5393	if (Result) {
5394	// If we generate a single load, we can use that for the chain. Otherwise,
5395	// build a factor node to remember the multiple loads are independent and
5396	// chain to that.
5397	SDValue NewChain;
5398	if (LdChain.size() == 1)
5399	NewChain = LdChain[0];
5400	else
5401	NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain);
5402
5403	// Modified the chain - switch anything that used the old chain to use
5404	// the new one.
5405	ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
5406
5407	return Result;
5408	}
5409
5410	report_fatal_error(reason: "Unable to widen vector load");
5411	}
5412
5413	SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD(VPLoadSDNode *N) {
5414	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5415	SDValue Mask = N->getMask();
5416	SDValue EVL = N->getVectorLength();
5417	ISD::LoadExtType ExtType = N->getExtensionType();
5418	SDLoc dl(N);
5419
5420	// The mask should be widened as well
5421	assert(getTypeAction(Mask.getValueType()) ==
5422	TargetLowering::TypeWidenVector &&
5423	"Unable to widen binary VP op");
5424	Mask = GetWidenedVector(Op: Mask);
5425	assert(Mask.getValueType().getVectorElementCount() ==
5426	TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
5427	.getVectorElementCount() &&
5428	"Unable to widen vector load");
5429
5430	SDValue Res =
5431	DAG.getLoadVP(AM: N->getAddressingMode(), ExtType, VT: WidenVT, dl, Chain: N->getChain(),
5432	Ptr: N->getBasePtr(), Offset: N->getOffset(), Mask, EVL,
5433	MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), IsExpanding: N->isExpandingLoad());
5434	// Legalize the chain result - switch anything that used the old chain to
5435	// use the new one.
5436	ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
5437	return Res;
5438	}
5439
5440	SDValue DAGTypeLegalizer::WidenVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *N) {
5441	SDLoc DL(N);
5442
5443	// The mask should be widened as well
5444	SDValue Mask = N->getMask();
5445	assert(getTypeAction(Mask.getValueType()) ==
5446	TargetLowering::TypeWidenVector &&
5447	"Unable to widen VP strided load");
5448	Mask = GetWidenedVector(Op: Mask);
5449
5450	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5451	assert(Mask.getValueType().getVectorElementCount() ==
5452	WidenVT.getVectorElementCount() &&
5453	"Data and mask vectors should have the same number of elements");
5454
5455	SDValue Res = DAG.getStridedLoadVP(
5456	AM: N->getAddressingMode(), ExtType: N->getExtensionType(), VT: WidenVT, DL, Chain: N->getChain(),
5457	Ptr: N->getBasePtr(), Offset: N->getOffset(), Stride: N->getStride(), Mask,
5458	EVL: N->getVectorLength(), MemVT: N->getMemoryVT(), MMO: N->getMemOperand(),
5459	IsExpanding: N->isExpandingLoad());
5460
5461	// Legalize the chain result - switch anything that used the old chain to
5462	// use the new one.
5463	ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
5464	return Res;
5465	}
5466
5467	SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
5468
5469	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: N->getValueType(ResNo: 0));
5470	SDValue Mask = N->getMask();
5471	EVT MaskVT = Mask.getValueType();
5472	SDValue PassThru = GetWidenedVector(Op: N->getPassThru());
5473	ISD::LoadExtType ExtType = N->getExtensionType();
5474	SDLoc dl(N);
5475
5476	// The mask should be widened as well
5477	EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
5478	VT: MaskVT.getVectorElementType(),
5479	NumElements: WidenVT.getVectorNumElements());
5480	Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
5481
5482	SDValue Res = DAG.getMaskedLoad(
5483	VT: WidenVT, dl, Chain: N->getChain(), Base: N->getBasePtr(), Offset: N->getOffset(), Mask,
5484	Src0: PassThru, MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), AM: N->getAddressingMode(),
5485	ExtType, IsExpanding: N->isExpandingLoad());
5486	// Legalize the chain result - switch anything that used the old chain to
5487	// use the new one.
5488	ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
5489	return Res;
5490	}
5491
5492	SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
5493
5494	EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5495	SDValue Mask = N->getMask();
5496	EVT MaskVT = Mask.getValueType();
5497	SDValue PassThru = GetWidenedVector(Op: N->getPassThru());
5498	SDValue Scale = N->getScale();
5499	unsigned NumElts = WideVT.getVectorNumElements();
5500	SDLoc dl(N);
5501
5502	// The mask should be widened as well
5503	EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
5504	VT: MaskVT.getVectorElementType(),
5505	NumElements: WideVT.getVectorNumElements());
5506	Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
5507
5508	// Widen the Index operand
5509	SDValue Index = N->getIndex();
5510	EVT WideIndexVT = EVT::getVectorVT(Context&: *DAG.getContext(),
5511	VT: Index.getValueType().getScalarType(),
5512	NumElements: NumElts);
5513	Index = ModifyToType(InOp: Index, NVT: WideIndexVT);
5514	SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
5515	Scale };
5516
5517	// Widen the MemoryType
5518	EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
5519	VT: N->getMemoryVT().getScalarType(), NumElements: NumElts);
5520	SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
5521	WideMemVT, dl, Ops, N->getMemOperand(),
5522	N->getIndexType(), N->getExtensionType());
5523
5524	// Legalize the chain result - switch anything that used the old chain to
5525	// use the new one.
5526	ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
5527	return Res;
5528	}
5529
5530	SDValue DAGTypeLegalizer::WidenVecRes_VP_GATHER(VPGatherSDNode *N) {
5531	EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5532	SDValue Mask = N->getMask();
5533	SDValue Scale = N->getScale();
5534	ElementCount WideEC = WideVT.getVectorElementCount();
5535	SDLoc dl(N);
5536
5537	SDValue Index = GetWidenedVector(Op: N->getIndex());
5538	EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
5539	VT: N->getMemoryVT().getScalarType(), EC: WideEC);
5540	Mask = GetWidenedMask(Mask, EC: WideEC);
5541
5542	SDValue Ops[] = {N->getChain(), N->getBasePtr(), Index, Scale,
5543	Mask, N->getVectorLength()};
5544	SDValue Res = DAG.getGatherVP(DAG.getVTList(WideVT, MVT::Other), WideMemVT,
5545	dl, Ops, N->getMemOperand(), N->getIndexType());
5546
5547	// Legalize the chain result - switch anything that used the old chain to
5548	// use the new one.
5549	ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
5550	return Res;
5551	}
5552
5553	SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) {
5554	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5555	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: N->getOperand(Num: 0));
5556	}
5557
5558	// Return true is this is a SETCC node or a strict version of it.
5559	static inline bool isSETCCOp(unsigned Opcode) {
5560	switch (Opcode) {
5561	case ISD::SETCC:
5562	case ISD::STRICT_FSETCC:
5563	case ISD::STRICT_FSETCCS:
5564	return true;
5565	}
5566	return false;
5567	}
5568
5569	// Return true if this is a node that could have two SETCCs as operands.
5570	static inline bool isLogicalMaskOp(unsigned Opcode) {
5571	switch (Opcode) {
5572	case ISD::AND:
5573	case ISD::OR:
5574	case ISD::XOR:
5575	return true;
5576	}
5577	return false;
5578	}
5579
5580	// If N is a SETCC or a strict variant of it, return the type
5581	// of the compare operands.
5582	static inline EVT getSETCCOperandType(SDValue N) {
5583	unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
5584	return N->getOperand(Num: OpNo).getValueType();
5585	}
5586
5587	// This is used just for the assert in convertMask(). Check that this either
5588	// a SETCC or a previously handled SETCC by convertMask().
5589	#ifndef NDEBUG
5590	static inline bool isSETCCorConvertedSETCC(SDValue N) {
5591	if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
5592	N = N.getOperand(i: 0);
5593	else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
5594	for (unsigned i = 1; i < N->getNumOperands(); ++i)
5595	if (!N->getOperand(Num: i)->isUndef())
5596	return false;
5597	N = N.getOperand(i: 0);
5598	}
5599
5600	if (N.getOpcode() == ISD::TRUNCATE)
5601	N = N.getOperand(i: 0);
5602	else if (N.getOpcode() == ISD::SIGN_EXTEND)
5603	N = N.getOperand(i: 0);
5604
5605	if (isLogicalMaskOp(Opcode: N.getOpcode()))
5606	return isSETCCorConvertedSETCC(N: N.getOperand(i: 0)) &&
5607	isSETCCorConvertedSETCC(N: N.getOperand(i: 1));
5608
5609	return (isSETCCOp(Opcode: N.getOpcode()) ||
5610	ISD::isBuildVectorOfConstantSDNodes(N: N.getNode()));
5611	}
5612	#endif
5613
5614	// Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
5615	// to ToMaskVT if needed with vector extension or truncation.
5616	SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
5617	EVT ToMaskVT) {
5618	// Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
5619	// FIXME: This code seems to be too restrictive, we might consider
5620	// generalizing it or dropping it.
5621	assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
5622
5623	// Make a new Mask node, with a legal result VT.
5624	SDValue Mask;
5625	SmallVector<SDValue, 4> Ops;
5626	for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
5627	Ops.push_back(Elt: InMask->getOperand(Num: i));
5628	if (InMask->isStrictFPOpcode()) {
5629	Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask),
5630	{ MaskVT, MVT::Other }, Ops);
5631	ReplaceValueWith(From: InMask.getValue(R: 1), To: Mask.getValue(R: 1));
5632	}
5633	else
5634	Mask = DAG.getNode(Opcode: InMask->getOpcode(), DL: SDLoc(InMask), VT: MaskVT, Ops);
5635
5636	// If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
5637	// extend or truncate is needed.
5638	LLVMContext &Ctx = *DAG.getContext();
5639	unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
5640	unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
5641	if (MaskScalarBits < ToMaskScalBits) {
5642	EVT ExtVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(),
5643	NumElements: MaskVT.getVectorNumElements());
5644	Mask = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: SDLoc(Mask), VT: ExtVT, Operand: Mask);
5645	} else if (MaskScalarBits > ToMaskScalBits) {
5646	EVT TruncVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(),
5647	NumElements: MaskVT.getVectorNumElements());
5648	Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(Mask), VT: TruncVT, Operand: Mask);
5649	}
5650
5651	assert(Mask->getValueType(0).getScalarSizeInBits() ==
5652	ToMaskVT.getScalarSizeInBits() &&
5653	"Mask should have the right element size by now.");
5654
5655	// Adjust Mask to the right number of elements.
5656	unsigned CurrMaskNumEls = Mask->getValueType(ResNo: 0).getVectorNumElements();
5657	if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
5658	SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL: SDLoc(Mask));
5659	Mask = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(Mask), VT: ToMaskVT, N1: Mask,
5660	N2: ZeroIdx);
5661	} else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
5662	unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
5663	EVT SubVT = Mask->getValueType(ResNo: 0);
5664	SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(VT: SubVT));
5665	SubOps[0] = Mask;
5666	Mask = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Mask), VT: ToMaskVT, Ops: SubOps);
5667	}
5668
5669	assert((Mask->getValueType(0) == ToMaskVT) &&
5670	"A mask of ToMaskVT should have been produced by now.");
5671
5672	return Mask;
5673	}
5674
5675	// This method tries to handle some special cases for the vselect mask
5676	// and if needed adjusting the mask vector type to match that of the VSELECT.
5677	// Without it, many cases end up with scalarization of the SETCC, with many
5678	// unnecessary instructions.
5679	SDValue DAGTypeLegalizer::WidenVSELECTMask(SDNode *N) {
5680	LLVMContext &Ctx = *DAG.getContext();
5681	SDValue Cond = N->getOperand(Num: 0);
5682
5683	if (N->getOpcode() != ISD::VSELECT)
5684	return SDValue();
5685
5686	if (!isSETCCOp(Opcode: Cond->getOpcode()) && !isLogicalMaskOp(Opcode: Cond->getOpcode()))
5687	return SDValue();
5688
5689	// If this is a splitted VSELECT that was previously already handled, do
5690	// nothing.
5691	EVT CondVT = Cond->getValueType(ResNo: 0);
5692	if (CondVT.getScalarSizeInBits() != 1)
5693	return SDValue();
5694
5695	EVT VSelVT = N->getValueType(ResNo: 0);
5696
5697	// This method can't handle scalable vector types.
5698	// FIXME: This support could be added in the future.
5699	if (VSelVT.isScalableVector())
5700	return SDValue();
5701
5702	// Only handle vector types which are a power of 2.
5703	if (!isPowerOf2_64(Value: VSelVT.getSizeInBits()))
5704	return SDValue();
5705
5706	// Don't touch if this will be scalarized.
5707	EVT FinalVT = VSelVT;
5708	while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector)
5709	FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: Ctx);
5710
5711	if (FinalVT.getVectorNumElements() == 1)
5712	return SDValue();
5713
5714	// If there is support for an i1 vector mask, don't touch.
5715	if (isSETCCOp(Opcode: Cond.getOpcode())) {
5716	EVT SetCCOpVT = getSETCCOperandType(N: Cond);
5717	while (TLI.getTypeAction(Context&: Ctx, VT: SetCCOpVT) != TargetLowering::TypeLegal)
5718	SetCCOpVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: SetCCOpVT);
5719	EVT SetCCResVT = getSetCCResultType(VT: SetCCOpVT);
5720	if (SetCCResVT.getScalarSizeInBits() == 1)
5721	return SDValue();
5722	} else if (CondVT.getScalarType() == MVT::i1) {
5723	// If there is support for an i1 vector mask (or only scalar i1 conditions),
5724	// don't touch.
5725	while (TLI.getTypeAction(Context&: Ctx, VT: CondVT) != TargetLowering::TypeLegal)
5726	CondVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: CondVT);
5727
5728	if (CondVT.getScalarType() == MVT::i1)
5729	return SDValue();
5730	}
5731
5732	// Widen the vselect result type if needed.
5733	if (getTypeAction(VT: VSelVT) == TargetLowering::TypeWidenVector)
5734	VSelVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: VSelVT);
5735
5736	// The mask of the VSELECT should have integer elements.
5737	EVT ToMaskVT = VSelVT;
5738	if (!ToMaskVT.getScalarType().isInteger())
5739	ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
5740
5741	SDValue Mask;
5742	if (isSETCCOp(Opcode: Cond->getOpcode())) {
5743	EVT MaskVT = getSetCCResultType(VT: getSETCCOperandType(N: Cond));
5744	Mask = convertMask(InMask: Cond, MaskVT, ToMaskVT);
5745	} else if (isLogicalMaskOp(Opcode: Cond->getOpcode()) &&
5746	isSETCCOp(Opcode: Cond->getOperand(Num: 0).getOpcode()) &&
5747	isSETCCOp(Opcode: Cond->getOperand(Num: 1).getOpcode())) {
5748	// Cond is (AND/OR/XOR (SETCC, SETCC))
5749	SDValue SETCC0 = Cond->getOperand(Num: 0);
5750	SDValue SETCC1 = Cond->getOperand(Num: 1);
5751	EVT VT0 = getSetCCResultType(VT: getSETCCOperandType(N: SETCC0));
5752	EVT VT1 = getSetCCResultType(VT: getSETCCOperandType(N: SETCC1));
5753	unsigned ScalarBits0 = VT0.getScalarSizeInBits();
5754	unsigned ScalarBits1 = VT1.getScalarSizeInBits();
5755	unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
5756	EVT MaskVT;
5757	// If the two SETCCs have different VTs, either extend/truncate one of
5758	// them to the other "towards" ToMaskVT, or truncate one and extend the
5759	// other to ToMaskVT.
5760	if (ScalarBits0 != ScalarBits1) {
5761	EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
5762	EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
5763	if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
5764	MaskVT = WideVT;
5765	else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
5766	MaskVT = NarrowVT;
5767	else
5768	MaskVT = ToMaskVT;
5769	} else
5770	// If the two SETCCs have the same VT, don't change it.
5771	MaskVT = VT0;
5772
5773	// Make new SETCCs and logical nodes.
5774	SETCC0 = convertMask(InMask: SETCC0, MaskVT: VT0, ToMaskVT: MaskVT);
5775	SETCC1 = convertMask(InMask: SETCC1, MaskVT: VT1, ToMaskVT: MaskVT);
5776	Cond = DAG.getNode(Opcode: Cond->getOpcode(), DL: SDLoc(Cond), VT: MaskVT, N1: SETCC0, N2: SETCC1);
5777
5778	// Convert the logical op for VSELECT if needed.
5779	Mask = convertMask(InMask: Cond, MaskVT, ToMaskVT);
5780	} else
5781	return SDValue();
5782
5783	return Mask;
5784	}
5785
5786	SDValue DAGTypeLegalizer::WidenVecRes_Select(SDNode *N) {
5787	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5788	ElementCount WidenEC = WidenVT.getVectorElementCount();
5789
5790	SDValue Cond1 = N->getOperand(Num: 0);
5791	EVT CondVT = Cond1.getValueType();
5792	unsigned Opcode = N->getOpcode();
5793	if (CondVT.isVector()) {
5794	if (SDValue WideCond = WidenVSELECTMask(N)) {
5795	SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
5796	SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2));
5797	assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
5798	return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: WideCond, N2: InOp1, N3: InOp2);
5799	}
5800
5801	EVT CondEltVT = CondVT.getVectorElementType();
5802	EVT CondWidenVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: CondEltVT, EC: WidenEC);
5803	if (getTypeAction(VT: CondVT) == TargetLowering::TypeWidenVector)
5804	Cond1 = GetWidenedVector(Op: Cond1);
5805
5806	// If we have to split the condition there is no point in widening the
5807	// select. This would result in an cycle of widening the select ->
5808	// widening the condition operand -> splitting the condition operand ->
5809	// splitting the select -> widening the select. Instead split this select
5810	// further and widen the resulting type.
5811	if (getTypeAction(VT: CondVT) == TargetLowering::TypeSplitVector) {
5812	SDValue SplitSelect = SplitVecOp_VSELECT(N, OpNo: 0);
5813	SDValue Res = ModifyToType(InOp: SplitSelect, NVT: WidenVT);
5814	return Res;
5815	}
5816
5817	if (Cond1.getValueType() != CondWidenVT)
5818	Cond1 = ModifyToType(InOp: Cond1, NVT: CondWidenVT);
5819	}
5820
5821	SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
5822	SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2));
5823	assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
5824	if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE)
5825	return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2,
5826	N4: N->getOperand(Num: 3));
5827	return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2);
5828	}
5829
5830	SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
5831	SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 2));
5832	SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 3));
5833	return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
5834	VT: InOp1.getValueType(), N1: N->getOperand(Num: 0),
5835	N2: N->getOperand(Num: 1), N3: InOp1, N4: InOp2, N5: N->getOperand(Num: 4));
5836	}
5837
5838	SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
5839	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5840	return DAG.getUNDEF(VT: WidenVT);
5841	}
5842
5843	SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
5844	EVT VT = N->getValueType(ResNo: 0);
5845	SDLoc dl(N);
5846
5847	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
5848	unsigned NumElts = VT.getVectorNumElements();
5849	unsigned WidenNumElts = WidenVT.getVectorNumElements();
5850
5851	SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5852	SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5853
5854	// Adjust mask based on new input vector length.
5855	SmallVector<int, 16> NewMask;
5856	for (unsigned i = 0; i != NumElts; ++i) {
5857	int Idx = N->getMaskElt(Idx: i);
5858	if (Idx < (int)NumElts)
5859	NewMask.push_back(Elt: Idx);
5860	else
5861	NewMask.push_back(Elt: Idx - NumElts + WidenNumElts);
5862	}
5863	for (unsigned i = NumElts; i != WidenNumElts; ++i)
5864	NewMask.push_back(Elt: -1);
5865	return DAG.getVectorShuffle(VT: WidenVT, dl, N1: InOp1, N2: InOp2, Mask: NewMask);
5866	}
5867
5868	SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_REVERSE(SDNode *N) {
5869	EVT VT = N->getValueType(ResNo: 0);
5870	EVT EltVT = VT.getVectorElementType();
5871	SDLoc dl(N);
5872
5873	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
5874	SDValue OpValue = GetWidenedVector(Op: N->getOperand(Num: 0));
5875	assert(WidenVT == OpValue.getValueType() && "Unexpected widened vector type");
5876
5877	SDValue ReverseVal = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL: dl, VT: WidenVT, Operand: OpValue);
5878	unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
5879	unsigned VTNumElts = VT.getVectorMinNumElements();
5880	unsigned IdxVal = WidenNumElts - VTNumElts;
5881
5882	if (VT.isScalableVector()) {
5883	// Try to split the 'Widen ReverseVal' into smaller extracts and concat the
5884	// results together, e.g.(nxv6i64 -> nxv8i64)
5885	// nxv8i64 vector_reverse
5886	// <->
5887	// nxv8i64 concat(
5888	// nxv2i64 extract_subvector(nxv8i64, 2)
5889	// nxv2i64 extract_subvector(nxv8i64, 4)
5890	// nxv2i64 extract_subvector(nxv8i64, 6)
5891	// nxv2i64 undef)
5892
5893	unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts);
5894	EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
5895	EC: ElementCount::getScalable(MinVal: GCD));
5896	assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
5897	"down type's element count");
5898	SmallVector<SDValue> Parts;
5899	unsigned i = 0;
5900	for (; i < VTNumElts / GCD; ++i)
5901	Parts.push_back(
5902	Elt: DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: PartVT, N1: ReverseVal,
5903	N2: DAG.getVectorIdxConstant(Val: IdxVal + i * GCD, DL: dl)));
5904	for (; i < WidenNumElts / GCD; ++i)
5905	Parts.push_back(Elt: DAG.getUNDEF(VT: PartVT));
5906
5907	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts);
5908	}
5909
5910	// Use VECTOR_SHUFFLE to combine new vector from 'ReverseVal' for
5911	// fixed-vectors.
5912	SmallVector<int, 16> Mask;
5913	for (unsigned i = 0; i != VTNumElts; ++i) {
5914	Mask.push_back(Elt: IdxVal + i);
5915	}
5916	for (unsigned i = VTNumElts; i != WidenNumElts; ++i)
5917	Mask.push_back(Elt: -1);
5918
5919	return DAG.getVectorShuffle(VT: WidenVT, dl, N1: ReverseVal, N2: DAG.getUNDEF(VT: WidenVT),
5920	Mask);
5921	}
5922
5923	SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
5924	assert(N->getValueType(0).isVector() &&
5925	N->getOperand(0).getValueType().isVector() &&
5926	"Operands must be vectors");
5927	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5928	ElementCount WidenEC = WidenVT.getVectorElementCount();
5929
5930	SDValue InOp1 = N->getOperand(Num: 0);
5931	EVT InVT = InOp1.getValueType();
5932	assert(InVT.isVector() && "can not widen non-vector type");
5933	EVT WidenInVT =
5934	EVT::getVectorVT(Context&: *DAG.getContext(), VT: InVT.getVectorElementType(), EC: WidenEC);
5935
5936	// The input and output types often differ here, and it could be that while
5937	// we'd prefer to widen the result type, the input operands have been split.
5938	// In this case, we also need to split the result of this node as well.
5939	if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector) {
5940	SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
5941	SDValue Res = ModifyToType(InOp: SplitVSetCC, NVT: WidenVT);
5942	return Res;
5943	}
5944
5945	// If the inputs also widen, handle them directly. Otherwise widen by hand.
5946	SDValue InOp2 = N->getOperand(Num: 1);
5947	if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
5948	InOp1 = GetWidenedVector(Op: InOp1);
5949	InOp2 = GetWidenedVector(Op: InOp2);
5950	} else {
5951	InOp1 = DAG.WidenVector(N: InOp1, DL: SDLoc(N));
5952	InOp2 = DAG.WidenVector(N: InOp2, DL: SDLoc(N));
5953	}
5954
5955	// Assume that the input and output will be widen appropriately. If not,
5956	// we will have to unroll it at some point.
5957	assert(InOp1.getValueType() == WidenInVT &&
5958	InOp2.getValueType() == WidenInVT &&
5959	"Input not widened to expected type!");
5960	(void)WidenInVT;
5961	if (N->getOpcode() == ISD::VP_SETCC) {
5962	SDValue Mask =
5963	GetWidenedMask(Mask: N->getOperand(Num: 3), EC: WidenVT.getVectorElementCount());
5964	return DAG.getNode(Opcode: ISD::VP_SETCC, DL: SDLoc(N), VT: WidenVT, N1: InOp1, N2: InOp2,
5965	N3: N->getOperand(Num: 2), N4: Mask, N5: N->getOperand(Num: 4));
5966	}
5967	return DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N), VT: WidenVT, N1: InOp1, N2: InOp2,
5968	N3: N->getOperand(Num: 2));
5969	}
5970
5971	SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) {
5972	assert(N->getValueType(0).isVector() &&
5973	N->getOperand(1).getValueType().isVector() &&
5974	"Operands must be vectors");
5975	EVT VT = N->getValueType(ResNo: 0);
5976	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
5977	unsigned WidenNumElts = WidenVT.getVectorNumElements();
5978	unsigned NumElts = VT.getVectorNumElements();
5979	EVT EltVT = VT.getVectorElementType();
5980
5981	SDLoc dl(N);
5982	SDValue Chain = N->getOperand(Num: 0);
5983	SDValue LHS = N->getOperand(Num: 1);
5984	SDValue RHS = N->getOperand(Num: 2);
5985	SDValue CC = N->getOperand(Num: 3);
5986	EVT TmpEltVT = LHS.getValueType().getVectorElementType();
5987
5988	// Fully unroll and reassemble.
5989	SmallVector<SDValue, 8> Scalars(WidenNumElts, DAG.getUNDEF(VT: EltVT));
5990	SmallVector<SDValue, 8> Chains(NumElts);
5991	for (unsigned i = 0; i != NumElts; ++i) {
5992	SDValue LHSElem = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: TmpEltVT, N1: LHS,
5993	N2: DAG.getVectorIdxConstant(Val: i, DL: dl));
5994	SDValue RHSElem = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: TmpEltVT, N1: RHS,
5995	N2: DAG.getVectorIdxConstant(Val: i, DL: dl));
5996
5997	Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other},
5998	{Chain, LHSElem, RHSElem, CC});
5999	Chains[i] = Scalars[i].getValue(R: 1);
6000	Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i],
6001	LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
6002	RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT));
6003	}
6004
6005	SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
6006	ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
6007
6008	return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: Scalars);
6009	}
6010
6011	//===----------------------------------------------------------------------===//
6012	// Widen Vector Operand
6013	//===----------------------------------------------------------------------===//
6014	bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
6015	LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG));
6016	SDValue Res = SDValue();
6017
6018	// See if the target wants to custom widen this node.
6019	if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
6020	return false;
6021
6022	switch (N->getOpcode()) {
6023	default:
6024	#ifndef NDEBUG
6025	dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
6026	N->dump(G: &DAG);
6027	dbgs() << "\n";
6028	#endif
6029	report_fatal_error(reason: "Do not know how to widen this operator's operand!");
6030
6031	case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
6032	case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
6033	case ISD::INSERT_SUBVECTOR: Res = WidenVecOp_INSERT_SUBVECTOR(N); break;
6034	case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
6035	case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
6036	case ISD::STORE: Res = WidenVecOp_STORE(N); break;
6037	case ISD::VP_STORE: Res = WidenVecOp_VP_STORE(N, OpNo); break;
6038	case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
6039	Res = WidenVecOp_VP_STRIDED_STORE(N, OpNo);
6040	break;
6041	case ISD::ANY_EXTEND_VECTOR_INREG:
6042	case ISD::SIGN_EXTEND_VECTOR_INREG:
6043	case ISD::ZERO_EXTEND_VECTOR_INREG:
6044	Res = WidenVecOp_EXTEND_VECTOR_INREG(N);
6045	break;
6046	case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
6047	case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break;
6048	case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
6049	case ISD::VP_SCATTER: Res = WidenVecOp_VP_SCATTER(N, OpNo); break;
6050	case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
6051	case ISD::STRICT_FSETCC:
6052	case ISD::STRICT_FSETCCS: Res = WidenVecOp_STRICT_FSETCC(N); break;
6053	case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break;
6054	case ISD::FLDEXP:
6055	case ISD::FCOPYSIGN:
6056	case ISD::LRINT:
6057	case ISD::LLRINT:
6058	Res = WidenVecOp_UnrollVectorOp(N);
6059	break;
6060	case ISD::IS_FPCLASS: Res = WidenVecOp_IS_FPCLASS(N); break;
6061
6062	case ISD::ANY_EXTEND:
6063	case ISD::SIGN_EXTEND:
6064	case ISD::ZERO_EXTEND:
6065	Res = WidenVecOp_EXTEND(N);
6066	break;
6067
6068	case ISD::FP_EXTEND:
6069	case ISD::STRICT_FP_EXTEND:
6070	case ISD::FP_ROUND:
6071	case ISD::STRICT_FP_ROUND:
6072	case ISD::FP_TO_SINT:
6073	case ISD::STRICT_FP_TO_SINT:
6074	case ISD::FP_TO_UINT:
6075	case ISD::STRICT_FP_TO_UINT:
6076	case ISD::SINT_TO_FP:
6077	case ISD::STRICT_SINT_TO_FP:
6078	case ISD::UINT_TO_FP:
6079	case ISD::STRICT_UINT_TO_FP:
6080	case ISD::TRUNCATE:
6081	Res = WidenVecOp_Convert(N);
6082	break;
6083
6084	case ISD::FP_TO_SINT_SAT:
6085	case ISD::FP_TO_UINT_SAT:
6086	Res = WidenVecOp_FP_TO_XINT_SAT(N);
6087	break;
6088
6089	case ISD::VECREDUCE_FADD:
6090	case ISD::VECREDUCE_FMUL:
6091	case ISD::VECREDUCE_ADD:
6092	case ISD::VECREDUCE_MUL:
6093	case ISD::VECREDUCE_AND:
6094	case ISD::VECREDUCE_OR:
6095	case ISD::VECREDUCE_XOR:
6096	case ISD::VECREDUCE_SMAX:
6097	case ISD::VECREDUCE_SMIN:
6098	case ISD::VECREDUCE_UMAX:
6099	case ISD::VECREDUCE_UMIN:
6100	case ISD::VECREDUCE_FMAX:
6101	case ISD::VECREDUCE_FMIN:
6102	case ISD::VECREDUCE_FMAXIMUM:
6103	case ISD::VECREDUCE_FMINIMUM:
6104	Res = WidenVecOp_VECREDUCE(N);
6105	break;
6106	case ISD::VECREDUCE_SEQ_FADD:
6107	case ISD::VECREDUCE_SEQ_FMUL:
6108	Res = WidenVecOp_VECREDUCE_SEQ(N);
6109	break;
6110	case ISD::VP_REDUCE_FADD:
6111	case ISD::VP_REDUCE_SEQ_FADD:
6112	case ISD::VP_REDUCE_FMUL:
6113	case ISD::VP_REDUCE_SEQ_FMUL:
6114	case ISD::VP_REDUCE_ADD:
6115	case ISD::VP_REDUCE_MUL:
6116	case ISD::VP_REDUCE_AND:
6117	case ISD::VP_REDUCE_OR:
6118	case ISD::VP_REDUCE_XOR:
6119	case ISD::VP_REDUCE_SMAX:
6120	case ISD::VP_REDUCE_SMIN:
6121	case ISD::VP_REDUCE_UMAX:
6122	case ISD::VP_REDUCE_UMIN:
6123	case ISD::VP_REDUCE_FMAX:
6124	case ISD::VP_REDUCE_FMIN:
6125	Res = WidenVecOp_VP_REDUCE(N);
6126	break;
6127	}
6128
6129	// If Res is null, the sub-method took care of registering the result.
6130	if (!Res.getNode()) return false;
6131
6132	// If the result is N, the sub-method updated N in place. Tell the legalizer
6133	// core about this.
6134	if (Res.getNode() == N)
6135	return true;
6136
6137
6138	if (N->isStrictFPOpcode())
6139	assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
6140	"Invalid operand expansion");
6141	else
6142	assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
6143	"Invalid operand expansion");
6144
6145	ReplaceValueWith(From: SDValue(N, 0), To: Res);
6146	return false;
6147	}
6148
6149	SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
6150	SDLoc DL(N);
6151	EVT VT = N->getValueType(ResNo: 0);
6152
6153	SDValue InOp = N->getOperand(Num: 0);
6154	assert(getTypeAction(InOp.getValueType()) ==
6155	TargetLowering::TypeWidenVector &&
6156	"Unexpected type action");
6157	InOp = GetWidenedVector(Op: InOp);
6158	assert(VT.getVectorNumElements() <
6159	InOp.getValueType().getVectorNumElements() &&
6160	"Input wasn't widened!");
6161
6162	// We may need to further widen the operand until it has the same total
6163	// vector size as the result.
6164	EVT InVT = InOp.getValueType();
6165	if (InVT.getSizeInBits() != VT.getSizeInBits()) {
6166	EVT InEltVT = InVT.getVectorElementType();
6167	for (EVT FixedVT : MVT::vector_valuetypes()) {
6168	EVT FixedEltVT = FixedVT.getVectorElementType();
6169	if (TLI.isTypeLegal(FixedVT) &&
6170	FixedVT.getSizeInBits() == VT.getSizeInBits() &&
6171	FixedEltVT == InEltVT) {
6172	assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
6173	"Not enough elements in the fixed type for the operand!");
6174	assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
6175	"We can't have the same type as we started with!");
6176	if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
6177	InOp = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, FixedVT,
6178	DAG.getUNDEF(FixedVT), InOp,
6179	DAG.getVectorIdxConstant(0, DL));
6180	else
6181	InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp,
6182	DAG.getVectorIdxConstant(0, DL));
6183	break;
6184	}
6185	}
6186	InVT = InOp.getValueType();
6187	if (InVT.getSizeInBits() != VT.getSizeInBits())
6188	// We couldn't find a legal vector type that was a widening of the input
6189	// and could be extended in-register to the result type, so we have to
6190	// scalarize.
6191	return WidenVecOp_Convert(N);
6192	}
6193
6194	// Use special DAG nodes to represent the operation of extending the
6195	// low lanes.
6196	switch (N->getOpcode()) {
6197	default:
6198	llvm_unreachable("Extend legalization on extend operation!");
6199	case ISD::ANY_EXTEND:
6200	return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
6201	case ISD::SIGN_EXTEND:
6202	return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
6203	case ISD::ZERO_EXTEND:
6204	return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
6205	}
6206	}
6207
6208	SDValue DAGTypeLegalizer::WidenVecOp_UnrollVectorOp(SDNode *N) {
6209	// The result (and first input) is legal, but the second input is illegal.
6210	// We can't do much to fix that, so just unroll and let the extracts off of
6211	// the second input be widened as needed later.
6212	return DAG.UnrollVectorOp(N);
6213	}
6214
6215	SDValue DAGTypeLegalizer::WidenVecOp_IS_FPCLASS(SDNode *N) {
6216	SDLoc DL(N);
6217	EVT ResultVT = N->getValueType(ResNo: 0);
6218	SDValue Test = N->getOperand(Num: 1);
6219	SDValue WideArg = GetWidenedVector(Op: N->getOperand(Num: 0));
6220
6221	// Process this node similarly to SETCC.
6222	EVT WideResultVT = getSetCCResultType(VT: WideArg.getValueType());
6223	if (ResultVT.getScalarType() == MVT::i1)
6224	WideResultVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
6225	WideResultVT.getVectorNumElements());
6226
6227	SDValue WideNode = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: WideResultVT,
6228	Ops: {WideArg, Test}, Flags: N->getFlags());
6229
6230	// Extract the needed results from the result vector.
6231	EVT ResVT =
6232	EVT::getVectorVT(Context&: *DAG.getContext(), VT: WideResultVT.getVectorElementType(),
6233	NumElements: ResultVT.getVectorNumElements());
6234	SDValue CC = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: ResVT, N1: WideNode,
6235	N2: DAG.getVectorIdxConstant(Val: 0, DL));
6236
6237	EVT OpVT = N->getOperand(Num: 0).getValueType();
6238	ISD::NodeType ExtendCode =
6239	TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
6240	return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: CC);
6241	}
6242
6243	SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
6244	// Since the result is legal and the input is illegal.
6245	EVT VT = N->getValueType(ResNo: 0);
6246	EVT EltVT = VT.getVectorElementType();
6247	SDLoc dl(N);
6248	SDValue InOp = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0);
6249	assert(getTypeAction(InOp.getValueType()) ==
6250	TargetLowering::TypeWidenVector &&
6251	"Unexpected type action");
6252	InOp = GetWidenedVector(Op: InOp);
6253	EVT InVT = InOp.getValueType();
6254	unsigned Opcode = N->getOpcode();
6255
6256	// See if a widened result type would be legal, if so widen the node.
6257	// FIXME: This isn't safe for StrictFP. Other optimization here is needed.
6258	EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
6259	EC: InVT.getVectorElementCount());
6260	if (TLI.isTypeLegal(VT: WideVT) && !N->isStrictFPOpcode()) {
6261	SDValue Res;
6262	if (N->isStrictFPOpcode()) {
6263	if (Opcode == ISD::STRICT_FP_ROUND)
6264	Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other },
6265	{ N->getOperand(0), InOp, N->getOperand(2) });
6266	else
6267	Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other },
6268	{ N->getOperand(0), InOp });
6269	// Legalize the chain result - switch anything that used the old chain to
6270	// use the new one.
6271	ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6272	} else {
6273	if (Opcode == ISD::FP_ROUND)
6274	Res = DAG.getNode(Opcode, DL: dl, VT: WideVT, N1: InOp, N2: N->getOperand(Num: 1));
6275	else
6276	Res = DAG.getNode(Opcode, DL: dl, VT: WideVT, Operand: InOp);
6277	}
6278	return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT, N1: Res,
6279	N2: DAG.getVectorIdxConstant(Val: 0, DL: dl));
6280	}
6281
6282	EVT InEltVT = InVT.getVectorElementType();
6283
6284	// Unroll the convert into some scalar code and create a nasty build vector.
6285	unsigned NumElts = VT.getVectorNumElements();
6286	SmallVector<SDValue, 16> Ops(NumElts);
6287	if (N->isStrictFPOpcode()) {
6288	SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
6289	SmallVector<SDValue, 32> OpChains;
6290	for (unsigned i=0; i < NumElts; ++i) {
6291	NewOps[1] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: InEltVT, N1: InOp,
6292	N2: DAG.getVectorIdxConstant(Val: i, DL: dl));
6293	Ops[i] = DAG.getNode(Opcode, dl, { EltVT, MVT::Other }, NewOps);
6294	OpChains.push_back(Elt: Ops[i].getValue(R: 1));
6295	}
6296	SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains);
6297	ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
6298	} else {
6299	for (unsigned i = 0; i < NumElts; ++i)
6300	Ops[i] = DAG.getNode(Opcode, DL: dl, VT: EltVT,
6301	Operand: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: InEltVT,
6302	N1: InOp, N2: DAG.getVectorIdxConstant(Val: i, DL: dl)));
6303	}
6304
6305	return DAG.getBuildVector(VT, DL: dl, Ops);
6306	}
6307
6308	SDValue DAGTypeLegalizer::WidenVecOp_FP_TO_XINT_SAT(SDNode *N) {
6309	EVT DstVT = N->getValueType(ResNo: 0);
6310	SDValue Src = GetWidenedVector(Op: N->getOperand(Num: 0));
6311	EVT SrcVT = Src.getValueType();
6312	ElementCount WideNumElts = SrcVT.getVectorElementCount();
6313	SDLoc dl(N);
6314
6315	// See if a widened result type would be legal, if so widen the node.
6316	EVT WideDstVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6317	VT: DstVT.getVectorElementType(), EC: WideNumElts);
6318	if (TLI.isTypeLegal(VT: WideDstVT)) {
6319	SDValue Res =
6320	DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WideDstVT, N1: Src, N2: N->getOperand(Num: 1));
6321	return DAG.getNode(
6322	Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DstVT, N1: Res,
6323	N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())));
6324	}
6325
6326	// Give up and unroll.
6327	return DAG.UnrollVectorOp(N);
6328	}
6329
6330	SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
6331	EVT VT = N->getValueType(ResNo: 0);
6332	SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6333	EVT InWidenVT = InOp.getValueType();
6334	SDLoc dl(N);
6335
6336	// Check if we can convert between two legal vector types and extract.
6337	TypeSize InWidenSize = InWidenVT.getSizeInBits();
6338	TypeSize Size = VT.getSizeInBits();
6339	// x86mmx is not an acceptable vector element type, so don't try.
6340	if (!VT.isVector() && VT != MVT::x86mmx &&
6341	InWidenSize.hasKnownScalarFactor(Size)) {
6342	unsigned NewNumElts = InWidenSize.getKnownScalarFactor(RHS: Size);
6343	EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: NewNumElts);
6344	if (TLI.isTypeLegal(VT: NewVT)) {
6345	SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp);
6346	return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT, N1: BitOp,
6347	N2: DAG.getVectorIdxConstant(Val: 0, DL: dl));
6348	}
6349	}
6350
6351	// Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened
6352	// to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not,
6353	// we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids
6354	// having to copy via memory.
6355	if (VT.isVector()) {
6356	EVT EltVT = VT.getVectorElementType();
6357	unsigned EltSize = EltVT.getFixedSizeInBits();
6358	if (InWidenSize.isKnownMultipleOf(RHS: EltSize)) {
6359	ElementCount NewNumElts =
6360	(InWidenVT.getVectorElementCount() * InWidenVT.getScalarSizeInBits())
6361	.divideCoefficientBy(RHS: EltSize);
6362	EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, EC: NewNumElts);
6363	if (TLI.isTypeLegal(VT: NewVT)) {
6364	SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp);
6365	return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT, N1: BitOp,
6366	N2: DAG.getVectorIdxConstant(Val: 0, DL: dl));
6367	}
6368	}
6369	}
6370
6371	return CreateStackStoreLoad(Op: InOp, DestVT: VT);
6372	}
6373
6374	SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
6375	EVT VT = N->getValueType(ResNo: 0);
6376	EVT EltVT = VT.getVectorElementType();
6377	EVT InVT = N->getOperand(Num: 0).getValueType();
6378	SDLoc dl(N);
6379
6380	// If the widen width for this operand is the same as the width of the concat
6381	// and all but the first operand is undef, just use the widened operand.
6382	unsigned NumOperands = N->getNumOperands();
6383	if (VT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) {
6384	unsigned i;
6385	for (i = 1; i < NumOperands; ++i)
6386	if (!N->getOperand(Num: i).isUndef())
6387	break;
6388
6389	if (i == NumOperands)
6390	return GetWidenedVector(Op: N->getOperand(Num: 0));
6391	}
6392
6393	// Otherwise, fall back to a nasty build vector.
6394	unsigned NumElts = VT.getVectorNumElements();
6395	SmallVector<SDValue, 16> Ops(NumElts);
6396
6397	unsigned NumInElts = InVT.getVectorNumElements();
6398
6399	unsigned Idx = 0;
6400	for (unsigned i=0; i < NumOperands; ++i) {
6401	SDValue InOp = N->getOperand(Num: i);
6402	assert(getTypeAction(InOp.getValueType()) ==
6403	TargetLowering::TypeWidenVector &&
6404	"Unexpected type action");
6405	InOp = GetWidenedVector(Op: InOp);
6406	for (unsigned j = 0; j < NumInElts; ++j)
6407	Ops[Idx++] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: InOp,
6408	N2: DAG.getVectorIdxConstant(Val: j, DL: dl));
6409	}
6410	return DAG.getBuildVector(VT, DL: dl, Ops);
6411	}
6412
6413	SDValue DAGTypeLegalizer::WidenVecOp_INSERT_SUBVECTOR(SDNode *N) {
6414	EVT VT = N->getValueType(ResNo: 0);
6415	SDValue SubVec = N->getOperand(Num: 1);
6416	SDValue InVec = N->getOperand(Num: 0);
6417
6418	if (getTypeAction(VT: SubVec.getValueType()) == TargetLowering::TypeWidenVector)
6419	SubVec = GetWidenedVector(Op: SubVec);
6420
6421	EVT SubVT = SubVec.getValueType();
6422
6423	// Whether or not all the elements of the widened SubVec will be inserted into
6424	// valid indices of VT.
6425	bool IndicesValid = false;
6426	// If we statically know that VT can fit SubVT, the indices are valid.
6427	if (VT.knownBitsGE(VT: SubVT))
6428	IndicesValid = true;
6429	else if (VT.isScalableVector() && SubVT.isFixedLengthVector()) {
6430	// Otherwise, if we're inserting a fixed vector into a scalable vector and
6431	// we know the minimum vscale we can work out if it's valid ourselves.
6432	Attribute Attr = DAG.getMachineFunction().getFunction().getFnAttribute(
6433	Attribute::VScaleRange);
6434	if (Attr.isValid()) {
6435	unsigned VScaleMin = Attr.getVScaleRangeMin();
6436	if (VT.getSizeInBits().getKnownMinValue() * VScaleMin >=
6437	SubVT.getFixedSizeInBits())
6438	IndicesValid = true;
6439	}
6440	}
6441
6442	// We need to make sure that the indices are still valid, otherwise we might
6443	// widen what was previously well-defined to something undefined.
6444	if (IndicesValid && InVec.isUndef() && N->getConstantOperandVal(Num: 2) == 0)
6445	return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT, N1: InVec, N2: SubVec,
6446	N3: N->getOperand(Num: 2));
6447
6448	report_fatal_error(reason: "Don't know how to widen the operands for "
6449	"INSERT_SUBVECTOR");
6450	}
6451
6452	SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
6453	SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6454	return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(N),
6455	VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1));
6456	}
6457
6458	SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
6459	SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6460	return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
6461	VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1));
6462	}
6463
6464	SDValue DAGTypeLegalizer::WidenVecOp_EXTEND_VECTOR_INREG(SDNode *N) {
6465	SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6466	return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: InOp);
6467	}
6468
6469	SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
6470	// We have to widen the value, but we want only to store the original
6471	// vector type.
6472	StoreSDNode *ST = cast<StoreSDNode>(Val: N);
6473
6474	if (!ST->getMemoryVT().getScalarType().isByteSized())
6475	return TLI.scalarizeVectorStore(ST, DAG);
6476
6477	if (ST->isTruncatingStore())
6478	return TLI.scalarizeVectorStore(ST, DAG);
6479
6480	// Generate a vector-predicated store if it is custom/legal on the target.
6481	// To avoid possible recursion, only do this if the widened mask type is
6482	// legal.
6483	// FIXME: Not all targets may support EVL in VP_STORE. These will have been
6484	// removed from the IR by the ExpandVectorPredication pass but we're
6485	// reintroducing them here.
6486	SDValue StVal = ST->getValue();
6487	EVT StVT = StVal.getValueType();
6488	EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StVT);
6489	EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
6490	WideVT.getVectorElementCount());
6491
6492	if (TLI.isOperationLegalOrCustom(Op: ISD::VP_STORE, VT: WideVT) &&
6493	TLI.isTypeLegal(VT: WideMaskVT)) {
6494	// Widen the value.
6495	SDLoc DL(N);
6496	StVal = GetWidenedVector(Op: StVal);
6497	SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT);
6498	SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(),
6499	EC: StVT.getVectorElementCount());
6500	return DAG.getStoreVP(Chain: ST->getChain(), dl: DL, Val: StVal, Ptr: ST->getBasePtr(),
6501	Offset: DAG.getUNDEF(VT: ST->getBasePtr().getValueType()), Mask,
6502	EVL, MemVT: StVT, MMO: ST->getMemOperand(),
6503	AM: ST->getAddressingMode());
6504	}
6505
6506	SmallVector<SDValue, 16> StChain;
6507	if (GenWidenVectorStores(StChain, ST)) {
6508	if (StChain.size() == 1)
6509	return StChain[0];
6510
6511	return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain);
6512	}
6513
6514	report_fatal_error(reason: "Unable to widen vector store");
6515	}
6516
6517	SDValue DAGTypeLegalizer::WidenVecOp_VP_STORE(SDNode *N, unsigned OpNo) {
6518	assert((OpNo == 1 || OpNo == 3) &&
6519	"Can widen only data or mask operand of vp_store");
6520	VPStoreSDNode *ST = cast<VPStoreSDNode>(Val: N);
6521	SDValue Mask = ST->getMask();
6522	SDValue StVal = ST->getValue();
6523	SDLoc dl(N);
6524
6525	if (OpNo == 1) {
6526	// Widen the value.
6527	StVal = GetWidenedVector(Op: StVal);
6528
6529	// We only handle the case where the mask needs widening to an
6530	// identically-sized type as the vector inputs.
6531	assert(getTypeAction(Mask.getValueType()) ==
6532	TargetLowering::TypeWidenVector &&
6533	"Unable to widen VP store");
6534	Mask = GetWidenedVector(Op: Mask);
6535	} else {
6536	Mask = GetWidenedVector(Op: Mask);
6537
6538	// We only handle the case where the stored value needs widening to an
6539	// identically-sized type as the mask.
6540	assert(getTypeAction(StVal.getValueType()) ==
6541	TargetLowering::TypeWidenVector &&
6542	"Unable to widen VP store");
6543	StVal = GetWidenedVector(Op: StVal);
6544	}
6545
6546	assert(Mask.getValueType().getVectorElementCount() ==
6547	StVal.getValueType().getVectorElementCount() &&
6548	"Mask and data vectors should have the same number of elements");
6549	return DAG.getStoreVP(Chain: ST->getChain(), dl, Val: StVal, Ptr: ST->getBasePtr(),
6550	Offset: ST->getOffset(), Mask, EVL: ST->getVectorLength(),
6551	MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(),
6552	AM: ST->getAddressingMode(), IsTruncating: ST->isTruncatingStore(),
6553	IsCompressing: ST->isCompressingStore());
6554	}
6555
6556	SDValue DAGTypeLegalizer::WidenVecOp_VP_STRIDED_STORE(SDNode *N,
6557	unsigned OpNo) {
6558	assert((OpNo == 1 || OpNo == 4) &&
6559	"Can widen only data or mask operand of vp_strided_store");
6560	VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N);
6561	SDValue Mask = SST->getMask();
6562	SDValue StVal = SST->getValue();
6563	SDLoc DL(N);
6564
6565	if (OpNo == 1)
6566	assert(getTypeAction(Mask.getValueType()) ==
6567	TargetLowering::TypeWidenVector &&
6568	"Unable to widen VP strided store");
6569	else
6570	assert(getTypeAction(StVal.getValueType()) ==
6571	TargetLowering::TypeWidenVector &&
6572	"Unable to widen VP strided store");
6573
6574	StVal = GetWidenedVector(Op: StVal);
6575	Mask = GetWidenedVector(Op: Mask);
6576
6577	assert(StVal.getValueType().getVectorElementCount() ==
6578	Mask.getValueType().getVectorElementCount() &&
6579	"Data and mask vectors should have the same number of elements");
6580
6581	return DAG.getStridedStoreVP(
6582	Chain: SST->getChain(), DL, Val: StVal, Ptr: SST->getBasePtr(), Offset: SST->getOffset(),
6583	Stride: SST->getStride(), Mask, EVL: SST->getVectorLength(), MemVT: SST->getMemoryVT(),
6584	MMO: SST->getMemOperand(), AM: SST->getAddressingMode(), IsTruncating: SST->isTruncatingStore(),
6585	IsCompressing: SST->isCompressingStore());
6586	}
6587
6588	SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
6589	assert((OpNo == 1 || OpNo == 4) &&
6590	"Can widen only data or mask operand of mstore");
6591	MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N);
6592	SDValue Mask = MST->getMask();
6593	EVT MaskVT = Mask.getValueType();
6594	SDValue StVal = MST->getValue();
6595	SDLoc dl(N);
6596
6597	if (OpNo == 1) {
6598	// Widen the value.
6599	StVal = GetWidenedVector(Op: StVal);
6600
6601	// The mask should be widened as well.
6602	EVT WideVT = StVal.getValueType();
6603	EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6604	VT: MaskVT.getVectorElementType(),
6605	NumElements: WideVT.getVectorNumElements());
6606	Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
6607	} else {
6608	// Widen the mask.
6609	EVT WideMaskVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MaskVT);
6610	Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
6611
6612	EVT ValueVT = StVal.getValueType();
6613	EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6614	VT: ValueVT.getVectorElementType(),
6615	NumElements: WideMaskVT.getVectorNumElements());
6616	StVal = ModifyToType(InOp: StVal, NVT: WideVT);
6617	}
6618
6619	assert(Mask.getValueType().getVectorNumElements() ==
6620	StVal.getValueType().getVectorNumElements() &&
6621	"Mask and data vectors should have the same number of elements");
6622	return DAG.getMaskedStore(Chain: MST->getChain(), dl, Val: StVal, Base: MST->getBasePtr(),
6623	Offset: MST->getOffset(), Mask, MemVT: MST->getMemoryVT(),
6624	MMO: MST->getMemOperand(), AM: MST->getAddressingMode(),
6625	IsTruncating: false, IsCompressing: MST->isCompressingStore());
6626	}
6627
6628	SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
6629	assert(OpNo == 4 && "Can widen only the index of mgather");
6630	auto *MG = cast<MaskedGatherSDNode>(Val: N);
6631	SDValue DataOp = MG->getPassThru();
6632	SDValue Mask = MG->getMask();
6633	SDValue Scale = MG->getScale();
6634
6635	// Just widen the index. It's allowed to have extra elements.
6636	SDValue Index = GetWidenedVector(Op: MG->getIndex());
6637
6638	SDLoc dl(N);
6639	SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
6640	Scale};
6641	SDValue Res = DAG.getMaskedGather(VTs: MG->getVTList(), MemVT: MG->getMemoryVT(), dl, Ops,
6642	MMO: MG->getMemOperand(), IndexType: MG->getIndexType(),
6643	ExtTy: MG->getExtensionType());
6644	ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6645	ReplaceValueWith(From: SDValue(N, 0), To: Res.getValue(R: 0));
6646	return SDValue();
6647	}
6648
6649	SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
6650	MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(Val: N);
6651	SDValue DataOp = MSC->getValue();
6652	SDValue Mask = MSC->getMask();
6653	SDValue Index = MSC->getIndex();
6654	SDValue Scale = MSC->getScale();
6655	EVT WideMemVT = MSC->getMemoryVT();
6656
6657	if (OpNo == 1) {
6658	DataOp = GetWidenedVector(Op: DataOp);
6659	unsigned NumElts = DataOp.getValueType().getVectorNumElements();
6660
6661	// Widen index.
6662	EVT IndexVT = Index.getValueType();
6663	EVT WideIndexVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6664	VT: IndexVT.getVectorElementType(), NumElements: NumElts);
6665	Index = ModifyToType(InOp: Index, NVT: WideIndexVT);
6666
6667	// The mask should be widened as well.
6668	EVT MaskVT = Mask.getValueType();
6669	EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6670	VT: MaskVT.getVectorElementType(), NumElements: NumElts);
6671	Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
6672
6673	// Widen the MemoryType
6674	WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6675	VT: MSC->getMemoryVT().getScalarType(), NumElements: NumElts);
6676	} else if (OpNo == 4) {
6677	// Just widen the index. It's allowed to have extra elements.
6678	Index = GetWidenedVector(Op: Index);
6679	} else
6680	llvm_unreachable("Can't widen this operand of mscatter");
6681
6682	SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
6683	Scale};
6684	return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), WideMemVT, SDLoc(N),
6685	Ops, MSC->getMemOperand(), MSC->getIndexType(),
6686	MSC->isTruncatingStore());
6687	}
6688
6689	SDValue DAGTypeLegalizer::WidenVecOp_VP_SCATTER(SDNode *N, unsigned OpNo) {
6690	VPScatterSDNode *VPSC = cast<VPScatterSDNode>(Val: N);
6691	SDValue DataOp = VPSC->getValue();
6692	SDValue Mask = VPSC->getMask();
6693	SDValue Index = VPSC->getIndex();
6694	SDValue Scale = VPSC->getScale();
6695	EVT WideMemVT = VPSC->getMemoryVT();
6696
6697	if (OpNo == 1) {
6698	DataOp = GetWidenedVector(Op: DataOp);
6699	Index = GetWidenedVector(Op: Index);
6700	const auto WideEC = DataOp.getValueType().getVectorElementCount();
6701	Mask = GetWidenedMask(Mask, EC: WideEC);
6702	WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6703	VT: VPSC->getMemoryVT().getScalarType(), EC: WideEC);
6704	} else if (OpNo == 3) {
6705	// Just widen the index. It's allowed to have extra elements.
6706	Index = GetWidenedVector(Op: Index);
6707	} else
6708	llvm_unreachable("Can't widen this operand of VP_SCATTER");
6709
6710	SDValue Ops[] = {
6711	VPSC->getChain(), DataOp, VPSC->getBasePtr(), Index, Scale, Mask,
6712	VPSC->getVectorLength()};
6713	return DAG.getScatterVP(DAG.getVTList(MVT::Other), WideMemVT, SDLoc(N), Ops,
6714	VPSC->getMemOperand(), VPSC->getIndexType());
6715	}
6716
6717	SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
6718	SDValue InOp0 = GetWidenedVector(Op: N->getOperand(Num: 0));
6719	SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
6720	SDLoc dl(N);
6721	EVT VT = N->getValueType(ResNo: 0);
6722
6723	// WARNING: In this code we widen the compare instruction with garbage.
6724	// This garbage may contain denormal floats which may be slow. Is this a real
6725	// concern ? Should we zero the unused lanes if this is a float compare ?
6726
6727	// Get a new SETCC node to compare the newly widened operands.
6728	// Only some of the compared elements are legal.
6729	EVT SVT = getSetCCResultType(VT: InOp0.getValueType());
6730	// The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
6731	if (VT.getScalarType() == MVT::i1)
6732	SVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
6733	SVT.getVectorElementCount());
6734
6735	SDValue WideSETCC = DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N),
6736	VT: SVT, N1: InOp0, N2: InOp1, N3: N->getOperand(Num: 2));
6737
6738	// Extract the needed results from the result vector.
6739	EVT ResVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6740	VT: SVT.getVectorElementType(),
6741	EC: VT.getVectorElementCount());
6742	SDValue CC = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: ResVT, N1: WideSETCC,
6743	N2: DAG.getVectorIdxConstant(Val: 0, DL: dl));
6744
6745	EVT OpVT = N->getOperand(Num: 0).getValueType();
6746	ISD::NodeType ExtendCode =
6747	TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
6748	return DAG.getNode(Opcode: ExtendCode, DL: dl, VT, Operand: CC);
6749	}
6750
6751	SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) {
6752	SDValue Chain = N->getOperand(Num: 0);
6753	SDValue LHS = GetWidenedVector(Op: N->getOperand(Num: 1));
6754	SDValue RHS = GetWidenedVector(Op: N->getOperand(Num: 2));
6755	SDValue CC = N->getOperand(Num: 3);
6756	SDLoc dl(N);
6757
6758	EVT VT = N->getValueType(ResNo: 0);
6759	EVT EltVT = VT.getVectorElementType();
6760	EVT TmpEltVT = LHS.getValueType().getVectorElementType();
6761	unsigned NumElts = VT.getVectorNumElements();
6762
6763	// Unroll into a build vector.
6764	SmallVector<SDValue, 8> Scalars(NumElts);
6765	SmallVector<SDValue, 8> Chains(NumElts);
6766
6767	for (unsigned i = 0; i != NumElts; ++i) {
6768	SDValue LHSElem = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: TmpEltVT, N1: LHS,
6769	N2: DAG.getVectorIdxConstant(Val: i, DL: dl));
6770	SDValue RHSElem = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: TmpEltVT, N1: RHS,
6771	N2: DAG.getVectorIdxConstant(Val: i, DL: dl));
6772
6773	Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other},
6774	{Chain, LHSElem, RHSElem, CC});
6775	Chains[i] = Scalars[i].getValue(R: 1);
6776	Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i],
6777	LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
6778	RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT));
6779	}
6780
6781	SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
6782	ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
6783
6784	return DAG.getBuildVector(VT, DL: dl, Ops: Scalars);
6785	}
6786
6787	SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
6788	SDLoc dl(N);
6789	SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 0));
6790	EVT OrigVT = N->getOperand(Num: 0).getValueType();
6791	EVT WideVT = Op.getValueType();
6792	EVT ElemVT = OrigVT.getVectorElementType();
6793	SDNodeFlags Flags = N->getFlags();
6794
6795	unsigned Opc = N->getOpcode();
6796	unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc);
6797	SDValue NeutralElem = DAG.getNeutralElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags);
6798	assert(NeutralElem && "Neutral element must exist");
6799
6800	// Pad the vector with the neutral element.
6801	unsigned OrigElts = OrigVT.getVectorMinNumElements();
6802	unsigned WideElts = WideVT.getVectorMinNumElements();
6803
6804	if (WideVT.isScalableVector()) {
6805	unsigned GCD = std::gcd(m: OrigElts, n: WideElts);
6806	EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT,
6807	EC: ElementCount::getScalable(MinVal: GCD));
6808	SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem);
6809	for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
6810	Op = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WideVT, N1: Op, N2: SplatNeutral,
6811	N3: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
6812	return DAG.getNode(Opcode: Opc, DL: dl, VT: N->getValueType(ResNo: 0), Operand: Op, Flags);
6813	}
6814
6815	for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
6816	Op = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: WideVT, N1: Op, N2: NeutralElem,
6817	N3: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
6818
6819	return DAG.getNode(Opcode: Opc, DL: dl, VT: N->getValueType(ResNo: 0), Operand: Op, Flags);
6820	}
6821
6822	SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) {
6823	SDLoc dl(N);
6824	SDValue AccOp = N->getOperand(Num: 0);
6825	SDValue VecOp = N->getOperand(Num: 1);
6826	SDValue Op = GetWidenedVector(Op: VecOp);
6827
6828	EVT OrigVT = VecOp.getValueType();
6829	EVT WideVT = Op.getValueType();
6830	EVT ElemVT = OrigVT.getVectorElementType();
6831	SDNodeFlags Flags = N->getFlags();
6832
6833	unsigned Opc = N->getOpcode();
6834	unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc);
6835	SDValue NeutralElem = DAG.getNeutralElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags);
6836
6837	// Pad the vector with the neutral element.
6838	unsigned OrigElts = OrigVT.getVectorMinNumElements();
6839	unsigned WideElts = WideVT.getVectorMinNumElements();
6840
6841	if (WideVT.isScalableVector()) {
6842	unsigned GCD = std::gcd(m: OrigElts, n: WideElts);
6843	EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT,
6844	EC: ElementCount::getScalable(MinVal: GCD));
6845	SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem);
6846	for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
6847	Op = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WideVT, N1: Op, N2: SplatNeutral,
6848	N3: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
6849	return DAG.getNode(Opcode: Opc, DL: dl, VT: N->getValueType(ResNo: 0), N1: AccOp, N2: Op, Flags);
6850	}
6851
6852	for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
6853	Op = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: WideVT, N1: Op, N2: NeutralElem,
6854	N3: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
6855
6856	return DAG.getNode(Opcode: Opc, DL: dl, VT: N->getValueType(ResNo: 0), N1: AccOp, N2: Op, Flags);
6857	}
6858
6859	SDValue DAGTypeLegalizer::WidenVecOp_VP_REDUCE(SDNode *N) {
6860	assert(N->isVPOpcode() && "Expected VP opcode");
6861
6862	SDLoc dl(N);
6863	SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 1));
6864	SDValue Mask = GetWidenedMask(Mask: N->getOperand(Num: 2),
6865	EC: Op.getValueType().getVectorElementCount());
6866
6867	return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0),
6868	Ops: {N->getOperand(Num: 0), Op, Mask, N->getOperand(Num: 3)},
6869	Flags: N->getFlags());
6870	}
6871
6872	SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
6873	// This only gets called in the case that the left and right inputs and
6874	// result are of a legal odd vector type, and the condition is illegal i1 of
6875	// the same odd width that needs widening.
6876	EVT VT = N->getValueType(ResNo: 0);
6877	assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT));
6878
6879	SDValue Cond = GetWidenedVector(Op: N->getOperand(Num: 0));
6880	SDValue LeftIn = DAG.WidenVector(N: N->getOperand(Num: 1), DL: SDLoc(N));
6881	SDValue RightIn = DAG.WidenVector(N: N->getOperand(Num: 2), DL: SDLoc(N));
6882	SDLoc DL(N);
6883
6884	SDValue Select = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LeftIn.getValueType(), N1: Cond,
6885	N2: LeftIn, N3: RightIn);
6886	return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: Select,
6887	N2: DAG.getVectorIdxConstant(Val: 0, DL));
6888	}
6889
6890	//===----------------------------------------------------------------------===//
6891	// Vector Widening Utilities
6892	//===----------------------------------------------------------------------===//
6893
6894	// Utility function to find the type to chop up a widen vector for load/store
6895	// TLI: Target lowering used to determine legal types.
6896	// Width: Width left need to load/store.
6897	// WidenVT: The widen vector type to load to/store from
6898	// Align: If 0, don't allow use of a wider type
6899	// WidenEx: If Align is not 0, the amount additional we can load/store from.
6900
6901	static std::optional<EVT> findMemType(SelectionDAG &DAG,
6902	const TargetLowering &TLI, unsigned Width,
6903	EVT WidenVT, unsigned Align = 0,
6904	unsigned WidenEx = 0) {
6905	EVT WidenEltVT = WidenVT.getVectorElementType();
6906	const bool Scalable = WidenVT.isScalableVector();
6907	unsigned WidenWidth = WidenVT.getSizeInBits().getKnownMinValue();
6908	unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
6909	unsigned AlignInBits = Align*8;
6910
6911	// If we have one element to load/store, return it.
6912	EVT RetVT = WidenEltVT;
6913	if (!Scalable && Width == WidenEltWidth)
6914	return RetVT;
6915
6916	// Don't bother looking for an integer type if the vector is scalable, skip
6917	// to vector types.
6918	if (!Scalable) {
6919	// See if there is larger legal integer than the element type to load/store.
6920	for (EVT MemVT : reverse(MVT::integer_valuetypes())) {
6921	unsigned MemVTWidth = MemVT.getSizeInBits();
6922	if (MemVT.getSizeInBits() <= WidenEltWidth)
6923	break;
6924	auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
6925	if ((Action == TargetLowering::TypeLegal ||
6926	Action == TargetLowering::TypePromoteInteger) &&
6927	(WidenWidth % MemVTWidth) == 0 &&
6928	isPowerOf2_32(WidenWidth / MemVTWidth) &&
6929	(MemVTWidth <= Width ||
6930	(Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
6931	if (MemVTWidth == WidenWidth)
6932	return MemVT;
6933	RetVT = MemVT;
6934	break;
6935	}
6936	}
6937	}
6938
6939	// See if there is a larger vector type to load/store that has the same vector
6940	// element type and is evenly divisible with the WidenVT.
6941	for (EVT MemVT : reverse(MVT::vector_valuetypes())) {
6942	// Skip vector MVTs which don't match the scalable property of WidenVT.
6943	if (Scalable != MemVT.isScalableVector())
6944	continue;
6945	unsigned MemVTWidth = MemVT.getSizeInBits().getKnownMinValue();
6946	auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
6947	if ((Action == TargetLowering::TypeLegal ||
6948	Action == TargetLowering::TypePromoteInteger) &&
6949	WidenEltVT == MemVT.getVectorElementType() &&
6950	(WidenWidth % MemVTWidth) == 0 &&
6951	isPowerOf2_32(WidenWidth / MemVTWidth) &&
6952	(MemVTWidth <= Width ||
6953	(Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
6954	if (RetVT.getFixedSizeInBits() < MemVTWidth || MemVT == WidenVT)
6955	return MemVT;
6956	}
6957	}
6958
6959	// Using element-wise loads and stores for widening operations is not
6960	// supported for scalable vectors
6961	if (Scalable)
6962	return std::nullopt;
6963
6964	return RetVT;
6965	}
6966
6967	// Builds a vector type from scalar loads
6968	// VecTy: Resulting Vector type
6969	// LDOps: Load operators to build a vector type
6970	// [Start,End) the list of loads to use.
6971	static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
6972	SmallVectorImpl<SDValue> &LdOps,
6973	unsigned Start, unsigned End) {
6974	SDLoc dl(LdOps[Start]);
6975	EVT LdTy = LdOps[Start].getValueType();
6976	unsigned Width = VecTy.getSizeInBits();
6977	unsigned NumElts = Width / LdTy.getSizeInBits();
6978	EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: LdTy, NumElements: NumElts);
6979
6980	unsigned Idx = 1;
6981	SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT,Operand: LdOps[Start]);
6982
6983	for (unsigned i = Start + 1; i != End; ++i) {
6984	EVT NewLdTy = LdOps[i].getValueType();
6985	if (NewLdTy != LdTy) {
6986	NumElts = Width / NewLdTy.getSizeInBits();
6987	NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewLdTy, NumElements: NumElts);
6988	VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: VecOp);
6989	// Readjust position and vector position based on new load type.
6990	Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
6991	LdTy = NewLdTy;
6992	}
6993	VecOp = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: NewVecVT, N1: VecOp, N2: LdOps[i],
6994	N3: DAG.getVectorIdxConstant(Val: Idx++, DL: dl));
6995	}
6996	return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecTy, Operand: VecOp);
6997	}
6998
6999	SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
7000	LoadSDNode *LD) {
7001	// The strategy assumes that we can efficiently load power-of-two widths.
7002	// The routine chops the vector into the largest vector loads with the same
7003	// element type or scalar loads and then recombines it to the widen vector
7004	// type.
7005	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0));
7006	EVT LdVT = LD->getMemoryVT();
7007	SDLoc dl(LD);
7008	assert(LdVT.isVector() && WidenVT.isVector());
7009	assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
7010	assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
7011
7012	// Load information
7013	SDValue Chain = LD->getChain();
7014	SDValue BasePtr = LD->getBasePtr();
7015	MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
7016	AAMDNodes AAInfo = LD->getAAInfo();
7017
7018	TypeSize LdWidth = LdVT.getSizeInBits();
7019	TypeSize WidenWidth = WidenVT.getSizeInBits();
7020	TypeSize WidthDiff = WidenWidth - LdWidth;
7021	// Allow wider loads if they are sufficiently aligned to avoid memory faults
7022	// and if the original load is simple.
7023	unsigned LdAlign =
7024	(!LD->isSimple() || LdVT.isScalableVector()) ? 0 : LD->getAlign().value();
7025
7026	// Find the vector type that can load from.
7027	std::optional<EVT> FirstVT =
7028	findMemType(DAG, TLI, Width: LdWidth.getKnownMinValue(), WidenVT, Align: LdAlign,
7029	WidenEx: WidthDiff.getKnownMinValue());
7030
7031	if (!FirstVT)
7032	return SDValue();
7033
7034	SmallVector<EVT, 8> MemVTs;
7035	TypeSize FirstVTWidth = FirstVT->getSizeInBits();
7036
7037	// Unless we're able to load in one instruction we must work out how to load
7038	// the remainder.
7039	if (!TypeSize::isKnownLE(LHS: LdWidth, RHS: FirstVTWidth)) {
7040	std::optional<EVT> NewVT = FirstVT;
7041	TypeSize RemainingWidth = LdWidth;
7042	TypeSize NewVTWidth = FirstVTWidth;
7043	do {
7044	RemainingWidth -= NewVTWidth;
7045	if (TypeSize::isKnownLT(LHS: RemainingWidth, RHS: NewVTWidth)) {
7046	// The current type we are using is too large. Find a better size.
7047	NewVT = findMemType(DAG, TLI, Width: RemainingWidth.getKnownMinValue(),
7048	WidenVT, Align: LdAlign, WidenEx: WidthDiff.getKnownMinValue());
7049	if (!NewVT)
7050	return SDValue();
7051	NewVTWidth = NewVT->getSizeInBits();
7052	}
7053	MemVTs.push_back(Elt: *NewVT);
7054	} while (TypeSize::isKnownGT(LHS: RemainingWidth, RHS: NewVTWidth));
7055	}
7056
7057	SDValue LdOp = DAG.getLoad(VT: *FirstVT, dl, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
7058	Alignment: LD->getOriginalAlign(), MMOFlags, AAInfo);
7059	LdChain.push_back(Elt: LdOp.getValue(R: 1));
7060
7061	// Check if we can load the element with one instruction.
7062	if (MemVTs.empty()) {
7063	assert(TypeSize::isKnownLE(LdWidth, FirstVTWidth));
7064	if (!FirstVT->isVector()) {
7065	unsigned NumElts =
7066	WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
7067	EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: *FirstVT, NumElements: NumElts);
7068	SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT, Operand: LdOp);
7069	return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: VecOp);
7070	}
7071	if (FirstVT == WidenVT)
7072	return LdOp;
7073
7074	// TODO: We don't currently have any tests that exercise this code path.
7075	assert(WidenWidth.getFixedValue() % FirstVTWidth.getFixedValue() == 0);
7076	unsigned NumConcat =
7077	WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
7078	SmallVector<SDValue, 16> ConcatOps(NumConcat);
7079	SDValue UndefVal = DAG.getUNDEF(VT: *FirstVT);
7080	ConcatOps[0] = LdOp;
7081	for (unsigned i = 1; i != NumConcat; ++i)
7082	ConcatOps[i] = UndefVal;
7083	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: ConcatOps);
7084	}
7085
7086	// Load vector by using multiple loads from largest vector to scalar.
7087	SmallVector<SDValue, 16> LdOps;
7088	LdOps.push_back(Elt: LdOp);
7089
7090	uint64_t ScaledOffset = 0;
7091	MachinePointerInfo MPI = LD->getPointerInfo();
7092
7093	// First incremement past the first load.
7094	IncrementPointer(N: cast<LoadSDNode>(Val&: LdOp), MemVT: *FirstVT, MPI, Ptr&: BasePtr,
7095	ScaledOffset: &ScaledOffset);
7096
7097	for (EVT MemVT : MemVTs) {
7098	Align NewAlign = ScaledOffset == 0
7099	? LD->getOriginalAlign()
7100	: commonAlignment(A: LD->getAlign(), Offset: ScaledOffset);
7101	SDValue L =
7102	DAG.getLoad(VT: MemVT, dl, Chain, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign, MMOFlags, AAInfo);
7103
7104	LdOps.push_back(Elt: L);
7105	LdChain.push_back(Elt: L.getValue(R: 1));
7106	IncrementPointer(N: cast<LoadSDNode>(Val&: L), MemVT, MPI, Ptr&: BasePtr, ScaledOffset: &ScaledOffset);
7107	}
7108
7109	// Build the vector from the load operations.
7110	unsigned End = LdOps.size();
7111	if (!LdOps[0].getValueType().isVector())
7112	// All the loads are scalar loads.
7113	return BuildVectorFromScalar(DAG, VecTy: WidenVT, LdOps, Start: 0, End);
7114
7115	// If the load contains vectors, build the vector using concat vector.
7116	// All of the vectors used to load are power-of-2, and the scalar loads can be
7117	// combined to make a power-of-2 vector.
7118	SmallVector<SDValue, 16> ConcatOps(End);
7119	int i = End - 1;
7120	int Idx = End;
7121	EVT LdTy = LdOps[i].getValueType();
7122	// First, combine the scalar loads to a vector.
7123	if (!LdTy.isVector()) {
7124	for (--i; i >= 0; --i) {
7125	LdTy = LdOps[i].getValueType();
7126	if (LdTy.isVector())
7127	break;
7128	}
7129	ConcatOps[--Idx] = BuildVectorFromScalar(DAG, VecTy: LdTy, LdOps, Start: i + 1, End);
7130	}
7131
7132	ConcatOps[--Idx] = LdOps[i];
7133	for (--i; i >= 0; --i) {
7134	EVT NewLdTy = LdOps[i].getValueType();
7135	if (NewLdTy != LdTy) {
7136	// Create a larger vector.
7137	TypeSize LdTySize = LdTy.getSizeInBits();
7138	TypeSize NewLdTySize = NewLdTy.getSizeInBits();
7139	assert(NewLdTySize.isScalable() == LdTySize.isScalable() &&
7140	NewLdTySize.isKnownMultipleOf(LdTySize.getKnownMinValue()));
7141	unsigned NumOps =
7142	NewLdTySize.getKnownMinValue() / LdTySize.getKnownMinValue();
7143	SmallVector<SDValue, 16> WidenOps(NumOps);
7144	unsigned j = 0;
7145	for (; j != End-Idx; ++j)
7146	WidenOps[j] = ConcatOps[Idx+j];
7147	for (; j != NumOps; ++j)
7148	WidenOps[j] = DAG.getUNDEF(VT: LdTy);
7149
7150	ConcatOps[End-1] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewLdTy,
7151	Ops: WidenOps);
7152	Idx = End - 1;
7153	LdTy = NewLdTy;
7154	}
7155	ConcatOps[--Idx] = LdOps[i];
7156	}
7157
7158	if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
7159	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT,
7160	Ops: ArrayRef(&ConcatOps[Idx], End - Idx));
7161
7162	// We need to fill the rest with undefs to build the vector.
7163	unsigned NumOps =
7164	WidenWidth.getKnownMinValue() / LdTy.getSizeInBits().getKnownMinValue();
7165	SmallVector<SDValue, 16> WidenOps(NumOps);
7166	SDValue UndefVal = DAG.getUNDEF(VT: LdTy);
7167	{
7168	unsigned i = 0;
7169	for (; i != End-Idx; ++i)
7170	WidenOps[i] = ConcatOps[Idx+i];
7171	for (; i != NumOps; ++i)
7172	WidenOps[i] = UndefVal;
7173	}
7174	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: WidenOps);
7175	}
7176
7177	SDValue
7178	DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
7179	LoadSDNode *LD,
7180	ISD::LoadExtType ExtType) {
7181	// For extension loads, it may not be more efficient to chop up the vector
7182	// and then extend it. Instead, we unroll the load and build a new vector.
7183	EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0));
7184	EVT LdVT = LD->getMemoryVT();
7185	SDLoc dl(LD);
7186	assert(LdVT.isVector() && WidenVT.isVector());
7187	assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
7188
7189	// Load information
7190	SDValue Chain = LD->getChain();
7191	SDValue BasePtr = LD->getBasePtr();
7192	MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
7193	AAMDNodes AAInfo = LD->getAAInfo();
7194
7195	if (LdVT.isScalableVector())
7196	report_fatal_error(reason: "Generating widen scalable extending vector loads is "
7197	"not yet supported");
7198
7199	EVT EltVT = WidenVT.getVectorElementType();
7200	EVT LdEltVT = LdVT.getVectorElementType();
7201	unsigned NumElts = LdVT.getVectorNumElements();
7202
7203	// Load each element and widen.
7204	unsigned WidenNumElts = WidenVT.getVectorNumElements();
7205	SmallVector<SDValue, 16> Ops(WidenNumElts);
7206	unsigned Increment = LdEltVT.getSizeInBits() / 8;
7207	Ops[0] =
7208	DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
7209	MemVT: LdEltVT, Alignment: LD->getOriginalAlign(), MMOFlags, AAInfo);
7210	LdChain.push_back(Elt: Ops[0].getValue(R: 1));
7211	unsigned i = 0, Offset = Increment;
7212	for (i=1; i < NumElts; ++i, Offset += Increment) {
7213	SDValue NewBasePtr =
7214	DAG.getObjectPtrOffset(SL: dl, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Offset));
7215	Ops[i] = DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: NewBasePtr,
7216	PtrInfo: LD->getPointerInfo().getWithOffset(O: Offset), MemVT: LdEltVT,
7217	Alignment: LD->getOriginalAlign(), MMOFlags, AAInfo);
7218	LdChain.push_back(Elt: Ops[i].getValue(R: 1));
7219	}
7220
7221	// Fill the rest with undefs.
7222	SDValue UndefVal = DAG.getUNDEF(VT: EltVT);
7223	for (; i != WidenNumElts; ++i)
7224	Ops[i] = UndefVal;
7225
7226	return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
7227	}
7228
7229	bool DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
7230	StoreSDNode *ST) {
7231	// The strategy assumes that we can efficiently store power-of-two widths.
7232	// The routine chops the vector into the largest vector stores with the same
7233	// element type or scalar stores.
7234	SDValue Chain = ST->getChain();
7235	SDValue BasePtr = ST->getBasePtr();
7236	MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
7237	AAMDNodes AAInfo = ST->getAAInfo();
7238	SDValue ValOp = GetWidenedVector(Op: ST->getValue());
7239	SDLoc dl(ST);
7240
7241	EVT StVT = ST->getMemoryVT();
7242	TypeSize StWidth = StVT.getSizeInBits();
7243	EVT ValVT = ValOp.getValueType();
7244	TypeSize ValWidth = ValVT.getSizeInBits();
7245	EVT ValEltVT = ValVT.getVectorElementType();
7246	unsigned ValEltWidth = ValEltVT.getFixedSizeInBits();
7247	assert(StVT.getVectorElementType() == ValEltVT);
7248	assert(StVT.isScalableVector() == ValVT.isScalableVector() &&
7249	"Mismatch between store and value types");
7250
7251	int Idx = 0; // current index to store
7252
7253	MachinePointerInfo MPI = ST->getPointerInfo();
7254	uint64_t ScaledOffset = 0;
7255
7256	// A breakdown of how to widen this vector store. Each element of the vector
7257	// is a memory VT combined with the number of times it is to be stored to,
7258	// e,g., v5i32 -> {{v2i32,2},{i32,1}}
7259	SmallVector<std::pair<EVT, unsigned>, 4> MemVTs;
7260
7261	while (StWidth.isNonZero()) {
7262	// Find the largest vector type we can store with.
7263	std::optional<EVT> NewVT =
7264	findMemType(DAG, TLI, Width: StWidth.getKnownMinValue(), WidenVT: ValVT);
7265	if (!NewVT)
7266	return false;
7267	MemVTs.push_back(Elt: {*NewVT, 0});
7268	TypeSize NewVTWidth = NewVT->getSizeInBits();
7269
7270	do {
7271	StWidth -= NewVTWidth;
7272	MemVTs.back().second++;
7273	} while (StWidth.isNonZero() && TypeSize::isKnownGE(LHS: StWidth, RHS: NewVTWidth));
7274	}
7275
7276	for (const auto &Pair : MemVTs) {
7277	EVT NewVT = Pair.first;
7278	unsigned Count = Pair.second;
7279	TypeSize NewVTWidth = NewVT.getSizeInBits();
7280
7281	if (NewVT.isVector()) {
7282	unsigned NumVTElts = NewVT.getVectorMinNumElements();
7283	do {
7284	Align NewAlign = ScaledOffset == 0
7285	? ST->getOriginalAlign()
7286	: commonAlignment(A: ST->getAlign(), Offset: ScaledOffset);
7287	SDValue EOp = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: NewVT, N1: ValOp,
7288	N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
7289	SDValue PartStore = DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign,
7290	MMOFlags, AAInfo);
7291	StChain.push_back(Elt: PartStore);
7292
7293	Idx += NumVTElts;
7294	IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr,
7295	ScaledOffset: &ScaledOffset);
7296	} while (--Count);
7297	} else {
7298	// Cast the vector to the scalar type we can store.
7299	unsigned NumElts = ValWidth.getFixedValue() / NewVTWidth.getFixedValue();
7300	EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewVT, NumElements: NumElts);
7301	SDValue VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: ValOp);
7302	// Readjust index position based on new vector type.
7303	Idx = Idx * ValEltWidth / NewVTWidth.getFixedValue();
7304	do {
7305	SDValue EOp = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: NewVT, N1: VecOp,
7306	N2: DAG.getVectorIdxConstant(Val: Idx++, DL: dl));
7307	SDValue PartStore =
7308	DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI, Alignment: ST->getOriginalAlign(),
7309	MMOFlags, AAInfo);
7310	StChain.push_back(Elt: PartStore);
7311
7312	IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr);
7313	} while (--Count);
7314	// Restore index back to be relative to the original widen element type.
7315	Idx = Idx * NewVTWidth.getFixedValue() / ValEltWidth;
7316	}
7317	}
7318
7319	return true;
7320	}
7321
7322	/// Modifies a vector input (widen or narrows) to a vector of NVT. The
7323	/// input vector must have the same element type as NVT.
7324	/// FillWithZeroes specifies that the vector should be widened with zeroes.
7325	SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
7326	bool FillWithZeroes) {
7327	// Note that InOp might have been widened so it might already have
7328	// the right width or it might need be narrowed.
7329	EVT InVT = InOp.getValueType();
7330	assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
7331	"input and widen element type must match");
7332	assert(InVT.isScalableVector() == NVT.isScalableVector() &&
7333	"cannot modify scalable vectors in this way");
7334	SDLoc dl(InOp);
7335
7336	// Check if InOp already has the right width.
7337	if (InVT == NVT)
7338	return InOp;
7339
7340	ElementCount InEC = InVT.getVectorElementCount();
7341	ElementCount WidenEC = NVT.getVectorElementCount();
7342	if (WidenEC.hasKnownScalarFactor(RHS: InEC)) {
7343	unsigned NumConcat = WidenEC.getKnownScalarFactor(RHS: InEC);
7344	SmallVector<SDValue, 16> Ops(NumConcat);
7345	SDValue FillVal = FillWithZeroes ? DAG.getConstant(Val: 0, DL: dl, VT: InVT) :
7346	DAG.getUNDEF(VT: InVT);
7347	Ops[0] = InOp;
7348	for (unsigned i = 1; i != NumConcat; ++i)
7349	Ops[i] = FillVal;
7350
7351	return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NVT, Ops);
7352	}
7353
7354	if (InEC.hasKnownScalarFactor(RHS: WidenEC))
7355	return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: NVT, N1: InOp,
7356	N2: DAG.getVectorIdxConstant(Val: 0, DL: dl));
7357
7358	assert(!InVT.isScalableVector() && !NVT.isScalableVector() &&
7359	"Scalable vectors should have been handled already.");
7360
7361	unsigned InNumElts = InEC.getFixedValue();
7362	unsigned WidenNumElts = WidenEC.getFixedValue();
7363
7364	// Fall back to extract and build (+ mask, if padding with zeros).
7365	SmallVector<SDValue, 16> Ops(WidenNumElts);
7366	EVT EltVT = NVT.getVectorElementType();
7367	unsigned MinNumElts = std::min(a: WidenNumElts, b: InNumElts);
7368	unsigned Idx;
7369	for (Idx = 0; Idx < MinNumElts; ++Idx)
7370	Ops[Idx] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: InOp,
7371	N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
7372
7373	SDValue UndefVal = DAG.getUNDEF(VT: EltVT);
7374	for (; Idx < WidenNumElts; ++Idx)
7375	Ops[Idx] = UndefVal;
7376
7377	SDValue Widened = DAG.getBuildVector(VT: NVT, DL: dl, Ops);
7378	if (!FillWithZeroes)
7379	return Widened;
7380
7381	assert(NVT.isInteger() &&
7382	"We expect to never want to FillWithZeroes for non-integral types.");
7383
7384	SmallVector<SDValue, 16> MaskOps;
7385	MaskOps.append(NumInputs: MinNumElts, Elt: DAG.getAllOnesConstant(DL: dl, VT: EltVT));
7386	MaskOps.append(NumInputs: WidenNumElts - MinNumElts, Elt: DAG.getConstant(Val: 0, DL: dl, VT: EltVT));
7387
7388	return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: NVT, N1: Widened,
7389	N2: DAG.getBuildVector(VT: NVT, DL: dl, Ops: MaskOps));
7390	}
7391