1	//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This implements the SelectionDAG class.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "llvm/CodeGen/SelectionDAG.h"
14	#include "SDNodeDbgValue.h"
15	#include "llvm/ADT/APFloat.h"
16	#include "llvm/ADT/APInt.h"
17	#include "llvm/ADT/APSInt.h"
18	#include "llvm/ADT/ArrayRef.h"
19	#include "llvm/ADT/BitVector.h"
20	#include "llvm/ADT/DenseSet.h"
21	#include "llvm/ADT/FoldingSet.h"
22	#include "llvm/ADT/STLExtras.h"
23	#include "llvm/ADT/SmallPtrSet.h"
24	#include "llvm/ADT/SmallVector.h"
25	#include "llvm/ADT/Twine.h"
26	#include "llvm/Analysis/AliasAnalysis.h"
27	#include "llvm/Analysis/MemoryLocation.h"
28	#include "llvm/Analysis/ValueTracking.h"
29	#include "llvm/Analysis/VectorUtils.h"
30	#include "llvm/BinaryFormat/Dwarf.h"
31	#include "llvm/CodeGen/Analysis.h"
32	#include "llvm/CodeGen/FunctionLoweringInfo.h"
33	#include "llvm/CodeGen/ISDOpcodes.h"
34	#include "llvm/CodeGen/MachineBasicBlock.h"
35	#include "llvm/CodeGen/MachineConstantPool.h"
36	#include "llvm/CodeGen/MachineFrameInfo.h"
37	#include "llvm/CodeGen/MachineFunction.h"
38	#include "llvm/CodeGen/MachineMemOperand.h"
39	#include "llvm/CodeGen/RuntimeLibcalls.h"
40	#include "llvm/CodeGen/SDPatternMatch.h"
41	#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
42	#include "llvm/CodeGen/SelectionDAGNodes.h"
43	#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
44	#include "llvm/CodeGen/TargetFrameLowering.h"
45	#include "llvm/CodeGen/TargetLowering.h"
46	#include "llvm/CodeGen/TargetRegisterInfo.h"
47	#include "llvm/CodeGen/TargetSubtargetInfo.h"
48	#include "llvm/CodeGen/ValueTypes.h"
49	#include "llvm/CodeGenTypes/MachineValueType.h"
50	#include "llvm/IR/Constant.h"
51	#include "llvm/IR/ConstantRange.h"
52	#include "llvm/IR/Constants.h"
53	#include "llvm/IR/DataLayout.h"
54	#include "llvm/IR/DebugInfoMetadata.h"
55	#include "llvm/IR/DebugLoc.h"
56	#include "llvm/IR/DerivedTypes.h"
57	#include "llvm/IR/Function.h"
58	#include "llvm/IR/GlobalValue.h"
59	#include "llvm/IR/Metadata.h"
60	#include "llvm/IR/Type.h"
61	#include "llvm/Support/Casting.h"
62	#include "llvm/Support/CodeGen.h"
63	#include "llvm/Support/Compiler.h"
64	#include "llvm/Support/Debug.h"
65	#include "llvm/Support/ErrorHandling.h"
66	#include "llvm/Support/KnownBits.h"
67	#include "llvm/Support/MathExtras.h"
68	#include "llvm/Support/Mutex.h"
69	#include "llvm/Support/raw_ostream.h"
70	#include "llvm/Target/TargetMachine.h"
71	#include "llvm/Target/TargetOptions.h"
72	#include "llvm/TargetParser/Triple.h"
73	#include "llvm/Transforms/Utils/SizeOpts.h"
74	#include <algorithm>
75	#include <cassert>
76	#include <cstdint>
77	#include <cstdlib>
78	#include <limits>
79	#include <set>
80	#include <string>
81	#include <utility>
82	#include <vector>
83
84	using namespace llvm;
85	using namespace llvm::SDPatternMatch;
86
87	/// makeVTList - Return an instance of the SDVTList struct initialized with the
88	/// specified members.
89	static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
90	SDVTList Res = {.VTs: VTs, .NumVTs: NumVTs};
91	return Res;
92	}
93
94	// Default null implementations of the callbacks.
95	void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
96	void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
97	void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {}
98
99	void SelectionDAG::DAGNodeDeletedListener::anchor() {}
100	void SelectionDAG::DAGNodeInsertedListener::anchor() {}
101
102	#define DEBUG_TYPE "selectiondag"
103
104	static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",
105	cl::Hidden, cl::init(Val: true),
106	cl::desc("Gang up loads and stores generated by inlining of memcpy"));
107
108	static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",
109	cl::desc("Number limit for gluing ld/st of memcpy."),
110	cl::Hidden, cl::init(Val: 0));
111
112	static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
113	LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G););
114	}
115
116	//===----------------------------------------------------------------------===//
117	// ConstantFPSDNode Class
118	//===----------------------------------------------------------------------===//
119
120	/// isExactlyValue - We don't rely on operator== working on double values, as
121	/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
122	/// As such, this method can be used to do an exact bit-for-bit comparison of
123	/// two floating point values.
124	bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
125	return getValueAPF().bitwiseIsEqual(RHS: V);
126	}
127
128	bool ConstantFPSDNode::isValueValidForType(EVT VT,
129	const APFloat& Val) {
130	assert(VT.isFloatingPoint() && "Can only convert between FP types");
131
132	// convert modifies in place, so make a copy.
133	APFloat Val2 = APFloat(Val);
134	bool losesInfo;
135	(void) Val2.convert(ToSemantics: SelectionDAG::EVTToAPFloatSemantics(VT),
136	RM: APFloat::rmNearestTiesToEven,
137	losesInfo: &losesInfo);
138	return !losesInfo;
139	}
140
141	//===----------------------------------------------------------------------===//
142	// ISD Namespace
143	//===----------------------------------------------------------------------===//
144
145	bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
146	if (N->getOpcode() == ISD::SPLAT_VECTOR) {
147	unsigned EltSize =
148	N->getValueType(ResNo: 0).getVectorElementType().getSizeInBits();
149	if (auto *Op0 = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0))) {
150	SplatVal = Op0->getAPIntValue().trunc(width: EltSize);
151	return true;
152	}
153	if (auto *Op0 = dyn_cast<ConstantFPSDNode>(Val: N->getOperand(Num: 0))) {
154	SplatVal = Op0->getValueAPF().bitcastToAPInt().trunc(width: EltSize);
155	return true;
156	}
157	}
158
159	auto *BV = dyn_cast<BuildVectorSDNode>(Val: N);
160	if (!BV)
161	return false;
162
163	APInt SplatUndef;
164	unsigned SplatBitSize;
165	bool HasUndefs;
166	unsigned EltSize = N->getValueType(ResNo: 0).getVectorElementType().getSizeInBits();
167	// Endianness does not matter here. We are checking for a splat given the
168	// element size of the vector, and if we find such a splat for little endian
169	// layout, then that should be valid also for big endian (as the full vector
170	// size is known to be a multiple of the element size).
171	const bool IsBigEndian = false;
172	return BV->isConstantSplat(SplatValue&: SplatVal, SplatUndef, SplatBitSize, HasAnyUndefs&: HasUndefs,
173	MinSplatBits: EltSize, isBigEndian: IsBigEndian) &&
174	EltSize == SplatBitSize;
175	}
176
177	// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
178	// specializations of the more general isConstantSplatVector()?
179
180	bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) {
181	// Look through a bit convert.
182	while (N->getOpcode() == ISD::BITCAST)
183	N = N->getOperand(Num: 0).getNode();
184
185	if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
186	APInt SplatVal;
187	return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnes();
188	}
189
190	if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
191
192	unsigned i = 0, e = N->getNumOperands();
193
194	// Skip over all of the undef values.
195	while (i != e && N->getOperand(Num: i).isUndef())
196	++i;
197
198	// Do not accept an all-undef vector.
199	if (i == e) return false;
200
201	// Do not accept build_vectors that aren't all constants or which have non-~0
202	// elements. We have to be a bit careful here, as the type of the constant
203	// may not be the same as the type of the vector elements due to type
204	// legalization (the elements are promoted to a legal type for the target and
205	// a vector of a type may be legal when the base element type is not).
206	// We only want to check enough bits to cover the vector elements, because
207	// we care if the resultant vector is all ones, not whether the individual
208	// constants are.
209	SDValue NotZero = N->getOperand(Num: i);
210	unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
211	if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: NotZero)) {
212	if (CN->getAPIntValue().countr_one() < EltSize)
213	return false;
214	} else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Val&: NotZero)) {
215	if (CFPN->getValueAPF().bitcastToAPInt().countr_one() < EltSize)
216	return false;
217	} else
218	return false;
219
220	// Okay, we have at least one ~0 value, check to see if the rest match or are
221	// undefs. Even with the above element type twiddling, this should be OK, as
222	// the same type legalization should have applied to all the elements.
223	for (++i; i != e; ++i)
224	if (N->getOperand(Num: i) != NotZero && !N->getOperand(Num: i).isUndef())
225	return false;
226	return true;
227	}
228
229	bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) {
230	// Look through a bit convert.
231	while (N->getOpcode() == ISD::BITCAST)
232	N = N->getOperand(Num: 0).getNode();
233
234	if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
235	APInt SplatVal;
236	return isConstantSplatVector(N, SplatVal) && SplatVal.isZero();
237	}
238
239	if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
240
241	bool IsAllUndef = true;
242	for (const SDValue &Op : N->op_values()) {
243	if (Op.isUndef())
244	continue;
245	IsAllUndef = false;
246	// Do not accept build_vectors that aren't all constants or which have non-0
247	// elements. We have to be a bit careful here, as the type of the constant
248	// may not be the same as the type of the vector elements due to type
249	// legalization (the elements are promoted to a legal type for the target
250	// and a vector of a type may be legal when the base element type is not).
251	// We only want to check enough bits to cover the vector elements, because
252	// we care if the resultant vector is all zeros, not whether the individual
253	// constants are.
254	unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
255	if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val: Op)) {
256	if (CN->getAPIntValue().countr_zero() < EltSize)
257	return false;
258	} else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Val: Op)) {
259	if (CFPN->getValueAPF().bitcastToAPInt().countr_zero() < EltSize)
260	return false;
261	} else
262	return false;
263	}
264
265	// Do not accept an all-undef vector.
266	if (IsAllUndef)
267	return false;
268	return true;
269	}
270
271	bool ISD::isBuildVectorAllOnes(const SDNode *N) {
272	return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true);
273	}
274
275	bool ISD::isBuildVectorAllZeros(const SDNode *N) {
276	return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true);
277	}
278
279	bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
280	if (N->getOpcode() != ISD::BUILD_VECTOR)
281	return false;
282
283	for (const SDValue &Op : N->op_values()) {
284	if (Op.isUndef())
285	continue;
286	if (!isa<ConstantSDNode>(Val: Op))
287	return false;
288	}
289	return true;
290	}
291
292	bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
293	if (N->getOpcode() != ISD::BUILD_VECTOR)
294	return false;
295
296	for (const SDValue &Op : N->op_values()) {
297	if (Op.isUndef())
298	continue;
299	if (!isa<ConstantFPSDNode>(Val: Op))
300	return false;
301	}
302	return true;
303	}
304
305	bool ISD::isVectorShrinkable(const SDNode *N, unsigned NewEltSize,
306	bool Signed) {
307	assert(N->getValueType(0).isVector() && "Expected a vector!");
308
309	unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
310	if (EltSize <= NewEltSize)
311	return false;
312
313	if (N->getOpcode() == ISD::ZERO_EXTEND) {
314	return (N->getOperand(Num: 0).getValueType().getScalarSizeInBits() <=
315	NewEltSize) &&
316	!Signed;
317	}
318	if (N->getOpcode() == ISD::SIGN_EXTEND) {
319	return (N->getOperand(Num: 0).getValueType().getScalarSizeInBits() <=
320	NewEltSize) &&
321	Signed;
322	}
323	if (N->getOpcode() != ISD::BUILD_VECTOR)
324	return false;
325
326	for (const SDValue &Op : N->op_values()) {
327	if (Op.isUndef())
328	continue;
329	if (!isa<ConstantSDNode>(Val: Op))
330	return false;
331
332	APInt C = Op->getAsAPIntVal().trunc(width: EltSize);
333	if (Signed && C.trunc(width: NewEltSize).sext(width: EltSize) != C)
334	return false;
335	if (!Signed && C.trunc(width: NewEltSize).zext(width: EltSize) != C)
336	return false;
337	}
338
339	return true;
340	}
341
342	bool ISD::allOperandsUndef(const SDNode *N) {
343	// Return false if the node has no operands.
344	// This is "logically inconsistent" with the definition of "all" but
345	// is probably the desired behavior.
346	if (N->getNumOperands() == 0)
347	return false;
348	return all_of(Range: N->op_values(), P: [](SDValue Op) { return Op.isUndef(); });
349	}
350
351	bool ISD::isFreezeUndef(const SDNode *N) {
352	return N->getOpcode() == ISD::FREEZE && N->getOperand(Num: 0).isUndef();
353	}
354
355	template <typename ConstNodeType>
356	bool ISD::matchUnaryPredicateImpl(SDValue Op,
357	std::function<bool(ConstNodeType *)> Match,
358	bool AllowUndefs) {
359	// FIXME: Add support for scalar UNDEF cases?
360	if (auto *C = dyn_cast<ConstNodeType>(Op))
361	return Match(C);
362
363	// FIXME: Add support for vector UNDEF cases?
364	if (ISD::BUILD_VECTOR != Op.getOpcode() &&
365	ISD::SPLAT_VECTOR != Op.getOpcode())
366	return false;
367
368	EVT SVT = Op.getValueType().getScalarType();
369	for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
370	if (AllowUndefs && Op.getOperand(i).isUndef()) {
371	if (!Match(nullptr))
372	return false;
373	continue;
374	}
375
376	auto *Cst = dyn_cast<ConstNodeType>(Op.getOperand(i));
377	if (!Cst || Cst->getValueType(0) != SVT || !Match(Cst))
378	return false;
379	}
380	return true;
381	}
382	// Build used template types.
383	template bool ISD::matchUnaryPredicateImpl<ConstantSDNode>(
384	SDValue, std::function<bool(ConstantSDNode *)>, bool);
385	template bool ISD::matchUnaryPredicateImpl<ConstantFPSDNode>(
386	SDValue, std::function<bool(ConstantFPSDNode *)>, bool);
387
388	bool ISD::matchBinaryPredicate(
389	SDValue LHS, SDValue RHS,
390	std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
391	bool AllowUndefs, bool AllowTypeMismatch) {
392	if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType())
393	return false;
394
395	// TODO: Add support for scalar UNDEF cases?
396	if (auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHS))
397	if (auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHS))
398	return Match(LHSCst, RHSCst);
399
400	// TODO: Add support for vector UNDEF cases?
401	if (LHS.getOpcode() != RHS.getOpcode() ||
402	(LHS.getOpcode() != ISD::BUILD_VECTOR &&
403	LHS.getOpcode() != ISD::SPLAT_VECTOR))
404	return false;
405
406	EVT SVT = LHS.getValueType().getScalarType();
407	for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
408	SDValue LHSOp = LHS.getOperand(i);
409	SDValue RHSOp = RHS.getOperand(i);
410	bool LHSUndef = AllowUndefs && LHSOp.isUndef();
411	bool RHSUndef = AllowUndefs && RHSOp.isUndef();
412	auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHSOp);
413	auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHSOp);
414	if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
415	return false;
416	if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT ||
417	LHSOp.getValueType() != RHSOp.getValueType()))
418	return false;
419	if (!Match(LHSCst, RHSCst))
420	return false;
421	}
422	return true;
423	}
424
425	ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) {
426	switch (VecReduceOpcode) {
427	default:
428	llvm_unreachable("Expected VECREDUCE opcode");
429	case ISD::VECREDUCE_FADD:
430	case ISD::VECREDUCE_SEQ_FADD:
431	case ISD::VP_REDUCE_FADD:
432	case ISD::VP_REDUCE_SEQ_FADD:
433	return ISD::FADD;
434	case ISD::VECREDUCE_FMUL:
435	case ISD::VECREDUCE_SEQ_FMUL:
436	case ISD::VP_REDUCE_FMUL:
437	case ISD::VP_REDUCE_SEQ_FMUL:
438	return ISD::FMUL;
439	case ISD::VECREDUCE_ADD:
440	case ISD::VP_REDUCE_ADD:
441	return ISD::ADD;
442	case ISD::VECREDUCE_MUL:
443	case ISD::VP_REDUCE_MUL:
444	return ISD::MUL;
445	case ISD::VECREDUCE_AND:
446	case ISD::VP_REDUCE_AND:
447	return ISD::AND;
448	case ISD::VECREDUCE_OR:
449	case ISD::VP_REDUCE_OR:
450	return ISD::OR;
451	case ISD::VECREDUCE_XOR:
452	case ISD::VP_REDUCE_XOR:
453	return ISD::XOR;
454	case ISD::VECREDUCE_SMAX:
455	case ISD::VP_REDUCE_SMAX:
456	return ISD::SMAX;
457	case ISD::VECREDUCE_SMIN:
458	case ISD::VP_REDUCE_SMIN:
459	return ISD::SMIN;
460	case ISD::VECREDUCE_UMAX:
461	case ISD::VP_REDUCE_UMAX:
462	return ISD::UMAX;
463	case ISD::VECREDUCE_UMIN:
464	case ISD::VP_REDUCE_UMIN:
465	return ISD::UMIN;
466	case ISD::VECREDUCE_FMAX:
467	case ISD::VP_REDUCE_FMAX:
468	return ISD::FMAXNUM;
469	case ISD::VECREDUCE_FMIN:
470	case ISD::VP_REDUCE_FMIN:
471	return ISD::FMINNUM;
472	case ISD::VECREDUCE_FMAXIMUM:
473	return ISD::FMAXIMUM;
474	case ISD::VECREDUCE_FMINIMUM:
475	return ISD::FMINIMUM;
476	}
477	}
478
479	bool ISD::isVPOpcode(unsigned Opcode) {
480	switch (Opcode) {
481	default:
482	return false;
483	#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) \
484	case ISD::VPSD: \
485	return true;
486	#include "llvm/IR/VPIntrinsics.def"
487	}
488	}
489
490	bool ISD::isVPBinaryOp(unsigned Opcode) {
491	switch (Opcode) {
492	default:
493	break;
494	#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
495	#define VP_PROPERTY_BINARYOP return true;
496	#define END_REGISTER_VP_SDNODE(VPSD) break;
497	#include "llvm/IR/VPIntrinsics.def"
498	}
499	return false;
500	}
501
502	bool ISD::isVPReduction(unsigned Opcode) {
503	switch (Opcode) {
504	default:
505	break;
506	#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
507	#define VP_PROPERTY_REDUCTION(STARTPOS, ...) return true;
508	#define END_REGISTER_VP_SDNODE(VPSD) break;
509	#include "llvm/IR/VPIntrinsics.def"
510	}
511	return false;
512	}
513
514	/// The operand position of the vector mask.
515	std::optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) {
516	switch (Opcode) {
517	default:
518	return std::nullopt;
519	#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...) \
520	case ISD::VPSD: \
521	return MASKPOS;
522	#include "llvm/IR/VPIntrinsics.def"
523	}
524	}
525
526	/// The operand position of the explicit vector length parameter.
527	std::optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) {
528	switch (Opcode) {
529	default:
530	return std::nullopt;
531	#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \
532	case ISD::VPSD: \
533	return EVLPOS;
534	#include "llvm/IR/VPIntrinsics.def"
535	}
536	}
537
538	std::optional<unsigned> ISD::getBaseOpcodeForVP(unsigned VPOpcode,
539	bool hasFPExcept) {
540	// FIXME: Return strict opcodes in case of fp exceptions.
541	switch (VPOpcode) {
542	default:
543	return std::nullopt;
544	#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC:
545	#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC;
546	#define END_REGISTER_VP_SDNODE(VPOPC) break;
547	#include "llvm/IR/VPIntrinsics.def"
548	}
549	return std::nullopt;
550	}
551
552	unsigned ISD::getVPForBaseOpcode(unsigned Opcode) {
553	switch (Opcode) {
554	default:
555	llvm_unreachable("can not translate this Opcode to VP.");
556	#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) break;
557	#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC:
558	#define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC;
559	#include "llvm/IR/VPIntrinsics.def"
560	}
561	}
562
563	ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
564	switch (ExtType) {
565	case ISD::EXTLOAD:
566	return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
567	case ISD::SEXTLOAD:
568	return ISD::SIGN_EXTEND;
569	case ISD::ZEXTLOAD:
570	return ISD::ZERO_EXTEND;
571	default:
572	break;
573	}
574
575	llvm_unreachable("Invalid LoadExtType");
576	}
577
578	ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
579	// To perform this operation, we just need to swap the L and G bits of the
580	// operation.
581	unsigned OldL = (Operation >> 2) & 1;
582	unsigned OldG = (Operation >> 1) & 1;
583	return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
584	(OldL << 1) | // New G bit
585	(OldG << 2)); // New L bit.
586	}
587
588	static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) {
589	unsigned Operation = Op;
590	if (isIntegerLike)
591	Operation ^= 7; // Flip L, G, E bits, but not U.
592	else
593	Operation ^= 15; // Flip all of the condition bits.
594
595	if (Operation > ISD::SETTRUE2)
596	Operation &= ~8; // Don't let N and U bits get set.
597
598	return ISD::CondCode(Operation);
599	}
600
601	ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) {
602	return getSetCCInverseImpl(Op, isIntegerLike: Type.isInteger());
603	}
604
605	ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op,
606	bool isIntegerLike) {
607	return getSetCCInverseImpl(Op, isIntegerLike);
608	}
609
610	/// For an integer comparison, return 1 if the comparison is a signed operation
611	/// and 2 if the result is an unsigned comparison. Return zero if the operation
612	/// does not depend on the sign of the input (setne and seteq).
613	static int isSignedOp(ISD::CondCode Opcode) {
614	switch (Opcode) {
615	default: llvm_unreachable("Illegal integer setcc operation!");
616	case ISD::SETEQ:
617	case ISD::SETNE: return 0;
618	case ISD::SETLT:
619	case ISD::SETLE:
620	case ISD::SETGT:
621	case ISD::SETGE: return 1;
622	case ISD::SETULT:
623	case ISD::SETULE:
624	case ISD::SETUGT:
625	case ISD::SETUGE: return 2;
626	}
627	}
628
629	ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
630	EVT Type) {
631	bool IsInteger = Type.isInteger();
632	if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3)
633	// Cannot fold a signed integer setcc with an unsigned integer setcc.
634	return ISD::SETCC_INVALID;
635
636	unsigned Op = Op1 | Op2; // Combine all of the condition bits.
637
638	// If the N and U bits get set, then the resultant comparison DOES suddenly
639	// care about orderedness, and it is true when ordered.
640	if (Op > ISD::SETTRUE2)
641	Op &= ~16; // Clear the U bit if the N bit is set.
642
643	// Canonicalize illegal integer setcc's.
644	if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
645	Op = ISD::SETNE;
646
647	return ISD::CondCode(Op);
648	}
649
650	ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
651	EVT Type) {
652	bool IsInteger = Type.isInteger();
653	if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3)
654	// Cannot fold a signed setcc with an unsigned setcc.
655	return ISD::SETCC_INVALID;
656
657	// Combine all of the condition bits.
658	ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
659
660	// Canonicalize illegal integer setcc's.
661	if (IsInteger) {
662	switch (Result) {
663	default: break;
664	case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
665	case ISD::SETOEQ: // SETEQ & SETU[LG]E
666	case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
667	case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
668	case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
669	}
670	}
671
672	return Result;
673	}
674
675	//===----------------------------------------------------------------------===//
676	// SDNode Profile Support
677	//===----------------------------------------------------------------------===//
678
679	/// AddNodeIDOpcode - Add the node opcode to the NodeID data.
680	static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) {
681	ID.AddInteger(I: OpC);
682	}
683
684	/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
685	/// solely with their pointer.
686	static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
687	ID.AddPointer(Ptr: VTList.VTs);
688	}
689
690	/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
691	static void AddNodeIDOperands(FoldingSetNodeID &ID,
692	ArrayRef<SDValue> Ops) {
693	for (const auto &Op : Ops) {
694	ID.AddPointer(Ptr: Op.getNode());
695	ID.AddInteger(I: Op.getResNo());
696	}
697	}
698
699	/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
700	static void AddNodeIDOperands(FoldingSetNodeID &ID,
701	ArrayRef<SDUse> Ops) {
702	for (const auto &Op : Ops) {
703	ID.AddPointer(Ptr: Op.getNode());
704	ID.AddInteger(I: Op.getResNo());
705	}
706	}
707
708	static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned OpC,
709	SDVTList VTList, ArrayRef<SDValue> OpList) {
710	AddNodeIDOpcode(ID, OpC);
711	AddNodeIDValueTypes(ID, VTList);
712	AddNodeIDOperands(ID, Ops: OpList);
713	}
714
715	/// If this is an SDNode with special info, add this info to the NodeID data.
716	static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
717	switch (N->getOpcode()) {
718	case ISD::TargetExternalSymbol:
719	case ISD::ExternalSymbol:
720	case ISD::MCSymbol:
721	llvm_unreachable("Should only be used on nodes with operands");
722	default: break; // Normal nodes don't need extra info.
723	case ISD::TargetConstant:
724	case ISD::Constant: {
725	const ConstantSDNode *C = cast<ConstantSDNode>(Val: N);
726	ID.AddPointer(Ptr: C->getConstantIntValue());
727	ID.AddBoolean(B: C->isOpaque());
728	break;
729	}
730	case ISD::TargetConstantFP:
731	case ISD::ConstantFP:
732	ID.AddPointer(Ptr: cast<ConstantFPSDNode>(Val: N)->getConstantFPValue());
733	break;
734	case ISD::TargetGlobalAddress:
735	case ISD::GlobalAddress:
736	case ISD::TargetGlobalTLSAddress:
737	case ISD::GlobalTLSAddress: {
738	const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val: N);
739	ID.AddPointer(Ptr: GA->getGlobal());
740	ID.AddInteger(I: GA->getOffset());
741	ID.AddInteger(I: GA->getTargetFlags());
742	break;
743	}
744	case ISD::BasicBlock:
745	ID.AddPointer(Ptr: cast<BasicBlockSDNode>(Val: N)->getBasicBlock());
746	break;
747	case ISD::Register:
748	ID.AddInteger(I: cast<RegisterSDNode>(Val: N)->getReg());
749	break;
750	case ISD::RegisterMask:
751	ID.AddPointer(Ptr: cast<RegisterMaskSDNode>(Val: N)->getRegMask());
752	break;
753	case ISD::SRCVALUE:
754	ID.AddPointer(Ptr: cast<SrcValueSDNode>(Val: N)->getValue());
755	break;
756	case ISD::FrameIndex:
757	case ISD::TargetFrameIndex:
758	ID.AddInteger(I: cast<FrameIndexSDNode>(Val: N)->getIndex());
759	break;
760	case ISD::LIFETIME_START:
761	case ISD::LIFETIME_END:
762	if (cast<LifetimeSDNode>(Val: N)->hasOffset()) {
763	ID.AddInteger(I: cast<LifetimeSDNode>(Val: N)->getSize());
764	ID.AddInteger(I: cast<LifetimeSDNode>(Val: N)->getOffset());
765	}
766	break;
767	case ISD::PSEUDO_PROBE:
768	ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getGuid());
769	ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getIndex());
770	ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getAttributes());
771	break;
772	case ISD::JumpTable:
773	case ISD::TargetJumpTable:
774	ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getIndex());
775	ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getTargetFlags());
776	break;
777	case ISD::ConstantPool:
778	case ISD::TargetConstantPool: {
779	const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Val: N);
780	ID.AddInteger(I: CP->getAlign().value());
781	ID.AddInteger(I: CP->getOffset());
782	if (CP->isMachineConstantPoolEntry())
783	CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
784	else
785	ID.AddPointer(Ptr: CP->getConstVal());
786	ID.AddInteger(I: CP->getTargetFlags());
787	break;
788	}
789	case ISD::TargetIndex: {
790	const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(Val: N);
791	ID.AddInteger(I: TI->getIndex());
792	ID.AddInteger(I: TI->getOffset());
793	ID.AddInteger(I: TI->getTargetFlags());
794	break;
795	}
796	case ISD::LOAD: {
797	const LoadSDNode *LD = cast<LoadSDNode>(Val: N);
798	ID.AddInteger(I: LD->getMemoryVT().getRawBits());
799	ID.AddInteger(I: LD->getRawSubclassData());
800	ID.AddInteger(I: LD->getPointerInfo().getAddrSpace());
801	ID.AddInteger(I: LD->getMemOperand()->getFlags());
802	break;
803	}
804	case ISD::STORE: {
805	const StoreSDNode *ST = cast<StoreSDNode>(Val: N);
806	ID.AddInteger(I: ST->getMemoryVT().getRawBits());
807	ID.AddInteger(I: ST->getRawSubclassData());
808	ID.AddInteger(I: ST->getPointerInfo().getAddrSpace());
809	ID.AddInteger(I: ST->getMemOperand()->getFlags());
810	break;
811	}
812	case ISD::VP_LOAD: {
813	const VPLoadSDNode *ELD = cast<VPLoadSDNode>(Val: N);
814	ID.AddInteger(I: ELD->getMemoryVT().getRawBits());
815	ID.AddInteger(I: ELD->getRawSubclassData());
816	ID.AddInteger(I: ELD->getPointerInfo().getAddrSpace());
817	ID.AddInteger(I: ELD->getMemOperand()->getFlags());
818	break;
819	}
820	case ISD::VP_STORE: {
821	const VPStoreSDNode *EST = cast<VPStoreSDNode>(Val: N);
822	ID.AddInteger(I: EST->getMemoryVT().getRawBits());
823	ID.AddInteger(I: EST->getRawSubclassData());
824	ID.AddInteger(I: EST->getPointerInfo().getAddrSpace());
825	ID.AddInteger(I: EST->getMemOperand()->getFlags());
826	break;
827	}
828	case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: {
829	const VPStridedLoadSDNode *SLD = cast<VPStridedLoadSDNode>(Val: N);
830	ID.AddInteger(I: SLD->getMemoryVT().getRawBits());
831	ID.AddInteger(I: SLD->getRawSubclassData());
832	ID.AddInteger(I: SLD->getPointerInfo().getAddrSpace());
833	break;
834	}
835	case ISD::EXPERIMENTAL_VP_STRIDED_STORE: {
836	const VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N);
837	ID.AddInteger(I: SST->getMemoryVT().getRawBits());
838	ID.AddInteger(I: SST->getRawSubclassData());
839	ID.AddInteger(I: SST->getPointerInfo().getAddrSpace());
840	break;
841	}
842	case ISD::VP_GATHER: {
843	const VPGatherSDNode *EG = cast<VPGatherSDNode>(Val: N);
844	ID.AddInteger(I: EG->getMemoryVT().getRawBits());
845	ID.AddInteger(I: EG->getRawSubclassData());
846	ID.AddInteger(I: EG->getPointerInfo().getAddrSpace());
847	ID.AddInteger(I: EG->getMemOperand()->getFlags());
848	break;
849	}
850	case ISD::VP_SCATTER: {
851	const VPScatterSDNode *ES = cast<VPScatterSDNode>(Val: N);
852	ID.AddInteger(I: ES->getMemoryVT().getRawBits());
853	ID.AddInteger(I: ES->getRawSubclassData());
854	ID.AddInteger(I: ES->getPointerInfo().getAddrSpace());
855	ID.AddInteger(I: ES->getMemOperand()->getFlags());
856	break;
857	}
858	case ISD::MLOAD: {
859	const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(Val: N);
860	ID.AddInteger(I: MLD->getMemoryVT().getRawBits());
861	ID.AddInteger(I: MLD->getRawSubclassData());
862	ID.AddInteger(I: MLD->getPointerInfo().getAddrSpace());
863	ID.AddInteger(I: MLD->getMemOperand()->getFlags());
864	break;
865	}
866	case ISD::MSTORE: {
867	const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N);
868	ID.AddInteger(I: MST->getMemoryVT().getRawBits());
869	ID.AddInteger(I: MST->getRawSubclassData());
870	ID.AddInteger(I: MST->getPointerInfo().getAddrSpace());
871	ID.AddInteger(I: MST->getMemOperand()->getFlags());
872	break;
873	}
874	case ISD::MGATHER: {
875	const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(Val: N);
876	ID.AddInteger(I: MG->getMemoryVT().getRawBits());
877	ID.AddInteger(I: MG->getRawSubclassData());
878	ID.AddInteger(I: MG->getPointerInfo().getAddrSpace());
879	ID.AddInteger(I: MG->getMemOperand()->getFlags());
880	break;
881	}
882	case ISD::MSCATTER: {
883	const MaskedScatterSDNode *MS = cast<MaskedScatterSDNode>(Val: N);
884	ID.AddInteger(I: MS->getMemoryVT().getRawBits());
885	ID.AddInteger(I: MS->getRawSubclassData());
886	ID.AddInteger(I: MS->getPointerInfo().getAddrSpace());
887	ID.AddInteger(I: MS->getMemOperand()->getFlags());
888	break;
889	}
890	case ISD::ATOMIC_CMP_SWAP:
891	case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
892	case ISD::ATOMIC_SWAP:
893	case ISD::ATOMIC_LOAD_ADD:
894	case ISD::ATOMIC_LOAD_SUB:
895	case ISD::ATOMIC_LOAD_AND:
896	case ISD::ATOMIC_LOAD_CLR:
897	case ISD::ATOMIC_LOAD_OR:
898	case ISD::ATOMIC_LOAD_XOR:
899	case ISD::ATOMIC_LOAD_NAND:
900	case ISD::ATOMIC_LOAD_MIN:
901	case ISD::ATOMIC_LOAD_MAX:
902	case ISD::ATOMIC_LOAD_UMIN:
903	case ISD::ATOMIC_LOAD_UMAX:
904	case ISD::ATOMIC_LOAD:
905	case ISD::ATOMIC_STORE: {
906	const AtomicSDNode *AT = cast<AtomicSDNode>(Val: N);
907	ID.AddInteger(I: AT->getMemoryVT().getRawBits());
908	ID.AddInteger(I: AT->getRawSubclassData());
909	ID.AddInteger(I: AT->getPointerInfo().getAddrSpace());
910	ID.AddInteger(I: AT->getMemOperand()->getFlags());
911	break;
912	}
913	case ISD::VECTOR_SHUFFLE: {
914	ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: N)->getMask();
915	for (int M : Mask)
916	ID.AddInteger(I: M);
917	break;
918	}
919	case ISD::TargetBlockAddress:
920	case ISD::BlockAddress: {
921	const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(Val: N);
922	ID.AddPointer(Ptr: BA->getBlockAddress());
923	ID.AddInteger(I: BA->getOffset());
924	ID.AddInteger(I: BA->getTargetFlags());
925	break;
926	}
927	case ISD::AssertAlign:
928	ID.AddInteger(I: cast<AssertAlignSDNode>(Val: N)->getAlign().value());
929	break;
930	case ISD::PREFETCH:
931	case ISD::INTRINSIC_VOID:
932	case ISD::INTRINSIC_W_CHAIN:
933	// Handled by MemIntrinsicSDNode check after the switch.
934	break;
935	} // end switch (N->getOpcode())
936
937	// MemIntrinsic nodes could also have subclass data, address spaces, and flags
938	// to check.
939	if (auto *MN = dyn_cast<MemIntrinsicSDNode>(Val: N)) {
940	ID.AddInteger(I: MN->getRawSubclassData());
941	ID.AddInteger(I: MN->getPointerInfo().getAddrSpace());
942	ID.AddInteger(I: MN->getMemOperand()->getFlags());
943	ID.AddInteger(I: MN->getMemoryVT().getRawBits());
944	}
945	}
946
947	/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
948	/// data.
949	static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
950	AddNodeIDOpcode(ID, OpC: N->getOpcode());
951	// Add the return value info.
952	AddNodeIDValueTypes(ID, VTList: N->getVTList());
953	// Add the operand info.
954	AddNodeIDOperands(ID, Ops: N->ops());
955
956	// Handle SDNode leafs with special info.
957	AddNodeIDCustom(ID, N);
958	}
959
960	//===----------------------------------------------------------------------===//
961	// SelectionDAG Class
962	//===----------------------------------------------------------------------===//
963
964	/// doNotCSE - Return true if CSE should not be performed for this node.
965	static bool doNotCSE(SDNode *N) {
966	if (N->getValueType(ResNo: 0) == MVT::Glue)
967	return true; // Never CSE anything that produces a glue result.
968
969	switch (N->getOpcode()) {
970	default: break;
971	case ISD::HANDLENODE:
972	case ISD::EH_LABEL:
973	return true; // Never CSE these nodes.
974	}
975
976	// Check that remaining values produced are not flags.
977	for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
978	if (N->getValueType(ResNo: i) == MVT::Glue)
979	return true; // Never CSE anything that produces a glue result.
980
981	return false;
982	}
983
984	/// RemoveDeadNodes - This method deletes all unreachable nodes in the
985	/// SelectionDAG.
986	void SelectionDAG::RemoveDeadNodes() {
987	// Create a dummy node (which is not added to allnodes), that adds a reference
988	// to the root node, preventing it from being deleted.
989	HandleSDNode Dummy(getRoot());
990
991	SmallVector<SDNode*, 128> DeadNodes;
992
993	// Add all obviously-dead nodes to the DeadNodes worklist.
994	for (SDNode &Node : allnodes())
995	if (Node.use_empty())
996	DeadNodes.push_back(Elt: &Node);
997
998	RemoveDeadNodes(DeadNodes);
999
1000	// If the root changed (e.g. it was a dead load, update the root).
1001	setRoot(Dummy.getValue());
1002	}
1003
1004	/// RemoveDeadNodes - This method deletes the unreachable nodes in the
1005	/// given list, and any nodes that become unreachable as a result.
1006	void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
1007
1008	// Process the worklist, deleting the nodes and adding their uses to the
1009	// worklist.
1010	while (!DeadNodes.empty()) {
1011	SDNode *N = DeadNodes.pop_back_val();
1012	// Skip to next node if we've already managed to delete the node. This could
1013	// happen if replacing a node causes a node previously added to the node to
1014	// be deleted.
1015	if (N->getOpcode() == ISD::DELETED_NODE)
1016	continue;
1017
1018	for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1019	DUL->NodeDeleted(N, nullptr);
1020
1021	// Take the node out of the appropriate CSE map.
1022	RemoveNodeFromCSEMaps(N);
1023
1024	// Next, brutally remove the operand list. This is safe to do, as there are
1025	// no cycles in the graph.
1026	for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
1027	SDUse &Use = *I++;
1028	SDNode *Operand = Use.getNode();
1029	Use.set(SDValue());
1030
1031	// Now that we removed this operand, see if there are no uses of it left.
1032	if (Operand->use_empty())
1033	DeadNodes.push_back(Elt: Operand);
1034	}
1035
1036	DeallocateNode(N);
1037	}
1038	}
1039
1040	void SelectionDAG::RemoveDeadNode(SDNode *N){
1041	SmallVector<SDNode*, 16> DeadNodes(1, N);
1042
1043	// Create a dummy node that adds a reference to the root node, preventing
1044	// it from being deleted. (This matters if the root is an operand of the
1045	// dead node.)
1046	HandleSDNode Dummy(getRoot());
1047
1048	RemoveDeadNodes(DeadNodes);
1049	}
1050
1051	void SelectionDAG::DeleteNode(SDNode *N) {
1052	// First take this out of the appropriate CSE map.
1053	RemoveNodeFromCSEMaps(N);
1054
1055	// Finally, remove uses due to operands of this node, remove from the
1056	// AllNodes list, and delete the node.
1057	DeleteNodeNotInCSEMaps(N);
1058	}
1059
1060	void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
1061	assert(N->getIterator() != AllNodes.begin() &&
1062	"Cannot delete the entry node!");
1063	assert(N->use_empty() && "Cannot delete a node that is not dead!");
1064
1065	// Drop all of the operands and decrement used node's use counts.
1066	N->DropOperands();
1067
1068	DeallocateNode(N);
1069	}
1070
1071	void SDDbgInfo::add(SDDbgValue *V, bool isParameter) {
1072	assert(!(V->isVariadic() && isParameter));
1073	if (isParameter)
1074	ByvalParmDbgValues.push_back(Elt: V);
1075	else
1076	DbgValues.push_back(Elt: V);
1077	for (const SDNode *Node : V->getSDNodes())
1078	if (Node)
1079	DbgValMap[Node].push_back(Elt: V);
1080	}
1081
1082	void SDDbgInfo::erase(const SDNode *Node) {
1083	DbgValMapType::iterator I = DbgValMap.find(Val: Node);
1084	if (I == DbgValMap.end())
1085	return;
1086	for (auto &Val: I->second)
1087	Val->setIsInvalidated();
1088	DbgValMap.erase(I);
1089	}
1090
1091	void SelectionDAG::DeallocateNode(SDNode *N) {
1092	// If we have operands, deallocate them.
1093	removeOperands(Node: N);
1094
1095	NodeAllocator.Deallocate(E: AllNodes.remove(IT: N));
1096
1097	// Set the opcode to DELETED_NODE to help catch bugs when node
1098	// memory is reallocated.
1099	// FIXME: There are places in SDag that have grown a dependency on the opcode
1100	// value in the released node.
1101	__asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
1102	N->NodeType = ISD::DELETED_NODE;
1103
1104	// If any of the SDDbgValue nodes refer to this SDNode, invalidate
1105	// them and forget about that node.
1106	DbgInfo->erase(Node: N);
1107
1108	// Invalidate extra info.
1109	SDEI.erase(Val: N);
1110	}
1111
1112	#ifndef NDEBUG
1113	/// VerifySDNode - Check the given SDNode. Aborts if it is invalid.
1114	static void VerifySDNode(SDNode *N, const TargetLowering *TLI) {
1115	switch (N->getOpcode()) {
1116	default:
1117	if (N->getOpcode() > ISD::BUILTIN_OP_END)
1118	TLI->verifyTargetSDNode(N);
1119	break;
1120	case ISD::BUILD_PAIR: {
1121	EVT VT = N->getValueType(ResNo: 0);
1122	assert(N->getNumValues() == 1 && "Too many results!");
1123	assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
1124	"Wrong return type!");
1125	assert(N->getNumOperands() == 2 && "Wrong number of operands!");
1126	assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1127	"Mismatched operand types!");
1128	assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
1129	"Wrong operand type!");
1130	assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
1131	"Wrong return type size");
1132	break;
1133	}
1134	case ISD::BUILD_VECTOR: {
1135	assert(N->getNumValues() == 1 && "Too many results!");
1136	assert(N->getValueType(0).isVector() && "Wrong return type!");
1137	assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
1138	"Wrong number of operands!");
1139	EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
1140	for (const SDUse &Op : N->ops()) {
1141	assert((Op.getValueType() == EltVT ||
1142	(EltVT.isInteger() && Op.getValueType().isInteger() &&
1143	EltVT.bitsLE(Op.getValueType()))) &&
1144	"Wrong operand type!");
1145	assert(Op.getValueType() == N->getOperand(0).getValueType() &&
1146	"Operands must all have the same type");
1147	}
1148	break;
1149	}
1150	}
1151	}
1152	#endif // NDEBUG
1153
1154	/// Insert a newly allocated node into the DAG.
1155	///
1156	/// Handles insertion into the all nodes list and CSE map, as well as
1157	/// verification and other common operations when a new node is allocated.
1158	void SelectionDAG::InsertNode(SDNode *N) {
1159	AllNodes.push_back(val: N);
1160	#ifndef NDEBUG
1161	N->PersistentId = NextPersistentId++;
1162	VerifySDNode(N, TLI);
1163	#endif
1164	for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1165	DUL->NodeInserted(N);
1166	}
1167
1168	/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
1169	/// correspond to it. This is useful when we're about to delete or repurpose
1170	/// the node. We don't want future request for structurally identical nodes
1171	/// to return N anymore.
1172	bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
1173	bool Erased = false;
1174	switch (N->getOpcode()) {
1175	case ISD::HANDLENODE: return false; // noop.
1176	case ISD::CONDCODE:
1177	assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
1178	"Cond code doesn't exist!");
1179	Erased = CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] != nullptr;
1180	CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] = nullptr;
1181	break;
1182	case ISD::ExternalSymbol:
1183	Erased = ExternalSymbols.erase(Key: cast<ExternalSymbolSDNode>(Val: N)->getSymbol());
1184	break;
1185	case ISD::TargetExternalSymbol: {
1186	ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(Val: N);
1187	Erased = TargetExternalSymbols.erase(x: std::pair<std::string, unsigned>(
1188	ESN->getSymbol(), ESN->getTargetFlags()));
1189	break;
1190	}
1191	case ISD::MCSymbol: {
1192	auto *MCSN = cast<MCSymbolSDNode>(Val: N);
1193	Erased = MCSymbols.erase(Val: MCSN->getMCSymbol());
1194	break;
1195	}
1196	case ISD::VALUETYPE: {
1197	EVT VT = cast<VTSDNode>(Val: N)->getVT();
1198	if (VT.isExtended()) {
1199	Erased = ExtendedValueTypeNodes.erase(x: VT);
1200	} else {
1201	Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
1202	ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
1203	}
1204	break;
1205	}
1206	default:
1207	// Remove it from the CSE Map.
1208	assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
1209	assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
1210	Erased = CSEMap.RemoveNode(N);
1211	break;
1212	}
1213	#ifndef NDEBUG
1214	// Verify that the node was actually in one of the CSE maps, unless it has a
1215	// glue result (which cannot be CSE'd) or is one of the special cases that are
1216	// not subject to CSE.
1217	if (!Erased && N->getValueType(ResNo: N->getNumValues()-1) != MVT::Glue &&
1218	!N->isMachineOpcode() && !doNotCSE(N)) {
1219	N->dump(G: this);
1220	dbgs() << "\n";
1221	llvm_unreachable("Node is not in map!");
1222	}
1223	#endif
1224	return Erased;
1225	}
1226
1227	/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
1228	/// maps and modified in place. Add it back to the CSE maps, unless an identical
1229	/// node already exists, in which case transfer all its users to the existing
1230	/// node. This transfer can potentially trigger recursive merging.
1231	void
1232	SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
1233	// For node types that aren't CSE'd, just act as if no identical node
1234	// already exists.
1235	if (!doNotCSE(N)) {
1236	SDNode *Existing = CSEMap.GetOrInsertNode(N);
1237	if (Existing != N) {
1238	// If there was already an existing matching node, use ReplaceAllUsesWith
1239	// to replace the dead one with the existing one. This can cause
1240	// recursive merging of other unrelated nodes down the line.
1241	ReplaceAllUsesWith(From: N, To: Existing);
1242
1243	// N is now dead. Inform the listeners and delete it.
1244	for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1245	DUL->NodeDeleted(N, Existing);
1246	DeleteNodeNotInCSEMaps(N);
1247	return;
1248	}
1249	}
1250
1251	// If the node doesn't already exist, we updated it. Inform listeners.
1252	for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1253	DUL->NodeUpdated(N);
1254	}
1255
1256	/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1257	/// were replaced with those specified. If this node is never memoized,
1258	/// return null, otherwise return a pointer to the slot it would take. If a
1259	/// node already exists with these operands, the slot will be non-null.
1260	SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
1261	void *&InsertPos) {
1262	if (doNotCSE(N))
1263	return nullptr;
1264
1265	SDValue Ops[] = { Op };
1266	FoldingSetNodeID ID;
1267	AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops);
1268	AddNodeIDCustom(ID, N);
1269	SDNode *Node = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos);
1270	if (Node)
1271	Node->intersectFlagsWith(Flags: N->getFlags());
1272	return Node;
1273	}
1274
1275	/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1276	/// were replaced with those specified. If this node is never memoized,
1277	/// return null, otherwise return a pointer to the slot it would take. If a
1278	/// node already exists with these operands, the slot will be non-null.
1279	SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
1280	SDValue Op1, SDValue Op2,
1281	void *&InsertPos) {
1282	if (doNotCSE(N))
1283	return nullptr;
1284
1285	SDValue Ops[] = { Op1, Op2 };
1286	FoldingSetNodeID ID;
1287	AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops);
1288	AddNodeIDCustom(ID, N);
1289	SDNode *Node = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos);
1290	if (Node)
1291	Node->intersectFlagsWith(Flags: N->getFlags());
1292	return Node;
1293	}
1294
1295	/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1296	/// were replaced with those specified. If this node is never memoized,
1297	/// return null, otherwise return a pointer to the slot it would take. If a
1298	/// node already exists with these operands, the slot will be non-null.
1299	SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
1300	void *&InsertPos) {
1301	if (doNotCSE(N))
1302	return nullptr;
1303
1304	FoldingSetNodeID ID;
1305	AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops);
1306	AddNodeIDCustom(ID, N);
1307	SDNode *Node = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos);
1308	if (Node)
1309	Node->intersectFlagsWith(Flags: N->getFlags());
1310	return Node;
1311	}
1312
1313	Align SelectionDAG::getEVTAlign(EVT VT) const {
1314	Type *Ty = VT == MVT::iPTR ? PointerType::get(C&: *getContext(), AddressSpace: 0)
1315	: VT.getTypeForEVT(Context&: *getContext());
1316
1317	return getDataLayout().getABITypeAlign(Ty);
1318	}
1319
1320	// EntryNode could meaningfully have debug info if we can find it...
1321	SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOptLevel OL)
1322	: TM(tm), OptLevel(OL), EntryNode(ISD::EntryToken, 0, DebugLoc(),
1323	getVTList(MVT::Other, MVT::Glue)),
1324	Root(getEntryNode()) {
1325	InsertNode(N: &EntryNode);
1326	DbgInfo = new SDDbgInfo();
1327	}
1328
1329	void SelectionDAG::init(MachineFunction &NewMF,
1330	OptimizationRemarkEmitter &NewORE, Pass *PassPtr,
1331	const TargetLibraryInfo *LibraryInfo,
1332	UniformityInfo *NewUA, ProfileSummaryInfo *PSIin,
1333	BlockFrequencyInfo *BFIin,
1334	FunctionVarLocs const *VarLocs) {
1335	MF = &NewMF;
1336	SDAGISelPass = PassPtr;
1337	ORE = &NewORE;
1338	TLI = getSubtarget().getTargetLowering();
1339	TSI = getSubtarget().getSelectionDAGInfo();
1340	LibInfo = LibraryInfo;
1341	Context = &MF->getFunction().getContext();
1342	UA = NewUA;
1343	PSI = PSIin;
1344	BFI = BFIin;
1345	FnVarLocs = VarLocs;
1346	}
1347
1348	SelectionDAG::~SelectionDAG() {
1349	assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
1350	allnodes_clear();
1351	OperandRecycler.clear(OperandAllocator);
1352	delete DbgInfo;
1353	}
1354
1355	bool SelectionDAG::shouldOptForSize() const {
1356	return MF->getFunction().hasOptSize() ||
1357	llvm::shouldOptimizeForSize(BB: FLI->MBB->getBasicBlock(), PSI, BFI);
1358	}
1359
1360	void SelectionDAG::allnodes_clear() {
1361	assert(&*AllNodes.begin() == &EntryNode);
1362	AllNodes.remove(IT: AllNodes.begin());
1363	while (!AllNodes.empty())
1364	DeallocateNode(N: &AllNodes.front());
1365	#ifndef NDEBUG
1366	NextPersistentId = 0;
1367	#endif
1368	}
1369
1370	SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1371	void *&InsertPos) {
1372	SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1373	if (N) {
1374	switch (N->getOpcode()) {
1375	default: break;
1376	case ISD::Constant:
1377	case ISD::ConstantFP:
1378	llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
1379	"debug location. Use another overload.");
1380	}
1381	}
1382	return N;
1383	}
1384
1385	SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1386	const SDLoc &DL, void *&InsertPos) {
1387	SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1388	if (N) {
1389	switch (N->getOpcode()) {
1390	case ISD::Constant:
1391	case ISD::ConstantFP:
1392	// Erase debug location from the node if the node is used at several
1393	// different places. Do not propagate one location to all uses as it
1394	// will cause a worse single stepping debugging experience.
1395	if (N->getDebugLoc() != DL.getDebugLoc())
1396	N->setDebugLoc(DebugLoc());
1397	break;
1398	default:
1399	// When the node's point of use is located earlier in the instruction
1400	// sequence than its prior point of use, update its debug info to the
1401	// earlier location.
1402	if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
1403	N->setDebugLoc(DL.getDebugLoc());
1404	break;
1405	}
1406	}
1407	return N;
1408	}
1409
1410	void SelectionDAG::clear() {
1411	allnodes_clear();
1412	OperandRecycler.clear(OperandAllocator);
1413	OperandAllocator.Reset();
1414	CSEMap.clear();
1415
1416	ExtendedValueTypeNodes.clear();
1417	ExternalSymbols.clear();
1418	TargetExternalSymbols.clear();
1419	MCSymbols.clear();
1420	SDEI.clear();
1421	std::fill(first: CondCodeNodes.begin(), last: CondCodeNodes.end(), value: nullptr);
1422	std::fill(first: ValueTypeNodes.begin(), last: ValueTypeNodes.end(), value: nullptr);
1423
1424	EntryNode.UseList = nullptr;
1425	InsertNode(N: &EntryNode);
1426	Root = getEntryNode();
1427	DbgInfo->clear();
1428	}
1429
1430	SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
1431	return VT.bitsGT(VT: Op.getValueType())
1432	? getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Op)
1433	: getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Op,
1434	N2: getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
1435	}
1436
1437	std::pair<SDValue, SDValue>
1438	SelectionDAG::getStrictFPExtendOrRound(SDValue Op, SDValue Chain,
1439	const SDLoc &DL, EVT VT) {
1440	assert(!VT.bitsEq(Op.getValueType()) &&
1441	"Strict no-op FP extend/round not allowed.");
1442	SDValue Res =
1443	VT.bitsGT(Op.getValueType())
1444	? getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op})
1445	: getNode(ISD::STRICT_FP_ROUND, DL, {VT, MVT::Other},
1446	{Chain, Op, getIntPtrConstant(0, DL)});
1447
1448	return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1));
1449	}
1450
1451	SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1452	return VT.bitsGT(VT: Op.getValueType()) ?
1453	getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Op) :
1454	getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1455	}
1456
1457	SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1458	return VT.bitsGT(VT: Op.getValueType()) ?
1459	getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: Op) :
1460	getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1461	}
1462
1463	SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1464	return VT.bitsGT(VT: Op.getValueType()) ?
1465	getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Op) :
1466	getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1467	}
1468
1469	SDValue SelectionDAG::getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL,
1470	EVT VT) {
1471	assert(!VT.isVector());
1472	auto Type = Op.getValueType();
1473	SDValue DestOp;
1474	if (Type == VT)
1475	return Op;
1476	auto Size = Op.getValueSizeInBits();
1477	DestOp = getBitcast(VT: MVT::getIntegerVT(BitWidth: Size), V: Op);
1478	if (DestOp.getValueType() == VT)
1479	return DestOp;
1480
1481	return getAnyExtOrTrunc(Op: DestOp, DL, VT);
1482	}
1483
1484	SDValue SelectionDAG::getBitcastedSExtOrTrunc(SDValue Op, const SDLoc &DL,
1485	EVT VT) {
1486	assert(!VT.isVector());
1487	auto Type = Op.getValueType();
1488	SDValue DestOp;
1489	if (Type == VT)
1490	return Op;
1491	auto Size = Op.getValueSizeInBits();
1492	DestOp = getBitcast(VT: MVT::getIntegerVT(BitWidth: Size), V: Op);
1493	if (DestOp.getValueType() == VT)
1494	return DestOp;
1495
1496	return getSExtOrTrunc(Op: DestOp, DL, VT);
1497	}
1498
1499	SDValue SelectionDAG::getBitcastedZExtOrTrunc(SDValue Op, const SDLoc &DL,
1500	EVT VT) {
1501	assert(!VT.isVector());
1502	auto Type = Op.getValueType();
1503	SDValue DestOp;
1504	if (Type == VT)
1505	return Op;
1506	auto Size = Op.getValueSizeInBits();
1507	DestOp = getBitcast(VT: MVT::getIntegerVT(BitWidth: Size), V: Op);
1508	if (DestOp.getValueType() == VT)
1509	return DestOp;
1510
1511	return getZExtOrTrunc(Op: DestOp, DL, VT);
1512	}
1513
1514	SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
1515	EVT OpVT) {
1516	if (VT.bitsLE(VT: Op.getValueType()))
1517	return getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Op);
1518
1519	TargetLowering::BooleanContent BType = TLI->getBooleanContents(Type: OpVT);
1520	return getNode(Opcode: TLI->getExtendForContent(Content: BType), DL: SL, VT, Operand: Op);
1521	}
1522
1523	SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1524	EVT OpVT = Op.getValueType();
1525	assert(VT.isInteger() && OpVT.isInteger() &&
1526	"Cannot getZeroExtendInReg FP types");
1527	assert(VT.isVector() == OpVT.isVector() &&
1528	"getZeroExtendInReg type should be vector iff the operand "
1529	"type is vector!");
1530	assert((!VT.isVector() ||
1531	VT.getVectorElementCount() == OpVT.getVectorElementCount()) &&
1532	"Vector element counts must match in getZeroExtendInReg");
1533	assert(VT.bitsLE(OpVT) && "Not extending!");
1534	if (OpVT == VT)
1535	return Op;
1536	APInt Imm = APInt::getLowBitsSet(numBits: OpVT.getScalarSizeInBits(),
1537	loBitsSet: VT.getScalarSizeInBits());
1538	return getNode(Opcode: ISD::AND, DL, VT: OpVT, N1: Op, N2: getConstant(Val: Imm, DL, VT: OpVT));
1539	}
1540
1541	SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1542	// Only unsigned pointer semantics are supported right now. In the future this
1543	// might delegate to TLI to check pointer signedness.
1544	return getZExtOrTrunc(Op, DL, VT);
1545	}
1546
1547	SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1548	// Only unsigned pointer semantics are supported right now. In the future this
1549	// might delegate to TLI to check pointer signedness.
1550	return getZeroExtendInReg(Op, DL, VT);
1551	}
1552
1553	SDValue SelectionDAG::getNegative(SDValue Val, const SDLoc &DL, EVT VT) {
1554	return getNode(Opcode: ISD::SUB, DL, VT, N1: getConstant(Val: 0, DL, VT), N2: Val);
1555	}
1556
1557	/// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1558	SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1559	return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: getAllOnesConstant(DL, VT));
1560	}
1561
1562	SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1563	SDValue TrueValue = getBoolConstant(V: true, DL, VT, OpVT: VT);
1564	return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: TrueValue);
1565	}
1566
1567	SDValue SelectionDAG::getVPLogicalNOT(const SDLoc &DL, SDValue Val,
1568	SDValue Mask, SDValue EVL, EVT VT) {
1569	SDValue TrueValue = getBoolConstant(V: true, DL, VT, OpVT: VT);
1570	return getNode(Opcode: ISD::VP_XOR, DL, VT, N1: Val, N2: TrueValue, N3: Mask, N4: EVL);
1571	}
1572
1573	SDValue SelectionDAG::getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op,
1574	SDValue Mask, SDValue EVL) {
1575	return getVPZExtOrTrunc(DL, VT, Op, Mask, EVL);
1576	}
1577
1578	SDValue SelectionDAG::getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op,
1579	SDValue Mask, SDValue EVL) {
1580	if (VT.bitsGT(VT: Op.getValueType()))
1581	return getNode(Opcode: ISD::VP_ZERO_EXTEND, DL, VT, N1: Op, N2: Mask, N3: EVL);
1582	if (VT.bitsLT(VT: Op.getValueType()))
1583	return getNode(Opcode: ISD::VP_TRUNCATE, DL, VT, N1: Op, N2: Mask, N3: EVL);
1584	return Op;
1585	}
1586
1587	SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT,
1588	EVT OpVT) {
1589	if (!V)
1590	return getConstant(Val: 0, DL, VT);
1591
1592	switch (TLI->getBooleanContents(Type: OpVT)) {
1593	case TargetLowering::ZeroOrOneBooleanContent:
1594	case TargetLowering::UndefinedBooleanContent:
1595	return getConstant(Val: 1, DL, VT);
1596	case TargetLowering::ZeroOrNegativeOneBooleanContent:
1597	return getAllOnesConstant(DL, VT);
1598	}
1599	llvm_unreachable("Unexpected boolean content enum!");
1600	}
1601
1602	SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
1603	bool isT, bool isO) {
1604	EVT EltVT = VT.getScalarType();
1605	assert((EltVT.getSizeInBits() >= 64 ||
1606	(uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
1607	"getConstant with a uint64_t value that doesn't fit in the type!");
1608	return getConstant(Val: APInt(EltVT.getSizeInBits(), Val), DL, VT, isTarget: isT, isOpaque: isO);
1609	}
1610
1611	SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
1612	bool isT, bool isO) {
1613	return getConstant(Val: *ConstantInt::get(Context&: *Context, V: Val), DL, VT, isTarget: isT, isOpaque: isO);
1614	}
1615
1616	SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
1617	EVT VT, bool isT, bool isO) {
1618	assert(VT.isInteger() && "Cannot create FP integer constant!");
1619
1620	EVT EltVT = VT.getScalarType();
1621	const ConstantInt *Elt = &Val;
1622
1623	// In some cases the vector type is legal but the element type is illegal and
1624	// needs to be promoted, for example v8i8 on ARM. In this case, promote the
1625	// inserted value (the type does not need to match the vector element type).
1626	// Any extra bits introduced will be truncated away.
1627	if (VT.isVector() && TLI->getTypeAction(Context&: *getContext(), VT: EltVT) ==
1628	TargetLowering::TypePromoteInteger) {
1629	EltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT);
1630	APInt NewVal;
1631	if (TLI->isSExtCheaperThanZExt(FromTy: VT.getScalarType(), ToTy: EltVT))
1632	NewVal = Elt->getValue().sextOrTrunc(width: EltVT.getSizeInBits());
1633	else
1634	NewVal = Elt->getValue().zextOrTrunc(width: EltVT.getSizeInBits());
1635	Elt = ConstantInt::get(Context&: *getContext(), V: NewVal);
1636	}
1637	// In other cases the element type is illegal and needs to be expanded, for
1638	// example v2i64 on MIPS32. In this case, find the nearest legal type, split
1639	// the value into n parts and use a vector type with n-times the elements.
1640	// Then bitcast to the type requested.
1641	// Legalizing constants too early makes the DAGCombiner's job harder so we
1642	// only legalize if the DAG tells us we must produce legal types.
1643	else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1644	TLI->getTypeAction(Context&: *getContext(), VT: EltVT) ==
1645	TargetLowering::TypeExpandInteger) {
1646	const APInt &NewVal = Elt->getValue();
1647	EVT ViaEltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT);
1648	unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1649
1650	// For scalable vectors, try to use a SPLAT_VECTOR_PARTS node.
1651	if (VT.isScalableVector() ||
1652	TLI->isOperationLegal(Op: ISD::SPLAT_VECTOR, VT)) {
1653	assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 &&
1654	"Can only handle an even split!");
1655	unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits;
1656
1657	SmallVector<SDValue, 2> ScalarParts;
1658	for (unsigned i = 0; i != Parts; ++i)
1659	ScalarParts.push_back(Elt: getConstant(
1660	Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL,
1661	VT: ViaEltVT, isT, isO));
1662
1663	return getNode(Opcode: ISD::SPLAT_VECTOR_PARTS, DL, VT, Ops: ScalarParts);
1664	}
1665
1666	unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1667	EVT ViaVecVT = EVT::getVectorVT(Context&: *getContext(), VT: ViaEltVT, NumElements: ViaVecNumElts);
1668
1669	// Check the temporary vector is the correct size. If this fails then
1670	// getTypeToTransformTo() probably returned a type whose size (in bits)
1671	// isn't a power-of-2 factor of the requested type size.
1672	assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1673
1674	SmallVector<SDValue, 2> EltParts;
1675	for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i)
1676	EltParts.push_back(Elt: getConstant(
1677	Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL,
1678	VT: ViaEltVT, isT, isO));
1679
1680	// EltParts is currently in little endian order. If we actually want
1681	// big-endian order then reverse it now.
1682	if (getDataLayout().isBigEndian())
1683	std::reverse(first: EltParts.begin(), last: EltParts.end());
1684
1685	// The elements must be reversed when the element order is different
1686	// to the endianness of the elements (because the BITCAST is itself a
1687	// vector shuffle in this situation). However, we do not need any code to
1688	// perform this reversal because getConstant() is producing a vector
1689	// splat.
1690	// This situation occurs in MIPS MSA.
1691
1692	SmallVector<SDValue, 8> Ops;
1693	for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1694	llvm::append_range(C&: Ops, R&: EltParts);
1695
1696	SDValue V =
1697	getNode(Opcode: ISD::BITCAST, DL, VT, Operand: getBuildVector(VT: ViaVecVT, DL, Ops));
1698	return V;
1699	}
1700
1701	assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1702	"APInt size does not match type size!");
1703	unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1704	FoldingSetNodeID ID;
1705	AddNodeIDNode(ID, OpC: Opc, VTList: getVTList(VT: EltVT), OpList: std::nullopt);
1706	ID.AddPointer(Ptr: Elt);
1707	ID.AddBoolean(B: isO);
1708	void *IP = nullptr;
1709	SDNode *N = nullptr;
1710	if ((N = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)))
1711	if (!VT.isVector())
1712	return SDValue(N, 0);
1713
1714	if (!N) {
1715	N = newSDNode<ConstantSDNode>(Args&: isT, Args&: isO, Args&: Elt, Args&: EltVT);
1716	CSEMap.InsertNode(N, InsertPos: IP);
1717	InsertNode(N);
1718	NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating constant: ", G: this);
1719	}
1720
1721	SDValue Result(N, 0);
1722	if (VT.isVector())
1723	Result = getSplat(VT, DL, Op: Result);
1724	return Result;
1725	}
1726
1727	SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
1728	bool isTarget) {
1729	return getConstant(Val, DL, VT: TLI->getPointerTy(DL: getDataLayout()), isT: isTarget);
1730	}
1731
1732	SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT,
1733	const SDLoc &DL, bool LegalTypes) {
1734	assert(VT.isInteger() && "Shift amount is not an integer type!");
1735	EVT ShiftVT = TLI->getShiftAmountTy(LHSTy: VT, DL: getDataLayout(), LegalTypes);
1736	return getConstant(Val, DL, VT: ShiftVT);
1737	}
1738
1739	SDValue SelectionDAG::getShiftAmountConstant(const APInt &Val, EVT VT,
1740	const SDLoc &DL, bool LegalTypes) {
1741	assert(Val.ult(VT.getScalarSizeInBits()) && "Out of range shift");
1742	return getShiftAmountConstant(Val: Val.getZExtValue(), VT, DL, LegalTypes);
1743	}
1744
1745	SDValue SelectionDAG::getVectorIdxConstant(uint64_t Val, const SDLoc &DL,
1746	bool isTarget) {
1747	return getConstant(Val, DL, VT: TLI->getVectorIdxTy(DL: getDataLayout()), isT: isTarget);
1748	}
1749
1750	SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
1751	bool isTarget) {
1752	return getConstantFP(V: *ConstantFP::get(Context&: *getContext(), V), DL, VT, isTarget);
1753	}
1754
1755	SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
1756	EVT VT, bool isTarget) {
1757	assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1758
1759	EVT EltVT = VT.getScalarType();
1760
1761	// Do the map lookup using the actual bit pattern for the floating point
1762	// value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1763	// we don't have issues with SNANs.
1764	unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1765	FoldingSetNodeID ID;
1766	AddNodeIDNode(ID, OpC: Opc, VTList: getVTList(VT: EltVT), OpList: std::nullopt);
1767	ID.AddPointer(Ptr: &V);
1768	void *IP = nullptr;
1769	SDNode *N = nullptr;
1770	if ((N = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)))
1771	if (!VT.isVector())
1772	return SDValue(N, 0);
1773
1774	if (!N) {
1775	N = newSDNode<ConstantFPSDNode>(Args&: isTarget, Args: &V, Args&: EltVT);
1776	CSEMap.InsertNode(N, InsertPos: IP);
1777	InsertNode(N);
1778	}
1779
1780	SDValue Result(N, 0);
1781	if (VT.isVector())
1782	Result = getSplat(VT, DL, Op: Result);
1783	NewSDValueDbgMsg(V: Result, Msg: "Creating fp constant: ", G: this);
1784	return Result;
1785	}
1786
1787	SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
1788	bool isTarget) {
1789	EVT EltVT = VT.getScalarType();
1790	if (EltVT == MVT::f32)
1791	return getConstantFP(V: APFloat((float)Val), DL, VT, isTarget);
1792	if (EltVT == MVT::f64)
1793	return getConstantFP(V: APFloat(Val), DL, VT, isTarget);
1794	if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1795	EltVT == MVT::f16 || EltVT == MVT::bf16) {
1796	bool Ignored;
1797	APFloat APF = APFloat(Val);
1798	APF.convert(ToSemantics: EVTToAPFloatSemantics(VT: EltVT), RM: APFloat::rmNearestTiesToEven,
1799	losesInfo: &Ignored);
1800	return getConstantFP(V: APF, DL, VT, isTarget);
1801	}
1802	llvm_unreachable("Unsupported type in getConstantFP");
1803	}
1804
1805	SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
1806	EVT VT, int64_t Offset, bool isTargetGA,
1807	unsigned TargetFlags) {
1808	assert((TargetFlags == 0 || isTargetGA) &&
1809	"Cannot set target flags on target-independent globals");
1810
1811	// Truncate (with sign-extension) the offset value to the pointer size.
1812	unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
1813	if (BitWidth < 64)
1814	Offset = SignExtend64(X: Offset, B: BitWidth);
1815
1816	unsigned Opc;
1817	if (GV->isThreadLocal())
1818	Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
1819	else
1820	Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1821
1822	FoldingSetNodeID ID;
1823	AddNodeIDNode(ID, OpC: Opc, VTList: getVTList(VT), OpList: std::nullopt);
1824	ID.AddPointer(Ptr: GV);
1825	ID.AddInteger(I: Offset);
1826	ID.AddInteger(I: TargetFlags);
1827	void *IP = nullptr;
1828	if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
1829	return SDValue(E, 0);
1830
1831	auto *N = newSDNode<GlobalAddressSDNode>(
1832	Args&: Opc, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: GV, Args&: VT, Args&: Offset, Args&: TargetFlags);
1833	CSEMap.InsertNode(N, InsertPos: IP);
1834	InsertNode(N);
1835	return SDValue(N, 0);
1836	}
1837
1838	SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1839	unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1840	FoldingSetNodeID ID;
1841	AddNodeIDNode(ID, OpC: Opc, VTList: getVTList(VT), OpList: std::nullopt);
1842	ID.AddInteger(I: FI);
1843	void *IP = nullptr;
1844	if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
1845	return SDValue(E, 0);
1846
1847	auto *N = newSDNode<FrameIndexSDNode>(Args&: FI, Args&: VT, Args&: isTarget);
1848	CSEMap.InsertNode(N, InsertPos: IP);
1849	InsertNode(N);
1850	return SDValue(N, 0);
1851	}
1852
1853	SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1854	unsigned TargetFlags) {
1855	assert((TargetFlags == 0 || isTarget) &&
1856	"Cannot set target flags on target-independent jump tables");
1857	unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
1858	FoldingSetNodeID ID;
1859	AddNodeIDNode(ID, OpC: Opc, VTList: getVTList(VT), OpList: std::nullopt);
1860	ID.AddInteger(I: JTI);
1861	ID.AddInteger(I: TargetFlags);
1862	void *IP = nullptr;
1863	if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
1864	return SDValue(E, 0);
1865
1866	auto *N = newSDNode<JumpTableSDNode>(Args&: JTI, Args&: VT, Args&: isTarget, Args&: TargetFlags);
1867	CSEMap.InsertNode(N, InsertPos: IP);
1868	InsertNode(N);
1869	return SDValue(N, 0);
1870	}
1871
1872	SDValue SelectionDAG::getJumpTableDebugInfo(int JTI, SDValue Chain,
1873	const SDLoc &DL) {
1874	EVT PTy = getTargetLoweringInfo().getPointerTy(DL: getDataLayout());
1875	return getNode(ISD::JUMP_TABLE_DEBUG_INFO, DL, MVT::Glue, Chain,
1876	getTargetConstant(Val: static_cast<uint64_t>(JTI), DL, VT: PTy, isOpaque: true));
1877	}
1878
1879	SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
1880	MaybeAlign Alignment, int Offset,
1881	bool isTarget, unsigned TargetFlags) {
1882	assert((TargetFlags == 0 || isTarget) &&
1883	"Cannot set target flags on target-independent globals");
1884	if (!Alignment)
1885	Alignment = shouldOptForSize()
1886	? getDataLayout().getABITypeAlign(Ty: C->getType())
1887	: getDataLayout().getPrefTypeAlign(Ty: C->getType());
1888	unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1889	FoldingSetNodeID ID;
1890	AddNodeIDNode(ID, OpC: Opc, VTList: getVTList(VT), OpList: std::nullopt);
1891	ID.AddInteger(I: Alignment->value());
1892	ID.AddInteger(I: Offset);
1893	ID.AddPointer(Ptr: C);
1894	ID.AddInteger(I: TargetFlags);
1895	void *IP = nullptr;
1896	if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
1897	return SDValue(E, 0);
1898
1899	auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VT, Args&: Offset, Args&: *Alignment,
1900	Args&: TargetFlags);
1901	CSEMap.InsertNode(N, InsertPos: IP);
1902	InsertNode(N);
1903	SDValue V = SDValue(N, 0);
1904	NewSDValueDbgMsg(V, Msg: "Creating new constant pool: ", G: this);
1905	return V;
1906	}
1907
1908	SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
1909	MaybeAlign Alignment, int Offset,
1910	bool isTarget, unsigned TargetFlags) {
1911	assert((TargetFlags == 0 || isTarget) &&
1912	"Cannot set target flags on target-independent globals");
1913	if (!Alignment)
1914	Alignment = getDataLayout().getPrefTypeAlign(Ty: C->getType());
1915	unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1916	FoldingSetNodeID ID;
1917	AddNodeIDNode(ID, OpC: Opc, VTList: getVTList(VT), OpList: std::nullopt);
1918	ID.AddInteger(I: Alignment->value());
1919	ID.AddInteger(I: Offset);
1920	C->addSelectionDAGCSEId(ID);
1921	ID.AddInteger(I: TargetFlags);
1922	void *IP = nullptr;
1923	if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
1924	return SDValue(E, 0);
1925
1926	auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VT, Args&: Offset, Args&: *Alignment,
1927	Args&: TargetFlags);
1928	CSEMap.InsertNode(N, InsertPos: IP);
1929	InsertNode(N);
1930	return SDValue(N, 0);
1931	}
1932
1933	SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
1934	FoldingSetNodeID ID;
1935	AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), std::nullopt);
1936	ID.AddPointer(Ptr: MBB);
1937	void *IP = nullptr;
1938	if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
1939	return SDValue(E, 0);
1940
1941	auto *N = newSDNode<BasicBlockSDNode>(Args&: MBB);
1942	CSEMap.InsertNode(N, InsertPos: IP);
1943	InsertNode(N);
1944	return SDValue(N, 0);
1945	}
1946
1947	SDValue SelectionDAG::getValueType(EVT VT) {
1948	if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
1949	ValueTypeNodes.size())
1950	ValueTypeNodes.resize(new_size: VT.getSimpleVT().SimpleTy+1);
1951
1952	SDNode *&N = VT.isExtended() ?
1953	ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
1954
1955	if (N) return SDValue(N, 0);
1956	N = newSDNode<VTSDNode>(Args&: VT);
1957	InsertNode(N);
1958	return SDValue(N, 0);
1959	}
1960
1961	SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
1962	SDNode *&N = ExternalSymbols[Sym];
1963	if (N) return SDValue(N, 0);
1964	N = newSDNode<ExternalSymbolSDNode>(Args: false, Args&: Sym, Args: 0, Args&: VT);
1965	InsertNode(N);
1966	return SDValue(N, 0);
1967	}
1968
1969	SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
1970	SDNode *&N = MCSymbols[Sym];
1971	if (N)
1972	return SDValue(N, 0);
1973	N = newSDNode<MCSymbolSDNode>(Args&: Sym, Args&: VT);
1974	InsertNode(N);
1975	return SDValue(N, 0);
1976	}
1977
1978	SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
1979	unsigned TargetFlags) {
1980	SDNode *&N =
1981	TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)];
1982	if (N) return SDValue(N, 0);
1983	N = newSDNode<ExternalSymbolSDNode>(Args: true, Args&: Sym, Args&: TargetFlags, Args&: VT);
1984	InsertNode(N);
1985	return SDValue(N, 0);
1986	}
1987
1988	SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
1989	if ((unsigned)Cond >= CondCodeNodes.size())
1990	CondCodeNodes.resize(new_size: Cond+1);
1991
1992	if (!CondCodeNodes[Cond]) {
1993	auto *N = newSDNode<CondCodeSDNode>(Args&: Cond);
1994	CondCodeNodes[Cond] = N;
1995	InsertNode(N);
1996	}
1997
1998	return SDValue(CondCodeNodes[Cond], 0);
1999	}
2000
2001	SDValue SelectionDAG::getVScale(const SDLoc &DL, EVT VT, APInt MulImm,
2002	bool ConstantFold) {
2003	assert(MulImm.getBitWidth() == VT.getSizeInBits() &&
2004	"APInt size does not match type size!");
2005
2006	if (MulImm == 0)
2007	return getConstant(Val: 0, DL, VT);
2008
2009	if (ConstantFold) {
2010	const MachineFunction &MF = getMachineFunction();
2011	const Function &F = MF.getFunction();
2012	ConstantRange CR = getVScaleRange(F: &F, BitWidth: 64);
2013	if (const APInt *C = CR.getSingleElement())
2014	return getConstant(Val: MulImm * C->getZExtValue(), DL, VT);
2015	}
2016
2017	return getNode(Opcode: ISD::VSCALE, DL, VT, Operand: getConstant(Val: MulImm, DL, VT));
2018	}
2019
2020	SDValue SelectionDAG::getElementCount(const SDLoc &DL, EVT VT, ElementCount EC,
2021	bool ConstantFold) {
2022	if (EC.isScalable())
2023	return getVScale(DL, VT,
2024	MulImm: APInt(VT.getSizeInBits(), EC.getKnownMinValue()));
2025
2026	return getConstant(Val: EC.getKnownMinValue(), DL, VT);
2027	}
2028
2029	SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) {
2030	APInt One(ResVT.getScalarSizeInBits(), 1);
2031	return getStepVector(DL, ResVT, StepVal: One);
2032	}
2033
2034	SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT,
2035	const APInt &StepVal) {
2036	assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth());
2037	if (ResVT.isScalableVector())
2038	return getNode(
2039	Opcode: ISD::STEP_VECTOR, DL, VT: ResVT,
2040	Operand: getTargetConstant(Val: StepVal, DL, VT: ResVT.getVectorElementType()));
2041
2042	SmallVector<SDValue, 16> OpsStepConstants;
2043	for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++)
2044	OpsStepConstants.push_back(
2045	Elt: getConstant(Val: StepVal * i, DL, VT: ResVT.getVectorElementType()));
2046	return getBuildVector(VT: ResVT, DL, Ops: OpsStepConstants);
2047	}
2048
2049	/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
2050	/// point at N1 to point at N2 and indices that point at N2 to point at N1.
2051	static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
2052	std::swap(a&: N1, b&: N2);
2053	ShuffleVectorSDNode::commuteMask(Mask: M);
2054	}
2055
2056	SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
2057	SDValue N2, ArrayRef<int> Mask) {
2058	assert(VT.getVectorNumElements() == Mask.size() &&
2059	"Must have the same number of vector elements as mask elements!");
2060	assert(VT == N1.getValueType() && VT == N2.getValueType() &&
2061	"Invalid VECTOR_SHUFFLE");
2062
2063	// Canonicalize shuffle undef, undef -> undef
2064	if (N1.isUndef() && N2.isUndef())
2065	return getUNDEF(VT);
2066
2067	// Validate that all indices in Mask are within the range of the elements
2068	// input to the shuffle.
2069	int NElts = Mask.size();
2070	assert(llvm::all_of(Mask,
2071	[&](int M) { return M < (NElts * 2) && M >= -1; }) &&
2072	"Index out of range");
2073
2074	// Copy the mask so we can do any needed cleanup.
2075	SmallVector<int, 8> MaskVec(Mask);
2076
2077	// Canonicalize shuffle v, v -> v, undef
2078	if (N1 == N2) {
2079	N2 = getUNDEF(VT);
2080	for (int i = 0; i != NElts; ++i)
2081	if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
2082	}
2083
2084	// Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
2085	if (N1.isUndef())
2086	commuteShuffle(N1, N2, M: MaskVec);
2087
2088	if (TLI->hasVectorBlend()) {
2089	// If shuffling a splat, try to blend the splat instead. We do this here so
2090	// that even when this arises during lowering we don't have to re-handle it.
2091	auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
2092	BitVector UndefElements;
2093	SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements);
2094	if (!Splat)
2095	return;
2096
2097	for (int i = 0; i < NElts; ++i) {
2098	if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
2099	continue;
2100
2101	// If this input comes from undef, mark it as such.
2102	if (UndefElements[MaskVec[i] - Offset]) {
2103	MaskVec[i] = -1;
2104	continue;
2105	}
2106
2107	// If we can blend a non-undef lane, use that instead.
2108	if (!UndefElements[i])
2109	MaskVec[i] = i + Offset;
2110	}
2111	};
2112	if (auto *N1BV = dyn_cast<BuildVectorSDNode>(Val&: N1))
2113	BlendSplat(N1BV, 0);
2114	if (auto *N2BV = dyn_cast<BuildVectorSDNode>(Val&: N2))
2115	BlendSplat(N2BV, NElts);
2116	}
2117
2118	// Canonicalize all index into lhs, -> shuffle lhs, undef
2119	// Canonicalize all index into rhs, -> shuffle rhs, undef
2120	bool AllLHS = true, AllRHS = true;
2121	bool N2Undef = N2.isUndef();
2122	for (int i = 0; i != NElts; ++i) {
2123	if (MaskVec[i] >= NElts) {
2124	if (N2Undef)
2125	MaskVec[i] = -1;
2126	else
2127	AllLHS = false;
2128	} else if (MaskVec[i] >= 0) {
2129	AllRHS = false;
2130	}
2131	}
2132	if (AllLHS && AllRHS)
2133	return getUNDEF(VT);
2134	if (AllLHS && !N2Undef)
2135	N2 = getUNDEF(VT);
2136	if (AllRHS) {
2137	N1 = getUNDEF(VT);
2138	commuteShuffle(N1, N2, M: MaskVec);
2139	}
2140	// Reset our undef status after accounting for the mask.
2141	N2Undef = N2.isUndef();
2142	// Re-check whether both sides ended up undef.
2143	if (N1.isUndef() && N2Undef)
2144	return getUNDEF(VT);
2145
2146	// If Identity shuffle return that node.
2147	bool Identity = true, AllSame = true;
2148	for (int i = 0; i != NElts; ++i) {
2149	if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
2150	if (MaskVec[i] != MaskVec[0]) AllSame = false;
2151	}
2152	if (Identity && NElts)
2153	return N1;
2154
2155	// Shuffling a constant splat doesn't change the result.
2156	if (N2Undef) {
2157	SDValue V = N1;
2158
2159	// Look through any bitcasts. We check that these don't change the number
2160	// (and size) of elements and just changes their types.
2161	while (V.getOpcode() == ISD::BITCAST)
2162	V = V->getOperand(Num: 0);
2163
2164	// A splat should always show up as a build vector node.
2165	if (auto *BV = dyn_cast<BuildVectorSDNode>(Val&: V)) {
2166	BitVector UndefElements;
2167	SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements);
2168	// If this is a splat of an undef, shuffling it is also undef.
2169	if (Splat && Splat.isUndef())
2170	return getUNDEF(VT);
2171
2172	bool SameNumElts =
2173	V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
2174
2175	// We only have a splat which can skip shuffles if there is a splatted
2176	// value and no undef lanes rearranged by the shuffle.
2177	if (Splat && UndefElements.none()) {
2178	// Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
2179	// number of elements match or the value splatted is a zero constant.
2180	if (SameNumElts || isNullConstant(V: Splat))
2181	return N1;
2182	}
2183
2184	// If the shuffle itself creates a splat, build the vector directly.
2185	if (AllSame && SameNumElts) {
2186	EVT BuildVT = BV->getValueType(ResNo: 0);
2187	const SDValue &Splatted = BV->getOperand(Num: MaskVec[0]);
2188	SDValue NewBV = getSplatBuildVector(VT: BuildVT, DL: dl, Op: Splatted);
2189
2190	// We may have jumped through bitcasts, so the type of the
2191	// BUILD_VECTOR may not match the type of the shuffle.
2192	if (BuildVT != VT)
2193	NewBV = getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: NewBV);
2194	return NewBV;
2195	}
2196	}
2197	}
2198
2199	FoldingSetNodeID ID;
2200	SDValue Ops[2] = { N1, N2 };
2201	AddNodeIDNode(ID, OpC: ISD::VECTOR_SHUFFLE, VTList: getVTList(VT), OpList: Ops);
2202	for (int i = 0; i != NElts; ++i)
2203	ID.AddInteger(I: MaskVec[i]);
2204
2205	void* IP = nullptr;
2206	if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
2207	return SDValue(E, 0);
2208
2209	// Allocate the mask array for the node out of the BumpPtrAllocator, since
2210	// SDNode doesn't have access to it. This memory will be "leaked" when
2211	// the node is deallocated, but recovered when the NodeAllocator is released.
2212	int *MaskAlloc = OperandAllocator.Allocate<int>(Num: NElts);
2213	llvm::copy(Range&: MaskVec, Out: MaskAlloc);
2214
2215	auto *N = newSDNode<ShuffleVectorSDNode>(Args&: VT, Args: dl.getIROrder(),
2216	Args: dl.getDebugLoc(), Args&: MaskAlloc);
2217	createOperands(Node: N, Vals: Ops);
2218
2219	CSEMap.InsertNode(N, InsertPos: IP);
2220	InsertNode(N);
2221	SDValue V = SDValue(N, 0);
2222	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
2223	return V;
2224	}
2225
2226	SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
2227	EVT VT = SV.getValueType(ResNo: 0);
2228	SmallVector<int, 8> MaskVec(SV.getMask());
2229	ShuffleVectorSDNode::commuteMask(Mask: MaskVec);
2230
2231	SDValue Op0 = SV.getOperand(Num: 0);
2232	SDValue Op1 = SV.getOperand(Num: 1);
2233	return getVectorShuffle(VT, dl: SDLoc(&SV), N1: Op1, N2: Op0, Mask: MaskVec);
2234	}
2235
2236	SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
2237	FoldingSetNodeID ID;
2238	AddNodeIDNode(ID, OpC: ISD::Register, VTList: getVTList(VT), OpList: std::nullopt);
2239	ID.AddInteger(I: RegNo);
2240	void *IP = nullptr;
2241	if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2242	return SDValue(E, 0);
2243
2244	auto *N = newSDNode<RegisterSDNode>(Args&: RegNo, Args&: VT);
2245	N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, UA);
2246	CSEMap.InsertNode(N, InsertPos: IP);
2247	InsertNode(N);
2248	return SDValue(N, 0);
2249	}
2250
2251	SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
2252	FoldingSetNodeID ID;
2253	AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), std::nullopt);
2254	ID.AddPointer(Ptr: RegMask);
2255	void *IP = nullptr;
2256	if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2257	return SDValue(E, 0);
2258
2259	auto *N = newSDNode<RegisterMaskSDNode>(Args&: RegMask);
2260	CSEMap.InsertNode(N, InsertPos: IP);
2261	InsertNode(N);
2262	return SDValue(N, 0);
2263	}
2264
2265	SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
2266	MCSymbol *Label) {
2267	return getLabelNode(Opcode: ISD::EH_LABEL, dl, Root, Label);
2268	}
2269
2270	SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
2271	SDValue Root, MCSymbol *Label) {
2272	FoldingSetNodeID ID;
2273	SDValue Ops[] = { Root };
2274	AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), Ops);
2275	ID.AddPointer(Ptr: Label);
2276	void *IP = nullptr;
2277	if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2278	return SDValue(E, 0);
2279
2280	auto *N =
2281	newSDNode<LabelSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: Label);
2282	createOperands(Node: N, Vals: Ops);
2283
2284	CSEMap.InsertNode(N, InsertPos: IP);
2285	InsertNode(N);
2286	return SDValue(N, 0);
2287	}
2288
2289	SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
2290	int64_t Offset, bool isTarget,
2291	unsigned TargetFlags) {
2292	unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
2293
2294	FoldingSetNodeID ID;
2295	AddNodeIDNode(ID, OpC: Opc, VTList: getVTList(VT), OpList: std::nullopt);
2296	ID.AddPointer(Ptr: BA);
2297	ID.AddInteger(I: Offset);
2298	ID.AddInteger(I: TargetFlags);
2299	void *IP = nullptr;
2300	if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2301	return SDValue(E, 0);
2302
2303	auto *N = newSDNode<BlockAddressSDNode>(Args&: Opc, Args&: VT, Args&: BA, Args&: Offset, Args&: TargetFlags);
2304	CSEMap.InsertNode(N, InsertPos: IP);
2305	InsertNode(N);
2306	return SDValue(N, 0);
2307	}
2308
2309	SDValue SelectionDAG::getSrcValue(const Value *V) {
2310	FoldingSetNodeID ID;
2311	AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), std::nullopt);
2312	ID.AddPointer(Ptr: V);
2313
2314	void *IP = nullptr;
2315	if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2316	return SDValue(E, 0);
2317
2318	auto *N = newSDNode<SrcValueSDNode>(Args&: V);
2319	CSEMap.InsertNode(N, InsertPos: IP);
2320	InsertNode(N);
2321	return SDValue(N, 0);
2322	}
2323
2324	SDValue SelectionDAG::getMDNode(const MDNode *MD) {
2325	FoldingSetNodeID ID;
2326	AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), std::nullopt);
2327	ID.AddPointer(Ptr: MD);
2328
2329	void *IP = nullptr;
2330	if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2331	return SDValue(E, 0);
2332
2333	auto *N = newSDNode<MDNodeSDNode>(Args&: MD);
2334	CSEMap.InsertNode(N, InsertPos: IP);
2335	InsertNode(N);
2336	return SDValue(N, 0);
2337	}
2338
2339	SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
2340	if (VT == V.getValueType())
2341	return V;
2342
2343	return getNode(Opcode: ISD::BITCAST, DL: SDLoc(V), VT, Operand: V);
2344	}
2345
2346	SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
2347	unsigned SrcAS, unsigned DestAS) {
2348	SDValue Ops[] = {Ptr};
2349	FoldingSetNodeID ID;
2350	AddNodeIDNode(ID, OpC: ISD::ADDRSPACECAST, VTList: getVTList(VT), OpList: Ops);
2351	ID.AddInteger(I: SrcAS);
2352	ID.AddInteger(I: DestAS);
2353
2354	void *IP = nullptr;
2355	if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
2356	return SDValue(E, 0);
2357
2358	auto *N = newSDNode<AddrSpaceCastSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
2359	Args&: VT, Args&: SrcAS, Args&: DestAS);
2360	createOperands(Node: N, Vals: Ops);
2361
2362	CSEMap.InsertNode(N, InsertPos: IP);
2363	InsertNode(N);
2364	return SDValue(N, 0);
2365	}
2366
2367	SDValue SelectionDAG::getFreeze(SDValue V) {
2368	return getNode(Opcode: ISD::FREEZE, DL: SDLoc(V), VT: V.getValueType(), Operand: V);
2369	}
2370
2371	/// getShiftAmountOperand - Return the specified value casted to
2372	/// the target's desired shift amount type.
2373	SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
2374	EVT OpTy = Op.getValueType();
2375	EVT ShTy = TLI->getShiftAmountTy(LHSTy, DL: getDataLayout());
2376	if (OpTy == ShTy || OpTy.isVector()) return Op;
2377
2378	return getZExtOrTrunc(Op, DL: SDLoc(Op), VT: ShTy);
2379	}
2380
2381	SDValue SelectionDAG::expandVAArg(SDNode *Node) {
2382	SDLoc dl(Node);
2383	const TargetLowering &TLI = getTargetLoweringInfo();
2384	const Value *V = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 2))->getValue();
2385	EVT VT = Node->getValueType(ResNo: 0);
2386	SDValue Tmp1 = Node->getOperand(Num: 0);
2387	SDValue Tmp2 = Node->getOperand(Num: 1);
2388	const MaybeAlign MA(Node->getConstantOperandVal(Num: 3));
2389
2390	SDValue VAListLoad = getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Tmp1,
2391	Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
2392	SDValue VAList = VAListLoad;
2393
2394	if (MA && *MA > TLI.getMinStackArgumentAlignment()) {
2395	VAList = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList,
2396	N2: getConstant(Val: MA->value() - 1, DL: dl, VT: VAList.getValueType()));
2397
2398	VAList =
2399	getNode(Opcode: ISD::AND, DL: dl, VT: VAList.getValueType(), N1: VAList,
2400	N2: getConstant(Val: -(int64_t)MA->value(), DL: dl, VT: VAList.getValueType()));
2401	}
2402
2403	// Increment the pointer, VAList, to the next vaarg
2404	Tmp1 = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList,
2405	N2: getConstant(Val: getDataLayout().getTypeAllocSize(
2406	Ty: VT.getTypeForEVT(Context&: *getContext())),
2407	DL: dl, VT: VAList.getValueType()));
2408	// Store the incremented VAList to the legalized pointer
2409	Tmp1 =
2410	getStore(Chain: VAListLoad.getValue(R: 1), dl, Val: Tmp1, Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
2411	// Load the actual argument out of the pointer VAList
2412	return getLoad(VT, dl, Chain: Tmp1, Ptr: VAList, PtrInfo: MachinePointerInfo());
2413	}
2414
2415	SDValue SelectionDAG::expandVACopy(SDNode *Node) {
2416	SDLoc dl(Node);
2417	const TargetLowering &TLI = getTargetLoweringInfo();
2418	// This defaults to loading a pointer from the input and storing it to the
2419	// output, returning the chain.
2420	const Value *VD = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 3))->getValue();
2421	const Value *VS = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 4))->getValue();
2422	SDValue Tmp1 =
2423	getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Node->getOperand(Num: 0),
2424	Ptr: Node->getOperand(Num: 2), PtrInfo: MachinePointerInfo(VS));
2425	return getStore(Chain: Tmp1.getValue(R: 1), dl, Val: Tmp1, Ptr: Node->getOperand(Num: 1),
2426	PtrInfo: MachinePointerInfo(VD));
2427	}
2428
2429	Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) {
2430	const DataLayout &DL = getDataLayout();
2431	Type *Ty = VT.getTypeForEVT(Context&: *getContext());
2432	Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2433
2434	if (TLI->isTypeLegal(VT) || !VT.isVector())
2435	return RedAlign;
2436
2437	const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2438	const Align StackAlign = TFI->getStackAlign();
2439
2440	// See if we can choose a smaller ABI alignment in cases where it's an
2441	// illegal vector type that will get broken down.
2442	if (RedAlign > StackAlign) {
2443	EVT IntermediateVT;
2444	MVT RegisterVT;
2445	unsigned NumIntermediates;
2446	TLI->getVectorTypeBreakdown(Context&: *getContext(), VT, IntermediateVT,
2447	NumIntermediates, RegisterVT);
2448	Ty = IntermediateVT.getTypeForEVT(Context&: *getContext());
2449	Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2450	if (RedAlign2 < RedAlign)
2451	RedAlign = RedAlign2;
2452	}
2453
2454	return RedAlign;
2455	}
2456
2457	SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) {
2458	MachineFrameInfo &MFI = MF->getFrameInfo();
2459	const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2460	int StackID = 0;
2461	if (Bytes.isScalable())
2462	StackID = TFI->getStackIDForScalableVectors();
2463	// The stack id gives an indication of whether the object is scalable or
2464	// not, so it's safe to pass in the minimum size here.
2465	int FrameIdx = MFI.CreateStackObject(Size: Bytes.getKnownMinValue(), Alignment,
2466	isSpillSlot: false, Alloca: nullptr, ID: StackID);
2467	return getFrameIndex(FI: FrameIdx, VT: TLI->getFrameIndexTy(DL: getDataLayout()));
2468	}
2469
2470	SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
2471	Type *Ty = VT.getTypeForEVT(Context&: *getContext());
2472	Align StackAlign =
2473	std::max(a: getDataLayout().getPrefTypeAlign(Ty), b: Align(minAlign));
2474	return CreateStackTemporary(Bytes: VT.getStoreSize(), Alignment: StackAlign);
2475	}
2476
2477	SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
2478	TypeSize VT1Size = VT1.getStoreSize();
2479	TypeSize VT2Size = VT2.getStoreSize();
2480	assert(VT1Size.isScalable() == VT2Size.isScalable() &&
2481	"Don't know how to choose the maximum size when creating a stack "
2482	"temporary");
2483	TypeSize Bytes = VT1Size.getKnownMinValue() > VT2Size.getKnownMinValue()
2484	? VT1Size
2485	: VT2Size;
2486
2487	Type *Ty1 = VT1.getTypeForEVT(Context&: *getContext());
2488	Type *Ty2 = VT2.getTypeForEVT(Context&: *getContext());
2489	const DataLayout &DL = getDataLayout();
2490	Align Align = std::max(a: DL.getPrefTypeAlign(Ty: Ty1), b: DL.getPrefTypeAlign(Ty: Ty2));
2491	return CreateStackTemporary(Bytes, Alignment: Align);
2492	}
2493
2494	SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
2495	ISD::CondCode Cond, const SDLoc &dl) {
2496	EVT OpVT = N1.getValueType();
2497
2498	auto GetUndefBooleanConstant = [&]() {
2499	if (VT.getScalarType() == MVT::i1 ||
2500	TLI->getBooleanContents(Type: OpVT) ==
2501	TargetLowering::UndefinedBooleanContent)
2502	return getUNDEF(VT);
2503	// ZeroOrOne / ZeroOrNegative require specific values for the high bits,
2504	// so we cannot use getUNDEF(). Return zero instead.
2505	return getConstant(Val: 0, DL: dl, VT);
2506	};
2507
2508	// These setcc operations always fold.
2509	switch (Cond) {
2510	default: break;
2511	case ISD::SETFALSE:
2512	case ISD::SETFALSE2: return getBoolConstant(V: false, DL: dl, VT, OpVT);
2513	case ISD::SETTRUE:
2514	case ISD::SETTRUE2: return getBoolConstant(V: true, DL: dl, VT, OpVT);
2515
2516	case ISD::SETOEQ:
2517	case ISD::SETOGT:
2518	case ISD::SETOGE:
2519	case ISD::SETOLT:
2520	case ISD::SETOLE:
2521	case ISD::SETONE:
2522	case ISD::SETO:
2523	case ISD::SETUO:
2524	case ISD::SETUEQ:
2525	case ISD::SETUNE:
2526	assert(!OpVT.isInteger() && "Illegal setcc for integer!");
2527	break;
2528	}
2529
2530	if (OpVT.isInteger()) {
2531	// For EQ and NE, we can always pick a value for the undef to make the
2532	// predicate pass or fail, so we can return undef.
2533	// Matches behavior in llvm::ConstantFoldCompareInstruction.
2534	// icmp eq/ne X, undef -> undef.
2535	if ((N1.isUndef() || N2.isUndef()) &&
2536	(Cond == ISD::SETEQ || Cond == ISD::SETNE))
2537	return GetUndefBooleanConstant();
2538
2539	// If both operands are undef, we can return undef for int comparison.
2540	// icmp undef, undef -> undef.
2541	if (N1.isUndef() && N2.isUndef())
2542	return GetUndefBooleanConstant();
2543
2544	// icmp X, X -> true/false
2545	// icmp X, undef -> true/false because undef could be X.
2546	if (N1.isUndef() || N2.isUndef() || N1 == N2)
2547	return getBoolConstant(V: ISD::isTrueWhenEqual(Cond), DL: dl, VT, OpVT);
2548	}
2549
2550	if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(Val&: N2)) {
2551	const APInt &C2 = N2C->getAPIntValue();
2552	if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Val&: N1)) {
2553	const APInt &C1 = N1C->getAPIntValue();
2554
2555	return getBoolConstant(V: ICmpInst::compare(LHS: C1, RHS: C2, Pred: getICmpCondCode(Pred: Cond)),
2556	DL: dl, VT, OpVT);
2557	}
2558	}
2559
2560	auto *N1CFP = dyn_cast<ConstantFPSDNode>(Val&: N1);
2561	auto *N2CFP = dyn_cast<ConstantFPSDNode>(Val&: N2);
2562
2563	if (N1CFP && N2CFP) {
2564	APFloat::cmpResult R = N1CFP->getValueAPF().compare(RHS: N2CFP->getValueAPF());
2565	switch (Cond) {
2566	default: break;
2567	case ISD::SETEQ: if (R==APFloat::cmpUnordered)
2568	return GetUndefBooleanConstant();
2569	[[fallthrough]];
2570	case ISD::SETOEQ: return getBoolConstant(V: R==APFloat::cmpEqual, DL: dl, VT,
2571	OpVT);
2572	case ISD::SETNE: if (R==APFloat::cmpUnordered)
2573	return GetUndefBooleanConstant();
2574	[[fallthrough]];
2575	case ISD::SETONE: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2576	R==APFloat::cmpLessThan, DL: dl, VT,
2577	OpVT);
2578	case ISD::SETLT: if (R==APFloat::cmpUnordered)
2579	return GetUndefBooleanConstant();
2580	[[fallthrough]];
2581	case ISD::SETOLT: return getBoolConstant(V: R==APFloat::cmpLessThan, DL: dl, VT,
2582	OpVT);
2583	case ISD::SETGT: if (R==APFloat::cmpUnordered)
2584	return GetUndefBooleanConstant();
2585	[[fallthrough]];
2586	case ISD::SETOGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan, DL: dl,
2587	VT, OpVT);
2588	case ISD::SETLE: if (R==APFloat::cmpUnordered)
2589	return GetUndefBooleanConstant();
2590	[[fallthrough]];
2591	case ISD::SETOLE: return getBoolConstant(V: R==APFloat::cmpLessThan ||
2592	R==APFloat::cmpEqual, DL: dl, VT,
2593	OpVT);
2594	case ISD::SETGE: if (R==APFloat::cmpUnordered)
2595	return GetUndefBooleanConstant();
2596	[[fallthrough]];
2597	case ISD::SETOGE: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2598	R==APFloat::cmpEqual, DL: dl, VT, OpVT);
2599	case ISD::SETO: return getBoolConstant(V: R!=APFloat::cmpUnordered, DL: dl, VT,
2600	OpVT);
2601	case ISD::SETUO: return getBoolConstant(V: R==APFloat::cmpUnordered, DL: dl, VT,
2602	OpVT);
2603	case ISD::SETUEQ: return getBoolConstant(V: R==APFloat::cmpUnordered ||
2604	R==APFloat::cmpEqual, DL: dl, VT,
2605	OpVT);
2606	case ISD::SETUNE: return getBoolConstant(V: R!=APFloat::cmpEqual, DL: dl, VT,
2607	OpVT);
2608	case ISD::SETULT: return getBoolConstant(V: R==APFloat::cmpUnordered ||
2609	R==APFloat::cmpLessThan, DL: dl, VT,
2610	OpVT);
2611	case ISD::SETUGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2612	R==APFloat::cmpUnordered, DL: dl, VT,
2613	OpVT);
2614	case ISD::SETULE: return getBoolConstant(V: R!=APFloat::cmpGreaterThan, DL: dl,
2615	VT, OpVT);
2616	case ISD::SETUGE: return getBoolConstant(V: R!=APFloat::cmpLessThan, DL: dl, VT,
2617	OpVT);
2618	}
2619	} else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) {
2620	// Ensure that the constant occurs on the RHS.
2621	ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Operation: Cond);
2622	if (!TLI->isCondCodeLegal(CC: SwappedCond, VT: OpVT.getSimpleVT()))
2623	return SDValue();
2624	return getSetCC(DL: dl, VT, LHS: N2, RHS: N1, Cond: SwappedCond);
2625	} else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2626	(OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) {
2627	// If an operand is known to be a nan (or undef that could be a nan), we can
2628	// fold it.
2629	// Choosing NaN for the undef will always make unordered comparison succeed
2630	// and ordered comparison fails.
2631	// Matches behavior in llvm::ConstantFoldCompareInstruction.
2632	switch (ISD::getUnorderedFlavor(Cond)) {
2633	default:
2634	llvm_unreachable("Unknown flavor!");
2635	case 0: // Known false.
2636	return getBoolConstant(V: false, DL: dl, VT, OpVT);
2637	case 1: // Known true.
2638	return getBoolConstant(V: true, DL: dl, VT, OpVT);
2639	case 2: // Undefined.
2640	return GetUndefBooleanConstant();
2641	}
2642	}
2643
2644	// Could not fold it.
2645	return SDValue();
2646	}
2647
2648	/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
2649	/// use this predicate to simplify operations downstream.
2650	bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
2651	unsigned BitWidth = Op.getScalarValueSizeInBits();
2652	return MaskedValueIsZero(Op, Mask: APInt::getSignMask(BitWidth), Depth);
2653	}
2654
2655	/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
2656	/// this predicate to simplify operations downstream. Mask is known to be zero
2657	/// for bits that V cannot have.
2658	bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2659	unsigned Depth) const {
2660	return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).Zero);
2661	}
2662
2663	/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in
2664	/// DemandedElts. We use this predicate to simplify operations downstream.
2665	/// Mask is known to be zero for bits that V cannot have.
2666	bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2667	const APInt &DemandedElts,
2668	unsigned Depth) const {
2669	return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, DemandedElts, Depth).Zero);
2670	}
2671
2672	/// MaskedVectorIsZero - Return true if 'Op' is known to be zero in
2673	/// DemandedElts. We use this predicate to simplify operations downstream.
2674	bool SelectionDAG::MaskedVectorIsZero(SDValue V, const APInt &DemandedElts,
2675	unsigned Depth /* = 0 */) const {
2676	return computeKnownBits(Op: V, DemandedElts, Depth).isZero();
2677	}
2678
2679	/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'.
2680	bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask,
2681	unsigned Depth) const {
2682	return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).One);
2683	}
2684
2685	APInt SelectionDAG::computeVectorKnownZeroElements(SDValue Op,
2686	const APInt &DemandedElts,
2687	unsigned Depth) const {
2688	EVT VT = Op.getValueType();
2689	assert(VT.isVector() && !VT.isScalableVector() && "Only for fixed vectors!");
2690
2691	unsigned NumElts = VT.getVectorNumElements();
2692	assert(DemandedElts.getBitWidth() == NumElts && "Unexpected demanded mask.");
2693
2694	APInt KnownZeroElements = APInt::getZero(numBits: NumElts);
2695	for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {
2696	if (!DemandedElts[EltIdx])
2697	continue; // Don't query elements that are not demanded.
2698	APInt Mask = APInt::getOneBitSet(numBits: NumElts, BitNo: EltIdx);
2699	if (MaskedVectorIsZero(V: Op, DemandedElts: Mask, Depth))
2700	KnownZeroElements.setBit(EltIdx);
2701	}
2702	return KnownZeroElements;
2703	}
2704
2705	/// isSplatValue - Return true if the vector V has the same value
2706	/// across all DemandedElts. For scalable vectors, we don't know the
2707	/// number of lanes at compile time. Instead, we use a 1 bit APInt
2708	/// to represent a conservative value for all lanes; that is, that
2709	/// one bit value is implicitly splatted across all lanes.
2710	bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
2711	APInt &UndefElts, unsigned Depth) const {
2712	unsigned Opcode = V.getOpcode();
2713	EVT VT = V.getValueType();
2714	assert(VT.isVector() && "Vector type expected");
2715	assert((!VT.isScalableVector() || DemandedElts.getBitWidth() == 1) &&
2716	"scalable demanded bits are ignored");
2717
2718	if (!DemandedElts)
2719	return false; // No demanded elts, better to assume we don't know anything.
2720
2721	if (Depth >= MaxRecursionDepth)
2722	return false; // Limit search depth.
2723
2724	// Deal with some common cases here that work for both fixed and scalable
2725	// vector types.
2726	switch (Opcode) {
2727	case ISD::SPLAT_VECTOR:
2728	UndefElts = V.getOperand(i: 0).isUndef()
2729	? APInt::getAllOnes(numBits: DemandedElts.getBitWidth())
2730	: APInt(DemandedElts.getBitWidth(), 0);
2731	return true;
2732	case ISD::ADD:
2733	case ISD::SUB:
2734	case ISD::AND:
2735	case ISD::XOR:
2736	case ISD::OR: {
2737	APInt UndefLHS, UndefRHS;
2738	SDValue LHS = V.getOperand(i: 0);
2739	SDValue RHS = V.getOperand(i: 1);
2740	if (isSplatValue(V: LHS, DemandedElts, UndefElts&: UndefLHS, Depth: Depth + 1) &&
2741	isSplatValue(V: RHS, DemandedElts, UndefElts&: UndefRHS, Depth: Depth + 1)) {
2742	UndefElts = UndefLHS | UndefRHS;
2743	return true;
2744	}
2745	return false;
2746	}
2747	case ISD::ABS:
2748	case ISD::TRUNCATE:
2749	case ISD::SIGN_EXTEND:
2750	case ISD::ZERO_EXTEND:
2751	return isSplatValue(V: V.getOperand(i: 0), DemandedElts, UndefElts, Depth: Depth + 1);
2752	default:
2753	if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
2754	Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
2755	return TLI->isSplatValueForTargetNode(Op: V, DemandedElts, UndefElts, DAG: *this,
2756	Depth);
2757	break;
2758	}
2759
2760	// We don't support other cases than those above for scalable vectors at
2761	// the moment.
2762	if (VT.isScalableVector())
2763	return false;
2764
2765	unsigned NumElts = VT.getVectorNumElements();
2766	assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch");
2767	UndefElts = APInt::getZero(numBits: NumElts);
2768
2769	switch (Opcode) {
2770	case ISD::BUILD_VECTOR: {
2771	SDValue Scl;
2772	for (unsigned i = 0; i != NumElts; ++i) {
2773	SDValue Op = V.getOperand(i);
2774	if (Op.isUndef()) {
2775	UndefElts.setBit(i);
2776	continue;
2777	}
2778	if (!DemandedElts[i])
2779	continue;
2780	if (Scl && Scl != Op)
2781	return false;
2782	Scl = Op;
2783	}
2784	return true;
2785	}
2786	case ISD::VECTOR_SHUFFLE: {
2787	// Check if this is a shuffle node doing a splat or a shuffle of a splat.
2788	APInt DemandedLHS = APInt::getZero(numBits: NumElts);
2789	APInt DemandedRHS = APInt::getZero(numBits: NumElts);
2790	ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val&: V)->getMask();
2791	for (int i = 0; i != (int)NumElts; ++i) {
2792	int M = Mask[i];
2793	if (M < 0) {
2794	UndefElts.setBit(i);
2795	continue;
2796	}
2797	if (!DemandedElts[i])
2798	continue;
2799	if (M < (int)NumElts)
2800	DemandedLHS.setBit(M);
2801	else
2802	DemandedRHS.setBit(M - NumElts);
2803	}
2804
2805	// If we aren't demanding either op, assume there's no splat.
2806	// If we are demanding both ops, assume there's no splat.
2807	if ((DemandedLHS.isZero() && DemandedRHS.isZero()) ||
2808	(!DemandedLHS.isZero() && !DemandedRHS.isZero()))
2809	return false;
2810
2811	// See if the demanded elts of the source op is a splat or we only demand
2812	// one element, which should always be a splat.
2813	// TODO: Handle source ops splats with undefs.
2814	auto CheckSplatSrc = [&](SDValue Src, const APInt &SrcElts) {
2815	APInt SrcUndefs;
2816	return (SrcElts.popcount() == 1) ||
2817	(isSplatValue(V: Src, DemandedElts: SrcElts, UndefElts&: SrcUndefs, Depth: Depth + 1) &&
2818	(SrcElts & SrcUndefs).isZero());
2819	};
2820	if (!DemandedLHS.isZero())
2821	return CheckSplatSrc(V.getOperand(i: 0), DemandedLHS);
2822	return CheckSplatSrc(V.getOperand(i: 1), DemandedRHS);
2823	}
2824	case ISD::EXTRACT_SUBVECTOR: {
2825	// Offset the demanded elts by the subvector index.
2826	SDValue Src = V.getOperand(i: 0);
2827	// We don't support scalable vectors at the moment.
2828	if (Src.getValueType().isScalableVector())
2829	return false;
2830	uint64_t Idx = V.getConstantOperandVal(i: 1);
2831	unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2832	APInt UndefSrcElts;
2833	APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
2834	if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) {
2835	UndefElts = UndefSrcElts.extractBits(numBits: NumElts, bitPosition: Idx);
2836	return true;
2837	}
2838	break;
2839	}
2840	case ISD::ANY_EXTEND_VECTOR_INREG:
2841	case ISD::SIGN_EXTEND_VECTOR_INREG:
2842	case ISD::ZERO_EXTEND_VECTOR_INREG: {
2843	// Widen the demanded elts by the src element count.
2844	SDValue Src = V.getOperand(i: 0);
2845	// We don't support scalable vectors at the moment.
2846	if (Src.getValueType().isScalableVector())
2847	return false;
2848	unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2849	APInt UndefSrcElts;
2850	APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts);
2851	if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) {
2852	UndefElts = UndefSrcElts.trunc(width: NumElts);
2853	return true;
2854	}
2855	break;
2856	}
2857	case ISD::BITCAST: {
2858	SDValue Src = V.getOperand(i: 0);
2859	EVT SrcVT = Src.getValueType();
2860	unsigned SrcBitWidth = SrcVT.getScalarSizeInBits();
2861	unsigned BitWidth = VT.getScalarSizeInBits();
2862
2863	// Ignore bitcasts from unsupported types.
2864	// TODO: Add fp support?
2865	if (!SrcVT.isVector() || !SrcVT.isInteger() || !VT.isInteger())
2866	break;
2867
2868	// Bitcast 'small element' vector to 'large element' vector.
2869	if ((BitWidth % SrcBitWidth) == 0) {
2870	// See if each sub element is a splat.
2871	unsigned Scale = BitWidth / SrcBitWidth;
2872	unsigned NumSrcElts = SrcVT.getVectorNumElements();
2873	APInt ScaledDemandedElts =
2874	APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
2875	for (unsigned I = 0; I != Scale; ++I) {
2876	APInt SubUndefElts;
2877	APInt SubDemandedElt = APInt::getOneBitSet(numBits: Scale, BitNo: I);
2878	APInt SubDemandedElts = APInt::getSplat(NewLen: NumSrcElts, V: SubDemandedElt);
2879	SubDemandedElts &= ScaledDemandedElts;
2880	if (!isSplatValue(V: Src, DemandedElts: SubDemandedElts, UndefElts&: SubUndefElts, Depth: Depth + 1))
2881	return false;
2882	// TODO: Add support for merging sub undef elements.
2883	if (!SubUndefElts.isZero())
2884	return false;
2885	}
2886	return true;
2887	}
2888	break;
2889	}
2890	}
2891
2892	return false;
2893	}
2894
2895	/// Helper wrapper to main isSplatValue function.
2896	bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) const {
2897	EVT VT = V.getValueType();
2898	assert(VT.isVector() && "Vector type expected");
2899
2900	APInt UndefElts;
2901	// Since the number of lanes in a scalable vector is unknown at compile time,
2902	// we track one bit which is implicitly broadcast to all lanes. This means
2903	// that all lanes in a scalable vector are considered demanded.
2904	APInt DemandedElts
2905	= APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements());
2906	return isSplatValue(V, DemandedElts, UndefElts) &&
2907	(AllowUndefs || !UndefElts);
2908	}
2909
2910	SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) {
2911	V = peekThroughExtractSubvectors(V);
2912
2913	EVT VT = V.getValueType();
2914	unsigned Opcode = V.getOpcode();
2915	switch (Opcode) {
2916	default: {
2917	APInt UndefElts;
2918	// Since the number of lanes in a scalable vector is unknown at compile time,
2919	// we track one bit which is implicitly broadcast to all lanes. This means
2920	// that all lanes in a scalable vector are considered demanded.
2921	APInt DemandedElts
2922	= APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements());
2923
2924	if (isSplatValue(V, DemandedElts, UndefElts)) {
2925	if (VT.isScalableVector()) {
2926	// DemandedElts and UndefElts are ignored for scalable vectors, since
2927	// the only supported cases are SPLAT_VECTOR nodes.
2928	SplatIdx = 0;
2929	} else {
2930	// Handle case where all demanded elements are UNDEF.
2931	if (DemandedElts.isSubsetOf(RHS: UndefElts)) {
2932	SplatIdx = 0;
2933	return getUNDEF(VT);
2934	}
2935	SplatIdx = (UndefElts & DemandedElts).countr_one();
2936	}
2937	return V;
2938	}
2939	break;
2940	}
2941	case ISD::SPLAT_VECTOR:
2942	SplatIdx = 0;
2943	return V;
2944	case ISD::VECTOR_SHUFFLE: {
2945	assert(!VT.isScalableVector());
2946	// Check if this is a shuffle node doing a splat.
2947	// TODO - remove this and rely purely on SelectionDAG::isSplatValue,
2948	// getTargetVShiftNode currently struggles without the splat source.
2949	auto *SVN = cast<ShuffleVectorSDNode>(Val&: V);
2950	if (!SVN->isSplat())
2951	break;
2952	int Idx = SVN->getSplatIndex();
2953	int NumElts = V.getValueType().getVectorNumElements();
2954	SplatIdx = Idx % NumElts;
2955	return V.getOperand(i: Idx / NumElts);
2956	}
2957	}
2958
2959	return SDValue();
2960	}
2961
2962	SDValue SelectionDAG::getSplatValue(SDValue V, bool LegalTypes) {
2963	int SplatIdx;
2964	if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) {
2965	EVT SVT = SrcVector.getValueType().getScalarType();
2966	EVT LegalSVT = SVT;
2967	if (LegalTypes && !TLI->isTypeLegal(VT: SVT)) {
2968	if (!SVT.isInteger())
2969	return SDValue();
2970	LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
2971	if (LegalSVT.bitsLT(VT: SVT))
2972	return SDValue();
2973	}
2974	return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(V), VT: LegalSVT, N1: SrcVector,
2975	N2: getVectorIdxConstant(Val: SplatIdx, DL: SDLoc(V)));
2976	}
2977	return SDValue();
2978	}
2979
2980	const APInt *
2981	SelectionDAG::getValidShiftAmountConstant(SDValue V,
2982	const APInt &DemandedElts) const {
2983	assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
2984	V.getOpcode() == ISD::SRA) &&
2985	"Unknown shift node");
2986	unsigned BitWidth = V.getScalarValueSizeInBits();
2987	if (ConstantSDNode *SA = isConstOrConstSplat(N: V.getOperand(i: 1), DemandedElts)) {
2988	// Shifting more than the bitwidth is not valid.
2989	const APInt &ShAmt = SA->getAPIntValue();
2990	if (ShAmt.ult(RHS: BitWidth))
2991	return &ShAmt;
2992	}
2993	return nullptr;
2994	}
2995
2996	const APInt *SelectionDAG::getValidShiftAmountConstant(SDValue V) const {
2997	EVT VT = V.getValueType();
2998	APInt DemandedElts = VT.isFixedLengthVector()
2999	? APInt::getAllOnes(numBits: VT.getVectorNumElements())
3000	: APInt(1, 1);
3001	return getValidShiftAmountConstant(V, DemandedElts);
3002	}
3003
3004	const APInt *SelectionDAG::getValidMinimumShiftAmountConstant(
3005	SDValue V, const APInt &DemandedElts) const {
3006	assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3007	V.getOpcode() == ISD::SRA) &&
3008	"Unknown shift node");
3009	if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts))
3010	return ValidAmt;
3011	unsigned BitWidth = V.getScalarValueSizeInBits();
3012	auto *BV = dyn_cast<BuildVectorSDNode>(Val: V.getOperand(i: 1));
3013	if (!BV)
3014	return nullptr;
3015	const APInt *MinShAmt = nullptr;
3016	for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3017	if (!DemandedElts[i])
3018	continue;
3019	auto *SA = dyn_cast<ConstantSDNode>(Val: BV->getOperand(Num: i));
3020	if (!SA)
3021	return nullptr;
3022	// Shifting more than the bitwidth is not valid.
3023	const APInt &ShAmt = SA->getAPIntValue();
3024	if (ShAmt.uge(RHS: BitWidth))
3025	return nullptr;
3026	if (MinShAmt && MinShAmt->ule(RHS: ShAmt))
3027	continue;
3028	MinShAmt = &ShAmt;
3029	}
3030	return MinShAmt;
3031	}
3032
3033	const APInt *SelectionDAG::getValidMinimumShiftAmountConstant(SDValue V) const {
3034	EVT VT = V.getValueType();
3035	APInt DemandedElts = VT.isFixedLengthVector()
3036	? APInt::getAllOnes(numBits: VT.getVectorNumElements())
3037	: APInt(1, 1);
3038	return getValidMinimumShiftAmountConstant(V, DemandedElts);
3039	}
3040
3041	const APInt *SelectionDAG::getValidMaximumShiftAmountConstant(
3042	SDValue V, const APInt &DemandedElts) const {
3043	assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3044	V.getOpcode() == ISD::SRA) &&
3045	"Unknown shift node");
3046	if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts))
3047	return ValidAmt;
3048	unsigned BitWidth = V.getScalarValueSizeInBits();
3049	auto *BV = dyn_cast<BuildVectorSDNode>(Val: V.getOperand(i: 1));
3050	if (!BV)
3051	return nullptr;
3052	const APInt *MaxShAmt = nullptr;
3053	for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3054	if (!DemandedElts[i])
3055	continue;
3056	auto *SA = dyn_cast<ConstantSDNode>(Val: BV->getOperand(Num: i));
3057	if (!SA)
3058	return nullptr;
3059	// Shifting more than the bitwidth is not valid.
3060	const APInt &ShAmt = SA->getAPIntValue();
3061	if (ShAmt.uge(RHS: BitWidth))
3062	return nullptr;
3063	if (MaxShAmt && MaxShAmt->uge(RHS: ShAmt))
3064	continue;
3065	MaxShAmt = &ShAmt;
3066	}
3067	return MaxShAmt;
3068	}
3069
3070	const APInt *SelectionDAG::getValidMaximumShiftAmountConstant(SDValue V) const {
3071	EVT VT = V.getValueType();
3072	APInt DemandedElts = VT.isFixedLengthVector()
3073	? APInt::getAllOnes(numBits: VT.getVectorNumElements())
3074	: APInt(1, 1);
3075	return getValidMaximumShiftAmountConstant(V, DemandedElts);
3076	}
3077
3078	/// Determine which bits of Op are known to be either zero or one and return
3079	/// them in Known. For vectors, the known bits are those that are shared by
3080	/// every vector element.
3081	KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const {
3082	EVT VT = Op.getValueType();
3083
3084	// Since the number of lanes in a scalable vector is unknown at compile time,
3085	// we track one bit which is implicitly broadcast to all lanes. This means
3086	// that all lanes in a scalable vector are considered demanded.
3087	APInt DemandedElts = VT.isFixedLengthVector()
3088	? APInt::getAllOnes(numBits: VT.getVectorNumElements())
3089	: APInt(1, 1);
3090	return computeKnownBits(Op, DemandedElts, Depth);
3091	}
3092
3093	/// Determine which bits of Op are known to be either zero or one and return
3094	/// them in Known. The DemandedElts argument allows us to only collect the known
3095	/// bits that are shared by the requested vector elements.
3096	KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
3097	unsigned Depth) const {
3098	unsigned BitWidth = Op.getScalarValueSizeInBits();
3099
3100	KnownBits Known(BitWidth); // Don't know anything.
3101
3102	if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
3103	// We know all of the bits for a constant!
3104	return KnownBits::makeConstant(C: C->getAPIntValue());
3105	}
3106	if (auto *C = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
3107	// We know all of the bits for a constant fp!
3108	return KnownBits::makeConstant(C: C->getValueAPF().bitcastToAPInt());
3109	}
3110
3111	if (Depth >= MaxRecursionDepth)
3112	return Known; // Limit search depth.
3113
3114	KnownBits Known2;
3115	unsigned NumElts = DemandedElts.getBitWidth();
3116	assert((!Op.getValueType().isFixedLengthVector() ||
3117	NumElts == Op.getValueType().getVectorNumElements()) &&
3118	"Unexpected vector size");
3119
3120	if (!DemandedElts)
3121	return Known; // No demanded elts, better to assume we don't know anything.
3122
3123	unsigned Opcode = Op.getOpcode();
3124	switch (Opcode) {
3125	case ISD::MERGE_VALUES:
3126	return computeKnownBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts,
3127	Depth: Depth + 1);
3128	case ISD::SPLAT_VECTOR: {
3129	SDValue SrcOp = Op.getOperand(i: 0);
3130	assert(SrcOp.getValueSizeInBits() >= BitWidth &&
3131	"Expected SPLAT_VECTOR implicit truncation");
3132	// Implicitly truncate the bits to match the official semantics of
3133	// SPLAT_VECTOR.
3134	Known = computeKnownBits(Op: SrcOp, Depth: Depth + 1).trunc(BitWidth);
3135	break;
3136	}
3137	case ISD::SPLAT_VECTOR_PARTS: {
3138	unsigned ScalarSize = Op.getOperand(i: 0).getScalarValueSizeInBits();
3139	assert(ScalarSize * Op.getNumOperands() == BitWidth &&
3140	"Expected SPLAT_VECTOR_PARTS scalars to cover element width");
3141	for (auto [I, SrcOp] : enumerate(First: Op->ops())) {
3142	Known.insertBits(SubBits: computeKnownBits(Op: SrcOp, Depth: Depth + 1), BitPosition: ScalarSize * I);
3143	}
3144	break;
3145	}
3146	case ISD::STEP_VECTOR: {
3147	const APInt &Step = Op.getConstantOperandAPInt(i: 0);
3148
3149	if (Step.isPowerOf2())
3150	Known.Zero.setLowBits(Step.logBase2());
3151
3152	const Function &F = getMachineFunction().getFunction();
3153
3154	if (!isUIntN(N: BitWidth, x: Op.getValueType().getVectorMinNumElements()))
3155	break;
3156	const APInt MinNumElts =
3157	APInt(BitWidth, Op.getValueType().getVectorMinNumElements());
3158
3159	bool Overflow;
3160	const APInt MaxNumElts = getVScaleRange(F: &F, BitWidth)
3161	.getUnsignedMax()
3162	.umul_ov(RHS: MinNumElts, Overflow);
3163	if (Overflow)
3164	break;
3165
3166	const APInt MaxValue = (MaxNumElts - 1).umul_ov(RHS: Step, Overflow);
3167	if (Overflow)
3168	break;
3169
3170	Known.Zero.setHighBits(MaxValue.countl_zero());
3171	break;
3172	}
3173	case ISD::BUILD_VECTOR:
3174	assert(!Op.getValueType().isScalableVector());
3175	// Collect the known bits that are shared by every demanded vector element.
3176	Known.Zero.setAllBits(); Known.One.setAllBits();
3177	for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
3178	if (!DemandedElts[i])
3179	continue;
3180
3181	SDValue SrcOp = Op.getOperand(i);
3182	Known2 = computeKnownBits(Op: SrcOp, Depth: Depth + 1);
3183
3184	// BUILD_VECTOR can implicitly truncate sources, we must handle this.
3185	if (SrcOp.getValueSizeInBits() != BitWidth) {
3186	assert(SrcOp.getValueSizeInBits() > BitWidth &&
3187	"Expected BUILD_VECTOR implicit truncation");
3188	Known2 = Known2.trunc(BitWidth);
3189	}
3190
3191	// Known bits are the values that are shared by every demanded element.
3192	Known = Known.intersectWith(RHS: Known2);
3193
3194	// If we don't know any bits, early out.
3195	if (Known.isUnknown())
3196	break;
3197	}
3198	break;
3199	case ISD::VECTOR_SHUFFLE: {
3200	assert(!Op.getValueType().isScalableVector());
3201	// Collect the known bits that are shared by every vector element referenced
3202	// by the shuffle.
3203	APInt DemandedLHS, DemandedRHS;
3204	const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
3205	assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
3206	if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
3207	DemandedLHS, DemandedRHS))
3208	break;
3209
3210	// Known bits are the values that are shared by every demanded element.
3211	Known.Zero.setAllBits(); Known.One.setAllBits();
3212	if (!!DemandedLHS) {
3213	SDValue LHS = Op.getOperand(i: 0);
3214	Known2 = computeKnownBits(Op: LHS, DemandedElts: DemandedLHS, Depth: Depth + 1);
3215	Known = Known.intersectWith(RHS: Known2);
3216	}
3217	// If we don't know any bits, early out.
3218	if (Known.isUnknown())
3219	break;
3220	if (!!DemandedRHS) {
3221	SDValue RHS = Op.getOperand(i: 1);
3222	Known2 = computeKnownBits(Op: RHS, DemandedElts: DemandedRHS, Depth: Depth + 1);
3223	Known = Known.intersectWith(RHS: Known2);
3224	}
3225	break;
3226	}
3227	case ISD::VSCALE: {
3228	const Function &F = getMachineFunction().getFunction();
3229	const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
3230	Known = getVScaleRange(F: &F, BitWidth).multiply(Other: Multiplier).toKnownBits();
3231	break;
3232	}
3233	case ISD::CONCAT_VECTORS: {
3234	if (Op.getValueType().isScalableVector())
3235	break;
3236	// Split DemandedElts and test each of the demanded subvectors.
3237	Known.Zero.setAllBits(); Known.One.setAllBits();
3238	EVT SubVectorVT = Op.getOperand(i: 0).getValueType();
3239	unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
3240	unsigned NumSubVectors = Op.getNumOperands();
3241	for (unsigned i = 0; i != NumSubVectors; ++i) {
3242	APInt DemandedSub =
3243	DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts);
3244	if (!!DemandedSub) {
3245	SDValue Sub = Op.getOperand(i);
3246	Known2 = computeKnownBits(Op: Sub, DemandedElts: DemandedSub, Depth: Depth + 1);
3247	Known = Known.intersectWith(RHS: Known2);
3248	}
3249	// If we don't know any bits, early out.
3250	if (Known.isUnknown())
3251	break;
3252	}
3253	break;
3254	}
3255	case ISD::INSERT_SUBVECTOR: {
3256	if (Op.getValueType().isScalableVector())
3257	break;
3258	// Demand any elements from the subvector and the remainder from the src its
3259	// inserted into.
3260	SDValue Src = Op.getOperand(i: 0);
3261	SDValue Sub = Op.getOperand(i: 1);
3262	uint64_t Idx = Op.getConstantOperandVal(i: 2);
3263	unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3264	APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
3265	APInt DemandedSrcElts = DemandedElts;
3266	DemandedSrcElts.insertBits(SubBits: APInt::getZero(numBits: NumSubElts), bitPosition: Idx);
3267
3268	Known.One.setAllBits();
3269	Known.Zero.setAllBits();
3270	if (!!DemandedSubElts) {
3271	Known = computeKnownBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1);
3272	if (Known.isUnknown())
3273	break; // early-out.
3274	}
3275	if (!!DemandedSrcElts) {
3276	Known2 = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3277	Known = Known.intersectWith(RHS: Known2);
3278	}
3279	break;
3280	}
3281	case ISD::EXTRACT_SUBVECTOR: {
3282	// Offset the demanded elts by the subvector index.
3283	SDValue Src = Op.getOperand(i: 0);
3284	// Bail until we can represent demanded elements for scalable vectors.
3285	if (Op.getValueType().isScalableVector() || Src.getValueType().isScalableVector())
3286	break;
3287	uint64_t Idx = Op.getConstantOperandVal(i: 1);
3288	unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3289	APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3290	Known = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3291	break;
3292	}
3293	case ISD::SCALAR_TO_VECTOR: {
3294	if (Op.getValueType().isScalableVector())
3295	break;
3296	// We know about scalar_to_vector as much as we know about it source,
3297	// which becomes the first element of otherwise unknown vector.
3298	if (DemandedElts != 1)
3299	break;
3300
3301	SDValue N0 = Op.getOperand(i: 0);
3302	Known = computeKnownBits(Op: N0, Depth: Depth + 1);
3303	if (N0.getValueSizeInBits() != BitWidth)
3304	Known = Known.trunc(BitWidth);
3305
3306	break;
3307	}
3308	case ISD::BITCAST: {
3309	if (Op.getValueType().isScalableVector())
3310	break;
3311
3312	SDValue N0 = Op.getOperand(i: 0);
3313	EVT SubVT = N0.getValueType();
3314	unsigned SubBitWidth = SubVT.getScalarSizeInBits();
3315
3316	// Ignore bitcasts from unsupported types.
3317	if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))
3318	break;
3319
3320	// Fast handling of 'identity' bitcasts.
3321	if (BitWidth == SubBitWidth) {
3322	Known = computeKnownBits(Op: N0, DemandedElts, Depth: Depth + 1);
3323	break;
3324	}
3325
3326	bool IsLE = getDataLayout().isLittleEndian();
3327
3328	// Bitcast 'small element' vector to 'large element' scalar/vector.
3329	if ((BitWidth % SubBitWidth) == 0) {
3330	assert(N0.getValueType().isVector() && "Expected bitcast from vector");
3331
3332	// Collect known bits for the (larger) output by collecting the known
3333	// bits from each set of sub elements and shift these into place.
3334	// We need to separately call computeKnownBits for each set of
3335	// sub elements as the knownbits for each is likely to be different.
3336	unsigned SubScale = BitWidth / SubBitWidth;
3337	APInt SubDemandedElts(NumElts * SubScale, 0);
3338	for (unsigned i = 0; i != NumElts; ++i)
3339	if (DemandedElts[i])
3340	SubDemandedElts.setBit(i * SubScale);
3341
3342	for (unsigned i = 0; i != SubScale; ++i) {
3343	Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts.shl(shiftAmt: i),
3344	Depth: Depth + 1);
3345	unsigned Shifts = IsLE ? i : SubScale - 1 - i;
3346	Known.insertBits(SubBits: Known2, BitPosition: SubBitWidth * Shifts);
3347	}
3348	}
3349
3350	// Bitcast 'large element' scalar/vector to 'small element' vector.
3351	if ((SubBitWidth % BitWidth) == 0) {
3352	assert(Op.getValueType().isVector() && "Expected bitcast to vector");
3353
3354	// Collect known bits for the (smaller) output by collecting the known
3355	// bits from the overlapping larger input elements and extracting the
3356	// sub sections we actually care about.
3357	unsigned SubScale = SubBitWidth / BitWidth;
3358	APInt SubDemandedElts =
3359	APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / SubScale);
3360	Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts, Depth: Depth + 1);
3361
3362	Known.Zero.setAllBits(); Known.One.setAllBits();
3363	for (unsigned i = 0; i != NumElts; ++i)
3364	if (DemandedElts[i]) {
3365	unsigned Shifts = IsLE ? i : NumElts - 1 - i;
3366	unsigned Offset = (Shifts % SubScale) * BitWidth;
3367	Known = Known.intersectWith(RHS: Known2.extractBits(NumBits: BitWidth, BitPosition: Offset));
3368	// If we don't know any bits, early out.
3369	if (Known.isUnknown())
3370	break;
3371	}
3372	}
3373	break;
3374	}
3375	case ISD::AND:
3376	Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3377	Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3378
3379	Known &= Known2;
3380	break;
3381	case ISD::OR:
3382	Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3383	Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3384
3385	Known |= Known2;
3386	break;
3387	case ISD::XOR:
3388	Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3389	Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3390
3391	Known ^= Known2;
3392	break;
3393	case ISD::MUL: {
3394	Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3395	Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3396	bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3397	// TODO: SelfMultiply can be poison, but not undef.
3398	if (SelfMultiply)
3399	SelfMultiply &= isGuaranteedNotToBeUndefOrPoison(
3400	Op: Op.getOperand(i: 0), DemandedElts, PoisonOnly: false, Depth: Depth + 1);
3401	Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3402
3403	// If the multiplication is known not to overflow, the product of a number
3404	// with itself is non-negative. Only do this if we didn't already computed
3405	// the opposite value for the sign bit.
3406	if (Op->getFlags().hasNoSignedWrap() &&
3407	Op.getOperand(i: 0) == Op.getOperand(i: 1) &&
3408	!Known.isNegative())
3409	Known.makeNonNegative();
3410	break;
3411	}
3412	case ISD::MULHU: {
3413	Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3414	Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3415	Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
3416	break;
3417	}
3418	case ISD::MULHS: {
3419	Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3420	Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3421	Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
3422	break;
3423	}
3424	case ISD::ABDU: {
3425	Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3426	Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3427	Known = KnownBits::abdu(LHS: Known, RHS: Known2);
3428	break;
3429	}
3430	case ISD::ABDS: {
3431	Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3432	Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3433	Known = KnownBits::abds(LHS: Known, RHS: Known2);
3434	break;
3435	}
3436	case ISD::UMUL_LOHI: {
3437	assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3438	Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3439	Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3440	bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3441	if (Op.getResNo() == 0)
3442	Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3443	else
3444	Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
3445	break;
3446	}
3447	case ISD::SMUL_LOHI: {
3448	assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3449	Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3450	Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3451	bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3452	if (Op.getResNo() == 0)
3453	Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3454	else
3455	Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
3456	break;
3457	}
3458	case ISD::AVGFLOORU:
3459	case ISD::AVGCEILU:
3460	case ISD::AVGFLOORS:
3461	case ISD::AVGCEILS: {
3462	bool IsCeil = Opcode == ISD::AVGCEILU || Opcode == ISD::AVGCEILS;
3463	bool IsSigned = Opcode == ISD::AVGFLOORS || Opcode == ISD::AVGCEILS;
3464	Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3465	Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3466	Known = IsSigned ? Known.sext(BitWidth: BitWidth + 1) : Known.zext(BitWidth: BitWidth + 1);
3467	Known2 = IsSigned ? Known2.sext(BitWidth: BitWidth + 1) : Known2.zext(BitWidth: BitWidth + 1);
3468	KnownBits Carry = KnownBits::makeConstant(C: APInt(1, IsCeil ? 1 : 0));
3469	Known = KnownBits::computeForAddCarry(LHS: Known, RHS: Known2, Carry);
3470	Known = Known.extractBits(NumBits: BitWidth, BitPosition: 1);
3471	break;
3472	}
3473	case ISD::SELECT:
3474	case ISD::VSELECT:
3475	Known = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
3476	// If we don't know any bits, early out.
3477	if (Known.isUnknown())
3478	break;
3479	Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
3480
3481	// Only known if known in both the LHS and RHS.
3482	Known = Known.intersectWith(RHS: Known2);
3483	break;
3484	case ISD::SELECT_CC:
3485	Known = computeKnownBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1);
3486	// If we don't know any bits, early out.
3487	if (Known.isUnknown())
3488	break;
3489	Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
3490
3491	// Only known if known in both the LHS and RHS.
3492	Known = Known.intersectWith(RHS: Known2);
3493	break;
3494	case ISD::SMULO:
3495	case ISD::UMULO:
3496	if (Op.getResNo() != 1)
3497	break;
3498	// The boolean result conforms to getBooleanContents.
3499	// If we know the result of a setcc has the top bits zero, use this info.
3500	// We know that we have an integer-based boolean since these operations
3501	// are only available for integer.
3502	if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) ==
3503	TargetLowering::ZeroOrOneBooleanContent &&
3504	BitWidth > 1)
3505	Known.Zero.setBitsFrom(1);
3506	break;
3507	case ISD::SETCC:
3508	case ISD::SETCCCARRY:
3509	case ISD::STRICT_FSETCC:
3510	case ISD::STRICT_FSETCCS: {
3511	unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
3512	// If we know the result of a setcc has the top bits zero, use this info.
3513	if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) ==
3514	TargetLowering::ZeroOrOneBooleanContent &&
3515	BitWidth > 1)
3516	Known.Zero.setBitsFrom(1);
3517	break;
3518	}
3519	case ISD::SHL:
3520	Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3521	Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3522	Known = KnownBits::shl(LHS: Known, RHS: Known2);
3523
3524	// Minimum shift low bits are known zero.
3525	if (const APInt *ShMinAmt =
3526	getValidMinimumShiftAmountConstant(V: Op, DemandedElts))
3527	Known.Zero.setLowBits(ShMinAmt->getZExtValue());
3528	break;
3529	case ISD::SRL:
3530	Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3531	Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3532	Known = KnownBits::lshr(LHS: Known, RHS: Known2, /*ShAmtNonZero=*/false,
3533	Exact: Op->getFlags().hasExact());
3534
3535	// Minimum shift high bits are known zero.
3536	if (const APInt *ShMinAmt =
3537	getValidMinimumShiftAmountConstant(V: Op, DemandedElts))
3538	Known.Zero.setHighBits(ShMinAmt->getZExtValue());
3539	break;
3540	case ISD::SRA:
3541	Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3542	Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3543	Known = KnownBits::ashr(LHS: Known, RHS: Known2, /*ShAmtNonZero=*/false,
3544	Exact: Op->getFlags().hasExact());
3545	break;
3546	case ISD::FSHL:
3547	case ISD::FSHR:
3548	if (ConstantSDNode *C = isConstOrConstSplat(N: Op.getOperand(i: 2), DemandedElts)) {
3549	unsigned Amt = C->getAPIntValue().urem(RHS: BitWidth);
3550
3551	// For fshl, 0-shift returns the 1st arg.
3552	// For fshr, 0-shift returns the 2nd arg.
3553	if (Amt == 0) {
3554	Known = computeKnownBits(Op: Op.getOperand(i: Opcode == ISD::FSHL ? 0 : 1),
3555	DemandedElts, Depth: Depth + 1);
3556	break;
3557	}
3558
3559	// fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
3560	// fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
3561	Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3562	Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3563	if (Opcode == ISD::FSHL) {
3564	Known.One <<= Amt;
3565	Known.Zero <<= Amt;
3566	Known2.One.lshrInPlace(ShiftAmt: BitWidth - Amt);
3567	Known2.Zero.lshrInPlace(ShiftAmt: BitWidth - Amt);
3568	} else {
3569	Known.One <<= BitWidth - Amt;
3570	Known.Zero <<= BitWidth - Amt;
3571	Known2.One.lshrInPlace(ShiftAmt: Amt);
3572	Known2.Zero.lshrInPlace(ShiftAmt: Amt);
3573	}
3574	Known = Known.unionWith(RHS: Known2);
3575	}
3576	break;
3577	case ISD::SHL_PARTS:
3578	case ISD::SRA_PARTS:
3579	case ISD::SRL_PARTS: {
3580	assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3581
3582	// Collect lo/hi source values and concatenate.
3583	unsigned LoBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
3584	unsigned HiBits = Op.getOperand(i: 1).getScalarValueSizeInBits();
3585	Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3586	Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3587	Known = Known2.concat(Lo: Known);
3588
3589	// Collect shift amount.
3590	Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
3591
3592	if (Opcode == ISD::SHL_PARTS)
3593	Known = KnownBits::shl(LHS: Known, RHS: Known2);
3594	else if (Opcode == ISD::SRA_PARTS)
3595	Known = KnownBits::ashr(LHS: Known, RHS: Known2);
3596	else // if (Opcode == ISD::SRL_PARTS)
3597	Known = KnownBits::lshr(LHS: Known, RHS: Known2);
3598
3599	// TODO: Minimum shift low/high bits are known zero.
3600
3601	if (Op.getResNo() == 0)
3602	Known = Known.extractBits(NumBits: LoBits, BitPosition: 0);
3603	else
3604	Known = Known.extractBits(NumBits: HiBits, BitPosition: LoBits);
3605	break;
3606	}
3607	case ISD::SIGN_EXTEND_INREG: {
3608	Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3609	EVT EVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
3610	Known = Known.sextInReg(SrcBitWidth: EVT.getScalarSizeInBits());
3611	break;
3612	}
3613	case ISD::CTTZ:
3614	case ISD::CTTZ_ZERO_UNDEF: {
3615	Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3616	// If we have a known 1, its position is our upper bound.
3617	unsigned PossibleTZ = Known2.countMaxTrailingZeros();
3618	unsigned LowBits = llvm::bit_width(Value: PossibleTZ);
3619	Known.Zero.setBitsFrom(LowBits);
3620	break;
3621	}
3622	case ISD::CTLZ:
3623	case ISD::CTLZ_ZERO_UNDEF: {
3624	Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3625	// If we have a known 1, its position is our upper bound.
3626	unsigned PossibleLZ = Known2.countMaxLeadingZeros();
3627	unsigned LowBits = llvm::bit_width(Value: PossibleLZ);
3628	Known.Zero.setBitsFrom(LowBits);
3629	break;
3630	}
3631	case ISD::CTPOP: {
3632	Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3633	// If we know some of the bits are zero, they can't be one.
3634	unsigned PossibleOnes = Known2.countMaxPopulation();
3635	Known.Zero.setBitsFrom(llvm::bit_width(Value: PossibleOnes));
3636	break;
3637	}
3638	case ISD::PARITY: {
3639	// Parity returns 0 everywhere but the LSB.
3640	Known.Zero.setBitsFrom(1);
3641	break;
3642	}
3643	case ISD::LOAD: {
3644	LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
3645	const Constant *Cst = TLI->getTargetConstantFromLoad(LD);
3646	if (ISD::isNON_EXTLoad(N: LD) && Cst) {
3647	// Determine any common known bits from the loaded constant pool value.
3648	Type *CstTy = Cst->getType();
3649	if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits() &&
3650	!Op.getValueType().isScalableVector()) {
3651	// If its a vector splat, then we can (quickly) reuse the scalar path.
3652	// NOTE: We assume all elements match and none are UNDEF.
3653	if (CstTy->isVectorTy()) {
3654	if (const Constant *Splat = Cst->getSplatValue()) {
3655	Cst = Splat;
3656	CstTy = Cst->getType();
3657	}
3658	}
3659	// TODO - do we need to handle different bitwidths?
3660	if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) {
3661	// Iterate across all vector elements finding common known bits.
3662	Known.One.setAllBits();
3663	Known.Zero.setAllBits();
3664	for (unsigned i = 0; i != NumElts; ++i) {
3665	if (!DemandedElts[i])
3666	continue;
3667	if (Constant *Elt = Cst->getAggregateElement(Elt: i)) {
3668	if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) {
3669	const APInt &Value = CInt->getValue();
3670	Known.One &= Value;
3671	Known.Zero &= ~Value;
3672	continue;
3673	}
3674	if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) {
3675	APInt Value = CFP->getValueAPF().bitcastToAPInt();
3676	Known.One &= Value;
3677	Known.Zero &= ~Value;
3678	continue;
3679	}
3680	}
3681	Known.One.clearAllBits();
3682	Known.Zero.clearAllBits();
3683	break;
3684	}
3685	} else if (BitWidth == CstTy->getPrimitiveSizeInBits()) {
3686	if (auto *CInt = dyn_cast<ConstantInt>(Val: Cst)) {
3687	Known = KnownBits::makeConstant(C: CInt->getValue());
3688	} else if (auto *CFP = dyn_cast<ConstantFP>(Val: Cst)) {
3689	Known =
3690	KnownBits::makeConstant(C: CFP->getValueAPF().bitcastToAPInt());
3691	}
3692	}
3693	}
3694	} else if (Op.getResNo() == 0) {
3695	KnownBits Known0(!LD->getMemoryVT().isScalableVT()
3696	? LD->getMemoryVT().getFixedSizeInBits()
3697	: BitWidth);
3698	EVT VT = Op.getValueType();
3699	// Fill in any known bits from range information. There are 3 types being
3700	// used. The results VT (same vector elt size as BitWidth), the loaded
3701	// MemoryVT (which may or may not be vector) and the range VTs original
3702	// type. The range matadata needs the full range (i.e
3703	// MemoryVT().getSizeInBits()), which is truncated to the correct elt size
3704	// if it is know. These are then extended to the original VT sizes below.
3705	if (const MDNode *MD = LD->getRanges()) {
3706	computeKnownBitsFromRangeMetadata(Ranges: *MD, Known&: Known0);
3707	if (VT.isVector()) {
3708	// Handle truncation to the first demanded element.
3709	// TODO: Figure out which demanded elements are covered
3710	if (DemandedElts != 1 || !getDataLayout().isLittleEndian())
3711	break;
3712	Known0 = Known0.trunc(BitWidth);
3713	}
3714	}
3715
3716	if (LD->getMemoryVT().isVector())
3717	Known0 = Known0.trunc(BitWidth: LD->getMemoryVT().getScalarSizeInBits());
3718
3719	// Extend the Known bits from memory to the size of the result.
3720	if (ISD::isZEXTLoad(N: Op.getNode()))
3721	Known = Known0.zext(BitWidth);
3722	else if (ISD::isSEXTLoad(N: Op.getNode()))
3723	Known = Known0.sext(BitWidth);
3724	else if (ISD::isEXTLoad(N: Op.getNode()))
3725	Known = Known0.anyext(BitWidth);
3726	else
3727	Known = Known0;
3728	assert(Known.getBitWidth() == BitWidth);
3729	return Known;
3730	}
3731	break;
3732	}
3733	case ISD::ZERO_EXTEND_VECTOR_INREG: {
3734	if (Op.getValueType().isScalableVector())
3735	break;
3736	EVT InVT = Op.getOperand(i: 0).getValueType();
3737	APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
3738	Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
3739	Known = Known.zext(BitWidth);
3740	break;
3741	}
3742	case ISD::ZERO_EXTEND: {
3743	Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3744	Known = Known.zext(BitWidth);
3745	break;
3746	}
3747	case ISD::SIGN_EXTEND_VECTOR_INREG: {
3748	if (Op.getValueType().isScalableVector())
3749	break;
3750	EVT InVT = Op.getOperand(i: 0).getValueType();
3751	APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
3752	Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
3753	// If the sign bit is known to be zero or one, then sext will extend
3754	// it to the top bits, else it will just zext.
3755	Known = Known.sext(BitWidth);
3756	break;
3757	}
3758	case ISD::SIGN_EXTEND: {
3759	Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3760	// If the sign bit is known to be zero or one, then sext will extend
3761	// it to the top bits, else it will just zext.
3762	Known = Known.sext(BitWidth);
3763	break;
3764	}
3765	case ISD::ANY_EXTEND_VECTOR_INREG: {
3766	if (Op.getValueType().isScalableVector())
3767	break;
3768	EVT InVT = Op.getOperand(i: 0).getValueType();
3769	APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
3770	Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
3771	Known = Known.anyext(BitWidth);
3772	break;
3773	}
3774	case ISD::ANY_EXTEND: {
3775	Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3776	Known = Known.anyext(BitWidth);
3777	break;
3778	}
3779	case ISD::TRUNCATE: {
3780	Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3781	Known = Known.trunc(BitWidth);
3782	break;
3783	}
3784	case ISD::AssertZext: {
3785	EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
3786	APInt InMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: VT.getSizeInBits());
3787	Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3788	Known.Zero |= (~InMask);
3789	Known.One &= (~Known.Zero);
3790	break;
3791	}
3792	case ISD::AssertAlign: {
3793	unsigned LogOfAlign = Log2(A: cast<AssertAlignSDNode>(Val&: Op)->getAlign());
3794	assert(LogOfAlign != 0);
3795
3796	// TODO: Should use maximum with source
3797	// If a node is guaranteed to be aligned, set low zero bits accordingly as
3798	// well as clearing one bits.
3799	Known.Zero.setLowBits(LogOfAlign);
3800	Known.One.clearLowBits(loBits: LogOfAlign);
3801	break;
3802	}
3803	case ISD::FGETSIGN:
3804	// All bits are zero except the low bit.
3805	Known.Zero.setBitsFrom(1);
3806	break;
3807	case ISD::ADD:
3808	case ISD::SUB: {
3809	SDNodeFlags Flags = Op.getNode()->getFlags();
3810	Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3811	Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3812	Known = KnownBits::computeForAddSub(
3813	Add: Op.getOpcode() == ISD::ADD, NSW: Flags.hasNoSignedWrap(),
3814	NUW: Flags.hasNoUnsignedWrap(), LHS: Known, RHS: Known2);
3815	break;
3816	}
3817	case ISD::USUBO:
3818	case ISD::SSUBO:
3819	case ISD::USUBO_CARRY:
3820	case ISD::SSUBO_CARRY:
3821	if (Op.getResNo() == 1) {
3822	// If we know the result of a setcc has the top bits zero, use this info.
3823	if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) ==
3824	TargetLowering::ZeroOrOneBooleanContent &&
3825	BitWidth > 1)
3826	Known.Zero.setBitsFrom(1);
3827	break;
3828	}
3829	[[fallthrough]];
3830	case ISD::SUBC: {
3831	assert(Op.getResNo() == 0 &&
3832	"We only compute knownbits for the difference here.");
3833
3834	// With USUBO_CARRY and SSUBO_CARRY a borrow bit may be added in.
3835	KnownBits Borrow(1);
3836	if (Opcode == ISD::USUBO_CARRY || Opcode == ISD::SSUBO_CARRY) {
3837	Borrow = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
3838	// Borrow has bit width 1
3839	Borrow = Borrow.trunc(BitWidth: 1);
3840	} else {
3841	Borrow.setAllZero();
3842	}
3843
3844	Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3845	Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3846	Known = KnownBits::computeForSubBorrow(LHS: Known, RHS: Known2, Borrow);
3847	break;
3848	}
3849	case ISD::UADDO:
3850	case ISD::SADDO:
3851	case ISD::UADDO_CARRY:
3852	case ISD::SADDO_CARRY:
3853	if (Op.getResNo() == 1) {
3854	// If we know the result of a setcc has the top bits zero, use this info.
3855	if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) ==
3856	TargetLowering::ZeroOrOneBooleanContent &&
3857	BitWidth > 1)
3858	Known.Zero.setBitsFrom(1);
3859	break;
3860	}
3861	[[fallthrough]];
3862	case ISD::ADDC:
3863	case ISD::ADDE: {
3864	assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.");
3865
3866	// With ADDE and UADDO_CARRY, a carry bit may be added in.
3867	KnownBits Carry(1);
3868	if (Opcode == ISD::ADDE)
3869	// Can't track carry from glue, set carry to unknown.
3870	Carry.resetAll();
3871	else if (Opcode == ISD::UADDO_CARRY || Opcode == ISD::SADDO_CARRY) {
3872	Carry = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
3873	// Carry has bit width 1
3874	Carry = Carry.trunc(BitWidth: 1);
3875	} else {
3876	Carry.setAllZero();
3877	}
3878
3879	Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3880	Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3881	Known = KnownBits::computeForAddCarry(LHS: Known, RHS: Known2, Carry);
3882	break;
3883	}
3884	case ISD::UDIV: {
3885	Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3886	Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3887	Known = KnownBits::udiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact());
3888	break;
3889	}
3890	case ISD::SDIV: {
3891	Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3892	Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3893	Known = KnownBits::sdiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact());
3894	break;
3895	}
3896	case ISD::SREM: {
3897	Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3898	Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3899	Known = KnownBits::srem(LHS: Known, RHS: Known2);
3900	break;
3901	}
3902	case ISD::UREM: {
3903	Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3904	Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3905	Known = KnownBits::urem(LHS: Known, RHS: Known2);
3906	break;
3907	}
3908	case ISD::EXTRACT_ELEMENT: {
3909	Known = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
3910	const unsigned Index = Op.getConstantOperandVal(i: 1);
3911	const unsigned EltBitWidth = Op.getValueSizeInBits();
3912
3913	// Remove low part of known bits mask
3914	Known.Zero = Known.Zero.getHiBits(numBits: Known.getBitWidth() - Index * EltBitWidth);
3915	Known.One = Known.One.getHiBits(numBits: Known.getBitWidth() - Index * EltBitWidth);
3916
3917	// Remove high part of known bit mask
3918	Known = Known.trunc(BitWidth: EltBitWidth);
3919	break;
3920	}
3921	case ISD::EXTRACT_VECTOR_ELT: {
3922	SDValue InVec = Op.getOperand(i: 0);
3923	SDValue EltNo = Op.getOperand(i: 1);
3924	EVT VecVT = InVec.getValueType();
3925	// computeKnownBits not yet implemented for scalable vectors.
3926	if (VecVT.isScalableVector())
3927	break;
3928	const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
3929	const unsigned NumSrcElts = VecVT.getVectorNumElements();
3930
3931	// If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
3932	// anything about the extended bits.
3933	if (BitWidth > EltBitWidth)
3934	Known = Known.trunc(BitWidth: EltBitWidth);
3935
3936	// If we know the element index, just demand that vector element, else for
3937	// an unknown element index, ignore DemandedElts and demand them all.
3938	APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
3939	auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
3940	if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
3941	DemandedSrcElts =
3942	APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
3943
3944	Known = computeKnownBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3945	if (BitWidth > EltBitWidth)
3946	Known = Known.anyext(BitWidth);
3947	break;
3948	}
3949	case ISD::INSERT_VECTOR_ELT: {
3950	if (Op.getValueType().isScalableVector())
3951	break;
3952
3953	// If we know the element index, split the demand between the
3954	// source vector and the inserted element, otherwise assume we need
3955	// the original demanded vector elements and the value.
3956	SDValue InVec = Op.getOperand(i: 0);
3957	SDValue InVal = Op.getOperand(i: 1);
3958	SDValue EltNo = Op.getOperand(i: 2);
3959	bool DemandedVal = true;
3960	APInt DemandedVecElts = DemandedElts;
3961	auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
3962	if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) {
3963	unsigned EltIdx = CEltNo->getZExtValue();
3964	DemandedVal = !!DemandedElts[EltIdx];
3965	DemandedVecElts.clearBit(BitPosition: EltIdx);
3966	}
3967	Known.One.setAllBits();
3968	Known.Zero.setAllBits();
3969	if (DemandedVal) {
3970	Known2 = computeKnownBits(Op: InVal, Depth: Depth + 1);
3971	Known = Known.intersectWith(RHS: Known2.zextOrTrunc(BitWidth));
3972	}
3973	if (!!DemandedVecElts) {
3974	Known2 = computeKnownBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1);
3975	Known = Known.intersectWith(RHS: Known2);
3976	}
3977	break;
3978	}
3979	case ISD::BITREVERSE: {
3980	Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3981	Known = Known2.reverseBits();
3982	break;
3983	}
3984	case ISD::BSWAP: {
3985	Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3986	Known = Known2.byteSwap();
3987	break;
3988	}
3989	case ISD::ABS: {
3990	Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3991	Known = Known2.abs();
3992	break;
3993	}
3994	case ISD::USUBSAT: {
3995	Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3996	Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3997	Known = KnownBits::usub_sat(LHS: Known, RHS: Known2);
3998	break;
3999	}
4000	case ISD::UMIN: {
4001	Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4002	Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4003	Known = KnownBits::umin(LHS: Known, RHS: Known2);
4004	break;
4005	}
4006	case ISD::UMAX: {
4007	Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4008	Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4009	Known = KnownBits::umax(LHS: Known, RHS: Known2);
4010	break;
4011	}
4012	case ISD::SMIN:
4013	case ISD::SMAX: {
4014	// If we have a clamp pattern, we know that the number of sign bits will be
4015	// the minimum of the clamp min/max range.
4016	bool IsMax = (Opcode == ISD::SMAX);
4017	ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
4018	if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)))
4019	if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
4020	CstHigh =
4021	isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts);
4022	if (CstLow && CstHigh) {
4023	if (!IsMax)
4024	std::swap(a&: CstLow, b&: CstHigh);
4025
4026	const APInt &ValueLow = CstLow->getAPIntValue();
4027	const APInt &ValueHigh = CstHigh->getAPIntValue();
4028	if (ValueLow.sle(RHS: ValueHigh)) {
4029	unsigned LowSignBits = ValueLow.getNumSignBits();
4030	unsigned HighSignBits = ValueHigh.getNumSignBits();
4031	unsigned MinSignBits = std::min(a: LowSignBits, b: HighSignBits);
4032	if (ValueLow.isNegative() && ValueHigh.isNegative()) {
4033	Known.One.setHighBits(MinSignBits);
4034	break;
4035	}
4036	if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {
4037	Known.Zero.setHighBits(MinSignBits);
4038	break;
4039	}
4040	}
4041	}
4042
4043	Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4044	Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4045	if (IsMax)
4046	Known = KnownBits::smax(LHS: Known, RHS: Known2);
4047	else
4048	Known = KnownBits::smin(LHS: Known, RHS: Known2);
4049
4050	// For SMAX, if CstLow is non-negative we know the result will be
4051	// non-negative and thus all sign bits are 0.
4052	// TODO: There's an equivalent of this for smin with negative constant for
4053	// known ones.
4054	if (IsMax && CstLow) {
4055	const APInt &ValueLow = CstLow->getAPIntValue();
4056	if (ValueLow.isNonNegative()) {
4057	unsigned SignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4058	Known.Zero.setHighBits(std::min(a: SignBits, b: ValueLow.getNumSignBits()));
4059	}
4060	}
4061
4062	break;
4063	}
4064	case ISD::UINT_TO_FP: {
4065	Known.makeNonNegative();
4066	break;
4067	}
4068	case ISD::SINT_TO_FP: {
4069	Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4070	if (Known2.isNonNegative())
4071	Known.makeNonNegative();
4072	else if (Known2.isNegative())
4073	Known.makeNegative();
4074	break;
4075	}
4076	case ISD::FP_TO_UINT_SAT: {
4077	// FP_TO_UINT_SAT produces an unsigned value that fits in the saturating VT.
4078	EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
4079	Known.Zero |= APInt::getBitsSetFrom(numBits: BitWidth, loBit: VT.getScalarSizeInBits());
4080	break;
4081	}
4082	case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
4083	if (Op.getResNo() == 1) {
4084	// The boolean result conforms to getBooleanContents.
4085	// If we know the result of a setcc has the top bits zero, use this info.
4086	// We know that we have an integer-based boolean since these operations
4087	// are only available for integer.
4088	if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) ==
4089	TargetLowering::ZeroOrOneBooleanContent &&
4090	BitWidth > 1)
4091	Known.Zero.setBitsFrom(1);
4092	break;
4093	}
4094	[[fallthrough]];
4095	case ISD::ATOMIC_CMP_SWAP:
4096	case ISD::ATOMIC_SWAP:
4097	case ISD::ATOMIC_LOAD_ADD:
4098	case ISD::ATOMIC_LOAD_SUB:
4099	case ISD::ATOMIC_LOAD_AND:
4100	case ISD::ATOMIC_LOAD_CLR:
4101	case ISD::ATOMIC_LOAD_OR:
4102	case ISD::ATOMIC_LOAD_XOR:
4103	case ISD::ATOMIC_LOAD_NAND:
4104	case ISD::ATOMIC_LOAD_MIN:
4105	case ISD::ATOMIC_LOAD_MAX:
4106	case ISD::ATOMIC_LOAD_UMIN:
4107	case ISD::ATOMIC_LOAD_UMAX:
4108	case ISD::ATOMIC_LOAD: {
4109	unsigned MemBits =
4110	cast<AtomicSDNode>(Val&: Op)->getMemoryVT().getScalarSizeInBits();
4111	// If we are looking at the loaded value.
4112	if (Op.getResNo() == 0) {
4113	if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
4114	Known.Zero.setBitsFrom(MemBits);
4115	else if (Op->getOpcode() == ISD::ATOMIC_LOAD &&
4116	cast<AtomicSDNode>(Val&: Op)->getExtensionType() == ISD::ZEXTLOAD)
4117	Known.Zero.setBitsFrom(MemBits);
4118	}
4119	break;
4120	}
4121	case ISD::FrameIndex:
4122	case ISD::TargetFrameIndex:
4123	TLI->computeKnownBitsForFrameIndex(FIOp: cast<FrameIndexSDNode>(Val&: Op)->getIndex(),
4124	Known, MF: getMachineFunction());
4125	break;
4126
4127	default:
4128	if (Opcode < ISD::BUILTIN_OP_END)
4129	break;
4130	[[fallthrough]];
4131	case ISD::INTRINSIC_WO_CHAIN:
4132	case ISD::INTRINSIC_W_CHAIN:
4133	case ISD::INTRINSIC_VOID:
4134	// TODO: Probably okay to remove after audit; here to reduce change size
4135	// in initial enablement patch for scalable vectors
4136	if (Op.getValueType().isScalableVector())
4137	break;
4138
4139	// Allow the target to implement this method for its nodes.
4140	TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, DAG: *this, Depth);
4141	break;
4142	}
4143
4144	assert(!Known.hasConflict() && "Bits known to be one AND zero?");
4145	return Known;
4146	}
4147
4148	/// Convert ConstantRange OverflowResult into SelectionDAG::OverflowKind.
4149	static SelectionDAG::OverflowKind mapOverflowResult(ConstantRange::OverflowResult OR) {
4150	switch (OR) {
4151	case ConstantRange::OverflowResult::MayOverflow:
4152	return SelectionDAG::OFK_Sometime;
4153	case ConstantRange::OverflowResult::AlwaysOverflowsLow:
4154	case ConstantRange::OverflowResult::AlwaysOverflowsHigh:
4155	return SelectionDAG::OFK_Always;
4156	case ConstantRange::OverflowResult::NeverOverflows:
4157	return SelectionDAG::OFK_Never;
4158	}
4159	llvm_unreachable("Unknown OverflowResult");
4160	}
4161
4162	SelectionDAG::OverflowKind
4163	SelectionDAG::computeOverflowForSignedAdd(SDValue N0, SDValue N1) const {
4164	// X + 0 never overflow
4165	if (isNullConstant(V: N1))
4166	return OFK_Never;
4167
4168	// If both operands each have at least two sign bits, the addition
4169	// cannot overflow.
4170	if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1)
4171	return OFK_Never;
4172
4173	// TODO: Add ConstantRange::signedAddMayOverflow handling.
4174	return OFK_Sometime;
4175	}
4176
4177	SelectionDAG::OverflowKind
4178	SelectionDAG::computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const {
4179	// X + 0 never overflow
4180	if (isNullConstant(V: N1))
4181	return OFK_Never;
4182
4183	// mulhi + 1 never overflow
4184	KnownBits N1Known = computeKnownBits(Op: N1);
4185	if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
4186	N1Known.getMaxValue().ult(RHS: 2))
4187	return OFK_Never;
4188
4189	KnownBits N0Known = computeKnownBits(Op: N0);
4190	if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1 &&
4191	N0Known.getMaxValue().ult(RHS: 2))
4192	return OFK_Never;
4193
4194	// Fallback to ConstantRange::unsignedAddMayOverflow handling.
4195	ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false);
4196	ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false);
4197	return mapOverflowResult(OR: N0Range.unsignedAddMayOverflow(Other: N1Range));
4198	}
4199
4200	SelectionDAG::OverflowKind
4201	SelectionDAG::computeOverflowForSignedSub(SDValue N0, SDValue N1) const {
4202	// X - 0 never overflow
4203	if (isNullConstant(V: N1))
4204	return OFK_Never;
4205
4206	// If both operands each have at least two sign bits, the subtraction
4207	// cannot overflow.
4208	if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1)
4209	return OFK_Never;
4210
4211	KnownBits N0Known = computeKnownBits(Op: N0);
4212	KnownBits N1Known = computeKnownBits(Op: N1);
4213	ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: true);
4214	ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: true);
4215	return mapOverflowResult(OR: N0Range.signedSubMayOverflow(Other: N1Range));
4216	}
4217
4218	SelectionDAG::OverflowKind
4219	SelectionDAG::computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const {
4220	// X - 0 never overflow
4221	if (isNullConstant(V: N1))
4222	return OFK_Never;
4223
4224	KnownBits N0Known = computeKnownBits(Op: N0);
4225	KnownBits N1Known = computeKnownBits(Op: N1);
4226	ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false);
4227	ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false);
4228	return mapOverflowResult(OR: N0Range.unsignedSubMayOverflow(Other: N1Range));
4229	}
4230
4231	SelectionDAG::OverflowKind
4232	SelectionDAG::computeOverflowForUnsignedMul(SDValue N0, SDValue N1) const {
4233	// X * 0 and X * 1 never overflow.
4234	if (isNullConstant(V: N1) || isOneConstant(V: N1))
4235	return OFK_Never;
4236
4237	KnownBits N0Known = computeKnownBits(Op: N0);
4238	KnownBits N1Known = computeKnownBits(Op: N1);
4239	ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false);
4240	ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false);
4241	return mapOverflowResult(OR: N0Range.unsignedMulMayOverflow(Other: N1Range));
4242	}
4243
4244	SelectionDAG::OverflowKind
4245	SelectionDAG::computeOverflowForSignedMul(SDValue N0, SDValue N1) const {
4246	// X * 0 and X * 1 never overflow.
4247	if (isNullConstant(V: N1) || isOneConstant(V: N1))
4248	return OFK_Never;
4249
4250	// Get the size of the result.
4251	unsigned BitWidth = N0.getScalarValueSizeInBits();
4252
4253	// Sum of the sign bits.
4254	unsigned SignBits = ComputeNumSignBits(Op: N0) + ComputeNumSignBits(Op: N1);
4255
4256	// If we have enough sign bits, then there's no overflow.
4257	if (SignBits > BitWidth + 1)
4258	return OFK_Never;
4259
4260	if (SignBits == BitWidth + 1) {
4261	// The overflow occurs when the true multiplication of the
4262	// the operands is the minimum negative number.
4263	KnownBits N0Known = computeKnownBits(Op: N0);
4264	KnownBits N1Known = computeKnownBits(Op: N1);
4265	// If one of the operands is non-negative, then there's no
4266	// overflow.
4267	if (N0Known.isNonNegative() || N1Known.isNonNegative())
4268	return OFK_Never;
4269	}
4270
4271	return OFK_Sometime;
4272	}
4273
4274	bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val, unsigned Depth) const {
4275	if (Depth >= MaxRecursionDepth)
4276	return false; // Limit search depth.
4277
4278	EVT OpVT = Val.getValueType();
4279	unsigned BitWidth = OpVT.getScalarSizeInBits();
4280
4281	// Is the constant a known power of 2?
4282	if (ISD::matchUnaryPredicate(Op: Val, Match: [BitWidth](ConstantSDNode *C) {
4283	return C->getAPIntValue().zextOrTrunc(width: BitWidth).isPowerOf2();
4284	}))
4285	return true;
4286
4287	// A left-shift of a constant one will have exactly one bit set because
4288	// shifting the bit off the end is undefined.
4289	if (Val.getOpcode() == ISD::SHL) {
4290	auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0));
4291	if (C && C->getAPIntValue() == 1)
4292	return true;
4293	return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1) &&
4294	isKnownNeverZero(Op: Val, Depth);
4295	}
4296
4297	// Similarly, a logical right-shift of a constant sign-bit will have exactly
4298	// one bit set.
4299	if (Val.getOpcode() == ISD::SRL) {
4300	auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0));
4301	if (C && C->getAPIntValue().isSignMask())
4302	return true;
4303	return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1) &&
4304	isKnownNeverZero(Op: Val, Depth);
4305	}
4306
4307	if (Val.getOpcode() == ISD::ROTL || Val.getOpcode() == ISD::ROTR)
4308	return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1);
4309
4310	// Are all operands of a build vector constant powers of two?
4311	if (Val.getOpcode() == ISD::BUILD_VECTOR)
4312	if (llvm::all_of(Range: Val->ops(), P: [BitWidth](SDValue E) {
4313	if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: E))
4314	return C->getAPIntValue().zextOrTrunc(width: BitWidth).isPowerOf2();
4315	return false;
4316	}))
4317	return true;
4318
4319	// Is the operand of a splat vector a constant power of two?
4320	if (Val.getOpcode() == ISD::SPLAT_VECTOR)
4321	if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: Val->getOperand(Num: 0)))
4322	if (C->getAPIntValue().zextOrTrunc(width: BitWidth).isPowerOf2())
4323	return true;
4324
4325	// vscale(power-of-two) is a power-of-two for some targets
4326	if (Val.getOpcode() == ISD::VSCALE &&
4327	getTargetLoweringInfo().isVScaleKnownToBeAPowerOfTwo() &&
4328	isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1))
4329	return true;
4330
4331	if (Val.getOpcode() == ISD::SMIN || Val.getOpcode() == ISD::SMAX ||
4332	Val.getOpcode() == ISD::UMIN || Val.getOpcode() == ISD::UMAX)
4333	return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), Depth: Depth + 1) &&
4334	isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1);
4335
4336	if (Val.getOpcode() == ISD::SELECT || Val.getOpcode() == ISD::VSELECT)
4337	return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 2), Depth: Depth + 1) &&
4338	isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), Depth: Depth + 1);
4339
4340	// Looking for `x & -x` pattern:
4341	// If x == 0:
4342	// x & -x -> 0
4343	// If x != 0:
4344	// x & -x -> non-zero pow2
4345	// so if we find the pattern return whether we know `x` is non-zero.
4346	SDValue X;
4347	if (sd_match(N: Val, P: m_And(L: m_Value(N&: X), R: m_Neg(V: m_Deferred(V&: X)))))
4348	return isKnownNeverZero(Op: X, Depth);
4349
4350	if (Val.getOpcode() == ISD::ZERO_EXTEND)
4351	return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1);
4352
4353	// More could be done here, though the above checks are enough
4354	// to handle some common cases.
4355	return false;
4356	}
4357
4358	unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
4359	EVT VT = Op.getValueType();
4360
4361	// Since the number of lanes in a scalable vector is unknown at compile time,
4362	// we track one bit which is implicitly broadcast to all lanes. This means
4363	// that all lanes in a scalable vector are considered demanded.
4364	APInt DemandedElts = VT.isFixedLengthVector()
4365	? APInt::getAllOnes(numBits: VT.getVectorNumElements())
4366	: APInt(1, 1);
4367	return ComputeNumSignBits(Op, DemandedElts, Depth);
4368	}
4369
4370	unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
4371	unsigned Depth) const {
4372	EVT VT = Op.getValueType();
4373	assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!");
4374	unsigned VTBits = VT.getScalarSizeInBits();
4375	unsigned NumElts = DemandedElts.getBitWidth();
4376	unsigned Tmp, Tmp2;
4377	unsigned FirstAnswer = 1;
4378
4379	if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
4380	const APInt &Val = C->getAPIntValue();
4381	return Val.getNumSignBits();
4382	}
4383
4384	if (Depth >= MaxRecursionDepth)
4385	return 1; // Limit search depth.
4386
4387	if (!DemandedElts)
4388	return 1; // No demanded elts, better to assume we don't know anything.
4389
4390	unsigned Opcode = Op.getOpcode();
4391	switch (Opcode) {
4392	default: break;
4393	case ISD::AssertSext:
4394	Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits();
4395	return VTBits-Tmp+1;
4396	case ISD::AssertZext:
4397	Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits();
4398	return VTBits-Tmp;
4399	case ISD::MERGE_VALUES:
4400	return ComputeNumSignBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts,
4401	Depth: Depth + 1);
4402	case ISD::SPLAT_VECTOR: {
4403	// Check if the sign bits of source go down as far as the truncated value.
4404	unsigned NumSrcBits = Op.getOperand(i: 0).getValueSizeInBits();
4405	unsigned NumSrcSignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4406	if (NumSrcSignBits > (NumSrcBits - VTBits))
4407	return NumSrcSignBits - (NumSrcBits - VTBits);
4408	break;
4409	}
4410	case ISD::BUILD_VECTOR:
4411	assert(!VT.isScalableVector());
4412	Tmp = VTBits;
4413	for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
4414	if (!DemandedElts[i])
4415	continue;
4416
4417	SDValue SrcOp = Op.getOperand(i);
4418	// BUILD_VECTOR can implicitly truncate sources, we handle this specially
4419	// for constant nodes to ensure we only look at the sign bits.
4420	if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: SrcOp)) {
4421	APInt T = C->getAPIntValue().trunc(width: VTBits);
4422	Tmp2 = T.getNumSignBits();
4423	} else {
4424	Tmp2 = ComputeNumSignBits(Op: SrcOp, Depth: Depth + 1);
4425
4426	if (SrcOp.getValueSizeInBits() != VTBits) {
4427	assert(SrcOp.getValueSizeInBits() > VTBits &&
4428	"Expected BUILD_VECTOR implicit truncation");
4429	unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
4430	Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
4431	}
4432	}
4433	Tmp = std::min(a: Tmp, b: Tmp2);
4434	}
4435	return Tmp;
4436
4437	case ISD::VECTOR_SHUFFLE: {
4438	// Collect the minimum number of sign bits that are shared by every vector
4439	// element referenced by the shuffle.
4440	APInt DemandedLHS, DemandedRHS;
4441	const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
4442	assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
4443	if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
4444	DemandedLHS, DemandedRHS))
4445	return 1;
4446
4447	Tmp = std::numeric_limits<unsigned>::max();
4448	if (!!DemandedLHS)
4449	Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS, Depth: Depth + 1);
4450	if (!!DemandedRHS) {
4451	Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS, Depth: Depth + 1);
4452	Tmp = std::min(a: Tmp, b: Tmp2);
4453	}
4454	// If we don't know anything, early out and try computeKnownBits fall-back.
4455	if (Tmp == 1)
4456	break;
4457	assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4458	return Tmp;
4459	}
4460
4461	case ISD::BITCAST: {
4462	if (VT.isScalableVector())
4463	break;
4464	SDValue N0 = Op.getOperand(i: 0);
4465	EVT SrcVT = N0.getValueType();
4466	unsigned SrcBits = SrcVT.getScalarSizeInBits();
4467
4468	// Ignore bitcasts from unsupported types..
4469	if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))
4470	break;
4471
4472	// Fast handling of 'identity' bitcasts.
4473	if (VTBits == SrcBits)
4474	return ComputeNumSignBits(Op: N0, DemandedElts, Depth: Depth + 1);
4475
4476	bool IsLE = getDataLayout().isLittleEndian();
4477
4478	// Bitcast 'large element' scalar/vector to 'small element' vector.
4479	if ((SrcBits % VTBits) == 0) {
4480	assert(VT.isVector() && "Expected bitcast to vector");
4481
4482	unsigned Scale = SrcBits / VTBits;
4483	APInt SrcDemandedElts =
4484	APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / Scale);
4485
4486	// Fast case - sign splat can be simply split across the small elements.
4487	Tmp = ComputeNumSignBits(Op: N0, DemandedElts: SrcDemandedElts, Depth: Depth + 1);
4488	if (Tmp == SrcBits)
4489	return VTBits;
4490
4491	// Slow case - determine how far the sign extends into each sub-element.
4492	Tmp2 = VTBits;
4493	for (unsigned i = 0; i != NumElts; ++i)
4494	if (DemandedElts[i]) {
4495	unsigned SubOffset = i % Scale;
4496	SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
4497	SubOffset = SubOffset * VTBits;
4498	if (Tmp <= SubOffset)
4499	return 1;
4500	Tmp2 = std::min(a: Tmp2, b: Tmp - SubOffset);
4501	}
4502	return Tmp2;
4503	}
4504	break;
4505	}
4506
4507	case ISD::FP_TO_SINT_SAT:
4508	// FP_TO_SINT_SAT produces a signed value that fits in the saturating VT.
4509	Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits();
4510	return VTBits - Tmp + 1;
4511	case ISD::SIGN_EXTEND:
4512	Tmp = VTBits - Op.getOperand(i: 0).getScalarValueSizeInBits();
4513	return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1) + Tmp;
4514	case ISD::SIGN_EXTEND_INREG:
4515	// Max of the input and what this extends.
4516	Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits();
4517	Tmp = VTBits-Tmp+1;
4518	Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1);
4519	return std::max(a: Tmp, b: Tmp2);
4520	case ISD::SIGN_EXTEND_VECTOR_INREG: {
4521	if (VT.isScalableVector())
4522	break;
4523	SDValue Src = Op.getOperand(i: 0);
4524	EVT SrcVT = Src.getValueType();
4525	APInt DemandedSrcElts = DemandedElts.zext(width: SrcVT.getVectorNumElements());
4526	Tmp = VTBits - SrcVT.getScalarSizeInBits();
4527	return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth+1) + Tmp;
4528	}
4529	case ISD::SRA:
4530	Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4531	// SRA X, C -> adds C sign bits.
4532	if (const APInt *ShAmt =
4533	getValidMinimumShiftAmountConstant(V: Op, DemandedElts))
4534	Tmp = std::min<uint64_t>(a: Tmp + ShAmt->getZExtValue(), b: VTBits);
4535	return Tmp;
4536	case ISD::SHL:
4537	if (const APInt *ShAmt =
4538	getValidMaximumShiftAmountConstant(V: Op, DemandedElts)) {
4539	// shl destroys sign bits, ensure it doesn't shift out all sign bits.
4540	Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4541	if (ShAmt->ult(RHS: Tmp))
4542	return Tmp - ShAmt->getZExtValue();
4543	}
4544	break;
4545	case ISD::AND:
4546	case ISD::OR:
4547	case ISD::XOR: // NOT is handled here.
4548	// Logical binary ops preserve the number of sign bits at the worst.
4549	Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1);
4550	if (Tmp != 1) {
4551	Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
4552	FirstAnswer = std::min(a: Tmp, b: Tmp2);
4553	// We computed what we know about the sign bits as our first
4554	// answer. Now proceed to the generic code that uses
4555	// computeKnownBits, and pick whichever answer is better.
4556	}
4557	break;
4558
4559	case ISD::SELECT:
4560	case ISD::VSELECT:
4561	Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
4562	if (Tmp == 1) return 1; // Early out.
4563	Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
4564	return std::min(a: Tmp, b: Tmp2);
4565	case ISD::SELECT_CC:
4566	Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
4567	if (Tmp == 1) return 1; // Early out.
4568	Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1);
4569	return std::min(a: Tmp, b: Tmp2);
4570
4571	case ISD::SMIN:
4572	case ISD::SMAX: {
4573	// If we have a clamp pattern, we know that the number of sign bits will be
4574	// the minimum of the clamp min/max range.
4575	bool IsMax = (Opcode == ISD::SMAX);
4576	ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
4577	if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)))
4578	if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
4579	CstHigh =
4580	isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts);
4581	if (CstLow && CstHigh) {
4582	if (!IsMax)
4583	std::swap(a&: CstLow, b&: CstHigh);
4584	if (CstLow->getAPIntValue().sle(RHS: CstHigh->getAPIntValue())) {
4585	Tmp = CstLow->getAPIntValue().getNumSignBits();
4586	Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
4587	return std::min(a: Tmp, b: Tmp2);
4588	}
4589	}
4590
4591	// Fallback - just get the minimum number of sign bits of the operands.
4592	Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4593	if (Tmp == 1)
4594	return 1; // Early out.
4595	Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4596	return std::min(a: Tmp, b: Tmp2);
4597	}
4598	case ISD::UMIN:
4599	case ISD::UMAX:
4600	Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4601	if (Tmp == 1)
4602	return 1; // Early out.
4603	Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4604	return std::min(a: Tmp, b: Tmp2);
4605	case ISD::SADDO:
4606	case ISD::UADDO:
4607	case ISD::SADDO_CARRY:
4608	case ISD::UADDO_CARRY:
4609	case ISD::SSUBO:
4610	case ISD::USUBO:
4611	case ISD::SSUBO_CARRY:
4612	case ISD::USUBO_CARRY:
4613	case ISD::SMULO:
4614	case ISD::UMULO:
4615	if (Op.getResNo() != 1)
4616	break;
4617	// The boolean result conforms to getBooleanContents. Fall through.
4618	// If setcc returns 0/-1, all bits are sign bits.
4619	// We know that we have an integer-based boolean since these operations
4620	// are only available for integer.
4621	if (TLI->getBooleanContents(isVec: VT.isVector(), isFloat: false) ==
4622	TargetLowering::ZeroOrNegativeOneBooleanContent)
4623	return VTBits;
4624	break;
4625	case ISD::SETCC:
4626	case ISD::SETCCCARRY:
4627	case ISD::STRICT_FSETCC:
4628	case ISD::STRICT_FSETCCS: {
4629	unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
4630	// If setcc returns 0/-1, all bits are sign bits.
4631	if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) ==
4632	TargetLowering::ZeroOrNegativeOneBooleanContent)
4633	return VTBits;
4634	break;
4635	}
4636	case ISD::ROTL:
4637	case ISD::ROTR:
4638	Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4639
4640	// If we're rotating an 0/-1 value, then it stays an 0/-1 value.
4641	if (Tmp == VTBits)
4642	return VTBits;
4643
4644	if (ConstantSDNode *C =
4645	isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)) {
4646	unsigned RotAmt = C->getAPIntValue().urem(RHS: VTBits);
4647
4648	// Handle rotate right by N like a rotate left by 32-N.
4649	if (Opcode == ISD::ROTR)
4650	RotAmt = (VTBits - RotAmt) % VTBits;
4651
4652	// If we aren't rotating out all of the known-in sign bits, return the
4653	// number that are left. This handles rotl(sext(x), 1) for example.
4654	if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt);
4655	}
4656	break;
4657	case ISD::ADD:
4658	case ISD::ADDC:
4659	// Add can have at most one carry bit. Thus we know that the output
4660	// is, at worst, one more bit than the inputs.
4661	Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4662	if (Tmp == 1) return 1; // Early out.
4663
4664	// Special case decrementing a value (ADD X, -1):
4665	if (ConstantSDNode *CRHS =
4666	isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts))
4667	if (CRHS->isAllOnes()) {
4668	KnownBits Known =
4669	computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4670
4671	// If the input is known to be 0 or 1, the output is 0/-1, which is all
4672	// sign bits set.
4673	if ((Known.Zero | 1).isAllOnes())
4674	return VTBits;
4675
4676	// If we are subtracting one from a positive number, there is no carry
4677	// out of the result.
4678	if (Known.isNonNegative())
4679	return Tmp;
4680	}
4681
4682	Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4683	if (Tmp2 == 1) return 1; // Early out.
4684	return std::min(a: Tmp, b: Tmp2) - 1;
4685	case ISD::SUB:
4686	Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4687	if (Tmp2 == 1) return 1; // Early out.
4688
4689	// Handle NEG.
4690	if (ConstantSDNode *CLHS =
4691	isConstOrConstSplat(N: Op.getOperand(i: 0), DemandedElts))
4692	if (CLHS->isZero()) {
4693	KnownBits Known =
4694	computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4695	// If the input is known to be 0 or 1, the output is 0/-1, which is all
4696	// sign bits set.
4697	if ((Known.Zero | 1).isAllOnes())
4698	return VTBits;
4699
4700	// If the input is known to be positive (the sign bit is known clear),
4701	// the output of the NEG has the same number of sign bits as the input.
4702	if (Known.isNonNegative())
4703	return Tmp2;
4704
4705	// Otherwise, we treat this like a SUB.
4706	}
4707
4708	// Sub can have at most one carry bit. Thus we know that the output
4709	// is, at worst, one more bit than the inputs.
4710	Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4711	if (Tmp == 1) return 1; // Early out.
4712	return std::min(a: Tmp, b: Tmp2) - 1;
4713	case ISD::MUL: {
4714	// The output of the Mul can be at most twice the valid bits in the inputs.
4715	unsigned SignBitsOp0 = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4716	if (SignBitsOp0 == 1)
4717	break;
4718	unsigned SignBitsOp1 = ComputeNumSignBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
4719	if (SignBitsOp1 == 1)
4720	break;
4721	unsigned OutValidBits =
4722	(VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
4723	return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
4724	}
4725	case ISD::SREM:
4726	// The sign bit is the LHS's sign bit, except when the result of the
4727	// remainder is zero. The magnitude of the result should be less than or
4728	// equal to the magnitude of the LHS. Therefore, the result should have
4729	// at least as many sign bits as the left hand side.
4730	return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4731	case ISD::TRUNCATE: {
4732	// Check if the sign bits of source go down as far as the truncated value.
4733	unsigned NumSrcBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
4734	unsigned NumSrcSignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4735	if (NumSrcSignBits > (NumSrcBits - VTBits))
4736	return NumSrcSignBits - (NumSrcBits - VTBits);
4737	break;
4738	}
4739	case ISD::EXTRACT_ELEMENT: {
4740	if (VT.isScalableVector())
4741	break;
4742	const int KnownSign = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
4743	const int BitWidth = Op.getValueSizeInBits();
4744	const int Items = Op.getOperand(i: 0).getValueSizeInBits() / BitWidth;
4745
4746	// Get reverse index (starting from 1), Op1 value indexes elements from
4747	// little end. Sign starts at big end.
4748	const int rIndex = Items - 1 - Op.getConstantOperandVal(i: 1);
4749
4750	// If the sign portion ends in our element the subtraction gives correct
4751	// result. Otherwise it gives either negative or > bitwidth result
4752	return std::clamp(val: KnownSign - rIndex * BitWidth, lo: 0, hi: BitWidth);
4753	}
4754	case ISD::INSERT_VECTOR_ELT: {
4755	if (VT.isScalableVector())
4756	break;
4757	// If we know the element index, split the demand between the
4758	// source vector and the inserted element, otherwise assume we need
4759	// the original demanded vector elements and the value.
4760	SDValue InVec = Op.getOperand(i: 0);
4761	SDValue InVal = Op.getOperand(i: 1);
4762	SDValue EltNo = Op.getOperand(i: 2);
4763	bool DemandedVal = true;
4764	APInt DemandedVecElts = DemandedElts;
4765	auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4766	if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) {
4767	unsigned EltIdx = CEltNo->getZExtValue();
4768	DemandedVal = !!DemandedElts[EltIdx];
4769	DemandedVecElts.clearBit(BitPosition: EltIdx);
4770	}
4771	Tmp = std::numeric_limits<unsigned>::max();
4772	if (DemandedVal) {
4773	// TODO - handle implicit truncation of inserted elements.
4774	if (InVal.getScalarValueSizeInBits() != VTBits)
4775	break;
4776	Tmp2 = ComputeNumSignBits(Op: InVal, Depth: Depth + 1);
4777	Tmp = std::min(a: Tmp, b: Tmp2);
4778	}
4779	if (!!DemandedVecElts) {
4780	Tmp2 = ComputeNumSignBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1);
4781	Tmp = std::min(a: Tmp, b: Tmp2);
4782	}
4783	assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4784	return Tmp;
4785	}
4786	case ISD::EXTRACT_VECTOR_ELT: {
4787	assert(!VT.isScalableVector());
4788	SDValue InVec = Op.getOperand(i: 0);
4789	SDValue EltNo = Op.getOperand(i: 1);
4790	EVT VecVT = InVec.getValueType();
4791	// ComputeNumSignBits not yet implemented for scalable vectors.
4792	if (VecVT.isScalableVector())
4793	break;
4794	const unsigned BitWidth = Op.getValueSizeInBits();
4795	const unsigned EltBitWidth = Op.getOperand(i: 0).getScalarValueSizeInBits();
4796	const unsigned NumSrcElts = VecVT.getVectorNumElements();
4797
4798	// If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
4799	// anything about sign bits. But if the sizes match we can derive knowledge
4800	// about sign bits from the vector operand.
4801	if (BitWidth != EltBitWidth)
4802	break;
4803
4804	// If we know the element index, just demand that vector element, else for
4805	// an unknown element index, ignore DemandedElts and demand them all.
4806	APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
4807	auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4808	if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
4809	DemandedSrcElts =
4810	APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
4811
4812	return ComputeNumSignBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
4813	}
4814	case ISD::EXTRACT_SUBVECTOR: {
4815	// Offset the demanded elts by the subvector index.
4816	SDValue Src = Op.getOperand(i: 0);
4817	// Bail until we can represent demanded elements for scalable vectors.
4818	if (Src.getValueType().isScalableVector())
4819	break;
4820	uint64_t Idx = Op.getConstantOperandVal(i: 1);
4821	unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
4822	APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
4823	return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
4824	}
4825	case ISD::CONCAT_VECTORS: {
4826	if (VT.isScalableVector())
4827	break;
4828	// Determine the minimum number of sign bits across all demanded
4829	// elts of the input vectors. Early out if the result is already 1.
4830	Tmp = std::numeric_limits<unsigned>::max();
4831	EVT SubVectorVT = Op.getOperand(i: 0).getValueType();
4832	unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
4833	unsigned NumSubVectors = Op.getNumOperands();
4834	for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
4835	APInt DemandedSub =
4836	DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts);
4837	if (!DemandedSub)
4838	continue;
4839	Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i), DemandedElts: DemandedSub, Depth: Depth + 1);
4840	Tmp = std::min(a: Tmp, b: Tmp2);
4841	}
4842	assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4843	return Tmp;
4844	}
4845	case ISD::INSERT_SUBVECTOR: {
4846	if (VT.isScalableVector())
4847	break;
4848	// Demand any elements from the subvector and the remainder from the src its
4849	// inserted into.
4850	SDValue Src = Op.getOperand(i: 0);
4851	SDValue Sub = Op.getOperand(i: 1);
4852	uint64_t Idx = Op.getConstantOperandVal(i: 2);
4853	unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
4854	APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
4855	APInt DemandedSrcElts = DemandedElts;
4856	DemandedSrcElts.insertBits(SubBits: APInt::getZero(numBits: NumSubElts), bitPosition: Idx);
4857
4858	Tmp = std::numeric_limits<unsigned>::max();
4859	if (!!DemandedSubElts) {
4860	Tmp = ComputeNumSignBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1);
4861	if (Tmp == 1)
4862	return 1; // early-out
4863	}
4864	if (!!DemandedSrcElts) {
4865	Tmp2 = ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
4866	Tmp = std::min(a: Tmp, b: Tmp2);
4867	}
4868	assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4869	return Tmp;
4870	}
4871	case ISD::LOAD: {
4872	LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
4873	if (const MDNode *Ranges = LD->getRanges()) {
4874	if (DemandedElts != 1)
4875	break;
4876
4877	ConstantRange CR = getConstantRangeFromMetadata(RangeMD: *Ranges);
4878	if (VTBits > CR.getBitWidth()) {
4879	switch (LD->getExtensionType()) {
4880	case ISD::SEXTLOAD:
4881	CR = CR.signExtend(BitWidth: VTBits);
4882	break;
4883	case ISD::ZEXTLOAD:
4884	CR = CR.zeroExtend(BitWidth: VTBits);
4885	break;
4886	default:
4887	break;
4888	}
4889	}
4890
4891	if (VTBits != CR.getBitWidth())
4892	break;
4893	return std::min(a: CR.getSignedMin().getNumSignBits(),
4894	b: CR.getSignedMax().getNumSignBits());
4895	}
4896
4897	break;
4898	}
4899	case ISD::ATOMIC_CMP_SWAP:
4900	case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
4901	case ISD::ATOMIC_SWAP:
4902	case ISD::ATOMIC_LOAD_ADD:
4903	case ISD::ATOMIC_LOAD_SUB:
4904	case ISD::ATOMIC_LOAD_AND:
4905	case ISD::ATOMIC_LOAD_CLR:
4906	case ISD::ATOMIC_LOAD_OR:
4907	case ISD::ATOMIC_LOAD_XOR:
4908	case ISD::ATOMIC_LOAD_NAND:
4909	case ISD::ATOMIC_LOAD_MIN:
4910	case ISD::ATOMIC_LOAD_MAX:
4911	case ISD::ATOMIC_LOAD_UMIN:
4912	case ISD::ATOMIC_LOAD_UMAX:
4913	case ISD::ATOMIC_LOAD: {
4914	Tmp = cast<AtomicSDNode>(Val&: Op)->getMemoryVT().getScalarSizeInBits();
4915	// If we are looking at the loaded value.
4916	if (Op.getResNo() == 0) {
4917	if (Tmp == VTBits)
4918	return 1; // early-out
4919	if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND)
4920	return VTBits - Tmp + 1;
4921	if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
4922	return VTBits - Tmp;
4923	if (Op->getOpcode() == ISD::ATOMIC_LOAD) {
4924	ISD::LoadExtType ETy = cast<AtomicSDNode>(Val&: Op)->getExtensionType();
4925	if (ETy == ISD::SEXTLOAD)
4926	return VTBits - Tmp + 1;
4927	if (ETy == ISD::ZEXTLOAD)
4928	return VTBits - Tmp;
4929	}
4930	}
4931	break;
4932	}
4933	}
4934
4935	// If we are looking at the loaded value of the SDNode.
4936	if (Op.getResNo() == 0) {
4937	// Handle LOADX separately here. EXTLOAD case will fallthrough.
4938	if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val&: Op)) {
4939	unsigned ExtType = LD->getExtensionType();
4940	switch (ExtType) {
4941	default: break;
4942	case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known.
4943	Tmp = LD->getMemoryVT().getScalarSizeInBits();
4944	return VTBits - Tmp + 1;
4945	case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known.
4946	Tmp = LD->getMemoryVT().getScalarSizeInBits();
4947	return VTBits - Tmp;
4948	case ISD::NON_EXTLOAD:
4949	if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
4950	// We only need to handle vectors - computeKnownBits should handle
4951	// scalar cases.
4952	Type *CstTy = Cst->getType();
4953	if (CstTy->isVectorTy() && !VT.isScalableVector() &&
4954	(NumElts * VTBits) == CstTy->getPrimitiveSizeInBits() &&
4955	VTBits == CstTy->getScalarSizeInBits()) {
4956	Tmp = VTBits;
4957	for (unsigned i = 0; i != NumElts; ++i) {
4958	if (!DemandedElts[i])
4959	continue;
4960	if (Constant *Elt = Cst->getAggregateElement(Elt: i)) {
4961	if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) {
4962	const APInt &Value = CInt->getValue();
4963	Tmp = std::min(a: Tmp, b: Value.getNumSignBits());
4964	continue;
4965	}
4966	if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) {
4967	APInt Value = CFP->getValueAPF().bitcastToAPInt();
4968	Tmp = std::min(a: Tmp, b: Value.getNumSignBits());
4969	continue;
4970	}
4971	}
4972	// Unknown type. Conservatively assume no bits match sign bit.
4973	return 1;
4974	}
4975	return Tmp;
4976	}
4977	}
4978	break;
4979	}
4980	}
4981	}
4982
4983	// Allow the target to implement this method for its nodes.
4984	if (Opcode >= ISD::BUILTIN_OP_END ||
4985	Opcode == ISD::INTRINSIC_WO_CHAIN ||
4986	Opcode == ISD::INTRINSIC_W_CHAIN ||
4987	Opcode == ISD::INTRINSIC_VOID) {
4988	// TODO: This can probably be removed once target code is audited. This
4989	// is here purely to reduce patch size and review complexity.
4990	if (!VT.isScalableVector()) {
4991	unsigned NumBits =
4992	TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, DAG: *this, Depth);
4993	if (NumBits > 1)
4994	FirstAnswer = std::max(a: FirstAnswer, b: NumBits);
4995	}
4996	}
4997
4998	// Finally, if we can prove that the top bits of the result are 0's or 1's,
4999	// use this information.
5000	KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
5001	return std::max(a: FirstAnswer, b: Known.countMinSignBits());
5002	}
5003
5004	unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
5005	unsigned Depth) const {
5006	unsigned SignBits = ComputeNumSignBits(Op, Depth);
5007	return Op.getScalarValueSizeInBits() - SignBits + 1;
5008	}
5009
5010	unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
5011	const APInt &DemandedElts,
5012	unsigned Depth) const {
5013	unsigned SignBits = ComputeNumSignBits(Op, DemandedElts, Depth);
5014	return Op.getScalarValueSizeInBits() - SignBits + 1;
5015	}
5016
5017	bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly,
5018	unsigned Depth) const {
5019	// Early out for FREEZE.
5020	if (Op.getOpcode() == ISD::FREEZE)
5021	return true;
5022
5023	// TODO: Assume we don't know anything for now.
5024	EVT VT = Op.getValueType();
5025	if (VT.isScalableVector())
5026	return false;
5027
5028	APInt DemandedElts = VT.isVector()
5029	? APInt::getAllOnes(numBits: VT.getVectorNumElements())
5030	: APInt(1, 1);
5031	return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, PoisonOnly, Depth);
5032	}
5033
5034	bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
5035	const APInt &DemandedElts,
5036	bool PoisonOnly,
5037	unsigned Depth) const {
5038	unsigned Opcode = Op.getOpcode();
5039
5040	// Early out for FREEZE.
5041	if (Opcode == ISD::FREEZE)
5042	return true;
5043
5044	if (Depth >= MaxRecursionDepth)
5045	return false; // Limit search depth.
5046
5047	if (isIntOrFPConstant(V: Op))
5048	return true;
5049
5050	switch (Opcode) {
5051	case ISD::CONDCODE:
5052	case ISD::VALUETYPE:
5053	case ISD::FrameIndex:
5054	case ISD::TargetFrameIndex:
5055	return true;
5056
5057	case ISD::UNDEF:
5058	return PoisonOnly;
5059
5060	case ISD::BUILD_VECTOR:
5061	// NOTE: BUILD_VECTOR has implicit truncation of wider scalar elements -
5062	// this shouldn't affect the result.
5063	for (unsigned i = 0, e = Op.getNumOperands(); i < e; ++i) {
5064	if (!DemandedElts[i])
5065	continue;
5066	if (!isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i), PoisonOnly,
5067	Depth: Depth + 1))
5068	return false;
5069	}
5070	return true;
5071
5072	// TODO: Search for noundef attributes from library functions.
5073
5074	// TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef.
5075
5076	default:
5077	// Allow the target to implement this method for its nodes.
5078	if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5079	Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
5080	return TLI->isGuaranteedNotToBeUndefOrPoisonForTargetNode(
5081	Op, DemandedElts, DAG: *this, PoisonOnly, Depth);
5082	break;
5083	}
5084
5085	// If Op can't create undef/poison and none of its operands are undef/poison
5086	// then Op is never undef/poison.
5087	// NOTE: TargetNodes can handle this in themselves in
5088	// isGuaranteedNotToBeUndefOrPoisonForTargetNode or let
5089	// TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode handle it.
5090	return !canCreateUndefOrPoison(Op, PoisonOnly, /*ConsiderFlags*/ true,
5091	Depth) &&
5092	all_of(Range: Op->ops(), P: [&](SDValue V) {
5093	return isGuaranteedNotToBeUndefOrPoison(Op: V, PoisonOnly, Depth: Depth + 1);
5094	});
5095	}
5096
5097	bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, bool PoisonOnly,
5098	bool ConsiderFlags,
5099	unsigned Depth) const {
5100	// TODO: Assume we don't know anything for now.
5101	EVT VT = Op.getValueType();
5102	if (VT.isScalableVector())
5103	return true;
5104
5105	APInt DemandedElts = VT.isVector()
5106	? APInt::getAllOnes(numBits: VT.getVectorNumElements())
5107	: APInt(1, 1);
5108	return canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly, ConsiderFlags,
5109	Depth);
5110	}
5111
5112	bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
5113	bool PoisonOnly, bool ConsiderFlags,
5114	unsigned Depth) const {
5115	// TODO: Assume we don't know anything for now.
5116	EVT VT = Op.getValueType();
5117	if (VT.isScalableVector())
5118	return true;
5119
5120	unsigned Opcode = Op.getOpcode();
5121	switch (Opcode) {
5122	case ISD::FREEZE:
5123	case ISD::CONCAT_VECTORS:
5124	case ISD::INSERT_SUBVECTOR:
5125	case ISD::AND:
5126	case ISD::XOR:
5127	case ISD::ROTL:
5128	case ISD::ROTR:
5129	case ISD::FSHL:
5130	case ISD::FSHR:
5131	case ISD::BSWAP:
5132	case ISD::CTPOP:
5133	case ISD::BITREVERSE:
5134	case ISD::PARITY:
5135	case ISD::SIGN_EXTEND:
5136	case ISD::TRUNCATE:
5137	case ISD::SIGN_EXTEND_INREG:
5138	case ISD::SIGN_EXTEND_VECTOR_INREG:
5139	case ISD::ZERO_EXTEND_VECTOR_INREG:
5140	case ISD::BITCAST:
5141	case ISD::BUILD_VECTOR:
5142	case ISD::BUILD_PAIR:
5143	return false;
5144
5145	case ISD::SETCC: {
5146	// Integer setcc cannot create undef or poison.
5147	if (Op.getOperand(i: 0).getValueType().isInteger())
5148	return false;
5149
5150	// FP compares are more complicated. They can create poison for nan/infinity
5151	// based on options and flags. The options and flags also cause special
5152	// nonan condition codes to be used. Those condition codes may be preserved
5153	// even if the nonan flag is dropped somewhere.
5154	ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
5155	if (((unsigned)CCCode & 0x10U))
5156	return true;
5157
5158	const TargetOptions &Options = getTarget().Options;
5159	return Options.NoNaNsFPMath || Options.NoInfsFPMath ||
5160	(ConsiderFlags &&
5161	(Op->getFlags().hasNoNaNs() || Op->getFlags().hasNoInfs()));
5162	}
5163
5164	// Matches hasPoisonGeneratingFlags().
5165	case ISD::ZERO_EXTEND:
5166	return ConsiderFlags && Op->getFlags().hasNonNeg();
5167
5168	case ISD::ADD:
5169	case ISD::SUB:
5170	case ISD::MUL:
5171	// Matches hasPoisonGeneratingFlags().
5172	return ConsiderFlags && (Op->getFlags().hasNoSignedWrap() ||
5173	Op->getFlags().hasNoUnsignedWrap());
5174
5175	case ISD::SHL:
5176	// If the max shift amount isn't in range, then the shift can create poison.
5177	if (!getValidMaximumShiftAmountConstant(V: Op, DemandedElts))
5178	return true;
5179
5180	// Matches hasPoisonGeneratingFlags().
5181	return ConsiderFlags && (Op->getFlags().hasNoSignedWrap() ||
5182	Op->getFlags().hasNoUnsignedWrap());
5183
5184	// Matches hasPoisonGeneratingFlags().
5185	case ISD::OR:
5186	return ConsiderFlags && Op->getFlags().hasDisjoint();
5187
5188	case ISD::SCALAR_TO_VECTOR:
5189	// Check if we demand any upper (undef) elements.
5190	return !PoisonOnly && DemandedElts.ugt(RHS: 1);
5191
5192	case ISD::EXTRACT_VECTOR_ELT: {
5193	// Ensure that the element index is in bounds.
5194	EVT VecVT = Op.getOperand(i: 0).getValueType();
5195	KnownBits KnownIdx = computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
5196	return KnownIdx.getMaxValue().uge(RHS: VecVT.getVectorMinNumElements());
5197	}
5198
5199	case ISD::INSERT_VECTOR_ELT:{
5200	// Ensure that the element index is in bounds.
5201	EVT VecVT = Op.getOperand(i: 0).getValueType();
5202	KnownBits KnownIdx = computeKnownBits(Op: Op.getOperand(i: 2), Depth: Depth + 1);
5203	return KnownIdx.getMaxValue().uge(RHS: VecVT.getVectorMinNumElements());
5204	}
5205
5206	default:
5207	// Allow the target to implement this method for its nodes.
5208	if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5209	Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
5210	return TLI->canCreateUndefOrPoisonForTargetNode(
5211	Op, DemandedElts, DAG: *this, PoisonOnly, ConsiderFlags, Depth);
5212	break;
5213	}
5214
5215	// Be conservative and return true.
5216	return true;
5217	}
5218
5219	bool SelectionDAG::isADDLike(SDValue Op) const {
5220	unsigned Opcode = Op.getOpcode();
5221	if (Opcode == ISD::OR)
5222	return Op->getFlags().hasDisjoint() ||
5223	haveNoCommonBitsSet(A: Op.getOperand(i: 0), B: Op.getOperand(i: 1));
5224	if (Opcode == ISD::XOR)
5225	return isMinSignedConstant(V: Op.getOperand(i: 1));
5226	return false;
5227	}
5228
5229	bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
5230	return Op.getNumOperands() == 2 && isa<ConstantSDNode>(Val: Op.getOperand(i: 1)) &&
5231	(Op.getOpcode() == ISD::ADD || isADDLike(Op));
5232	}
5233
5234	bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const {
5235	// If we're told that NaNs won't happen, assume they won't.
5236	if (getTarget().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs())
5237	return true;
5238
5239	if (Depth >= MaxRecursionDepth)
5240	return false; // Limit search depth.
5241
5242	// If the value is a constant, we can obviously see if it is a NaN or not.
5243	if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
5244	return !C->getValueAPF().isNaN() ||
5245	(SNaN && !C->getValueAPF().isSignaling());
5246	}
5247
5248	unsigned Opcode = Op.getOpcode();
5249	switch (Opcode) {
5250	case ISD::FADD:
5251	case ISD::FSUB:
5252	case ISD::FMUL:
5253	case ISD::FDIV:
5254	case ISD::FREM:
5255	case ISD::FSIN:
5256	case ISD::FCOS:
5257	case ISD::FMA:
5258	case ISD::FMAD: {
5259	if (SNaN)
5260	return true;
5261	// TODO: Need isKnownNeverInfinity
5262	return false;
5263	}
5264	case ISD::FCANONICALIZE:
5265	case ISD::FEXP:
5266	case ISD::FEXP2:
5267	case ISD::FEXP10:
5268	case ISD::FTRUNC:
5269	case ISD::FFLOOR:
5270	case ISD::FCEIL:
5271	case ISD::FROUND:
5272	case ISD::FROUNDEVEN:
5273	case ISD::FRINT:
5274	case ISD::LRINT:
5275	case ISD::LLRINT:
5276	case ISD::FNEARBYINT:
5277	case ISD::FLDEXP: {
5278	if (SNaN)
5279	return true;
5280	return isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
5281	}
5282	case ISD::FABS:
5283	case ISD::FNEG:
5284	case ISD::FCOPYSIGN: {
5285	return isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
5286	}
5287	case ISD::SELECT:
5288	return isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1) &&
5289	isKnownNeverNaN(Op: Op.getOperand(i: 2), SNaN, Depth: Depth + 1);
5290	case ISD::FP_EXTEND:
5291	case ISD::FP_ROUND: {
5292	if (SNaN)
5293	return true;
5294	return isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
5295	}
5296	case ISD::SINT_TO_FP:
5297	case ISD::UINT_TO_FP:
5298	return true;
5299	case ISD::FSQRT: // Need is known positive
5300	case ISD::FLOG:
5301	case ISD::FLOG2:
5302	case ISD::FLOG10:
5303	case ISD::FPOWI:
5304	case ISD::FPOW: {
5305	if (SNaN)
5306	return true;
5307	// TODO: Refine on operand
5308	return false;
5309	}
5310	case ISD::FMINNUM:
5311	case ISD::FMAXNUM: {
5312	// Only one needs to be known not-nan, since it will be returned if the
5313	// other ends up being one.
5314	return isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1) ||
5315	isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1);
5316	}
5317	case ISD::FMINNUM_IEEE:
5318	case ISD::FMAXNUM_IEEE: {
5319	if (SNaN)
5320	return true;
5321	// This can return a NaN if either operand is an sNaN, or if both operands
5322	// are NaN.
5323	return (isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN: false, Depth: Depth + 1) &&
5324	isKnownNeverSNaN(Op: Op.getOperand(i: 1), Depth: Depth + 1)) ||
5325	(isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN: false, Depth: Depth + 1) &&
5326	isKnownNeverSNaN(Op: Op.getOperand(i: 0), Depth: Depth + 1));
5327	}
5328	case ISD::FMINIMUM:
5329	case ISD::FMAXIMUM: {
5330	// TODO: Does this quiet or return the origina NaN as-is?
5331	return isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1) &&
5332	isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1);
5333	}
5334	case ISD::EXTRACT_VECTOR_ELT: {
5335	return isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
5336	}
5337	case ISD::BUILD_VECTOR: {
5338	for (const SDValue &Opnd : Op->ops())
5339	if (!isKnownNeverNaN(Op: Opnd, SNaN, Depth: Depth + 1))
5340	return false;
5341	return true;
5342	}
5343	default:
5344	if (Opcode >= ISD::BUILTIN_OP_END ||
5345	Opcode == ISD::INTRINSIC_WO_CHAIN ||
5346	Opcode == ISD::INTRINSIC_W_CHAIN ||
5347	Opcode == ISD::INTRINSIC_VOID) {
5348	return TLI->isKnownNeverNaNForTargetNode(Op, DAG: *this, SNaN, Depth);
5349	}
5350
5351	return false;
5352	}
5353	}
5354
5355	bool SelectionDAG::isKnownNeverZeroFloat(SDValue Op) const {
5356	assert(Op.getValueType().isFloatingPoint() &&
5357	"Floating point type expected");
5358
5359	// If the value is a constant, we can obviously see if it is a zero or not.
5360	return ISD::matchUnaryFpPredicate(
5361	Op, Match: [](ConstantFPSDNode *C) { return !C->isZero(); });
5362	}
5363
5364	bool SelectionDAG::isKnownNeverZero(SDValue Op, unsigned Depth) const {
5365	if (Depth >= MaxRecursionDepth)
5366	return false; // Limit search depth.
5367
5368	assert(!Op.getValueType().isFloatingPoint() &&
5369	"Floating point types unsupported - use isKnownNeverZeroFloat");
5370
5371	// If the value is a constant, we can obviously see if it is a zero or not.
5372	if (ISD::matchUnaryPredicate(Op,
5373	Match: [](ConstantSDNode *C) { return !C->isZero(); }))
5374	return true;
5375
5376	// TODO: Recognize more cases here. Most of the cases are also incomplete to
5377	// some degree.
5378	switch (Op.getOpcode()) {
5379	default:
5380	break;
5381
5382	case ISD::OR:
5383	return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) ||
5384	isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5385
5386	case ISD::VSELECT:
5387	case ISD::SELECT:
5388	return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
5389	isKnownNeverZero(Op: Op.getOperand(i: 2), Depth: Depth + 1);
5390
5391	case ISD::SHL: {
5392	if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
5393	return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5394	KnownBits ValKnown = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5395	// 1 << X is never zero.
5396	if (ValKnown.One[0])
5397	return true;
5398	// If max shift cnt of known ones is non-zero, result is non-zero.
5399	APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1).getMaxValue();
5400	if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) &&
5401	!ValKnown.One.shl(ShiftAmt: MaxCnt).isZero())
5402	return true;
5403	break;
5404	}
5405	case ISD::UADDSAT:
5406	case ISD::UMAX:
5407	return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) ||
5408	isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5409
5410	// For smin/smax: If either operand is known negative/positive
5411	// respectively we don't need the other to be known at all.
5412	case ISD::SMAX: {
5413	KnownBits Op1 = computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
5414	if (Op1.isStrictlyPositive())
5415	return true;
5416
5417	KnownBits Op0 = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5418	if (Op0.isStrictlyPositive())
5419	return true;
5420
5421	if (Op1.isNonZero() && Op0.isNonZero())
5422	return true;
5423
5424	return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
5425	isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5426	}
5427	case ISD::SMIN: {
5428	KnownBits Op1 = computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
5429	if (Op1.isNegative())
5430	return true;
5431
5432	KnownBits Op0 = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5433	if (Op0.isNegative())
5434	return true;
5435
5436	if (Op1.isNonZero() && Op0.isNonZero())
5437	return true;
5438
5439	return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
5440	isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5441	}
5442	case ISD::UMIN:
5443	return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
5444	isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5445
5446	case ISD::ROTL:
5447	case ISD::ROTR:
5448	case ISD::BITREVERSE:
5449	case ISD::BSWAP:
5450	case ISD::CTPOP:
5451	case ISD::ABS:
5452	return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5453
5454	case ISD::SRA:
5455	case ISD::SRL: {
5456	if (Op->getFlags().hasExact())
5457	return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5458	KnownBits ValKnown = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5459	if (ValKnown.isNegative())
5460	return true;
5461	// If max shift cnt of known ones is non-zero, result is non-zero.
5462	APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1).getMaxValue();
5463	if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) &&
5464	!ValKnown.One.lshr(ShiftAmt: MaxCnt).isZero())
5465	return true;
5466	break;
5467	}
5468	case ISD::UDIV:
5469	case ISD::SDIV:
5470	// div exact can only produce a zero if the dividend is zero.
5471	// TODO: For udiv this is also true if Op1 u<= Op0
5472	if (Op->getFlags().hasExact())
5473	return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5474	break;
5475
5476	case ISD::ADD:
5477	if (Op->getFlags().hasNoUnsignedWrap())
5478	if (isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) ||
5479	isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1))
5480	return true;
5481	// TODO: There are a lot more cases we can prove for add.
5482	break;
5483
5484	case ISD::SUB: {
5485	if (isNullConstant(V: Op.getOperand(i: 0)))
5486	return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1);
5487
5488	std::optional<bool> ne =
5489	KnownBits::ne(LHS: computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1),
5490	RHS: computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1));
5491	return ne && *ne;
5492	}
5493
5494	case ISD::MUL:
5495	if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
5496	if (isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
5497	isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1))
5498	return true;
5499	break;
5500
5501	case ISD::ZERO_EXTEND:
5502	case ISD::SIGN_EXTEND:
5503	return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5504	}
5505
5506	return computeKnownBits(Op, Depth).isNonZero();
5507	}
5508
5509	bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
5510	// Check the obvious case.
5511	if (A == B) return true;
5512
5513	// For negative and positive zero.
5514	if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(Val&: A))
5515	if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(Val&: B))
5516	if (CA->isZero() && CB->isZero()) return true;
5517
5518	// Otherwise they may not be equal.
5519	return false;
5520	}
5521
5522	// Only bits set in Mask must be negated, other bits may be arbitrary.
5523	SDValue llvm::getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs) {
5524	if (isBitwiseNot(V, AllowUndefs))
5525	return V.getOperand(i: 0);
5526
5527	// Handle any_extend (not (truncate X)) pattern, where Mask only sets
5528	// bits in the non-extended part.
5529	ConstantSDNode *MaskC = isConstOrConstSplat(N: Mask);
5530	if (!MaskC || V.getOpcode() != ISD::ANY_EXTEND)
5531	return SDValue();
5532	SDValue ExtArg = V.getOperand(i: 0);
5533	if (ExtArg.getScalarValueSizeInBits() >=
5534	MaskC->getAPIntValue().getActiveBits() &&
5535	isBitwiseNot(V: ExtArg, AllowUndefs) &&
5536	ExtArg.getOperand(i: 0).getOpcode() == ISD::TRUNCATE &&
5537	ExtArg.getOperand(i: 0).getOperand(i: 0).getValueType() == V.getValueType())
5538	return ExtArg.getOperand(i: 0).getOperand(i: 0);
5539	return SDValue();
5540	}
5541
5542	static bool haveNoCommonBitsSetCommutative(SDValue A, SDValue B) {
5543	// Match masked merge pattern (X & ~M) op (Y & M)
5544	// Including degenerate case (X & ~M) op M
5545	auto MatchNoCommonBitsPattern = [&](SDValue Not, SDValue Mask,
5546	SDValue Other) {
5547	if (SDValue NotOperand =
5548	getBitwiseNotOperand(V: Not, Mask, /* AllowUndefs */ true)) {
5549	if (NotOperand->getOpcode() == ISD::ZERO_EXTEND ||
5550	NotOperand->getOpcode() == ISD::TRUNCATE)
5551	NotOperand = NotOperand->getOperand(Num: 0);
5552
5553	if (Other == NotOperand)
5554	return true;
5555	if (Other->getOpcode() == ISD::AND)
5556	return NotOperand == Other->getOperand(Num: 0) ||
5557	NotOperand == Other->getOperand(Num: 1);
5558	}
5559	return false;
5560	};
5561
5562	if (A->getOpcode() == ISD::ZERO_EXTEND || A->getOpcode() == ISD::TRUNCATE)
5563	A = A->getOperand(Num: 0);
5564
5565	if (B->getOpcode() == ISD::ZERO_EXTEND || B->getOpcode() == ISD::TRUNCATE)
5566	B = B->getOperand(Num: 0);
5567
5568	if (A->getOpcode() == ISD::AND)
5569	return MatchNoCommonBitsPattern(A->getOperand(Num: 0), A->getOperand(Num: 1), B) ||
5570	MatchNoCommonBitsPattern(A->getOperand(Num: 1), A->getOperand(Num: 0), B);
5571	return false;
5572	}
5573
5574	// FIXME: unify with llvm::haveNoCommonBitsSet.
5575	bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
5576	assert(A.getValueType() == B.getValueType() &&
5577	"Values must have the same type");
5578	if (haveNoCommonBitsSetCommutative(A, B) ||
5579	haveNoCommonBitsSetCommutative(A: B, B: A))
5580	return true;
5581	return KnownBits::haveNoCommonBitsSet(LHS: computeKnownBits(Op: A),
5582	RHS: computeKnownBits(Op: B));
5583	}
5584
5585	static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step,
5586	SelectionDAG &DAG) {
5587	if (cast<ConstantSDNode>(Val&: Step)->isZero())
5588	return DAG.getConstant(Val: 0, DL, VT);
5589
5590	return SDValue();
5591	}
5592
5593	static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT,
5594	ArrayRef<SDValue> Ops,
5595	SelectionDAG &DAG) {
5596	int NumOps = Ops.size();
5597	assert(NumOps != 0 && "Can't build an empty vector!");
5598	assert(!VT.isScalableVector() &&
5599	"BUILD_VECTOR cannot be used with scalable types");
5600	assert(VT.getVectorNumElements() == (unsigned)NumOps &&
5601	"Incorrect element count in BUILD_VECTOR!");
5602
5603	// BUILD_VECTOR of UNDEFs is UNDEF.
5604	if (llvm::all_of(Range&: Ops, P: [](SDValue Op) { return Op.isUndef(); }))
5605	return DAG.getUNDEF(VT);
5606
5607	// BUILD_VECTOR of seq extract/insert from the same vector + type is Identity.
5608	SDValue IdentitySrc;
5609	bool IsIdentity = true;
5610	for (int i = 0; i != NumOps; ++i) {
5611	if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
5612	Ops[i].getOperand(i: 0).getValueType() != VT ||
5613	(IdentitySrc && Ops[i].getOperand(i: 0) != IdentitySrc) ||
5614	!isa<ConstantSDNode>(Val: Ops[i].getOperand(i: 1)) ||
5615	Ops[i].getConstantOperandAPInt(i: 1) != i) {
5616	IsIdentity = false;
5617	break;
5618	}
5619	IdentitySrc = Ops[i].getOperand(i: 0);
5620	}
5621	if (IsIdentity)
5622	return IdentitySrc;
5623
5624	return SDValue();
5625	}
5626
5627	/// Try to simplify vector concatenation to an input value, undef, or build
5628	/// vector.
5629	static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
5630	ArrayRef<SDValue> Ops,
5631	SelectionDAG &DAG) {
5632	assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");
5633	assert(llvm::all_of(Ops,
5634	[Ops](SDValue Op) {
5635	return Ops[0].getValueType() == Op.getValueType();
5636	}) &&
5637	"Concatenation of vectors with inconsistent value types!");
5638	assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) ==
5639	VT.getVectorElementCount() &&
5640	"Incorrect element count in vector concatenation!");
5641
5642	if (Ops.size() == 1)
5643	return Ops[0];
5644
5645	// Concat of UNDEFs is UNDEF.
5646	if (llvm::all_of(Range&: Ops, P: [](SDValue Op) { return Op.isUndef(); }))
5647	return DAG.getUNDEF(VT);
5648
5649	// Scan the operands and look for extract operations from a single source
5650	// that correspond to insertion at the same location via this concatenation:
5651	// concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ...
5652	SDValue IdentitySrc;
5653	bool IsIdentity = true;
5654	for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
5655	SDValue Op = Ops[i];
5656	unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements();
5657	if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
5658	Op.getOperand(i: 0).getValueType() != VT ||
5659	(IdentitySrc && Op.getOperand(i: 0) != IdentitySrc) ||
5660	Op.getConstantOperandVal(i: 1) != IdentityIndex) {
5661	IsIdentity = false;
5662	break;
5663	}
5664	assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&
5665	"Unexpected identity source vector for concat of extracts");
5666	IdentitySrc = Op.getOperand(i: 0);
5667	}
5668	if (IsIdentity) {
5669	assert(IdentitySrc && "Failed to set source vector of extracts");
5670	return IdentitySrc;
5671	}
5672
5673	// The code below this point is only designed to work for fixed width
5674	// vectors, so we bail out for now.
5675	if (VT.isScalableVector())
5676	return SDValue();
5677
5678	// A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be
5679	// simplified to one big BUILD_VECTOR.
5680	// FIXME: Add support for SCALAR_TO_VECTOR as well.
5681	EVT SVT = VT.getScalarType();
5682	SmallVector<SDValue, 16> Elts;
5683	for (SDValue Op : Ops) {
5684	EVT OpVT = Op.getValueType();
5685	if (Op.isUndef())
5686	Elts.append(NumInputs: OpVT.getVectorNumElements(), Elt: DAG.getUNDEF(VT: SVT));
5687	else if (Op.getOpcode() == ISD::BUILD_VECTOR)
5688	Elts.append(in_start: Op->op_begin(), in_end: Op->op_end());
5689	else
5690	return SDValue();
5691	}
5692
5693	// BUILD_VECTOR requires all inputs to be of the same type, find the
5694	// maximum type and extend them all.
5695	for (SDValue Op : Elts)
5696	SVT = (SVT.bitsLT(VT: Op.getValueType()) ? Op.getValueType() : SVT);
5697
5698	if (SVT.bitsGT(VT: VT.getScalarType())) {
5699	for (SDValue &Op : Elts) {
5700	if (Op.isUndef())
5701	Op = DAG.getUNDEF(VT: SVT);
5702	else
5703	Op = DAG.getTargetLoweringInfo().isZExtFree(FromTy: Op.getValueType(), ToTy: SVT)
5704	? DAG.getZExtOrTrunc(Op, DL, VT: SVT)
5705	: DAG.getSExtOrTrunc(Op, DL, VT: SVT);
5706	}
5707	}
5708
5709	SDValue V = DAG.getBuildVector(VT, DL, Ops: Elts);
5710	NewSDValueDbgMsg(V, Msg: "New node fold concat vectors: ", G: &DAG);
5711	return V;
5712	}
5713
5714	/// Gets or creates the specified node.
5715	SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
5716	FoldingSetNodeID ID;
5717	AddNodeIDNode(ID, OpC: Opcode, VTList: getVTList(VT), OpList: std::nullopt);
5718	void *IP = nullptr;
5719	if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
5720	return SDValue(E, 0);
5721
5722	auto *N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(),
5723	Args: getVTList(VT));
5724	CSEMap.InsertNode(N, InsertPos: IP);
5725
5726	InsertNode(N);
5727	SDValue V = SDValue(N, 0);
5728	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
5729	return V;
5730	}
5731
5732	SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5733	SDValue N1) {
5734	SDNodeFlags Flags;
5735	if (Inserter)
5736	Flags = Inserter->getFlags();
5737	return getNode(Opcode, DL, VT, Operand: N1, Flags);
5738	}
5739
5740	SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5741	SDValue N1, const SDNodeFlags Flags) {
5742	assert(N1.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!");
5743
5744	// Constant fold unary operations with a vector integer or float operand.
5745	switch (Opcode) {
5746	default:
5747	// FIXME: Entirely reasonable to perform folding of other unary
5748	// operations here as the need arises.
5749	break;
5750	case ISD::FNEG:
5751	case ISD::FABS:
5752	case ISD::FCEIL:
5753	case ISD::FTRUNC:
5754	case ISD::FFLOOR:
5755	case ISD::FP_EXTEND:
5756	case ISD::FP_TO_SINT:
5757	case ISD::FP_TO_UINT:
5758	case ISD::FP_TO_FP16:
5759	case ISD::FP_TO_BF16:
5760	case ISD::TRUNCATE:
5761	case ISD::ANY_EXTEND:
5762	case ISD::ZERO_EXTEND:
5763	case ISD::SIGN_EXTEND:
5764	case ISD::UINT_TO_FP:
5765	case ISD::SINT_TO_FP:
5766	case ISD::FP16_TO_FP:
5767	case ISD::BF16_TO_FP:
5768	case ISD::BITCAST:
5769	case ISD::ABS:
5770	case ISD::BITREVERSE:
5771	case ISD::BSWAP:
5772	case ISD::CTLZ:
5773	case ISD::CTLZ_ZERO_UNDEF:
5774	case ISD::CTTZ:
5775	case ISD::CTTZ_ZERO_UNDEF:
5776	case ISD::CTPOP:
5777	case ISD::STEP_VECTOR: {
5778	SDValue Ops = {N1};
5779	if (SDValue Fold = FoldConstantArithmetic(Opcode, DL, VT, Ops))
5780	return Fold;
5781	}
5782	}
5783
5784	unsigned OpOpcode = N1.getNode()->getOpcode();
5785	switch (Opcode) {
5786	case ISD::STEP_VECTOR:
5787	assert(VT.isScalableVector() &&
5788	"STEP_VECTOR can only be used with scalable types");
5789	assert(OpOpcode == ISD::TargetConstant &&
5790	VT.getVectorElementType() == N1.getValueType() &&
5791	"Unexpected step operand");
5792	break;
5793	case ISD::FREEZE:
5794	assert(VT == N1.getValueType() && "Unexpected VT!");
5795	if (isGuaranteedNotToBeUndefOrPoison(Op: N1, /*PoisonOnly*/ false,
5796	/*Depth*/ 1))
5797	return N1;
5798	break;
5799	case ISD::TokenFactor:
5800	case ISD::MERGE_VALUES:
5801	case ISD::CONCAT_VECTORS:
5802	return N1; // Factor, merge or concat of one node? No need.
5803	case ISD::BUILD_VECTOR: {
5804	// Attempt to simplify BUILD_VECTOR.
5805	SDValue Ops[] = {N1};
5806	if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
5807	return V;
5808	break;
5809	}
5810	case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
5811	case ISD::FP_EXTEND:
5812	assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&
5813	"Invalid FP cast!");
5814	if (N1.getValueType() == VT) return N1; // noop conversion.
5815	assert((!VT.isVector() || VT.getVectorElementCount() ==
5816	N1.getValueType().getVectorElementCount()) &&
5817	"Vector element count mismatch!");
5818	assert(N1.getValueType().bitsLT(VT) && "Invalid fpext node, dst < src!");
5819	if (N1.isUndef())
5820	return getUNDEF(VT);
5821	break;
5822	case ISD::FP_TO_SINT:
5823	case ISD::FP_TO_UINT:
5824	if (N1.isUndef())
5825	return getUNDEF(VT);
5826	break;
5827	case ISD::SINT_TO_FP:
5828	case ISD::UINT_TO_FP:
5829	// [us]itofp(undef) = 0, because the result value is bounded.
5830	if (N1.isUndef())
5831	return getConstantFP(Val: 0.0, DL, VT);
5832	break;
5833	case ISD::SIGN_EXTEND:
5834	assert(VT.isInteger() && N1.getValueType().isInteger() &&
5835	"Invalid SIGN_EXTEND!");
5836	assert(VT.isVector() == N1.getValueType().isVector() &&
5837	"SIGN_EXTEND result type type should be vector iff the operand "
5838	"type is vector!");
5839	if (N1.getValueType() == VT) return N1; // noop extension
5840	assert((!VT.isVector() || VT.getVectorElementCount() ==
5841	N1.getValueType().getVectorElementCount()) &&
5842	"Vector element count mismatch!");
5843	assert(N1.getValueType().bitsLT(VT) && "Invalid sext node, dst < src!");
5844	if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) {
5845	SDNodeFlags Flags;
5846	if (OpOpcode == ISD::ZERO_EXTEND)
5847	Flags.setNonNeg(N1->getFlags().hasNonNeg());
5848	return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
5849	}
5850	if (OpOpcode == ISD::UNDEF)
5851	// sext(undef) = 0, because the top bits will all be the same.
5852	return getConstant(Val: 0, DL, VT);
5853	break;
5854	case ISD::ZERO_EXTEND:
5855	assert(VT.isInteger() && N1.getValueType().isInteger() &&
5856	"Invalid ZERO_EXTEND!");
5857	assert(VT.isVector() == N1.getValueType().isVector() &&
5858	"ZERO_EXTEND result type type should be vector iff the operand "
5859	"type is vector!");
5860	if (N1.getValueType() == VT) return N1; // noop extension
5861	assert((!VT.isVector() || VT.getVectorElementCount() ==
5862	N1.getValueType().getVectorElementCount()) &&
5863	"Vector element count mismatch!");
5864	assert(N1.getValueType().bitsLT(VT) && "Invalid zext node, dst < src!");
5865	if (OpOpcode == ISD::ZERO_EXTEND) { // (zext (zext x)) -> (zext x)
5866	SDNodeFlags Flags;
5867	Flags.setNonNeg(N1->getFlags().hasNonNeg());
5868	return getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, N1: N1.getOperand(i: 0), Flags);
5869	}
5870	if (OpOpcode == ISD::UNDEF)
5871	// zext(undef) = 0, because the top bits will be zero.
5872	return getConstant(Val: 0, DL, VT);
5873
5874	// Skip unnecessary zext_inreg pattern:
5875	// (zext (trunc x)) -> x iff the upper bits are known zero.
5876	// TODO: Remove (zext (trunc (and x, c))) exception which some targets
5877	// use to recognise zext_inreg patterns.
5878	if (OpOpcode == ISD::TRUNCATE) {
5879	SDValue OpOp = N1.getOperand(i: 0);
5880	if (OpOp.getValueType() == VT) {
5881	if (OpOp.getOpcode() != ISD::AND) {
5882	APInt HiBits = APInt::getBitsSetFrom(numBits: VT.getScalarSizeInBits(),
5883	loBit: N1.getScalarValueSizeInBits());
5884	if (MaskedValueIsZero(V: OpOp, Mask: HiBits)) {
5885	transferDbgValues(From: N1, To: OpOp);
5886	return OpOp;
5887	}
5888	}
5889	}
5890	}
5891	break;
5892	case ISD::ANY_EXTEND:
5893	assert(VT.isInteger() && N1.getValueType().isInteger() &&
5894	"Invalid ANY_EXTEND!");
5895	assert(VT.isVector() == N1.getValueType().isVector() &&
5896	"ANY_EXTEND result type type should be vector iff the operand "
5897	"type is vector!");
5898	if (N1.getValueType() == VT) return N1; // noop extension
5899	assert((!VT.isVector() || VT.getVectorElementCount() ==
5900	N1.getValueType().getVectorElementCount()) &&
5901	"Vector element count mismatch!");
5902	assert(N1.getValueType().bitsLT(VT) && "Invalid anyext node, dst < src!");
5903
5904	if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
5905	OpOpcode == ISD::ANY_EXTEND) {
5906	SDNodeFlags Flags;
5907	if (OpOpcode == ISD::ZERO_EXTEND)
5908	Flags.setNonNeg(N1->getFlags().hasNonNeg());
5909	// (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
5910	return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
5911	}
5912	if (OpOpcode == ISD::UNDEF)
5913	return getUNDEF(VT);
5914
5915	// (ext (trunc x)) -> x
5916	if (OpOpcode == ISD::TRUNCATE) {
5917	SDValue OpOp = N1.getOperand(i: 0);
5918	if (OpOp.getValueType() == VT) {
5919	transferDbgValues(From: N1, To: OpOp);
5920	return OpOp;
5921	}
5922	}
5923	break;
5924	case ISD::TRUNCATE:
5925	assert(VT.isInteger() && N1.getValueType().isInteger() &&
5926	"Invalid TRUNCATE!");
5927	assert(VT.isVector() == N1.getValueType().isVector() &&
5928	"TRUNCATE result type type should be vector iff the operand "
5929	"type is vector!");
5930	if (N1.getValueType() == VT) return N1; // noop truncate
5931	assert((!VT.isVector() || VT.getVectorElementCount() ==
5932	N1.getValueType().getVectorElementCount()) &&
5933	"Vector element count mismatch!");
5934	assert(N1.getValueType().bitsGT(VT) && "Invalid truncate node, src < dst!");
5935	if (OpOpcode == ISD::TRUNCATE)
5936	return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0));
5937	if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
5938	OpOpcode == ISD::ANY_EXTEND) {
5939	// If the source is smaller than the dest, we still need an extend.
5940	if (N1.getOperand(i: 0).getValueType().getScalarType().bitsLT(
5941	VT: VT.getScalarType()))
5942	return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0));
5943	if (N1.getOperand(i: 0).getValueType().bitsGT(VT))
5944	return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0));
5945	return N1.getOperand(i: 0);
5946	}
5947	if (OpOpcode == ISD::UNDEF)
5948	return getUNDEF(VT);
5949	if (OpOpcode == ISD::VSCALE && !NewNodesMustHaveLegalTypes)
5950	return getVScale(DL, VT,
5951	MulImm: N1.getConstantOperandAPInt(i: 0).trunc(width: VT.getSizeInBits()));
5952	break;
5953	case ISD::ANY_EXTEND_VECTOR_INREG:
5954	case ISD::ZERO_EXTEND_VECTOR_INREG:
5955	case ISD::SIGN_EXTEND_VECTOR_INREG:
5956	assert(VT.isVector() && "This DAG node is restricted to vector types.");
5957	assert(N1.getValueType().bitsLE(VT) &&
5958	"The input must be the same size or smaller than the result.");
5959	assert(VT.getVectorMinNumElements() <
5960	N1.getValueType().getVectorMinNumElements() &&
5961	"The destination vector type must have fewer lanes than the input.");
5962	break;
5963	case ISD::ABS:
5964	assert(VT.isInteger() && VT == N1.getValueType() && "Invalid ABS!");
5965	if (OpOpcode == ISD::UNDEF)
5966	return getConstant(Val: 0, DL, VT);
5967	break;
5968	case ISD::BSWAP:
5969	assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BSWAP!");
5970	assert((VT.getScalarSizeInBits() % 16 == 0) &&
5971	"BSWAP types must be a multiple of 16 bits!");
5972	if (OpOpcode == ISD::UNDEF)
5973	return getUNDEF(VT);
5974	// bswap(bswap(X)) -> X.
5975	if (OpOpcode == ISD::BSWAP)
5976	return N1.getOperand(i: 0);
5977	break;
5978	case ISD::BITREVERSE:
5979	assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BITREVERSE!");
5980	if (OpOpcode == ISD::UNDEF)
5981	return getUNDEF(VT);
5982	break;
5983	case ISD::BITCAST:
5984	assert(VT.getSizeInBits() == N1.getValueSizeInBits() &&
5985	"Cannot BITCAST between types of different sizes!");
5986	if (VT == N1.getValueType()) return N1; // noop conversion.
5987	if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
5988	return getNode(Opcode: ISD::BITCAST, DL, VT, N1: N1.getOperand(i: 0));
5989	if (OpOpcode == ISD::UNDEF)
5990	return getUNDEF(VT);
5991	break;
5992	case ISD::SCALAR_TO_VECTOR:
5993	assert(VT.isVector() && !N1.getValueType().isVector() &&
5994	(VT.getVectorElementType() == N1.getValueType() ||
5995	(VT.getVectorElementType().isInteger() &&
5996	N1.getValueType().isInteger() &&
5997	VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
5998	"Illegal SCALAR_TO_VECTOR node!");
5999	if (OpOpcode == ISD::UNDEF)
6000	return getUNDEF(VT);
6001	// scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
6002	if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
6003	isa<ConstantSDNode>(Val: N1.getOperand(i: 1)) &&
6004	N1.getConstantOperandVal(i: 1) == 0 &&
6005	N1.getOperand(i: 0).getValueType() == VT)
6006	return N1.getOperand(i: 0);
6007	break;
6008	case ISD::FNEG:
6009	// Negation of an unknown bag of bits is still completely undefined.
6010	if (OpOpcode == ISD::UNDEF)
6011	return getUNDEF(VT);
6012
6013	if (OpOpcode == ISD::FNEG) // --X -> X
6014	return N1.getOperand(i: 0);
6015	break;
6016	case ISD::FABS:
6017	if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
6018	return getNode(Opcode: ISD::FABS, DL, VT, N1: N1.getOperand(i: 0));
6019	break;
6020	case ISD::VSCALE:
6021	assert(VT == N1.getValueType() && "Unexpected VT!");
6022	break;
6023	case ISD::CTPOP:
6024	if (N1.getValueType().getScalarType() == MVT::i1)
6025	return N1;
6026	break;
6027	case ISD::CTLZ:
6028	case ISD::CTTZ:
6029	if (N1.getValueType().getScalarType() == MVT::i1)
6030	return getNOT(DL, Val: N1, VT: N1.getValueType());
6031	break;
6032	case ISD::VECREDUCE_ADD:
6033	if (N1.getValueType().getScalarType() == MVT::i1)
6034	return getNode(Opcode: ISD::VECREDUCE_XOR, DL, VT, N1);
6035	break;
6036	case ISD::VECREDUCE_SMIN:
6037	case ISD::VECREDUCE_UMAX:
6038	if (N1.getValueType().getScalarType() == MVT::i1)
6039	return getNode(Opcode: ISD::VECREDUCE_OR, DL, VT, N1);
6040	break;
6041	case ISD::VECREDUCE_SMAX:
6042	case ISD::VECREDUCE_UMIN:
6043	if (N1.getValueType().getScalarType() == MVT::i1)
6044	return getNode(Opcode: ISD::VECREDUCE_AND, DL, VT, N1);
6045	break;
6046	case ISD::SPLAT_VECTOR:
6047	assert(VT.isVector() && "Wrong return type!");
6048	// FIXME: Hexagon uses i32 scalar for a floating point zero vector so allow
6049	// that for now.
6050	assert((VT.getVectorElementType() == N1.getValueType() ||
6051	(VT.isFloatingPoint() && N1.getValueType() == MVT::i32) ||
6052	(VT.getVectorElementType().isInteger() &&
6053	N1.getValueType().isInteger() &&
6054	VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
6055	"Wrong operand type!");
6056	break;
6057	}
6058
6059	SDNode *N;
6060	SDVTList VTs = getVTList(VT);
6061	SDValue Ops[] = {N1};
6062	if (VT != MVT::Glue) { // Don't CSE glue producing nodes
6063	FoldingSetNodeID ID;
6064	AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
6065	void *IP = nullptr;
6066	if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
6067	E->intersectFlagsWith(Flags);
6068	return SDValue(E, 0);
6069	}
6070
6071	N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
6072	N->setFlags(Flags);
6073	createOperands(Node: N, Vals: Ops);
6074	CSEMap.InsertNode(N, InsertPos: IP);
6075	} else {
6076	N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
6077	createOperands(Node: N, Vals: Ops);
6078	}
6079
6080	InsertNode(N);
6081	SDValue V = SDValue(N, 0);
6082	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
6083	return V;
6084	}
6085
6086	static std::optional<APInt> FoldValue(unsigned Opcode, const APInt &C1,
6087	const APInt &C2) {
6088	switch (Opcode) {
6089	case ISD::ADD: return C1 + C2;
6090	case ISD::SUB: return C1 - C2;
6091	case ISD::MUL: return C1 * C2;
6092	case ISD::AND: return C1 & C2;
6093	case ISD::OR: return C1 | C2;
6094	case ISD::XOR: return C1 ^ C2;
6095	case ISD::SHL: return C1 << C2;
6096	case ISD::SRL: return C1.lshr(ShiftAmt: C2);
6097	case ISD::SRA: return C1.ashr(ShiftAmt: C2);
6098	case ISD::ROTL: return C1.rotl(rotateAmt: C2);
6099	case ISD::ROTR: return C1.rotr(rotateAmt: C2);
6100	case ISD::SMIN: return C1.sle(RHS: C2) ? C1 : C2;
6101	case ISD::SMAX: return C1.sge(RHS: C2) ? C1 : C2;
6102	case ISD::UMIN: return C1.ule(RHS: C2) ? C1 : C2;
6103	case ISD::UMAX: return C1.uge(RHS: C2) ? C1 : C2;
6104	case ISD::SADDSAT: return C1.sadd_sat(RHS: C2);
6105	case ISD::UADDSAT: return C1.uadd_sat(RHS: C2);
6106	case ISD::SSUBSAT: return C1.ssub_sat(RHS: C2);
6107	case ISD::USUBSAT: return C1.usub_sat(RHS: C2);
6108	case ISD::SSHLSAT: return C1.sshl_sat(RHS: C2);
6109	case ISD::USHLSAT: return C1.ushl_sat(RHS: C2);
6110	case ISD::UDIV:
6111	if (!C2.getBoolValue())
6112	break;
6113	return C1.udiv(RHS: C2);
6114	case ISD::UREM:
6115	if (!C2.getBoolValue())
6116	break;
6117	return C1.urem(RHS: C2);
6118	case ISD::SDIV:
6119	if (!C2.getBoolValue())
6120	break;
6121	return C1.sdiv(RHS: C2);
6122	case ISD::SREM:
6123	if (!C2.getBoolValue())
6124	break;
6125	return C1.srem(RHS: C2);
6126	case ISD::AVGFLOORS:
6127	return APIntOps::avgFloorS(C1, C2);
6128	case ISD::AVGFLOORU:
6129	return APIntOps::avgFloorU(C1, C2);
6130	case ISD::AVGCEILS:
6131	return APIntOps::avgCeilS(C1, C2);
6132	case ISD::AVGCEILU:
6133	return APIntOps::avgCeilU(C1, C2);
6134	case ISD::ABDS:
6135	return APIntOps::abds(A: C1, B: C2);
6136	case ISD::ABDU:
6137	return APIntOps::abdu(A: C1, B: C2);
6138	case ISD::MULHS:
6139	return APIntOps::mulhs(C1, C2);
6140	case ISD::MULHU:
6141	return APIntOps::mulhu(C1, C2);
6142	}
6143	return std::nullopt;
6144	}
6145	// Handle constant folding with UNDEF.
6146	// TODO: Handle more cases.
6147	static std::optional<APInt> FoldValueWithUndef(unsigned Opcode, const APInt &C1,
6148	bool IsUndef1, const APInt &C2,
6149	bool IsUndef2) {
6150	if (!(IsUndef1 || IsUndef2))
6151	return FoldValue(Opcode, C1, C2);
6152
6153	// Fold and(x, undef) -> 0
6154	// Fold mul(x, undef) -> 0
6155	if (Opcode == ISD::AND || Opcode == ISD::MUL)
6156	return APInt::getZero(numBits: C1.getBitWidth());
6157
6158	return std::nullopt;
6159	}
6160
6161	SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
6162	const GlobalAddressSDNode *GA,
6163	const SDNode *N2) {
6164	if (GA->getOpcode() != ISD::GlobalAddress)
6165	return SDValue();
6166	if (!TLI->isOffsetFoldingLegal(GA))
6167	return SDValue();
6168	auto *C2 = dyn_cast<ConstantSDNode>(Val: N2);
6169	if (!C2)
6170	return SDValue();
6171	int64_t Offset = C2->getSExtValue();
6172	switch (Opcode) {
6173	case ISD::ADD: break;
6174	case ISD::SUB: Offset = -uint64_t(Offset); break;
6175	default: return SDValue();
6176	}
6177	return getGlobalAddress(GV: GA->getGlobal(), DL: SDLoc(C2), VT,
6178	Offset: GA->getOffset() + uint64_t(Offset));
6179	}
6180
6181	bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
6182	switch (Opcode) {
6183	case ISD::SDIV:
6184	case ISD::UDIV:
6185	case ISD::SREM:
6186	case ISD::UREM: {
6187	// If a divisor is zero/undef or any element of a divisor vector is
6188	// zero/undef, the whole op is undef.
6189	assert(Ops.size() == 2 && "Div/rem should have 2 operands");
6190	SDValue Divisor = Ops[1];
6191	if (Divisor.isUndef() || isNullConstant(V: Divisor))
6192	return true;
6193
6194	return ISD::isBuildVectorOfConstantSDNodes(N: Divisor.getNode()) &&
6195	llvm::any_of(Range: Divisor->op_values(),
6196	P: [](SDValue V) { return V.isUndef() ||
6197	isNullConstant(V); });
6198	// TODO: Handle signed overflow.
6199	}
6200	// TODO: Handle oversized shifts.
6201	default:
6202	return false;
6203	}
6204	}
6205
6206	SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
6207	EVT VT, ArrayRef<SDValue> Ops) {
6208	// If the opcode is a target-specific ISD node, there's nothing we can
6209	// do here and the operand rules may not line up with the below, so
6210	// bail early.
6211	// We can't create a scalar CONCAT_VECTORS so skip it. It will break
6212	// for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by
6213	// foldCONCAT_VECTORS in getNode before this is called.
6214	if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS)
6215	return SDValue();
6216
6217	unsigned NumOps = Ops.size();
6218	if (NumOps == 0)
6219	return SDValue();
6220
6221	if (isUndef(Opcode, Ops))
6222	return getUNDEF(VT);
6223
6224	// Handle unary special cases.
6225	if (NumOps == 1) {
6226	SDValue N1 = Ops[0];
6227
6228	// Constant fold unary operations with an integer constant operand. Even
6229	// opaque constant will be folded, because the folding of unary operations
6230	// doesn't create new constants with different values. Nevertheless, the
6231	// opaque flag is preserved during folding to prevent future folding with
6232	// other constants.
6233	if (auto *C = dyn_cast<ConstantSDNode>(Val&: N1)) {
6234	const APInt &Val = C->getAPIntValue();
6235	switch (Opcode) {
6236	case ISD::SIGN_EXTEND:
6237	return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT,
6238	isT: C->isTargetOpcode(), isO: C->isOpaque());
6239	case ISD::TRUNCATE:
6240	if (C->isOpaque())
6241	break;
6242	[[fallthrough]];
6243	case ISD::ZERO_EXTEND:
6244	return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT,
6245	isT: C->isTargetOpcode(), isO: C->isOpaque());
6246	case ISD::ANY_EXTEND:
6247	// Some targets like RISCV prefer to sign extend some types.
6248	if (TLI->isSExtCheaperThanZExt(FromTy: N1.getValueType(), ToTy: VT))
6249	return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT,
6250	isT: C->isTargetOpcode(), isO: C->isOpaque());
6251	return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT,
6252	isT: C->isTargetOpcode(), isO: C->isOpaque());
6253	case ISD::ABS:
6254	return getConstant(Val: Val.abs(), DL, VT, isT: C->isTargetOpcode(),
6255	isO: C->isOpaque());
6256	case ISD::BITREVERSE:
6257	return getConstant(Val: Val.reverseBits(), DL, VT, isT: C->isTargetOpcode(),
6258	isO: C->isOpaque());
6259	case ISD::BSWAP:
6260	return getConstant(Val: Val.byteSwap(), DL, VT, isT: C->isTargetOpcode(),
6261	isO: C->isOpaque());
6262	case ISD::CTPOP:
6263	return getConstant(Val: Val.popcount(), DL, VT, isT: C->isTargetOpcode(),
6264	isO: C->isOpaque());
6265	case ISD::CTLZ:
6266	case ISD::CTLZ_ZERO_UNDEF:
6267	return getConstant(Val: Val.countl_zero(), DL, VT, isT: C->isTargetOpcode(),
6268	isO: C->isOpaque());
6269	case ISD::CTTZ:
6270	case ISD::CTTZ_ZERO_UNDEF:
6271	return getConstant(Val: Val.countr_zero(), DL, VT, isT: C->isTargetOpcode(),
6272	isO: C->isOpaque());
6273	case ISD::UINT_TO_FP:
6274	case ISD::SINT_TO_FP: {
6275	APFloat apf(EVTToAPFloatSemantics(VT),
6276	APInt::getZero(numBits: VT.getSizeInBits()));
6277	(void)apf.convertFromAPInt(Input: Val, IsSigned: Opcode == ISD::SINT_TO_FP,
6278	RM: APFloat::rmNearestTiesToEven);
6279	return getConstantFP(V: apf, DL, VT);
6280	}
6281	case ISD::FP16_TO_FP:
6282	case ISD::BF16_TO_FP: {
6283	bool Ignored;
6284	APFloat FPV(Opcode == ISD::FP16_TO_FP ? APFloat::IEEEhalf()
6285	: APFloat::BFloat(),
6286	(Val.getBitWidth() == 16) ? Val : Val.trunc(width: 16));
6287
6288	// This can return overflow, underflow, or inexact; we don't care.
6289	// FIXME need to be more flexible about rounding mode.
6290	(void)FPV.convert(ToSemantics: EVTToAPFloatSemantics(VT),
6291	RM: APFloat::rmNearestTiesToEven, losesInfo: &Ignored);
6292	return getConstantFP(V: FPV, DL, VT);
6293	}
6294	case ISD::STEP_VECTOR:
6295	if (SDValue V = FoldSTEP_VECTOR(DL, VT, Step: N1, DAG&: *this))
6296	return V;
6297	break;
6298	case ISD::BITCAST:
6299	if (VT == MVT::f16 && C->getValueType(ResNo: 0) == MVT::i16)
6300	return getConstantFP(V: APFloat(APFloat::IEEEhalf(), Val), DL, VT);
6301	if (VT == MVT::f32 && C->getValueType(ResNo: 0) == MVT::i32)
6302	return getConstantFP(V: APFloat(APFloat::IEEEsingle(), Val), DL, VT);
6303	if (VT == MVT::f64 && C->getValueType(ResNo: 0) == MVT::i64)
6304	return getConstantFP(V: APFloat(APFloat::IEEEdouble(), Val), DL, VT);
6305	if (VT == MVT::f128 && C->getValueType(ResNo: 0) == MVT::i128)
6306	return getConstantFP(V: APFloat(APFloat::IEEEquad(), Val), DL, VT);
6307	break;
6308	}
6309	}
6310
6311	// Constant fold unary operations with a floating point constant operand.
6312	if (auto *C = dyn_cast<ConstantFPSDNode>(Val&: N1)) {
6313	APFloat V = C->getValueAPF(); // make copy
6314	switch (Opcode) {
6315	case ISD::FNEG:
6316	V.changeSign();
6317	return getConstantFP(V, DL, VT);
6318	case ISD::FABS:
6319	V.clearSign();
6320	return getConstantFP(V, DL, VT);
6321	case ISD::FCEIL: {
6322	APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardPositive);
6323	if (fs == APFloat::opOK || fs == APFloat::opInexact)
6324	return getConstantFP(V, DL, VT);
6325	return SDValue();
6326	}
6327	case ISD::FTRUNC: {
6328	APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardZero);
6329	if (fs == APFloat::opOK || fs == APFloat::opInexact)
6330	return getConstantFP(V, DL, VT);
6331	return SDValue();
6332	}
6333	case ISD::FFLOOR: {
6334	APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardNegative);
6335	if (fs == APFloat::opOK || fs == APFloat::opInexact)
6336	return getConstantFP(V, DL, VT);
6337	return SDValue();
6338	}
6339	case ISD::FP_EXTEND: {
6340	bool ignored;
6341	// This can return overflow, underflow, or inexact; we don't care.
6342	// FIXME need to be more flexible about rounding mode.
6343	(void)V.convert(ToSemantics: EVTToAPFloatSemantics(VT), RM: APFloat::rmNearestTiesToEven,
6344	losesInfo: &ignored);
6345	return getConstantFP(V, DL, VT);
6346	}
6347	case ISD::FP_TO_SINT:
6348	case ISD::FP_TO_UINT: {
6349	bool ignored;
6350	APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
6351	// FIXME need to be more flexible about rounding mode.
6352	APFloat::opStatus s =
6353	V.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &ignored);
6354	if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
6355	break;
6356	return getConstant(Val: IntVal, DL, VT);
6357	}
6358	case ISD::FP_TO_FP16:
6359	case ISD::FP_TO_BF16: {
6360	bool Ignored;
6361	// This can return overflow, underflow, or inexact; we don't care.
6362	// FIXME need to be more flexible about rounding mode.
6363	(void)V.convert(ToSemantics: Opcode == ISD::FP_TO_FP16 ? APFloat::IEEEhalf()
6364	: APFloat::BFloat(),
6365	RM: APFloat::rmNearestTiesToEven, losesInfo: &Ignored);
6366	return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT);
6367	}
6368	case ISD::BITCAST:
6369	if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::f16)
6370	return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
6371	VT);
6372	if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::bf16)
6373	return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
6374	VT);
6375	if (VT == MVT::i32 && C->getValueType(ResNo: 0) == MVT::f32)
6376	return getConstant(Val: (uint32_t)V.bitcastToAPInt().getZExtValue(), DL,
6377	VT);
6378	if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
6379	return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT);
6380	break;
6381	}
6382	}
6383
6384	// Early-out if we failed to constant fold a bitcast.
6385	if (Opcode == ISD::BITCAST)
6386	return SDValue();
6387	}
6388
6389	// Handle binops special cases.
6390	if (NumOps == 2) {
6391	if (SDValue CFP = foldConstantFPMath(Opcode, DL, VT, Ops))
6392	return CFP;
6393
6394	if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0])) {
6395	if (auto *C2 = dyn_cast<ConstantSDNode>(Val: Ops[1])) {
6396	if (C1->isOpaque() || C2->isOpaque())
6397	return SDValue();
6398
6399	std::optional<APInt> FoldAttempt =
6400	FoldValue(Opcode, C1: C1->getAPIntValue(), C2: C2->getAPIntValue());
6401	if (!FoldAttempt)
6402	return SDValue();
6403
6404	SDValue Folded = getConstant(Val: *FoldAttempt, DL, VT);
6405	assert((!Folded || !VT.isVector()) &&
6406	"Can't fold vectors ops with scalar operands");
6407	return Folded;
6408	}
6409	}
6410
6411	// fold (add Sym, c) -> Sym+c
6412	if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[0]))
6413	return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[1].getNode());
6414	if (TLI->isCommutativeBinOp(Opcode))
6415	if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[1]))
6416	return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[0].getNode());
6417	}
6418
6419	// This is for vector folding only from here on.
6420	if (!VT.isVector())
6421	return SDValue();
6422
6423	ElementCount NumElts = VT.getVectorElementCount();
6424
6425	// See if we can fold through bitcasted integer ops.
6426	if (NumOps == 2 && VT.isFixedLengthVector() && VT.isInteger() &&
6427	Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
6428	Ops[0].getOpcode() == ISD::BITCAST &&
6429	Ops[1].getOpcode() == ISD::BITCAST) {
6430	SDValue N1 = peekThroughBitcasts(V: Ops[0]);
6431	SDValue N2 = peekThroughBitcasts(V: Ops[1]);
6432	auto *BV1 = dyn_cast<BuildVectorSDNode>(Val&: N1);
6433	auto *BV2 = dyn_cast<BuildVectorSDNode>(Val&: N2);
6434	EVT BVVT = N1.getValueType();
6435	if (BV1 && BV2 && BVVT.isInteger() && BVVT == N2.getValueType()) {
6436	bool IsLE = getDataLayout().isLittleEndian();
6437	unsigned EltBits = VT.getScalarSizeInBits();
6438	SmallVector<APInt> RawBits1, RawBits2;
6439	BitVector UndefElts1, UndefElts2;
6440	if (BV1->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits1, UndefElements&: UndefElts1) &&
6441	BV2->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits2, UndefElements&: UndefElts2)) {
6442	SmallVector<APInt> RawBits;
6443	for (unsigned I = 0, E = NumElts.getFixedValue(); I != E; ++I) {
6444	std::optional<APInt> Fold = FoldValueWithUndef(
6445	Opcode, C1: RawBits1[I], IsUndef1: UndefElts1[I], C2: RawBits2[I], IsUndef2: UndefElts2[I]);
6446	if (!Fold)
6447	break;
6448	RawBits.push_back(Elt: *Fold);
6449	}
6450	if (RawBits.size() == NumElts.getFixedValue()) {
6451	// We have constant folded, but we need to cast this again back to
6452	// the original (possibly legalized) type.
6453	SmallVector<APInt> DstBits;
6454	BitVector DstUndefs;
6455	BuildVectorSDNode::recastRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: BVVT.getScalarSizeInBits(),
6456	DstBitElements&: DstBits, SrcBitElements: RawBits, DstUndefElements&: DstUndefs,
6457	SrcUndefElements: BitVector(RawBits.size(), false));
6458	EVT BVEltVT = BV1->getOperand(Num: 0).getValueType();
6459	unsigned BVEltBits = BVEltVT.getSizeInBits();
6460	SmallVector<SDValue> Ops(DstBits.size(), getUNDEF(VT: BVEltVT));
6461	for (unsigned I = 0, E = DstBits.size(); I != E; ++I) {
6462	if (DstUndefs[I])
6463	continue;
6464	Ops[I] = getConstant(Val: DstBits[I].sext(width: BVEltBits), DL, VT: BVEltVT);
6465	}
6466	return getBitcast(VT, V: getBuildVector(VT: BVVT, DL, Ops));
6467	}
6468	}
6469	}
6470	}
6471
6472	// Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)).
6473	// (shl step_vector(C0), C1) -> (step_vector(C0 << C1))
6474	if ((Opcode == ISD::MUL || Opcode == ISD::SHL) &&
6475	Ops[0].getOpcode() == ISD::STEP_VECTOR) {
6476	APInt RHSVal;
6477	if (ISD::isConstantSplatVector(N: Ops[1].getNode(), SplatVal&: RHSVal)) {
6478	APInt NewStep = Opcode == ISD::MUL
6479	? Ops[0].getConstantOperandAPInt(i: 0) * RHSVal
6480	: Ops[0].getConstantOperandAPInt(i: 0) << RHSVal;
6481	return getStepVector(DL, ResVT: VT, StepVal: NewStep);
6482	}
6483	}
6484
6485	auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) {
6486	return !Op.getValueType().isVector() ||
6487	Op.getValueType().getVectorElementCount() == NumElts;
6488	};
6489
6490	auto IsBuildVectorSplatVectorOrUndef = [](const SDValue &Op) {
6491	return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE ||
6492	Op.getOpcode() == ISD::BUILD_VECTOR ||
6493	Op.getOpcode() == ISD::SPLAT_VECTOR;
6494	};
6495
6496	// All operands must be vector types with the same number of elements as
6497	// the result type and must be either UNDEF or a build/splat vector
6498	// or UNDEF scalars.
6499	if (!llvm::all_of(Range&: Ops, P: IsBuildVectorSplatVectorOrUndef) ||
6500	!llvm::all_of(Range&: Ops, P: IsScalarOrSameVectorSize))
6501	return SDValue();
6502
6503	// If we are comparing vectors, then the result needs to be a i1 boolean that
6504	// is then extended back to the legal result type depending on how booleans
6505	// are represented.
6506	EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
6507	ISD::NodeType ExtendCode =
6508	(Opcode == ISD::SETCC && SVT != VT.getScalarType())
6509	? TargetLowering::getExtendForContent(Content: TLI->getBooleanContents(Type: VT))
6510	: ISD::SIGN_EXTEND;
6511
6512	// Find legal integer scalar type for constant promotion and
6513	// ensure that its scalar size is at least as large as source.
6514	EVT LegalSVT = VT.getScalarType();
6515	if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
6516	LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
6517	if (LegalSVT.bitsLT(VT: VT.getScalarType()))
6518	return SDValue();
6519	}
6520
6521	// For scalable vector types we know we're dealing with SPLAT_VECTORs. We
6522	// only have one operand to check. For fixed-length vector types we may have
6523	// a combination of BUILD_VECTOR and SPLAT_VECTOR.
6524	unsigned NumVectorElts = NumElts.isScalable() ? 1 : NumElts.getFixedValue();
6525
6526	// Constant fold each scalar lane separately.
6527	SmallVector<SDValue, 4> ScalarResults;
6528	for (unsigned I = 0; I != NumVectorElts; I++) {
6529	SmallVector<SDValue, 4> ScalarOps;
6530	for (SDValue Op : Ops) {
6531	EVT InSVT = Op.getValueType().getScalarType();
6532	if (Op.getOpcode() != ISD::BUILD_VECTOR &&
6533	Op.getOpcode() != ISD::SPLAT_VECTOR) {
6534	if (Op.isUndef())
6535	ScalarOps.push_back(Elt: getUNDEF(VT: InSVT));
6536	else
6537	ScalarOps.push_back(Elt: Op);
6538	continue;
6539	}
6540
6541	SDValue ScalarOp =
6542	Op.getOperand(i: Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I);
6543	EVT ScalarVT = ScalarOp.getValueType();
6544
6545	// Build vector (integer) scalar operands may need implicit
6546	// truncation - do this before constant folding.
6547	if (ScalarVT.isInteger() && ScalarVT.bitsGT(VT: InSVT)) {
6548	// Don't create illegally-typed nodes unless they're constants or undef
6549	// - if we fail to constant fold we can't guarantee the (dead) nodes
6550	// we're creating will be cleaned up before being visited for
6551	// legalization.
6552	if (NewNodesMustHaveLegalTypes && !ScalarOp.isUndef() &&
6553	!isa<ConstantSDNode>(Val: ScalarOp) &&
6554	TLI->getTypeAction(Context&: *getContext(), VT: InSVT) !=
6555	TargetLowering::TypeLegal)
6556	return SDValue();
6557	ScalarOp = getNode(Opcode: ISD::TRUNCATE, DL, VT: InSVT, N1: ScalarOp);
6558	}
6559
6560	ScalarOps.push_back(Elt: ScalarOp);
6561	}
6562
6563	// Constant fold the scalar operands.
6564	SDValue ScalarResult = getNode(Opcode, DL, VT: SVT, Ops: ScalarOps);
6565
6566	// Legalize the (integer) scalar constant if necessary.
6567	if (LegalSVT != SVT)
6568	ScalarResult = getNode(Opcode: ExtendCode, DL, VT: LegalSVT, N1: ScalarResult);
6569
6570	// Scalar folding only succeeded if the result is a constant or UNDEF.
6571	if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
6572	ScalarResult.getOpcode() != ISD::ConstantFP)
6573	return SDValue();
6574	ScalarResults.push_back(Elt: ScalarResult);
6575	}
6576
6577	SDValue V = NumElts.isScalable() ? getSplatVector(VT, DL, Op: ScalarResults[0])
6578	: getBuildVector(VT, DL, Ops: ScalarResults);
6579	NewSDValueDbgMsg(V, Msg: "New node fold constant vector: ", G: this);
6580	return V;
6581	}
6582
6583	SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
6584	EVT VT, ArrayRef<SDValue> Ops) {
6585	// TODO: Add support for unary/ternary fp opcodes.
6586	if (Ops.size() != 2)
6587	return SDValue();
6588
6589	// TODO: We don't do any constant folding for strict FP opcodes here, but we
6590	// should. That will require dealing with a potentially non-default
6591	// rounding mode, checking the "opStatus" return value from the APFloat
6592	// math calculations, and possibly other variations.
6593	SDValue N1 = Ops[0];
6594	SDValue N2 = Ops[1];
6595	ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ false);
6596	ConstantFPSDNode *N2CFP = isConstOrConstSplatFP(N: N2, /*AllowUndefs*/ false);
6597	if (N1CFP && N2CFP) {
6598	APFloat C1 = N1CFP->getValueAPF(); // make copy
6599	const APFloat &C2 = N2CFP->getValueAPF();
6600	switch (Opcode) {
6601	case ISD::FADD:
6602	C1.add(RHS: C2, RM: APFloat::rmNearestTiesToEven);
6603	return getConstantFP(V: C1, DL, VT);
6604	case ISD::FSUB:
6605	C1.subtract(RHS: C2, RM: APFloat::rmNearestTiesToEven);
6606	return getConstantFP(V: C1, DL, VT);
6607	case ISD::FMUL:
6608	C1.multiply(RHS: C2, RM: APFloat::rmNearestTiesToEven);
6609	return getConstantFP(V: C1, DL, VT);
6610	case ISD::FDIV:
6611	C1.divide(RHS: C2, RM: APFloat::rmNearestTiesToEven);
6612	return getConstantFP(V: C1, DL, VT);
6613	case ISD::FREM:
6614	C1.mod(RHS: C2);
6615	return getConstantFP(V: C1, DL, VT);
6616	case ISD::FCOPYSIGN:
6617	C1.copySign(RHS: C2);
6618	return getConstantFP(V: C1, DL, VT);
6619	case ISD::FMINNUM:
6620	return getConstantFP(V: minnum(A: C1, B: C2), DL, VT);
6621	case ISD::FMAXNUM:
6622	return getConstantFP(V: maxnum(A: C1, B: C2), DL, VT);
6623	case ISD::FMINIMUM:
6624	return getConstantFP(V: minimum(A: C1, B: C2), DL, VT);
6625	case ISD::FMAXIMUM:
6626	return getConstantFP(V: maximum(A: C1, B: C2), DL, VT);
6627	default: break;
6628	}
6629	}
6630	if (N1CFP && Opcode == ISD::FP_ROUND) {
6631	APFloat C1 = N1CFP->getValueAPF(); // make copy
6632	bool Unused;
6633	// This can return overflow, underflow, or inexact; we don't care.
6634	// FIXME need to be more flexible about rounding mode.
6635	(void) C1.convert(ToSemantics: EVTToAPFloatSemantics(VT), RM: APFloat::rmNearestTiesToEven,
6636	losesInfo: &Unused);
6637	return getConstantFP(V: C1, DL, VT);
6638	}
6639
6640	switch (Opcode) {
6641	case ISD::FSUB:
6642	// -0.0 - undef --> undef (consistent with "fneg undef")
6643	if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ true))
6644	if (N1C && N1C->getValueAPF().isNegZero() && N2.isUndef())
6645	return getUNDEF(VT);
6646	[[fallthrough]];
6647
6648	case ISD::FADD:
6649	case ISD::FMUL:
6650	case ISD::FDIV:
6651	case ISD::FREM:
6652	// If both operands are undef, the result is undef. If 1 operand is undef,
6653	// the result is NaN. This should match the behavior of the IR optimizer.
6654	if (N1.isUndef() && N2.isUndef())
6655	return getUNDEF(VT);
6656	if (N1.isUndef() || N2.isUndef())
6657	return getConstantFP(V: APFloat::getNaN(Sem: EVTToAPFloatSemantics(VT)), DL, VT);
6658	}
6659	return SDValue();
6660	}
6661
6662	SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) {
6663	assert(Val.getValueType().isInteger() && "Invalid AssertAlign!");
6664
6665	// There's no need to assert on a byte-aligned pointer. All pointers are at
6666	// least byte aligned.
6667	if (A == Align(1))
6668	return Val;
6669
6670	FoldingSetNodeID ID;
6671	AddNodeIDNode(ID, OpC: ISD::AssertAlign, VTList: getVTList(VT: Val.getValueType()), OpList: {Val});
6672	ID.AddInteger(I: A.value());
6673
6674	void *IP = nullptr;
6675	if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
6676	return SDValue(E, 0);
6677
6678	auto *N = newSDNode<AssertAlignSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(),
6679	Args: Val.getValueType(), Args&: A);
6680	createOperands(Node: N, Vals: {Val});
6681
6682	CSEMap.InsertNode(N, InsertPos: IP);
6683	InsertNode(N);
6684
6685	SDValue V(N, 0);
6686	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
6687	return V;
6688	}
6689
6690	SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6691	SDValue N1, SDValue N2) {
6692	SDNodeFlags Flags;
6693	if (Inserter)
6694	Flags = Inserter->getFlags();
6695	return getNode(Opcode, DL, VT, N1, N2, Flags);
6696	}
6697
6698	void SelectionDAG::canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1,
6699	SDValue &N2) const {
6700	if (!TLI->isCommutativeBinOp(Opcode))
6701	return;
6702
6703	// Canonicalize:
6704	// binop(const, nonconst) -> binop(nonconst, const)
6705	SDNode *N1C = isConstantIntBuildVectorOrConstantInt(N: N1);
6706	SDNode *N2C = isConstantIntBuildVectorOrConstantInt(N: N2);
6707	SDNode *N1CFP = isConstantFPBuildVectorOrConstantFP(N: N1);
6708	SDNode *N2CFP = isConstantFPBuildVectorOrConstantFP(N: N2);
6709	if ((N1C && !N2C) || (N1CFP && !N2CFP))
6710	std::swap(a&: N1, b&: N2);
6711
6712	// Canonicalize:
6713	// binop(splat(x), step_vector) -> binop(step_vector, splat(x))
6714	else if (N1.getOpcode() == ISD::SPLAT_VECTOR &&
6715	N2.getOpcode() == ISD::STEP_VECTOR)
6716	std::swap(a&: N1, b&: N2);
6717	}
6718
6719	SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6720	SDValue N1, SDValue N2, const SDNodeFlags Flags) {
6721	assert(N1.getOpcode() != ISD::DELETED_NODE &&
6722	N2.getOpcode() != ISD::DELETED_NODE &&
6723	"Operand is DELETED_NODE!");
6724
6725	canonicalizeCommutativeBinop(Opcode, N1, N2);
6726
6727	auto *N1C = dyn_cast<ConstantSDNode>(Val&: N1);
6728	auto *N2C = dyn_cast<ConstantSDNode>(Val&: N2);
6729
6730	// Don't allow undefs in vector splats - we might be returning N2 when folding
6731	// to zero etc.
6732	ConstantSDNode *N2CV =
6733	isConstOrConstSplat(N: N2, /*AllowUndefs*/ false, /*AllowTruncation*/ true);
6734
6735	switch (Opcode) {
6736	default: break;
6737	case ISD::TokenFactor:
6738	assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
6739	N2.getValueType() == MVT::Other && "Invalid token factor!");
6740	// Fold trivial token factors.
6741	if (N1.getOpcode() == ISD::EntryToken) return N2;
6742	if (N2.getOpcode() == ISD::EntryToken) return N1;
6743	if (N1 == N2) return N1;
6744	break;
6745	case ISD::BUILD_VECTOR: {
6746	// Attempt to simplify BUILD_VECTOR.
6747	SDValue Ops[] = {N1, N2};
6748	if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
6749	return V;
6750	break;
6751	}
6752	case ISD::CONCAT_VECTORS: {
6753	SDValue Ops[] = {N1, N2};
6754	if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
6755	return V;
6756	break;
6757	}
6758	case ISD::AND:
6759	assert(VT.isInteger() && "This operator does not apply to FP types!");
6760	assert(N1.getValueType() == N2.getValueType() &&
6761	N1.getValueType() == VT && "Binary operator types must match!");
6762	// (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
6763	// worth handling here.
6764	if (N2CV && N2CV->isZero())
6765	return N2;
6766	if (N2CV && N2CV->isAllOnes()) // X & -1 -> X
6767	return N1;
6768	break;
6769	case ISD::OR:
6770	case ISD::XOR:
6771	case ISD::ADD:
6772	case ISD::SUB:
6773	assert(VT.isInteger() && "This operator does not apply to FP types!");
6774	assert(N1.getValueType() == N2.getValueType() &&
6775	N1.getValueType() == VT && "Binary operator types must match!");
6776	// (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
6777	// it's worth handling here.
6778	if (N2CV && N2CV->isZero())
6779	return N1;
6780	if ((Opcode == ISD::ADD || Opcode == ISD::SUB) && VT.isVector() &&
6781	VT.getVectorElementType() == MVT::i1)
6782	return getNode(Opcode: ISD::XOR, DL, VT, N1, N2);
6783	break;
6784	case ISD::MUL:
6785	assert(VT.isInteger() && "This operator does not apply to FP types!");
6786	assert(N1.getValueType() == N2.getValueType() &&
6787	N1.getValueType() == VT && "Binary operator types must match!");
6788	if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
6789	return getNode(Opcode: ISD::AND, DL, VT, N1, N2);
6790	if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
6791	const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0);
6792	const APInt &N2CImm = N2C->getAPIntValue();
6793	return getVScale(DL, VT, MulImm: MulImm * N2CImm);
6794	}
6795	break;
6796	case ISD::UDIV:
6797	case ISD::UREM:
6798	case ISD::MULHU:
6799	case ISD::MULHS:
6800	case ISD::SDIV:
6801	case ISD::SREM:
6802	case ISD::SADDSAT:
6803	case ISD::SSUBSAT:
6804	case ISD::UADDSAT:
6805	case ISD::USUBSAT:
6806	assert(VT.isInteger() && "This operator does not apply to FP types!");
6807	assert(N1.getValueType() == N2.getValueType() &&
6808	N1.getValueType() == VT && "Binary operator types must match!");
6809	if (VT.isVector() && VT.getVectorElementType() == MVT::i1) {
6810	// fold (add_sat x, y) -> (or x, y) for bool types.
6811	if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT)
6812	return getNode(Opcode: ISD::OR, DL, VT, N1, N2);
6813	// fold (sub_sat x, y) -> (and x, ~y) for bool types.
6814	if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT)
6815	return getNode(Opcode: ISD::AND, DL, VT, N1, N2: getNOT(DL, Val: N2, VT));
6816	}
6817	break;
6818	case ISD::ABDS:
6819	case ISD::ABDU:
6820	assert(VT.isInteger() && "This operator does not apply to FP types!");
6821	assert(N1.getValueType() == N2.getValueType() &&
6822	N1.getValueType() == VT && "Binary operator types must match!");
6823	break;
6824	case ISD::SMIN:
6825	case ISD::UMAX:
6826	assert(VT.isInteger() && "This operator does not apply to FP types!");
6827	assert(N1.getValueType() == N2.getValueType() &&
6828	N1.getValueType() == VT && "Binary operator types must match!");
6829	if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
6830	return getNode(Opcode: ISD::OR, DL, VT, N1, N2);
6831	break;
6832	case ISD::SMAX:
6833	case ISD::UMIN:
6834	assert(VT.isInteger() && "This operator does not apply to FP types!");
6835	assert(N1.getValueType() == N2.getValueType() &&
6836	N1.getValueType() == VT && "Binary operator types must match!");
6837	if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
6838	return getNode(Opcode: ISD::AND, DL, VT, N1, N2);
6839	break;
6840	case ISD::FADD:
6841	case ISD::FSUB:
6842	case ISD::FMUL:
6843	case ISD::FDIV:
6844	case ISD::FREM:
6845	assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
6846	assert(N1.getValueType() == N2.getValueType() &&
6847	N1.getValueType() == VT && "Binary operator types must match!");
6848	if (SDValue V = simplifyFPBinop(Opcode, X: N1, Y: N2, Flags))
6849	return V;
6850	break;
6851	case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
6852	assert(N1.getValueType() == VT &&
6853	N1.getValueType().isFloatingPoint() &&
6854	N2.getValueType().isFloatingPoint() &&
6855	"Invalid FCOPYSIGN!");
6856	break;
6857	case ISD::SHL:
6858	if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
6859	const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0);
6860	const APInt &ShiftImm = N2C->getAPIntValue();
6861	return getVScale(DL, VT, MulImm: MulImm << ShiftImm);
6862	}
6863	[[fallthrough]];
6864	case ISD::SRA:
6865	case ISD::SRL:
6866	if (SDValue V = simplifyShift(X: N1, Y: N2))
6867	return V;
6868	[[fallthrough]];
6869	case ISD::ROTL:
6870	case ISD::ROTR:
6871	assert(VT == N1.getValueType() &&
6872	"Shift operators return type must be the same as their first arg");
6873	assert(VT.isInteger() && N2.getValueType().isInteger() &&
6874	"Shifts only work on integers");
6875	assert((!VT.isVector() || VT == N2.getValueType()) &&
6876	"Vector shift amounts must be in the same as their first arg");
6877	// Verify that the shift amount VT is big enough to hold valid shift
6878	// amounts. This catches things like trying to shift an i1024 value by an
6879	// i8, which is easy to fall into in generic code that uses
6880	// TLI.getShiftAmount().
6881	assert(N2.getValueType().getScalarSizeInBits() >=
6882	Log2_32_Ceil(VT.getScalarSizeInBits()) &&
6883	"Invalid use of small shift amount with oversized value!");
6884
6885	// Always fold shifts of i1 values so the code generator doesn't need to
6886	// handle them. Since we know the size of the shift has to be less than the
6887	// size of the value, the shift/rotate count is guaranteed to be zero.
6888	if (VT == MVT::i1)
6889	return N1;
6890	if (N2CV && N2CV->isZero())
6891	return N1;
6892	break;
6893	case ISD::FP_ROUND:
6894	assert(VT.isFloatingPoint() &&
6895	N1.getValueType().isFloatingPoint() &&
6896	VT.bitsLE(N1.getValueType()) &&
6897	N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
6898	"Invalid FP_ROUND!");
6899	if (N1.getValueType() == VT) return N1; // noop conversion.
6900	break;
6901	case ISD::AssertSext:
6902	case ISD::AssertZext: {
6903	EVT EVT = cast<VTSDNode>(Val&: N2)->getVT();
6904	assert(VT == N1.getValueType() && "Not an inreg extend!");
6905	assert(VT.isInteger() && EVT.isInteger() &&
6906	"Cannot *_EXTEND_INREG FP types");
6907	assert(!EVT.isVector() &&
6908	"AssertSExt/AssertZExt type should be the vector element type "
6909	"rather than the vector type!");
6910	assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!");
6911	if (VT.getScalarType() == EVT) return N1; // noop assertion.
6912	break;
6913	}
6914	case ISD::SIGN_EXTEND_INREG: {
6915	EVT EVT = cast<VTSDNode>(Val&: N2)->getVT();
6916	assert(VT == N1.getValueType() && "Not an inreg extend!");
6917	assert(VT.isInteger() && EVT.isInteger() &&
6918	"Cannot *_EXTEND_INREG FP types");
6919	assert(EVT.isVector() == VT.isVector() &&
6920	"SIGN_EXTEND_INREG type should be vector iff the operand "
6921	"type is vector!");
6922	assert((!EVT.isVector() ||
6923	EVT.getVectorElementCount() == VT.getVectorElementCount()) &&
6924	"Vector element counts must match in SIGN_EXTEND_INREG");
6925	assert(EVT.bitsLE(VT) && "Not extending!");
6926	if (EVT == VT) return N1; // Not actually extending
6927
6928	auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {
6929	unsigned FromBits = EVT.getScalarSizeInBits();
6930	Val <<= Val.getBitWidth() - FromBits;
6931	Val.ashrInPlace(ShiftAmt: Val.getBitWidth() - FromBits);
6932	return getConstant(Val, DL, VT: ConstantVT);
6933	};
6934
6935	if (N1C) {
6936	const APInt &Val = N1C->getAPIntValue();
6937	return SignExtendInReg(Val, VT);
6938	}
6939
6940	if (ISD::isBuildVectorOfConstantSDNodes(N: N1.getNode())) {
6941	SmallVector<SDValue, 8> Ops;
6942	llvm::EVT OpVT = N1.getOperand(i: 0).getValueType();
6943	for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
6944	SDValue Op = N1.getOperand(i);
6945	if (Op.isUndef()) {
6946	Ops.push_back(Elt: getUNDEF(VT: OpVT));
6947	continue;
6948	}
6949	ConstantSDNode *C = cast<ConstantSDNode>(Val&: Op);
6950	APInt Val = C->getAPIntValue();
6951	Ops.push_back(Elt: SignExtendInReg(Val, OpVT));
6952	}
6953	return getBuildVector(VT, DL, Ops);
6954	}
6955
6956	if (N1.getOpcode() == ISD::SPLAT_VECTOR &&
6957	isa<ConstantSDNode>(Val: N1.getOperand(i: 0)))
6958	return getNode(
6959	Opcode: ISD::SPLAT_VECTOR, DL, VT,
6960	N1: SignExtendInReg(N1.getConstantOperandAPInt(i: 0),
6961	N1.getOperand(i: 0).getValueType()));
6962	break;
6963	}
6964	case ISD::FP_TO_SINT_SAT:
6965	case ISD::FP_TO_UINT_SAT: {
6966	assert(VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() &&
6967	N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT");
6968	assert(N1.getValueType().isVector() == VT.isVector() &&
6969	"FP_TO_*INT_SAT type should be vector iff the operand type is "
6970	"vector!");
6971	assert((!VT.isVector() || VT.getVectorElementCount() ==
6972	N1.getValueType().getVectorElementCount()) &&
6973	"Vector element counts must match in FP_TO_*INT_SAT");
6974	assert(!cast<VTSDNode>(N2)->getVT().isVector() &&
6975	"Type to saturate to must be a scalar.");
6976	assert(cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) &&
6977	"Not extending!");
6978	break;
6979	}
6980	case ISD::EXTRACT_VECTOR_ELT:
6981	assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() &&
6982	"The result of EXTRACT_VECTOR_ELT must be at least as wide as the \
6983	element type of the vector.");
6984
6985	// Extract from an undefined value or using an undefined index is undefined.
6986	if (N1.isUndef() || N2.isUndef())
6987	return getUNDEF(VT);
6988
6989	// EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF for fixed length
6990	// vectors. For scalable vectors we will provide appropriate support for
6991	// dealing with arbitrary indices.
6992	if (N2C && N1.getValueType().isFixedLengthVector() &&
6993	N2C->getAPIntValue().uge(RHS: N1.getValueType().getVectorNumElements()))
6994	return getUNDEF(VT);
6995
6996	// EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
6997	// expanding copies of large vectors from registers. This only works for
6998	// fixed length vectors, since we need to know the exact number of
6999	// elements.
7000	if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS &&
7001	N1.getOperand(i: 0).getValueType().isFixedLengthVector()) {
7002	unsigned Factor =
7003	N1.getOperand(i: 0).getValueType().getVectorNumElements();
7004	return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT,
7005	N1: N1.getOperand(i: N2C->getZExtValue() / Factor),
7006	N2: getVectorIdxConstant(Val: N2C->getZExtValue() % Factor, DL));
7007	}
7008
7009	// EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while
7010	// lowering is expanding large vector constants.
7011	if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR ||
7012	N1.getOpcode() == ISD::SPLAT_VECTOR)) {
7013	assert((N1.getOpcode() != ISD::BUILD_VECTOR ||
7014	N1.getValueType().isFixedLengthVector()) &&
7015	"BUILD_VECTOR used for scalable vectors");
7016	unsigned Index =
7017	N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0;
7018	SDValue Elt = N1.getOperand(i: Index);
7019
7020	if (VT != Elt.getValueType())
7021	// If the vector element type is not legal, the BUILD_VECTOR operands
7022	// are promoted and implicitly truncated, and the result implicitly
7023	// extended. Make that explicit here.
7024	Elt = getAnyExtOrTrunc(Op: Elt, DL, VT);
7025
7026	return Elt;
7027	}
7028
7029	// EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
7030	// operations are lowered to scalars.
7031	if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
7032	// If the indices are the same, return the inserted element else
7033	// if the indices are known different, extract the element from
7034	// the original vector.
7035	SDValue N1Op2 = N1.getOperand(i: 2);
7036	ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(Val&: N1Op2);
7037
7038	if (N1Op2C && N2C) {
7039	if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
7040	if (VT == N1.getOperand(i: 1).getValueType())
7041	return N1.getOperand(i: 1);
7042	if (VT.isFloatingPoint()) {
7043	assert(VT.getSizeInBits() > N1.getOperand(1).getValueType().getSizeInBits());
7044	return getFPExtendOrRound(Op: N1.getOperand(i: 1), DL, VT);
7045	}
7046	return getSExtOrTrunc(Op: N1.getOperand(i: 1), DL, VT);
7047	}
7048	return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0), N2);
7049	}
7050	}
7051
7052	// EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
7053	// when vector types are scalarized and v1iX is legal.
7054	// vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx).
7055	// Here we are completely ignoring the extract element index (N2),
7056	// which is fine for fixed width vectors, since any index other than 0
7057	// is undefined anyway. However, this cannot be ignored for scalable
7058	// vectors - in theory we could support this, but we don't want to do this
7059	// without a profitability check.
7060	if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
7061	N1.getValueType().isFixedLengthVector() &&
7062	N1.getValueType().getVectorNumElements() == 1) {
7063	return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0),
7064	N2: N1.getOperand(i: 1));
7065	}
7066	break;
7067	case ISD::EXTRACT_ELEMENT:
7068	assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
7069	assert(!N1.getValueType().isVector() && !VT.isVector() &&
7070	(N1.getValueType().isInteger() == VT.isInteger()) &&
7071	N1.getValueType() != VT &&
7072	"Wrong types for EXTRACT_ELEMENT!");
7073
7074	// EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
7075	// 64-bit integers into 32-bit parts. Instead of building the extract of
7076	// the BUILD_PAIR, only to have legalize rip it apart, just do it now.
7077	if (N1.getOpcode() == ISD::BUILD_PAIR)
7078	return N1.getOperand(i: N2C->getZExtValue());
7079
7080	// EXTRACT_ELEMENT of a constant int is also very common.
7081	if (N1C) {
7082	unsigned ElementSize = VT.getSizeInBits();
7083	unsigned Shift = ElementSize * N2C->getZExtValue();
7084	const APInt &Val = N1C->getAPIntValue();
7085	return getConstant(Val: Val.extractBits(numBits: ElementSize, bitPosition: Shift), DL, VT);
7086	}
7087	break;
7088	case ISD::EXTRACT_SUBVECTOR: {
7089	EVT N1VT = N1.getValueType();
7090	assert(VT.isVector() && N1VT.isVector() &&
7091	"Extract subvector VTs must be vectors!");
7092	assert(VT.getVectorElementType() == N1VT.getVectorElementType() &&
7093	"Extract subvector VTs must have the same element type!");
7094	assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) &&
7095	"Cannot extract a scalable vector from a fixed length vector!");
7096	assert((VT.isScalableVector() != N1VT.isScalableVector() ||
7097	VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements()) &&
7098	"Extract subvector must be from larger vector to smaller vector!");
7099	assert(N2C && "Extract subvector index must be a constant");
7100	assert((VT.isScalableVector() != N1VT.isScalableVector() ||
7101	(VT.getVectorMinNumElements() + N2C->getZExtValue()) <=
7102	N1VT.getVectorMinNumElements()) &&
7103	"Extract subvector overflow!");
7104	assert(N2C->getAPIntValue().getBitWidth() ==
7105	TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() &&
7106	"Constant index for EXTRACT_SUBVECTOR has an invalid size");
7107
7108	// Trivial extraction.
7109	if (VT == N1VT)
7110	return N1;
7111
7112	// EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
7113	if (N1.isUndef())
7114	return getUNDEF(VT);
7115
7116	// EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
7117	// the concat have the same type as the extract.
7118	if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
7119	VT == N1.getOperand(i: 0).getValueType()) {
7120	unsigned Factor = VT.getVectorMinNumElements();
7121	return N1.getOperand(i: N2C->getZExtValue() / Factor);
7122	}
7123
7124	// EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
7125	// during shuffle legalization.
7126	if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(i: 2) &&
7127	VT == N1.getOperand(i: 1).getValueType())
7128	return N1.getOperand(i: 1);
7129	break;
7130	}
7131	}
7132
7133	// Perform trivial constant folding.
7134	if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, Ops: {N1, N2}))
7135	return SV;
7136
7137	// Canonicalize an UNDEF to the RHS, even over a constant.
7138	if (N1.isUndef()) {
7139	if (TLI->isCommutativeBinOp(Opcode)) {
7140	std::swap(a&: N1, b&: N2);
7141	} else {
7142	switch (Opcode) {
7143	case ISD::SUB:
7144	return getUNDEF(VT); // fold op(undef, arg2) -> undef
7145	case ISD::SIGN_EXTEND_INREG:
7146	case ISD::UDIV:
7147	case ISD::SDIV:
7148	case ISD::UREM:
7149	case ISD::SREM:
7150	case ISD::SSUBSAT:
7151	case ISD::USUBSAT:
7152	return getConstant(Val: 0, DL, VT); // fold op(undef, arg2) -> 0
7153	}
7154	}
7155	}
7156
7157	// Fold a bunch of operators when the RHS is undef.
7158	if (N2.isUndef()) {
7159	switch (Opcode) {
7160	case ISD::XOR:
7161	if (N1.isUndef())
7162	// Handle undef ^ undef -> 0 special case. This is a common
7163	// idiom (misuse).
7164	return getConstant(Val: 0, DL, VT);
7165	[[fallthrough]];
7166	case ISD::ADD:
7167	case ISD::SUB:
7168	case ISD::UDIV:
7169	case ISD::SDIV:
7170	case ISD::UREM:
7171	case ISD::SREM:
7172	return getUNDEF(VT); // fold op(arg1, undef) -> undef
7173	case ISD::MUL:
7174	case ISD::AND:
7175	case ISD::SSUBSAT:
7176	case ISD::USUBSAT:
7177	return getConstant(Val: 0, DL, VT); // fold op(arg1, undef) -> 0
7178	case ISD::OR:
7179	case ISD::SADDSAT:
7180	case ISD::UADDSAT:
7181	return getAllOnesConstant(DL, VT);
7182	}
7183	}
7184
7185	// Memoize this node if possible.
7186	SDNode *N;
7187	SDVTList VTs = getVTList(VT);
7188	SDValue Ops[] = {N1, N2};
7189	if (VT != MVT::Glue) {
7190	FoldingSetNodeID ID;
7191	AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
7192	void *IP = nullptr;
7193	if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
7194	E->intersectFlagsWith(Flags);
7195	return SDValue(E, 0);
7196	}
7197
7198	N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7199	N->setFlags(Flags);
7200	createOperands(Node: N, Vals: Ops);
7201	CSEMap.InsertNode(N, InsertPos: IP);
7202	} else {
7203	N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7204	createOperands(Node: N, Vals: Ops);
7205	}
7206
7207	InsertNode(N);
7208	SDValue V = SDValue(N, 0);
7209	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
7210	return V;
7211	}
7212
7213	SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7214	SDValue N1, SDValue N2, SDValue N3) {
7215	SDNodeFlags Flags;
7216	if (Inserter)
7217	Flags = Inserter->getFlags();
7218	return getNode(Opcode, DL, VT, N1, N2, N3, Flags);
7219	}
7220
7221	SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7222	SDValue N1, SDValue N2, SDValue N3,
7223	const SDNodeFlags Flags) {
7224	assert(N1.getOpcode() != ISD::DELETED_NODE &&
7225	N2.getOpcode() != ISD::DELETED_NODE &&
7226	N3.getOpcode() != ISD::DELETED_NODE &&
7227	"Operand is DELETED_NODE!");
7228	// Perform various simplifications.
7229	switch (Opcode) {
7230	case ISD::FMA:
7231	case ISD::FMAD: {
7232	assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
7233	assert(N1.getValueType() == VT && N2.getValueType() == VT &&
7234	N3.getValueType() == VT && "FMA types must match!");
7235	ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(Val&: N1);
7236	ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(Val&: N2);
7237	ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(Val&: N3);
7238	if (N1CFP && N2CFP && N3CFP) {
7239	APFloat V1 = N1CFP->getValueAPF();
7240	const APFloat &V2 = N2CFP->getValueAPF();
7241	const APFloat &V3 = N3CFP->getValueAPF();
7242	if (Opcode == ISD::FMAD) {
7243	V1.multiply(RHS: V2, RM: APFloat::rmNearestTiesToEven);
7244	V1.add(RHS: V3, RM: APFloat::rmNearestTiesToEven);
7245	} else
7246	V1.fusedMultiplyAdd(Multiplicand: V2, Addend: V3, RM: APFloat::rmNearestTiesToEven);
7247	return getConstantFP(V: V1, DL, VT);
7248	}
7249	break;
7250	}
7251	case ISD::BUILD_VECTOR: {
7252	// Attempt to simplify BUILD_VECTOR.
7253	SDValue Ops[] = {N1, N2, N3};
7254	if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
7255	return V;
7256	break;
7257	}
7258	case ISD::CONCAT_VECTORS: {
7259	SDValue Ops[] = {N1, N2, N3};
7260	if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
7261	return V;
7262	break;
7263	}
7264	case ISD::SETCC: {
7265	assert(VT.isInteger() && "SETCC result type must be an integer!");
7266	assert(N1.getValueType() == N2.getValueType() &&
7267	"SETCC operands must have the same type!");
7268	assert(VT.isVector() == N1.getValueType().isVector() &&
7269	"SETCC type should be vector iff the operand type is vector!");
7270	assert((!VT.isVector() || VT.getVectorElementCount() ==
7271	N1.getValueType().getVectorElementCount()) &&
7272	"SETCC vector element counts must match!");
7273	// Use FoldSetCC to simplify SETCC's.
7274	if (SDValue V = FoldSetCC(VT, N1, N2, Cond: cast<CondCodeSDNode>(Val&: N3)->get(), dl: DL))
7275	return V;
7276	// Vector constant folding.
7277	SDValue Ops[] = {N1, N2, N3};
7278	if (SDValue V = FoldConstantArithmetic(Opcode, DL, VT, Ops)) {
7279	NewSDValueDbgMsg(V, Msg: "New node vector constant folding: ", G: this);
7280	return V;
7281	}
7282	break;
7283	}
7284	case ISD::SELECT:
7285	case ISD::VSELECT:
7286	if (SDValue V = simplifySelect(Cond: N1, TVal: N2, FVal: N3))
7287	return V;
7288	break;
7289	case ISD::VECTOR_SHUFFLE:
7290	llvm_unreachable("should use getVectorShuffle constructor!");
7291	case ISD::VECTOR_SPLICE: {
7292	if (cast<ConstantSDNode>(Val&: N3)->isZero())
7293	return N1;
7294	break;
7295	}
7296	case ISD::INSERT_VECTOR_ELT: {
7297	ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(Val&: N3);
7298	// INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except
7299	// for scalable vectors where we will generate appropriate code to
7300	// deal with out-of-bounds cases correctly.
7301	if (N3C && N1.getValueType().isFixedLengthVector() &&
7302	N3C->getZExtValue() >= N1.getValueType().getVectorNumElements())
7303	return getUNDEF(VT);
7304
7305	// Undefined index can be assumed out-of-bounds, so that's UNDEF too.
7306	if (N3.isUndef())
7307	return getUNDEF(VT);
7308
7309	// If the inserted element is an UNDEF, just use the input vector.
7310	if (N2.isUndef())
7311	return N1;
7312
7313	break;
7314	}
7315	case ISD::INSERT_SUBVECTOR: {
7316	// Inserting undef into undef is still undef.
7317	if (N1.isUndef() && N2.isUndef())
7318	return getUNDEF(VT);
7319
7320	EVT N2VT = N2.getValueType();
7321	assert(VT == N1.getValueType() &&
7322	"Dest and insert subvector source types must match!");
7323	assert(VT.isVector() && N2VT.isVector() &&
7324	"Insert subvector VTs must be vectors!");
7325	assert(VT.getVectorElementType() == N2VT.getVectorElementType() &&
7326	"Insert subvector VTs must have the same element type!");
7327	assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) &&
7328	"Cannot insert a scalable vector into a fixed length vector!");
7329	assert((VT.isScalableVector() != N2VT.isScalableVector() ||
7330	VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements()) &&
7331	"Insert subvector must be from smaller vector to larger vector!");
7332	assert(isa<ConstantSDNode>(N3) &&
7333	"Insert subvector index must be constant");
7334	assert((VT.isScalableVector() != N2VT.isScalableVector() ||
7335	(N2VT.getVectorMinNumElements() + N3->getAsZExtVal()) <=
7336	VT.getVectorMinNumElements()) &&
7337	"Insert subvector overflow!");
7338	assert(N3->getAsAPIntVal().getBitWidth() ==
7339	TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() &&
7340	"Constant index for INSERT_SUBVECTOR has an invalid size");
7341
7342	// Trivial insertion.
7343	if (VT == N2VT)
7344	return N2;
7345
7346	// If this is an insert of an extracted vector into an undef vector, we
7347	// can just use the input to the extract.
7348	if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
7349	N2.getOperand(i: 1) == N3 && N2.getOperand(i: 0).getValueType() == VT)
7350	return N2.getOperand(i: 0);
7351	break;
7352	}
7353	case ISD::BITCAST:
7354	// Fold bit_convert nodes from a type to themselves.
7355	if (N1.getValueType() == VT)
7356	return N1;
7357	break;
7358	case ISD::VP_TRUNCATE:
7359	case ISD::VP_SIGN_EXTEND:
7360	case ISD::VP_ZERO_EXTEND:
7361	// Don't create noop casts.
7362	if (N1.getValueType() == VT)
7363	return N1;
7364	break;
7365	}
7366
7367	// Memoize node if it doesn't produce a glue result.
7368	SDNode *N;
7369	SDVTList VTs = getVTList(VT);
7370	SDValue Ops[] = {N1, N2, N3};
7371	if (VT != MVT::Glue) {
7372	FoldingSetNodeID ID;
7373	AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
7374	void *IP = nullptr;
7375	if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
7376	E->intersectFlagsWith(Flags);
7377	return SDValue(E, 0);
7378	}
7379
7380	N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7381	N->setFlags(Flags);
7382	createOperands(Node: N, Vals: Ops);
7383	CSEMap.InsertNode(N, InsertPos: IP);
7384	} else {
7385	N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7386	createOperands(Node: N, Vals: Ops);
7387	}
7388
7389	InsertNode(N);
7390	SDValue V = SDValue(N, 0);
7391	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
7392	return V;
7393	}
7394
7395	SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7396	SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
7397	SDValue Ops[] = { N1, N2, N3, N4 };
7398	return getNode(Opcode, DL, VT, Ops);
7399	}
7400
7401	SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7402	SDValue N1, SDValue N2, SDValue N3, SDValue N4,
7403	SDValue N5) {
7404	SDValue Ops[] = { N1, N2, N3, N4, N5 };
7405	return getNode(Opcode, DL, VT, Ops);
7406	}
7407
7408	/// getStackArgumentTokenFactor - Compute a TokenFactor to force all
7409	/// the incoming stack arguments to be loaded from the stack.
7410	SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
7411	SmallVector<SDValue, 8> ArgChains;
7412
7413	// Include the original chain at the beginning of the list. When this is
7414	// used by target LowerCall hooks, this helps legalize find the
7415	// CALLSEQ_BEGIN node.
7416	ArgChains.push_back(Elt: Chain);
7417
7418	// Add a chain value for each stack argument.
7419	for (SDNode *U : getEntryNode().getNode()->uses())
7420	if (LoadSDNode *L = dyn_cast<LoadSDNode>(Val: U))
7421	if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: L->getBasePtr()))
7422	if (FI->getIndex() < 0)
7423	ArgChains.push_back(Elt: SDValue(L, 1));
7424
7425	// Build a tokenfactor for all the chains.
7426	return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
7427	}
7428
7429	/// getMemsetValue - Vectorized representation of the memset value
7430	/// operand.
7431	static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
7432	const SDLoc &dl) {
7433	assert(!Value.isUndef());
7434
7435	unsigned NumBits = VT.getScalarSizeInBits();
7436	if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Value)) {
7437	assert(C->getAPIntValue().getBitWidth() == 8);
7438	APInt Val = APInt::getSplat(NewLen: NumBits, V: C->getAPIntValue());
7439	if (VT.isInteger()) {
7440	bool IsOpaque = VT.getSizeInBits() > 64 ||
7441	!DAG.getTargetLoweringInfo().isLegalStoreImmediate(Value: C->getSExtValue());
7442	return DAG.getConstant(Val, DL: dl, VT, isT: false, isO: IsOpaque);
7443	}
7444	return DAG.getConstantFP(V: APFloat(DAG.EVTToAPFloatSemantics(VT), Val), DL: dl,
7445	VT);
7446	}
7447
7448	assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
7449	EVT IntVT = VT.getScalarType();
7450	if (!IntVT.isInteger())
7451	IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: IntVT.getSizeInBits());
7452
7453	Value = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, N1: Value);
7454	if (NumBits > 8) {
7455	// Use a multiplication with 0x010101... to extend the input to the
7456	// required length.
7457	APInt Magic = APInt::getSplat(NewLen: NumBits, V: APInt(8, 0x01));
7458	Value = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: IntVT, N1: Value,
7459	N2: DAG.getConstant(Val: Magic, DL: dl, VT: IntVT));
7460	}
7461
7462	if (VT != Value.getValueType() && !VT.isInteger())
7463	Value = DAG.getBitcast(VT: VT.getScalarType(), V: Value);
7464	if (VT != Value.getValueType())
7465	Value = DAG.getSplatBuildVector(VT, DL: dl, Op: Value);
7466
7467	return Value;
7468	}
7469
7470	/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
7471	/// used when a memcpy is turned into a memset when the source is a constant
7472	/// string ptr.
7473	static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG,
7474	const TargetLowering &TLI,
7475	const ConstantDataArraySlice &Slice) {
7476	// Handle vector with all elements zero.
7477	if (Slice.Array == nullptr) {
7478	if (VT.isInteger())
7479	return DAG.getConstant(Val: 0, DL: dl, VT);
7480	if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
7481	return DAG.getConstantFP(Val: 0.0, DL: dl, VT);
7482	if (VT.isVector()) {
7483	unsigned NumElts = VT.getVectorNumElements();
7484	MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
7485	return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT,
7486	N1: DAG.getConstant(Val: 0, DL: dl,
7487	VT: EVT::getVectorVT(Context&: *DAG.getContext(),
7488	VT: EltVT, NumElements: NumElts)));
7489	}
7490	llvm_unreachable("Expected type!");
7491	}
7492
7493	assert(!VT.isVector() && "Can't handle vector type here!");
7494	unsigned NumVTBits = VT.getSizeInBits();
7495	unsigned NumVTBytes = NumVTBits / 8;
7496	unsigned NumBytes = std::min(a: NumVTBytes, b: unsigned(Slice.Length));
7497
7498	APInt Val(NumVTBits, 0);
7499	if (DAG.getDataLayout().isLittleEndian()) {
7500	for (unsigned i = 0; i != NumBytes; ++i)
7501	Val |= (uint64_t)(unsigned char)Slice[i] << i*8;
7502	} else {
7503	for (unsigned i = 0; i != NumBytes; ++i)
7504	Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
7505	}
7506
7507	// If the "cost" of materializing the integer immediate is less than the cost
7508	// of a load, then it is cost effective to turn the load into the immediate.
7509	Type *Ty = VT.getTypeForEVT(Context&: *DAG.getContext());
7510	if (TLI.shouldConvertConstantLoadToIntImm(Imm: Val, Ty))
7511	return DAG.getConstant(Val, DL: dl, VT);
7512	return SDValue();
7513	}
7514
7515	SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset,
7516	const SDLoc &DL,
7517	const SDNodeFlags Flags) {
7518	EVT VT = Base.getValueType();
7519	SDValue Index;
7520
7521	if (Offset.isScalable())
7522	Index = getVScale(DL, VT: Base.getValueType(),
7523	MulImm: APInt(Base.getValueSizeInBits().getFixedValue(),
7524	Offset.getKnownMinValue()));
7525	else
7526	Index = getConstant(Val: Offset.getFixedValue(), DL, VT);
7527
7528	return getMemBasePlusOffset(Base, Offset: Index, DL, Flags);
7529	}
7530
7531	SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset,
7532	const SDLoc &DL,
7533	const SDNodeFlags Flags) {
7534	assert(Offset.getValueType().isInteger());
7535	EVT BasePtrVT = Ptr.getValueType();
7536	return getNode(Opcode: ISD::ADD, DL, VT: BasePtrVT, N1: Ptr, N2: Offset, Flags);
7537	}
7538
7539	/// Returns true if memcpy source is constant data.
7540	static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) {
7541	uint64_t SrcDelta = 0;
7542	GlobalAddressSDNode *G = nullptr;
7543	if (Src.getOpcode() == ISD::GlobalAddress)
7544	G = cast<GlobalAddressSDNode>(Val&: Src);
7545	else if (Src.getOpcode() == ISD::ADD &&
7546	Src.getOperand(i: 0).getOpcode() == ISD::GlobalAddress &&
7547	Src.getOperand(i: 1).getOpcode() == ISD::Constant) {
7548	G = cast<GlobalAddressSDNode>(Val: Src.getOperand(i: 0));
7549	SrcDelta = Src.getConstantOperandVal(i: 1);
7550	}
7551	if (!G)
7552	return false;
7553
7554	return getConstantDataArrayInfo(V: G->getGlobal(), Slice, ElementSize: 8,
7555	Offset: SrcDelta + G->getOffset());
7556	}
7557
7558	static bool shouldLowerMemFuncForSize(const MachineFunction &MF,
7559	SelectionDAG &DAG) {
7560	// On Darwin, -Os means optimize for size without hurting performance, so
7561	// only really optimize for size when -Oz (MinSize) is used.
7562	if (MF.getTarget().getTargetTriple().isOSDarwin())
7563	return MF.getFunction().hasMinSize();
7564	return DAG.shouldOptForSize();
7565	}
7566
7567	static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
7568	SmallVector<SDValue, 32> &OutChains, unsigned From,
7569	unsigned To, SmallVector<SDValue, 16> &OutLoadChains,
7570	SmallVector<SDValue, 16> &OutStoreChains) {
7571	assert(OutLoadChains.size() && "Missing loads in memcpy inlining");
7572	assert(OutStoreChains.size() && "Missing stores in memcpy inlining");
7573	SmallVector<SDValue, 16> GluedLoadChains;
7574	for (unsigned i = From; i < To; ++i) {
7575	OutChains.push_back(Elt: OutLoadChains[i]);
7576	GluedLoadChains.push_back(Elt: OutLoadChains[i]);
7577	}
7578
7579	// Chain for all loads.
7580	SDValue LoadToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
7581	GluedLoadChains);
7582
7583	for (unsigned i = From; i < To; ++i) {
7584	StoreSDNode *ST = dyn_cast<StoreSDNode>(Val&: OutStoreChains[i]);
7585	SDValue NewStore = DAG.getTruncStore(Chain: LoadToken, dl, Val: ST->getValue(),
7586	Ptr: ST->getBasePtr(), SVT: ST->getMemoryVT(),
7587	MMO: ST->getMemOperand());
7588	OutChains.push_back(Elt: NewStore);
7589	}
7590	}
7591
7592	static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
7593	SDValue Chain, SDValue Dst, SDValue Src,
7594	uint64_t Size, Align Alignment,
7595	bool isVol, bool AlwaysInline,
7596	MachinePointerInfo DstPtrInfo,
7597	MachinePointerInfo SrcPtrInfo,
7598	const AAMDNodes &AAInfo, AAResults *AA) {
7599	// Turn a memcpy of undef to nop.
7600	// FIXME: We need to honor volatile even is Src is undef.
7601	if (Src.isUndef())
7602	return Chain;
7603
7604	// Expand memcpy to a series of load and store ops if the size operand falls
7605	// below a certain threshold.
7606	// TODO: In the AlwaysInline case, if the size is big then generate a loop
7607	// rather than maybe a humongous number of loads and stores.
7608	const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7609	const DataLayout &DL = DAG.getDataLayout();
7610	LLVMContext &C = *DAG.getContext();
7611	std::vector<EVT> MemOps;
7612	bool DstAlignCanChange = false;
7613	MachineFunction &MF = DAG.getMachineFunction();
7614	MachineFrameInfo &MFI = MF.getFrameInfo();
7615	bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
7616	FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
7617	if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
7618	DstAlignCanChange = true;
7619	MaybeAlign SrcAlign = DAG.InferPtrAlign(Ptr: Src);
7620	if (!SrcAlign || Alignment > *SrcAlign)
7621	SrcAlign = Alignment;
7622	assert(SrcAlign && "SrcAlign must be set");
7623	ConstantDataArraySlice Slice;
7624	// If marked as volatile, perform a copy even when marked as constant.
7625	bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice);
7626	bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
7627	unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
7628	const MemOp Op = isZeroConstant
7629	? MemOp::Set(Size, DstAlignCanChange, DstAlign: Alignment,
7630	/*IsZeroMemset*/ true, IsVolatile: isVol)
7631	: MemOp::Copy(Size, DstAlignCanChange, DstAlign: Alignment,
7632	SrcAlign: *SrcAlign, IsVolatile: isVol, MemcpyStrSrc: CopyFromConstant);
7633	if (!TLI.findOptimalMemOpLowering(
7634	MemOps, Limit, Op, DstAS: DstPtrInfo.getAddrSpace(),
7635	SrcAS: SrcPtrInfo.getAddrSpace(), FuncAttributes: MF.getFunction().getAttributes()))
7636	return SDValue();
7637
7638	if (DstAlignCanChange) {
7639	Type *Ty = MemOps[0].getTypeForEVT(Context&: C);
7640	Align NewAlign = DL.getABITypeAlign(Ty);
7641
7642	// Don't promote to an alignment that would require dynamic stack
7643	// realignment which may conflict with optimizations such as tail call
7644	// optimization.
7645	const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
7646	if (!TRI->hasStackRealignment(MF))
7647	while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(Alignment: NewAlign))
7648	NewAlign = NewAlign.previous();
7649
7650	if (NewAlign > Alignment) {
7651	// Give the stack frame object a larger alignment if needed.
7652	if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign)
7653	MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign);
7654	Alignment = NewAlign;
7655	}
7656	}
7657
7658	// Prepare AAInfo for loads/stores after lowering this memcpy.
7659	AAMDNodes NewAAInfo = AAInfo;
7660	NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
7661
7662	const Value *SrcVal = dyn_cast_if_present<const Value *>(Val&: SrcPtrInfo.V);
7663	bool isConstant =
7664	AA && SrcVal &&
7665	AA->pointsToConstantMemory(Loc: MemoryLocation(SrcVal, Size, AAInfo));
7666
7667	MachineMemOperand::Flags MMOFlags =
7668	isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
7669	SmallVector<SDValue, 16> OutLoadChains;
7670	SmallVector<SDValue, 16> OutStoreChains;
7671	SmallVector<SDValue, 32> OutChains;
7672	unsigned NumMemOps = MemOps.size();
7673	uint64_t SrcOff = 0, DstOff = 0;
7674	for (unsigned i = 0; i != NumMemOps; ++i) {
7675	EVT VT = MemOps[i];
7676	unsigned VTSize = VT.getSizeInBits() / 8;
7677	SDValue Value, Store;
7678
7679	if (VTSize > Size) {
7680	// Issuing an unaligned load / store pair that overlaps with the previous
7681	// pair. Adjust the offset accordingly.
7682	assert(i == NumMemOps-1 && i != 0);
7683	SrcOff -= VTSize - Size;
7684	DstOff -= VTSize - Size;
7685	}
7686
7687	if (CopyFromConstant &&
7688	(isZeroConstant || (VT.isInteger() && !VT.isVector()))) {
7689	// It's unlikely a store of a vector immediate can be done in a single
7690	// instruction. It would require a load from a constantpool first.
7691	// We only handle zero vectors here.
7692	// FIXME: Handle other cases where store of vector immediate is done in
7693	// a single instruction.
7694	ConstantDataArraySlice SubSlice;
7695	if (SrcOff < Slice.Length) {
7696	SubSlice = Slice;
7697	SubSlice.move(Delta: SrcOff);
7698	} else {
7699	// This is an out-of-bounds access and hence UB. Pretend we read zero.
7700	SubSlice.Array = nullptr;
7701	SubSlice.Offset = 0;
7702	SubSlice.Length = VTSize;
7703	}
7704	Value = getMemsetStringVal(VT, dl, DAG, TLI, Slice: SubSlice);
7705	if (Value.getNode()) {
7706	Store = DAG.getStore(
7707	Chain, dl, Val: Value,
7708	Ptr: DAG.getMemBasePlusOffset(Base: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff), DL: dl),
7709	PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment, MMOFlags, AAInfo: NewAAInfo);
7710	OutChains.push_back(Elt: Store);
7711	}
7712	}
7713
7714	if (!Store.getNode()) {
7715	// The type might not be legal for the target. This should only happen
7716	// if the type is smaller than a legal type, as on PPC, so the right
7717	// thing to do is generate a LoadExt/StoreTrunc pair. These simplify
7718	// to Load/Store if NVT==VT.
7719	// FIXME does the case above also need this?
7720	EVT NVT = TLI.getTypeToTransformTo(Context&: C, VT);
7721	assert(NVT.bitsGE(VT));
7722
7723	bool isDereferenceable =
7724	SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL);
7725	MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
7726	if (isDereferenceable)
7727	SrcMMOFlags |= MachineMemOperand::MODereferenceable;
7728	if (isConstant)
7729	SrcMMOFlags |= MachineMemOperand::MOInvariant;
7730
7731	Value = DAG.getExtLoad(
7732	ExtType: ISD::EXTLOAD, dl, VT: NVT, Chain,
7733	Ptr: DAG.getMemBasePlusOffset(Base: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff), DL: dl),
7734	PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), MemVT: VT,
7735	Alignment: commonAlignment(A: *SrcAlign, Offset: SrcOff), MMOFlags: SrcMMOFlags, AAInfo: NewAAInfo);
7736	OutLoadChains.push_back(Elt: Value.getValue(R: 1));
7737
7738	Store = DAG.getTruncStore(
7739	Chain, dl, Val: Value,
7740	Ptr: DAG.getMemBasePlusOffset(Base: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff), DL: dl),
7741	PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), SVT: VT, Alignment, MMOFlags, AAInfo: NewAAInfo);
7742	OutStoreChains.push_back(Elt: Store);
7743	}
7744	SrcOff += VTSize;
7745	DstOff += VTSize;
7746	Size -= VTSize;
7747	}
7748
7749	unsigned GluedLdStLimit = MaxLdStGlue == 0 ?
7750	TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue;
7751	unsigned NumLdStInMemcpy = OutStoreChains.size();
7752
7753	if (NumLdStInMemcpy) {
7754	// It may be that memcpy might be converted to memset if it's memcpy
7755	// of constants. In such a case, we won't have loads and stores, but
7756	// just stores. In the absence of loads, there is nothing to gang up.
7757	if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) {
7758	// If target does not care, just leave as it.
7759	for (unsigned i = 0; i < NumLdStInMemcpy; ++i) {
7760	OutChains.push_back(Elt: OutLoadChains[i]);
7761	OutChains.push_back(Elt: OutStoreChains[i]);
7762	}
7763	} else {
7764	// Ld/St less than/equal limit set by target.
7765	if (NumLdStInMemcpy <= GluedLdStLimit) {
7766	chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: 0,
7767	To: NumLdStInMemcpy, OutLoadChains,
7768	OutStoreChains);
7769	} else {
7770	unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit;
7771	unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
7772	unsigned GlueIter = 0;
7773
7774	for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
7775	unsigned IndexFrom = NumLdStInMemcpy - GlueIter - GluedLdStLimit;
7776	unsigned IndexTo = NumLdStInMemcpy - GlueIter;
7777
7778	chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: IndexFrom, To: IndexTo,
7779	OutLoadChains, OutStoreChains);
7780	GlueIter += GluedLdStLimit;
7781	}
7782
7783	// Residual ld/st.
7784	if (RemainingLdStInMemcpy) {
7785	chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: 0,
7786	To: RemainingLdStInMemcpy, OutLoadChains,
7787	OutStoreChains);
7788	}
7789	}
7790	}
7791	}
7792	return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
7793	}
7794
7795	static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
7796	SDValue Chain, SDValue Dst, SDValue Src,
7797	uint64_t Size, Align Alignment,
7798	bool isVol, bool AlwaysInline,
7799	MachinePointerInfo DstPtrInfo,
7800	MachinePointerInfo SrcPtrInfo,
7801	const AAMDNodes &AAInfo) {
7802	// Turn a memmove of undef to nop.
7803	// FIXME: We need to honor volatile even is Src is undef.
7804	if (Src.isUndef())
7805	return Chain;
7806
7807	// Expand memmove to a series of load and store ops if the size operand falls
7808	// below a certain threshold.
7809	const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7810	const DataLayout &DL = DAG.getDataLayout();
7811	LLVMContext &C = *DAG.getContext();
7812	std::vector<EVT> MemOps;
7813	bool DstAlignCanChange = false;
7814	MachineFunction &MF = DAG.getMachineFunction();
7815	MachineFrameInfo &MFI = MF.getFrameInfo();
7816	bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
7817	FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
7818	if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
7819	DstAlignCanChange = true;
7820	MaybeAlign SrcAlign = DAG.InferPtrAlign(Ptr: Src);
7821	if (!SrcAlign || Alignment > *SrcAlign)
7822	SrcAlign = Alignment;
7823	assert(SrcAlign && "SrcAlign must be set");
7824	unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
7825	if (!TLI.findOptimalMemOpLowering(
7826	MemOps, Limit,
7827	Op: MemOp::Copy(Size, DstAlignCanChange, DstAlign: Alignment, SrcAlign: *SrcAlign,
7828	/*IsVolatile*/ true),
7829	DstAS: DstPtrInfo.getAddrSpace(), SrcAS: SrcPtrInfo.getAddrSpace(),
7830	FuncAttributes: MF.getFunction().getAttributes()))
7831	return SDValue();
7832
7833	if (DstAlignCanChange) {
7834	Type *Ty = MemOps[0].getTypeForEVT(Context&: C);
7835	Align NewAlign = DL.getABITypeAlign(Ty);
7836
7837	// Don't promote to an alignment that would require dynamic stack
7838	// realignment which may conflict with optimizations such as tail call
7839	// optimization.
7840	const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
7841	if (!TRI->hasStackRealignment(MF))
7842	while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(Alignment: NewAlign))
7843	NewAlign = NewAlign.previous();
7844
7845	if (NewAlign > Alignment) {
7846	// Give the stack frame object a larger alignment if needed.
7847	if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign)
7848	MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign);
7849	Alignment = NewAlign;
7850	}
7851	}
7852
7853	// Prepare AAInfo for loads/stores after lowering this memmove.
7854	AAMDNodes NewAAInfo = AAInfo;
7855	NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
7856
7857	MachineMemOperand::Flags MMOFlags =
7858	isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
7859	uint64_t SrcOff = 0, DstOff = 0;
7860	SmallVector<SDValue, 8> LoadValues;
7861	SmallVector<SDValue, 8> LoadChains;
7862	SmallVector<SDValue, 8> OutChains;
7863	unsigned NumMemOps = MemOps.size();
7864	for (unsigned i = 0; i < NumMemOps; i++) {
7865	EVT VT = MemOps[i];
7866	unsigned VTSize = VT.getSizeInBits() / 8;
7867	SDValue Value;
7868
7869	bool isDereferenceable =
7870	SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL);
7871	MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
7872	if (isDereferenceable)
7873	SrcMMOFlags |= MachineMemOperand::MODereferenceable;
7874
7875	Value = DAG.getLoad(
7876	VT, dl, Chain,
7877	Ptr: DAG.getMemBasePlusOffset(Base: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff), DL: dl),
7878	PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), Alignment: *SrcAlign, MMOFlags: SrcMMOFlags, AAInfo: NewAAInfo);
7879	LoadValues.push_back(Elt: Value);
7880	LoadChains.push_back(Elt: Value.getValue(R: 1));
7881	SrcOff += VTSize;
7882	}
7883	Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
7884	OutChains.clear();
7885	for (unsigned i = 0; i < NumMemOps; i++) {
7886	EVT VT = MemOps[i];
7887	unsigned VTSize = VT.getSizeInBits() / 8;
7888	SDValue Store;
7889
7890	Store = DAG.getStore(
7891	Chain, dl, Val: LoadValues[i],
7892	Ptr: DAG.getMemBasePlusOffset(Base: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff), DL: dl),
7893	PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment, MMOFlags, AAInfo: NewAAInfo);
7894	OutChains.push_back(Elt: Store);
7895	DstOff += VTSize;
7896	}
7897
7898	return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
7899	}
7900
7901	/// Lower the call to 'memset' intrinsic function into a series of store
7902	/// operations.
7903	///
7904	/// \param DAG Selection DAG where lowered code is placed.
7905	/// \param dl Link to corresponding IR location.
7906	/// \param Chain Control flow dependency.
7907	/// \param Dst Pointer to destination memory location.
7908	/// \param Src Value of byte to write into the memory.
7909	/// \param Size Number of bytes to write.
7910	/// \param Alignment Alignment of the destination in bytes.
7911	/// \param isVol True if destination is volatile.
7912	/// \param AlwaysInline Makes sure no function call is generated.
7913	/// \param DstPtrInfo IR information on the memory pointer.
7914	/// \returns New head in the control flow, if lowering was successful, empty
7915	/// SDValue otherwise.
7916	///
7917	/// The function tries to replace 'llvm.memset' intrinsic with several store
7918	/// operations and value calculation code. This is usually profitable for small
7919	/// memory size or when the semantic requires inlining.
7920	static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
7921	SDValue Chain, SDValue Dst, SDValue Src,
7922	uint64_t Size, Align Alignment, bool isVol,
7923	bool AlwaysInline, MachinePointerInfo DstPtrInfo,
7924	const AAMDNodes &AAInfo) {
7925	// Turn a memset of undef to nop.
7926	// FIXME: We need to honor volatile even is Src is undef.
7927	if (Src.isUndef())
7928	return Chain;
7929
7930	// Expand memset to a series of load/store ops if the size operand
7931	// falls below a certain threshold.
7932	const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7933	std::vector<EVT> MemOps;
7934	bool DstAlignCanChange = false;
7935	MachineFunction &MF = DAG.getMachineFunction();
7936	MachineFrameInfo &MFI = MF.getFrameInfo();
7937	bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
7938	FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
7939	if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
7940	DstAlignCanChange = true;
7941	bool IsZeroVal = isNullConstant(V: Src);
7942	unsigned Limit = AlwaysInline ? ~0 : TLI.getMaxStoresPerMemset(OptSize);
7943
7944	if (!TLI.findOptimalMemOpLowering(
7945	MemOps, Limit,
7946	Op: MemOp::Set(Size, DstAlignCanChange, DstAlign: Alignment, IsZeroMemset: IsZeroVal, IsVolatile: isVol),
7947	DstAS: DstPtrInfo.getAddrSpace(), SrcAS: ~0u, FuncAttributes: MF.getFunction().getAttributes()))
7948	return SDValue();
7949
7950	if (DstAlignCanChange) {
7951	Type *Ty = MemOps[0].getTypeForEVT(Context&: *DAG.getContext());
7952	const DataLayout &DL = DAG.getDataLayout();
7953	Align NewAlign = DL.getABITypeAlign(Ty);
7954
7955	// Don't promote to an alignment that would require dynamic stack
7956	// realignment which may conflict with optimizations such as tail call
7957	// optimization.
7958	const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
7959	if (!TRI->hasStackRealignment(MF))
7960	while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(Alignment: NewAlign))
7961	NewAlign = NewAlign.previous();
7962
7963	if (NewAlign > Alignment) {
7964	// Give the stack frame object a larger alignment if needed.
7965	if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign)
7966	MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign);
7967	Alignment = NewAlign;
7968	}
7969	}
7970
7971	SmallVector<SDValue, 8> OutChains;
7972	uint64_t DstOff = 0;
7973	unsigned NumMemOps = MemOps.size();
7974
7975	// Find the largest store and generate the bit pattern for it.
7976	EVT LargestVT = MemOps[0];
7977	for (unsigned i = 1; i < NumMemOps; i++)
7978	if (MemOps[i].bitsGT(VT: LargestVT))
7979	LargestVT = MemOps[i];
7980	SDValue MemSetValue = getMemsetValue(Value: Src, VT: LargestVT, DAG, dl);
7981
7982	// Prepare AAInfo for loads/stores after lowering this memset.
7983	AAMDNodes NewAAInfo = AAInfo;
7984	NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
7985
7986	for (unsigned i = 0; i < NumMemOps; i++) {
7987	EVT VT = MemOps[i];
7988	unsigned VTSize = VT.getSizeInBits() / 8;
7989	if (VTSize > Size) {
7990	// Issuing an unaligned load / store pair that overlaps with the previous
7991	// pair. Adjust the offset accordingly.
7992	assert(i == NumMemOps-1 && i != 0);
7993	DstOff -= VTSize - Size;
7994	}
7995
7996	// If this store is smaller than the largest store see whether we can get
7997	// the smaller value for free with a truncate or extract vector element and
7998	// then store.
7999	SDValue Value = MemSetValue;
8000	if (VT.bitsLT(VT: LargestVT)) {
8001	unsigned Index;
8002	unsigned NElts = LargestVT.getSizeInBits() / VT.getSizeInBits();
8003	EVT SVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getScalarType(), NumElements: NElts);
8004	if (!LargestVT.isVector() && !VT.isVector() &&
8005	TLI.isTruncateFree(FromVT: LargestVT, ToVT: VT))
8006	Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, N1: MemSetValue);
8007	else if (LargestVT.isVector() && !VT.isVector() &&
8008	TLI.shallExtractConstSplatVectorElementToStore(
8009	VectorTy: LargestVT.getTypeForEVT(Context&: *DAG.getContext()),
8010	ElemSizeInBits: VT.getSizeInBits(), Index) &&
8011	TLI.isTypeLegal(VT: SVT) &&
8012	LargestVT.getSizeInBits() == SVT.getSizeInBits()) {
8013	// Target which can combine store(extractelement VectorTy, Idx) can get
8014	// the smaller value for free.
8015	SDValue TailValue = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: SVT, N1: MemSetValue);
8016	Value = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT, N1: TailValue,
8017	N2: DAG.getVectorIdxConstant(Val: Index, DL: dl));
8018	} else
8019	Value = getMemsetValue(Value: Src, VT, DAG, dl);
8020	}
8021	assert(Value.getValueType() == VT && "Value with wrong type.");
8022	SDValue Store = DAG.getStore(
8023	Chain, dl, Val: Value,
8024	Ptr: DAG.getMemBasePlusOffset(Base: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff), DL: dl),
8025	PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment,
8026	MMOFlags: isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone,
8027	AAInfo: NewAAInfo);
8028	OutChains.push_back(Elt: Store);
8029	DstOff += VT.getSizeInBits() / 8;
8030	Size -= VTSize;
8031	}
8032
8033	return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
8034	}
8035
8036	static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
8037	unsigned AS) {
8038	// Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
8039	// pointer operands can be losslessly bitcasted to pointers of address space 0
8040	if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(SrcAS: AS, DestAS: 0)) {
8041	report_fatal_error(reason: "cannot lower memory intrinsic in address space " +
8042	Twine(AS));
8043	}
8044	}
8045
8046	SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
8047	SDValue Src, SDValue Size, Align Alignment,
8048	bool isVol, bool AlwaysInline, bool isTailCall,
8049	MachinePointerInfo DstPtrInfo,
8050	MachinePointerInfo SrcPtrInfo,
8051	const AAMDNodes &AAInfo, AAResults *AA) {
8052	// Check to see if we should lower the memcpy to loads and stores first.
8053	// For cases within the target-specified limits, this is the best choice.
8054	ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
8055	if (ConstantSize) {
8056	// Memcpy with size zero? Just return the original chain.
8057	if (ConstantSize->isZero())
8058	return Chain;
8059
8060	SDValue Result = getMemcpyLoadsAndStores(
8061	DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), Alignment,
8062	isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo, AA);
8063	if (Result.getNode())
8064	return Result;
8065	}
8066
8067	// Then check to see if we should lower the memcpy with target-specific
8068	// code. If the target chooses to do this, this is the next best.
8069	if (TSI) {
8070	SDValue Result = TSI->EmitTargetCodeForMemcpy(
8071	DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, Alignment, isVolatile: isVol, AlwaysInline,
8072	DstPtrInfo, SrcPtrInfo);
8073	if (Result.getNode())
8074	return Result;
8075	}
8076
8077	// If we really need inline code and the target declined to provide it,
8078	// use a (potentially long) sequence of loads and stores.
8079	if (AlwaysInline) {
8080	assert(ConstantSize && "AlwaysInline requires a constant size!");
8081	return getMemcpyLoadsAndStores(
8082	DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), Alignment,
8083	isVol, AlwaysInline: true, DstPtrInfo, SrcPtrInfo, AAInfo, AA);
8084	}
8085
8086	checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
8087	checkAddrSpaceIsValidForLibcall(TLI, AS: SrcPtrInfo.getAddrSpace());
8088
8089	// FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
8090	// memcpy is not guaranteed to be safe. libc memcpys aren't required to
8091	// respect volatile, so they may do things like read or write memory
8092	// beyond the given memory regions. But fixing this isn't easy, and most
8093	// people don't care.
8094
8095	// Emit a library call.
8096	TargetLowering::ArgListTy Args;
8097	TargetLowering::ArgListEntry Entry;
8098	Entry.Ty = PointerType::getUnqual(C&: *getContext());
8099	Entry.Node = Dst; Args.push_back(x: Entry);
8100	Entry.Node = Src; Args.push_back(x: Entry);
8101
8102	Entry.Ty = getDataLayout().getIntPtrType(C&: *getContext());
8103	Entry.Node = Size; Args.push_back(x: Entry);
8104	// FIXME: pass in SDLoc
8105	TargetLowering::CallLoweringInfo CLI(*this);
8106	CLI.setDebugLoc(dl)
8107	.setChain(Chain)
8108	.setLibCallee(CC: TLI->getLibcallCallingConv(Call: RTLIB::MEMCPY),
8109	ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()),
8110	Target: getExternalSymbol(Sym: TLI->getLibcallName(Call: RTLIB::MEMCPY),
8111	VT: TLI->getPointerTy(DL: getDataLayout())),
8112	ArgsList: std::move(Args))
8113	.setDiscardResult()
8114	.setTailCall(isTailCall);
8115
8116	std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
8117	return CallResult.second;
8118	}
8119
8120	SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
8121	SDValue Dst, SDValue Src, SDValue Size,
8122	Type *SizeTy, unsigned ElemSz,
8123	bool isTailCall,
8124	MachinePointerInfo DstPtrInfo,
8125	MachinePointerInfo SrcPtrInfo) {
8126	// Emit a library call.
8127	TargetLowering::ArgListTy Args;
8128	TargetLowering::ArgListEntry Entry;
8129	Entry.Ty = getDataLayout().getIntPtrType(C&: *getContext());
8130	Entry.Node = Dst;
8131	Args.push_back(x: Entry);
8132
8133	Entry.Node = Src;
8134	Args.push_back(x: Entry);
8135
8136	Entry.Ty = SizeTy;
8137	Entry.Node = Size;
8138	Args.push_back(x: Entry);
8139
8140	RTLIB::Libcall LibraryCall =
8141	RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
8142	if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
8143	report_fatal_error(reason: "Unsupported element size");
8144
8145	TargetLowering::CallLoweringInfo CLI(*this);
8146	CLI.setDebugLoc(dl)
8147	.setChain(Chain)
8148	.setLibCallee(CC: TLI->getLibcallCallingConv(Call: LibraryCall),
8149	ResultType: Type::getVoidTy(C&: *getContext()),
8150	Target: getExternalSymbol(Sym: TLI->getLibcallName(Call: LibraryCall),
8151	VT: TLI->getPointerTy(DL: getDataLayout())),
8152	ArgsList: std::move(Args))
8153	.setDiscardResult()
8154	.setTailCall(isTailCall);
8155
8156	std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8157	return CallResult.second;
8158	}
8159
8160	SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
8161	SDValue Src, SDValue Size, Align Alignment,
8162	bool isVol, bool isTailCall,
8163	MachinePointerInfo DstPtrInfo,
8164	MachinePointerInfo SrcPtrInfo,
8165	const AAMDNodes &AAInfo, AAResults *AA) {
8166	// Check to see if we should lower the memmove to loads and stores first.
8167	// For cases within the target-specified limits, this is the best choice.
8168	ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
8169	if (ConstantSize) {
8170	// Memmove with size zero? Just return the original chain.
8171	if (ConstantSize->isZero())
8172	return Chain;
8173
8174	SDValue Result = getMemmoveLoadsAndStores(
8175	DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), Alignment,
8176	isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo);
8177	if (Result.getNode())
8178	return Result;
8179	}
8180
8181	// Then check to see if we should lower the memmove with target-specific
8182	// code. If the target chooses to do this, this is the next best.
8183	if (TSI) {
8184	SDValue Result =
8185	TSI->EmitTargetCodeForMemmove(DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size,
8186	Alignment, isVolatile: isVol, DstPtrInfo, SrcPtrInfo);
8187	if (Result.getNode())
8188	return Result;
8189	}
8190
8191	checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
8192	checkAddrSpaceIsValidForLibcall(TLI, AS: SrcPtrInfo.getAddrSpace());
8193
8194	// FIXME: If the memmove is volatile, lowering it to plain libc memmove may
8195	// not be safe. See memcpy above for more details.
8196
8197	// Emit a library call.
8198	TargetLowering::ArgListTy Args;
8199	TargetLowering::ArgListEntry Entry;
8200	Entry.Ty = PointerType::getUnqual(C&: *getContext());
8201	Entry.Node = Dst; Args.push_back(x: Entry);
8202	Entry.Node = Src; Args.push_back(x: Entry);
8203
8204	Entry.Ty = getDataLayout().getIntPtrType(C&: *getContext());
8205	Entry.Node = Size; Args.push_back(x: Entry);
8206	// FIXME: pass in SDLoc
8207	TargetLowering::CallLoweringInfo CLI(*this);
8208	CLI.setDebugLoc(dl)
8209	.setChain(Chain)
8210	.setLibCallee(CC: TLI->getLibcallCallingConv(Call: RTLIB::MEMMOVE),
8211	ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()),
8212	Target: getExternalSymbol(Sym: TLI->getLibcallName(Call: RTLIB::MEMMOVE),
8213	VT: TLI->getPointerTy(DL: getDataLayout())),
8214	ArgsList: std::move(Args))
8215	.setDiscardResult()
8216	.setTailCall(isTailCall);
8217
8218	std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
8219	return CallResult.second;
8220	}
8221
8222	SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
8223	SDValue Dst, SDValue Src, SDValue Size,
8224	Type *SizeTy, unsigned ElemSz,
8225	bool isTailCall,
8226	MachinePointerInfo DstPtrInfo,
8227	MachinePointerInfo SrcPtrInfo) {
8228	// Emit a library call.
8229	TargetLowering::ArgListTy Args;
8230	TargetLowering::ArgListEntry Entry;
8231	Entry.Ty = getDataLayout().getIntPtrType(C&: *getContext());
8232	Entry.Node = Dst;
8233	Args.push_back(x: Entry);
8234
8235	Entry.Node = Src;
8236	Args.push_back(x: Entry);
8237
8238	Entry.Ty = SizeTy;
8239	Entry.Node = Size;
8240	Args.push_back(x: Entry);
8241
8242	RTLIB::Libcall LibraryCall =
8243	RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
8244	if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
8245	report_fatal_error(reason: "Unsupported element size");
8246
8247	TargetLowering::CallLoweringInfo CLI(*this);
8248	CLI.setDebugLoc(dl)
8249	.setChain(Chain)
8250	.setLibCallee(CC: TLI->getLibcallCallingConv(Call: LibraryCall),
8251	ResultType: Type::getVoidTy(C&: *getContext()),
8252	Target: getExternalSymbol(Sym: TLI->getLibcallName(Call: LibraryCall),
8253	VT: TLI->getPointerTy(DL: getDataLayout())),
8254	ArgsList: std::move(Args))
8255	.setDiscardResult()
8256	.setTailCall(isTailCall);
8257
8258	std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8259	return CallResult.second;
8260	}
8261
8262	SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
8263	SDValue Src, SDValue Size, Align Alignment,
8264	bool isVol, bool AlwaysInline, bool isTailCall,
8265	MachinePointerInfo DstPtrInfo,
8266	const AAMDNodes &AAInfo) {
8267	// Check to see if we should lower the memset to stores first.
8268	// For cases within the target-specified limits, this is the best choice.
8269	ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
8270	if (ConstantSize) {
8271	// Memset with size zero? Just return the original chain.
8272	if (ConstantSize->isZero())
8273	return Chain;
8274
8275	SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src,
8276	Size: ConstantSize->getZExtValue(), Alignment,
8277	isVol, AlwaysInline: false, DstPtrInfo, AAInfo);
8278
8279	if (Result.getNode())
8280	return Result;
8281	}
8282
8283	// Then check to see if we should lower the memset with target-specific
8284	// code. If the target chooses to do this, this is the next best.
8285	if (TSI) {
8286	SDValue Result = TSI->EmitTargetCodeForMemset(
8287	DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, Alignment, isVolatile: isVol, AlwaysInline, DstPtrInfo);
8288	if (Result.getNode())
8289	return Result;
8290	}
8291
8292	// If we really need inline code and the target declined to provide it,
8293	// use a (potentially long) sequence of loads and stores.
8294	if (AlwaysInline) {
8295	assert(ConstantSize && "AlwaysInline requires a constant size!");
8296	SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src,
8297	Size: ConstantSize->getZExtValue(), Alignment,
8298	isVol, AlwaysInline: true, DstPtrInfo, AAInfo);
8299	assert(Result &&
8300	"getMemsetStores must return a valid sequence when AlwaysInline");
8301	return Result;
8302	}
8303
8304	checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
8305
8306	// Emit a library call.
8307	auto &Ctx = *getContext();
8308	const auto& DL = getDataLayout();
8309
8310	TargetLowering::CallLoweringInfo CLI(*this);
8311	// FIXME: pass in SDLoc
8312	CLI.setDebugLoc(dl).setChain(Chain);
8313
8314	const char *BzeroName = getTargetLoweringInfo().getLibcallName(Call: RTLIB::BZERO);
8315
8316	// Helper function to create an Entry from Node and Type.
8317	const auto CreateEntry = [](SDValue Node, Type *Ty) {
8318	TargetLowering::ArgListEntry Entry;
8319	Entry.Node = Node;
8320	Entry.Ty = Ty;
8321	return Entry;
8322	};
8323
8324	// If zeroing out and bzero is present, use it.
8325	if (isNullConstant(V: Src) && BzeroName) {
8326	TargetLowering::ArgListTy Args;
8327	Args.push_back(x: CreateEntry(Dst, PointerType::getUnqual(C&: Ctx)));
8328	Args.push_back(x: CreateEntry(Size, DL.getIntPtrType(C&: Ctx)));
8329	CLI.setLibCallee(
8330	CC: TLI->getLibcallCallingConv(Call: RTLIB::BZERO), ResultType: Type::getVoidTy(C&: Ctx),
8331	Target: getExternalSymbol(Sym: BzeroName, VT: TLI->getPointerTy(DL)), ArgsList: std::move(Args));
8332	} else {
8333	TargetLowering::ArgListTy Args;
8334	Args.push_back(x: CreateEntry(Dst, PointerType::getUnqual(C&: Ctx)));
8335	Args.push_back(x: CreateEntry(Src, Src.getValueType().getTypeForEVT(Context&: Ctx)));
8336	Args.push_back(x: CreateEntry(Size, DL.getIntPtrType(C&: Ctx)));
8337	CLI.setLibCallee(CC: TLI->getLibcallCallingConv(Call: RTLIB::MEMSET),
8338	ResultType: Dst.getValueType().getTypeForEVT(Context&: Ctx),
8339	Target: getExternalSymbol(Sym: TLI->getLibcallName(Call: RTLIB::MEMSET),
8340	VT: TLI->getPointerTy(DL)),
8341	ArgsList: std::move(Args));
8342	}
8343
8344	CLI.setDiscardResult().setTailCall(isTailCall);
8345
8346	std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8347	return CallResult.second;
8348	}
8349
8350	SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl,
8351	SDValue Dst, SDValue Value, SDValue Size,
8352	Type *SizeTy, unsigned ElemSz,
8353	bool isTailCall,
8354	MachinePointerInfo DstPtrInfo) {
8355	// Emit a library call.
8356	TargetLowering::ArgListTy Args;
8357	TargetLowering::ArgListEntry Entry;
8358	Entry.Ty = getDataLayout().getIntPtrType(C&: *getContext());
8359	Entry.Node = Dst;
8360	Args.push_back(x: Entry);
8361
8362	Entry.Ty = Type::getInt8Ty(C&: *getContext());
8363	Entry.Node = Value;
8364	Args.push_back(x: Entry);
8365
8366	Entry.Ty = SizeTy;
8367	Entry.Node = Size;
8368	Args.push_back(x: Entry);
8369
8370	RTLIB::Libcall LibraryCall =
8371	RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
8372	if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
8373	report_fatal_error(reason: "Unsupported element size");
8374
8375	TargetLowering::CallLoweringInfo CLI(*this);
8376	CLI.setDebugLoc(dl)
8377	.setChain(Chain)
8378	.setLibCallee(CC: TLI->getLibcallCallingConv(Call: LibraryCall),
8379	ResultType: Type::getVoidTy(C&: *getContext()),
8380	Target: getExternalSymbol(Sym: TLI->getLibcallName(Call: LibraryCall),
8381	VT: TLI->getPointerTy(DL: getDataLayout())),
8382	ArgsList: std::move(Args))
8383	.setDiscardResult()
8384	.setTailCall(isTailCall);
8385
8386	std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8387	return CallResult.second;
8388	}
8389
8390	SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
8391	SDVTList VTList, ArrayRef<SDValue> Ops,
8392	MachineMemOperand *MMO) {
8393	FoldingSetNodeID ID;
8394	ID.AddInteger(I: MemVT.getRawBits());
8395	AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
8396	ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
8397	ID.AddInteger(I: MMO->getFlags());
8398	void* IP = nullptr;
8399	if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
8400	cast<AtomicSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
8401	return SDValue(E, 0);
8402	}
8403
8404	auto *N = newSDNode<AtomicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
8405	Args&: VTList, Args&: MemVT, Args&: MMO);
8406	createOperands(Node: N, Vals: Ops);
8407
8408	CSEMap.InsertNode(N, InsertPos: IP);
8409	InsertNode(N);
8410	return SDValue(N, 0);
8411	}
8412
8413	SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
8414	EVT MemVT, SDVTList VTs, SDValue Chain,
8415	SDValue Ptr, SDValue Cmp, SDValue Swp,
8416	MachineMemOperand *MMO) {
8417	assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
8418	Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
8419	assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
8420
8421	SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
8422	return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
8423	}
8424
8425	SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
8426	SDValue Chain, SDValue Ptr, SDValue Val,
8427	MachineMemOperand *MMO) {
8428	assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
8429	Opcode == ISD::ATOMIC_LOAD_SUB ||
8430	Opcode == ISD::ATOMIC_LOAD_AND ||
8431	Opcode == ISD::ATOMIC_LOAD_CLR ||
8432	Opcode == ISD::ATOMIC_LOAD_OR ||
8433	Opcode == ISD::ATOMIC_LOAD_XOR ||
8434	Opcode == ISD::ATOMIC_LOAD_NAND ||
8435	Opcode == ISD::ATOMIC_LOAD_MIN ||
8436	Opcode == ISD::ATOMIC_LOAD_MAX ||
8437	Opcode == ISD::ATOMIC_LOAD_UMIN ||
8438	Opcode == ISD::ATOMIC_LOAD_UMAX ||
8439	Opcode == ISD::ATOMIC_LOAD_FADD ||
8440	Opcode == ISD::ATOMIC_LOAD_FSUB ||
8441	Opcode == ISD::ATOMIC_LOAD_FMAX ||
8442	Opcode == ISD::ATOMIC_LOAD_FMIN ||
8443	Opcode == ISD::ATOMIC_LOAD_UINC_WRAP ||
8444	Opcode == ISD::ATOMIC_LOAD_UDEC_WRAP ||
8445	Opcode == ISD::ATOMIC_SWAP ||
8446	Opcode == ISD::ATOMIC_STORE) &&
8447	"Invalid Atomic Op");
8448
8449	EVT VT = Val.getValueType();
8450
8451	SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
8452	getVTList(VT, MVT::Other);
8453	SDValue Ops[] = {Chain, Ptr, Val};
8454	return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
8455	}
8456
8457	SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
8458	EVT VT, SDValue Chain, SDValue Ptr,
8459	MachineMemOperand *MMO) {
8460	assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
8461
8462	SDVTList VTs = getVTList(VT, MVT::Other);
8463	SDValue Ops[] = {Chain, Ptr};
8464	return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
8465	}
8466
8467	/// getMergeValues - Create a MERGE_VALUES node from the given operands.
8468	SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
8469	if (Ops.size() == 1)
8470	return Ops[0];
8471
8472	SmallVector<EVT, 4> VTs;
8473	VTs.reserve(N: Ops.size());
8474	for (const SDValue &Op : Ops)
8475	VTs.push_back(Elt: Op.getValueType());
8476	return getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: getVTList(VTs), Ops);
8477	}
8478
8479	SDValue SelectionDAG::getMemIntrinsicNode(
8480	unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
8481	EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
8482	MachineMemOperand::Flags Flags, LocationSize Size,
8483	const AAMDNodes &AAInfo) {
8484	if (Size.hasValue() && !Size.getValue())
8485	Size = LocationSize::precise(Value: MemVT.getStoreSize());
8486
8487	MachineFunction &MF = getMachineFunction();
8488	MachineMemOperand *MMO =
8489	MF.getMachineMemOperand(PtrInfo, F: Flags, Size, BaseAlignment: Alignment, AAInfo);
8490
8491	return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
8492	}
8493
8494	SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
8495	SDVTList VTList,
8496	ArrayRef<SDValue> Ops, EVT MemVT,
8497	MachineMemOperand *MMO) {
8498	assert((Opcode == ISD::INTRINSIC_VOID ||
8499	Opcode == ISD::INTRINSIC_W_CHAIN ||
8500	Opcode == ISD::PREFETCH ||
8501	(Opcode <= (unsigned)std::numeric_limits<int>::max() &&
8502	(int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
8503	"Opcode is not a memory-accessing opcode!");
8504
8505	// Memoize the node unless it returns a glue result.
8506	MemIntrinsicSDNode *N;
8507	if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
8508	FoldingSetNodeID ID;
8509	AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
8510	ID.AddInteger(I: getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
8511	Opc: Opcode, Order: dl.getIROrder(), VTs: VTList, MemoryVT: MemVT, MMO));
8512	ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
8513	ID.AddInteger(I: MMO->getFlags());
8514	ID.AddInteger(I: MemVT.getRawBits());
8515	void *IP = nullptr;
8516	if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
8517	cast<MemIntrinsicSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
8518	return SDValue(E, 0);
8519	}
8520
8521	N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
8522	Args&: VTList, Args&: MemVT, Args&: MMO);
8523	createOperands(Node: N, Vals: Ops);
8524
8525	CSEMap.InsertNode(N, InsertPos: IP);
8526	} else {
8527	N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
8528	Args&: VTList, Args&: MemVT, Args&: MMO);
8529	createOperands(Node: N, Vals: Ops);
8530	}
8531	InsertNode(N);
8532	SDValue V(N, 0);
8533	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8534	return V;
8535	}
8536
8537	SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl,
8538	SDValue Chain, int FrameIndex,
8539	int64_t Size, int64_t Offset) {
8540	const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END;
8541	const auto VTs = getVTList(MVT::Other);
8542	SDValue Ops[2] = {
8543	Chain,
8544	getFrameIndex(FI: FrameIndex,
8545	VT: getTargetLoweringInfo().getFrameIndexTy(DL: getDataLayout()),
8546	isTarget: true)};
8547
8548	FoldingSetNodeID ID;
8549	AddNodeIDNode(ID, Opcode, VTs, Ops);
8550	ID.AddInteger(I: FrameIndex);
8551	ID.AddInteger(I: Size);
8552	ID.AddInteger(I: Offset);
8553	void *IP = nullptr;
8554	if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
8555	return SDValue(E, 0);
8556
8557	LifetimeSDNode *N = newSDNode<LifetimeSDNode>(
8558	Opcode, dl.getIROrder(), dl.getDebugLoc(), VTs, Size, Offset);
8559	createOperands(Node: N, Vals: Ops);
8560	CSEMap.InsertNode(N, InsertPos: IP);
8561	InsertNode(N);
8562	SDValue V(N, 0);
8563	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8564	return V;
8565	}
8566
8567	SDValue SelectionDAG::getPseudoProbeNode(const SDLoc &Dl, SDValue Chain,
8568	uint64_t Guid, uint64_t Index,
8569	uint32_t Attr) {
8570	const unsigned Opcode = ISD::PSEUDO_PROBE;
8571	const auto VTs = getVTList(MVT::Other);
8572	SDValue Ops[] = {Chain};
8573	FoldingSetNodeID ID;
8574	AddNodeIDNode(ID, Opcode, VTs, Ops);
8575	ID.AddInteger(I: Guid);
8576	ID.AddInteger(I: Index);
8577	void *IP = nullptr;
8578	if (SDNode *E = FindNodeOrInsertPos(ID, DL: Dl, InsertPos&: IP))
8579	return SDValue(E, 0);
8580
8581	auto *N = newSDNode<PseudoProbeSDNode>(
8582	Opcode, Dl.getIROrder(), Dl.getDebugLoc(), VTs, Guid, Index, Attr);
8583	createOperands(Node: N, Vals: Ops);
8584	CSEMap.InsertNode(N, IP);
8585	InsertNode(N: N);
8586	SDValue V(N, 0);
8587	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8588	return V;
8589	}
8590
8591	/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
8592	/// MachinePointerInfo record from it. This is particularly useful because the
8593	/// code generator has many cases where it doesn't bother passing in a
8594	/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
8595	static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
8596	SelectionDAG &DAG, SDValue Ptr,
8597	int64_t Offset = 0) {
8598	// If this is FI+Offset, we can model it.
8599	if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr))
8600	return MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(),
8601	FI: FI->getIndex(), Offset);
8602
8603	// If this is (FI+Offset1)+Offset2, we can model it.
8604	if (Ptr.getOpcode() != ISD::ADD ||
8605	!isa<ConstantSDNode>(Val: Ptr.getOperand(i: 1)) ||
8606	!isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0)))
8607	return Info;
8608
8609	int FI = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex();
8610	return MachinePointerInfo::getFixedStack(
8611	MF&: DAG.getMachineFunction(), FI,
8612	Offset: Offset + cast<ConstantSDNode>(Val: Ptr.getOperand(i: 1))->getSExtValue());
8613	}
8614
8615	/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
8616	/// MachinePointerInfo record from it. This is particularly useful because the
8617	/// code generator has many cases where it doesn't bother passing in a
8618	/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
8619	static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
8620	SelectionDAG &DAG, SDValue Ptr,
8621	SDValue OffsetOp) {
8622	// If the 'Offset' value isn't a constant, we can't handle this.
8623	if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(Val&: OffsetOp))
8624	return InferPointerInfo(Info, DAG, Ptr, Offset: OffsetNode->getSExtValue());
8625	if (OffsetOp.isUndef())
8626	return InferPointerInfo(Info, DAG, Ptr);
8627	return Info;
8628	}
8629
8630	SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
8631	EVT VT, const SDLoc &dl, SDValue Chain,
8632	SDValue Ptr, SDValue Offset,
8633	MachinePointerInfo PtrInfo, EVT MemVT,
8634	Align Alignment,
8635	MachineMemOperand::Flags MMOFlags,
8636	const AAMDNodes &AAInfo, const MDNode *Ranges) {
8637	assert(Chain.getValueType() == MVT::Other &&
8638	"Invalid chain type");
8639
8640	MMOFlags |= MachineMemOperand::MOLoad;
8641	assert((MMOFlags & MachineMemOperand::MOStore) == 0);
8642	// If we don't have a PtrInfo, infer the trivial frame index case to simplify
8643	// clients.
8644	if (PtrInfo.V.isNull())
8645	PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset);
8646
8647	LocationSize Size = LocationSize::precise(Value: MemVT.getStoreSize());
8648	MachineFunction &MF = getMachineFunction();
8649	MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size,
8650	BaseAlignment: Alignment, AAInfo, Ranges);
8651	return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
8652	}
8653
8654	SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
8655	EVT VT, const SDLoc &dl, SDValue Chain,
8656	SDValue Ptr, SDValue Offset, EVT MemVT,
8657	MachineMemOperand *MMO) {
8658	if (VT == MemVT) {
8659	ExtType = ISD::NON_EXTLOAD;
8660	} else if (ExtType == ISD::NON_EXTLOAD) {
8661	assert(VT == MemVT && "Non-extending load from different memory type!");
8662	} else {
8663	// Extending load.
8664	assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
8665	"Should only be an extending load, not truncating!");
8666	assert(VT.isInteger() == MemVT.isInteger() &&
8667	"Cannot convert from FP to Int or Int -> FP!");
8668	assert(VT.isVector() == MemVT.isVector() &&
8669	"Cannot use an ext load to convert to or from a vector!");
8670	assert((!VT.isVector() ||
8671	VT.getVectorElementCount() == MemVT.getVectorElementCount()) &&
8672	"Cannot use an ext load to change the number of vector elements!");
8673	}
8674
8675	bool Indexed = AM != ISD::UNINDEXED;
8676	assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
8677
8678	SDVTList VTs = Indexed ?
8679	getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
8680	SDValue Ops[] = { Chain, Ptr, Offset };
8681	FoldingSetNodeID ID;
8682	AddNodeIDNode(ID, OpC: ISD::LOAD, VTList: VTs, OpList: Ops);
8683	ID.AddInteger(I: MemVT.getRawBits());
8684	ID.AddInteger(I: getSyntheticNodeSubclassData<LoadSDNode>(
8685	IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: MemVT, Args&: MMO));
8686	ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
8687	ID.AddInteger(I: MMO->getFlags());
8688	void *IP = nullptr;
8689	if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
8690	cast<LoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
8691	return SDValue(E, 0);
8692	}
8693	auto *N = newSDNode<LoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
8694	Args&: ExtType, Args&: MemVT, Args&: MMO);
8695	createOperands(Node: N, Vals: Ops);
8696
8697	CSEMap.InsertNode(N, InsertPos: IP);
8698	InsertNode(N);
8699	SDValue V(N, 0);
8700	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8701	return V;
8702	}
8703
8704	SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
8705	SDValue Ptr, MachinePointerInfo PtrInfo,
8706	MaybeAlign Alignment,
8707	MachineMemOperand::Flags MMOFlags,
8708	const AAMDNodes &AAInfo, const MDNode *Ranges) {
8709	SDValue Undef = getUNDEF(VT: Ptr.getValueType());
8710	return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
8711	PtrInfo, MemVT: VT, Alignment, MMOFlags, AAInfo, Ranges);
8712	}
8713
8714	SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
8715	SDValue Ptr, MachineMemOperand *MMO) {
8716	SDValue Undef = getUNDEF(VT: Ptr.getValueType());
8717	return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
8718	MemVT: VT, MMO);
8719	}
8720
8721	SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
8722	EVT VT, SDValue Chain, SDValue Ptr,
8723	MachinePointerInfo PtrInfo, EVT MemVT,
8724	MaybeAlign Alignment,
8725	MachineMemOperand::Flags MMOFlags,
8726	const AAMDNodes &AAInfo) {
8727	SDValue Undef = getUNDEF(VT: Ptr.getValueType());
8728	return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, PtrInfo,
8729	MemVT, Alignment, MMOFlags, AAInfo);
8730	}
8731
8732	SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
8733	EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
8734	MachineMemOperand *MMO) {
8735	SDValue Undef = getUNDEF(VT: Ptr.getValueType());
8736	return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef,
8737	MemVT, MMO);
8738	}
8739
8740	SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
8741	SDValue Base, SDValue Offset,
8742	ISD::MemIndexedMode AM) {
8743	LoadSDNode *LD = cast<LoadSDNode>(Val&: OrigLoad);
8744	assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
8745	// Don't propagate the invariant or dereferenceable flags.
8746	auto MMOFlags =
8747	LD->getMemOperand()->getFlags() &
8748	~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
8749	return getLoad(AM, ExtType: LD->getExtensionType(), VT: OrigLoad.getValueType(), dl,
8750	Chain: LD->getChain(), Ptr: Base, Offset, PtrInfo: LD->getPointerInfo(),
8751	MemVT: LD->getMemoryVT(), Alignment: LD->getAlign(), MMOFlags, AAInfo: LD->getAAInfo());
8752	}
8753
8754	SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
8755	SDValue Ptr, MachinePointerInfo PtrInfo,
8756	Align Alignment,
8757	MachineMemOperand::Flags MMOFlags,
8758	const AAMDNodes &AAInfo) {
8759	assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
8760
8761	MMOFlags |= MachineMemOperand::MOStore;
8762	assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
8763
8764	if (PtrInfo.V.isNull())
8765	PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
8766
8767	MachineFunction &MF = getMachineFunction();
8768	LocationSize Size = LocationSize::precise(Value: Val.getValueType().getStoreSize());
8769	MachineMemOperand *MMO =
8770	MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size, BaseAlignment: Alignment, AAInfo);
8771	return getStore(Chain, dl, Val, Ptr, MMO);
8772	}
8773
8774	SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
8775	SDValue Ptr, MachineMemOperand *MMO) {
8776	assert(Chain.getValueType() == MVT::Other &&
8777	"Invalid chain type");
8778	EVT VT = Val.getValueType();
8779	SDVTList VTs = getVTList(MVT::Other);
8780	SDValue Undef = getUNDEF(VT: Ptr.getValueType());
8781	SDValue Ops[] = { Chain, Val, Ptr, Undef };
8782	FoldingSetNodeID ID;
8783	AddNodeIDNode(ID, OpC: ISD::STORE, VTList: VTs, OpList: Ops);
8784	ID.AddInteger(I: VT.getRawBits());
8785	ID.AddInteger(I: getSyntheticNodeSubclassData<StoreSDNode>(
8786	IROrder: dl.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: false, Args&: VT, Args&: MMO));
8787	ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
8788	ID.AddInteger(I: MMO->getFlags());
8789	void *IP = nullptr;
8790	if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
8791	cast<StoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
8792	return SDValue(E, 0);
8793	}
8794	auto *N = newSDNode<StoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
8795	Args: ISD::UNINDEXED, Args: false, Args&: VT, Args&: MMO);
8796	createOperands(Node: N, Vals: Ops);
8797
8798	CSEMap.InsertNode(N, InsertPos: IP);
8799	InsertNode(N);
8800	SDValue V(N, 0);
8801	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8802	return V;
8803	}
8804
8805	SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
8806	SDValue Ptr, MachinePointerInfo PtrInfo,
8807	EVT SVT, Align Alignment,
8808	MachineMemOperand::Flags MMOFlags,
8809	const AAMDNodes &AAInfo) {
8810	assert(Chain.getValueType() == MVT::Other &&
8811	"Invalid chain type");
8812
8813	MMOFlags |= MachineMemOperand::MOStore;
8814	assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
8815
8816	if (PtrInfo.V.isNull())
8817	PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
8818
8819	MachineFunction &MF = getMachineFunction();
8820	MachineMemOperand *MMO = MF.getMachineMemOperand(
8821	PtrInfo, F: MMOFlags, Size: LocationSize::precise(Value: SVT.getStoreSize()), BaseAlignment: Alignment,
8822	AAInfo);
8823	return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
8824	}
8825
8826	SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
8827	SDValue Ptr, EVT SVT,
8828	MachineMemOperand *MMO) {
8829	EVT VT = Val.getValueType();
8830
8831	assert(Chain.getValueType() == MVT::Other &&
8832	"Invalid chain type");
8833	if (VT == SVT)
8834	return getStore(Chain, dl, Val, Ptr, MMO);
8835
8836	assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
8837	"Should only be a truncating store, not extending!");
8838	assert(VT.isInteger() == SVT.isInteger() &&
8839	"Can't do FP-INT conversion!");
8840	assert(VT.isVector() == SVT.isVector() &&
8841	"Cannot use trunc store to convert to or from a vector!");
8842	assert((!VT.isVector() ||
8843	VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
8844	"Cannot use trunc store to change the number of vector elements!");
8845
8846	SDVTList VTs = getVTList(MVT::Other);
8847	SDValue Undef = getUNDEF(VT: Ptr.getValueType());
8848	SDValue Ops[] = { Chain, Val, Ptr, Undef };
8849	FoldingSetNodeID ID;
8850	AddNodeIDNode(ID, OpC: ISD::STORE, VTList: VTs, OpList: Ops);
8851	ID.AddInteger(I: SVT.getRawBits());
8852	ID.AddInteger(I: getSyntheticNodeSubclassData<StoreSDNode>(
8853	IROrder: dl.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: true, Args&: SVT, Args&: MMO));
8854	ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
8855	ID.AddInteger(I: MMO->getFlags());
8856	void *IP = nullptr;
8857	if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
8858	cast<StoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
8859	return SDValue(E, 0);
8860	}
8861	auto *N = newSDNode<StoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
8862	Args: ISD::UNINDEXED, Args: true, Args&: SVT, Args&: MMO);
8863	createOperands(Node: N, Vals: Ops);
8864
8865	CSEMap.InsertNode(N, InsertPos: IP);
8866	InsertNode(N);
8867	SDValue V(N, 0);
8868	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8869	return V;
8870	}
8871
8872	SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
8873	SDValue Base, SDValue Offset,
8874	ISD::MemIndexedMode AM) {
8875	StoreSDNode *ST = cast<StoreSDNode>(Val&: OrigStore);
8876	assert(ST->getOffset().isUndef() && "Store is already a indexed store!");
8877	SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
8878	SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
8879	FoldingSetNodeID ID;
8880	AddNodeIDNode(ID, OpC: ISD::STORE, VTList: VTs, OpList: Ops);
8881	ID.AddInteger(I: ST->getMemoryVT().getRawBits());
8882	ID.AddInteger(I: ST->getRawSubclassData());
8883	ID.AddInteger(I: ST->getPointerInfo().getAddrSpace());
8884	ID.AddInteger(I: ST->getMemOperand()->getFlags());
8885	void *IP = nullptr;
8886	if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
8887	return SDValue(E, 0);
8888
8889	auto *N = newSDNode<StoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
8890	Args: ST->isTruncatingStore(), Args: ST->getMemoryVT(),
8891	Args: ST->getMemOperand());
8892	createOperands(Node: N, Vals: Ops);
8893
8894	CSEMap.InsertNode(N, InsertPos: IP);
8895	InsertNode(N);
8896	SDValue V(N, 0);
8897	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8898	return V;
8899	}
8900
8901	SDValue SelectionDAG::getLoadVP(
8902	ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
8903	SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL,
8904	MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
8905	MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
8906	const MDNode *Ranges, bool IsExpanding) {
8907	assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
8908
8909	MMOFlags |= MachineMemOperand::MOLoad;
8910	assert((MMOFlags & MachineMemOperand::MOStore) == 0);
8911	// If we don't have a PtrInfo, infer the trivial frame index case to simplify
8912	// clients.
8913	if (PtrInfo.V.isNull())
8914	PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset);
8915
8916	LocationSize Size = LocationSize::precise(Value: MemVT.getStoreSize());
8917	MachineFunction &MF = getMachineFunction();
8918	MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size,
8919	BaseAlignment: Alignment, AAInfo, Ranges);
8920	return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT,
8921	MMO, IsExpanding);
8922	}
8923
8924	SDValue SelectionDAG::getLoadVP(ISD::MemIndexedMode AM,
8925	ISD::LoadExtType ExtType, EVT VT,
8926	const SDLoc &dl, SDValue Chain, SDValue Ptr,
8927	SDValue Offset, SDValue Mask, SDValue EVL,
8928	EVT MemVT, MachineMemOperand *MMO,
8929	bool IsExpanding) {
8930	bool Indexed = AM != ISD::UNINDEXED;
8931	assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
8932
8933	SDVTList VTs = Indexed ? getVTList(VT, Ptr.getValueType(), MVT::Other)
8934	: getVTList(VT, MVT::Other);
8935	SDValue Ops[] = {Chain, Ptr, Offset, Mask, EVL};
8936	FoldingSetNodeID ID;
8937	AddNodeIDNode(ID, OpC: ISD::VP_LOAD, VTList: VTs, OpList: Ops);
8938	ID.AddInteger(I: MemVT.getRawBits());
8939	ID.AddInteger(I: getSyntheticNodeSubclassData<VPLoadSDNode>(
8940	IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO));
8941	ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
8942	ID.AddInteger(I: MMO->getFlags());
8943	void *IP = nullptr;
8944	if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
8945	cast<VPLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
8946	return SDValue(E, 0);
8947	}
8948	auto *N = newSDNode<VPLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
8949	Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO);
8950	createOperands(Node: N, Vals: Ops);
8951
8952	CSEMap.InsertNode(N, InsertPos: IP);
8953	InsertNode(N);
8954	SDValue V(N, 0);
8955	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8956	return V;
8957	}
8958
8959	SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
8960	SDValue Ptr, SDValue Mask, SDValue EVL,
8961	MachinePointerInfo PtrInfo,
8962	MaybeAlign Alignment,
8963	MachineMemOperand::Flags MMOFlags,
8964	const AAMDNodes &AAInfo, const MDNode *Ranges,
8965	bool IsExpanding) {
8966	SDValue Undef = getUNDEF(VT: Ptr.getValueType());
8967	return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
8968	Mask, EVL, PtrInfo, MemVT: VT, Alignment, MMOFlags, AAInfo, Ranges,
8969	IsExpanding);
8970	}
8971
8972	SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
8973	SDValue Ptr, SDValue Mask, SDValue EVL,
8974	MachineMemOperand *MMO, bool IsExpanding) {
8975	SDValue Undef = getUNDEF(VT: Ptr.getValueType());
8976	return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
8977	Mask, EVL, MemVT: VT, MMO, IsExpanding);
8978	}
8979
8980	SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
8981	EVT VT, SDValue Chain, SDValue Ptr,
8982	SDValue Mask, SDValue EVL,
8983	MachinePointerInfo PtrInfo, EVT MemVT,
8984	MaybeAlign Alignment,
8985	MachineMemOperand::Flags MMOFlags,
8986	const AAMDNodes &AAInfo, bool IsExpanding) {
8987	SDValue Undef = getUNDEF(VT: Ptr.getValueType());
8988	return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask,
8989	EVL, PtrInfo, MemVT, Alignment, MMOFlags, AAInfo, Ranges: nullptr,
8990	IsExpanding);
8991	}
8992
8993	SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
8994	EVT VT, SDValue Chain, SDValue Ptr,
8995	SDValue Mask, SDValue EVL, EVT MemVT,
8996	MachineMemOperand *MMO, bool IsExpanding) {
8997	SDValue Undef = getUNDEF(VT: Ptr.getValueType());
8998	return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask,
8999	EVL, MemVT, MMO, IsExpanding);
9000	}
9001
9002	SDValue SelectionDAG::getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl,
9003	SDValue Base, SDValue Offset,
9004	ISD::MemIndexedMode AM) {
9005	auto *LD = cast<VPLoadSDNode>(Val&: OrigLoad);
9006	assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
9007	// Don't propagate the invariant or dereferenceable flags.
9008	auto MMOFlags =
9009	LD->getMemOperand()->getFlags() &
9010	~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
9011	return getLoadVP(AM, ExtType: LD->getExtensionType(), VT: OrigLoad.getValueType(), dl,
9012	Chain: LD->getChain(), Ptr: Base, Offset, Mask: LD->getMask(),
9013	EVL: LD->getVectorLength(), PtrInfo: LD->getPointerInfo(),
9014	MemVT: LD->getMemoryVT(), Alignment: LD->getAlign(), MMOFlags, AAInfo: LD->getAAInfo(),
9015	Ranges: nullptr, IsExpanding: LD->isExpandingLoad());
9016	}
9017
9018	SDValue SelectionDAG::getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val,
9019	SDValue Ptr, SDValue Offset, SDValue Mask,
9020	SDValue EVL, EVT MemVT, MachineMemOperand *MMO,
9021	ISD::MemIndexedMode AM, bool IsTruncating,
9022	bool IsCompressing) {
9023	assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9024	bool Indexed = AM != ISD::UNINDEXED;
9025	assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
9026	SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other)
9027	: getVTList(MVT::Other);
9028	SDValue Ops[] = {Chain, Val, Ptr, Offset, Mask, EVL};
9029	FoldingSetNodeID ID;
9030	AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops);
9031	ID.AddInteger(I: MemVT.getRawBits());
9032	ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>(
9033	IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
9034	ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9035	ID.AddInteger(I: MMO->getFlags());
9036	void *IP = nullptr;
9037	if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
9038	cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
9039	return SDValue(E, 0);
9040	}
9041	auto *N = newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
9042	Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO);
9043	createOperands(Node: N, Vals: Ops);
9044
9045	CSEMap.InsertNode(N, InsertPos: IP);
9046	InsertNode(N);
9047	SDValue V(N, 0);
9048	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9049	return V;
9050	}
9051
9052	SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
9053	SDValue Val, SDValue Ptr, SDValue Mask,
9054	SDValue EVL, MachinePointerInfo PtrInfo,
9055	EVT SVT, Align Alignment,
9056	MachineMemOperand::Flags MMOFlags,
9057	const AAMDNodes &AAInfo,
9058	bool IsCompressing) {
9059	assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9060
9061	MMOFlags |= MachineMemOperand::MOStore;
9062	assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
9063
9064	if (PtrInfo.V.isNull())
9065	PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
9066
9067	MachineFunction &MF = getMachineFunction();
9068	MachineMemOperand *MMO = MF.getMachineMemOperand(
9069	PtrInfo, F: MMOFlags, Size: LocationSize::precise(Value: SVT.getStoreSize()), BaseAlignment: Alignment,
9070	AAInfo);
9071	return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO,
9072	IsCompressing);
9073	}
9074
9075	SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
9076	SDValue Val, SDValue Ptr, SDValue Mask,
9077	SDValue EVL, EVT SVT,
9078	MachineMemOperand *MMO,
9079	bool IsCompressing) {
9080	EVT VT = Val.getValueType();
9081
9082	assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9083	if (VT == SVT)
9084	return getStoreVP(Chain, dl, Val, Ptr, Offset: getUNDEF(VT: Ptr.getValueType()), Mask,
9085	EVL, MemVT: VT, MMO, AM: ISD::UNINDEXED,
9086	/*IsTruncating*/ false, IsCompressing);
9087
9088	assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
9089	"Should only be a truncating store, not extending!");
9090	assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
9091	assert(VT.isVector() == SVT.isVector() &&
9092	"Cannot use trunc store to convert to or from a vector!");
9093	assert((!VT.isVector() ||
9094	VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
9095	"Cannot use trunc store to change the number of vector elements!");
9096
9097	SDVTList VTs = getVTList(MVT::Other);
9098	SDValue Undef = getUNDEF(VT: Ptr.getValueType());
9099	SDValue Ops[] = {Chain, Val, Ptr, Undef, Mask, EVL};
9100	FoldingSetNodeID ID;
9101	AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops);
9102	ID.AddInteger(I: SVT.getRawBits());
9103	ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>(
9104	IROrder: dl.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO));
9105	ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9106	ID.AddInteger(I: MMO->getFlags());
9107	void *IP = nullptr;
9108	if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
9109	cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
9110	return SDValue(E, 0);
9111	}
9112	auto *N =
9113	newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
9114	Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO);
9115	createOperands(Node: N, Vals: Ops);
9116
9117	CSEMap.InsertNode(N, InsertPos: IP);
9118	InsertNode(N);
9119	SDValue V(N, 0);
9120	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9121	return V;
9122	}
9123
9124	SDValue SelectionDAG::getIndexedStoreVP(SDValue OrigStore, const SDLoc &dl,
9125	SDValue Base, SDValue Offset,
9126	ISD::MemIndexedMode AM) {
9127	auto *ST = cast<VPStoreSDNode>(Val&: OrigStore);
9128	assert(ST->getOffset().isUndef() && "Store is already an indexed store!");
9129	SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
9130	SDValue Ops[] = {ST->getChain(), ST->getValue(), Base,
9131	Offset, ST->getMask(), ST->getVectorLength()};
9132	FoldingSetNodeID ID;
9133	AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops);
9134	ID.AddInteger(I: ST->getMemoryVT().getRawBits());
9135	ID.AddInteger(I: ST->getRawSubclassData());
9136	ID.AddInteger(I: ST->getPointerInfo().getAddrSpace());
9137	ID.AddInteger(I: ST->getMemOperand()->getFlags());
9138	void *IP = nullptr;
9139	if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
9140	return SDValue(E, 0);
9141
9142	auto *N = newSDNode<VPStoreSDNode>(
9143	Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM, Args: ST->isTruncatingStore(),
9144	Args: ST->isCompressingStore(), Args: ST->getMemoryVT(), Args: ST->getMemOperand());
9145	createOperands(Node: N, Vals: Ops);
9146
9147	CSEMap.InsertNode(N, InsertPos: IP);
9148	InsertNode(N);
9149	SDValue V(N, 0);
9150	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9151	return V;
9152	}
9153
9154	SDValue SelectionDAG::getStridedLoadVP(
9155	ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL,
9156	SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask,
9157	SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding) {
9158	bool Indexed = AM != ISD::UNINDEXED;
9159	assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
9160
9161	SDValue Ops[] = {Chain, Ptr, Offset, Stride, Mask, EVL};
9162	SDVTList VTs = Indexed ? getVTList(VT, Ptr.getValueType(), MVT::Other)
9163	: getVTList(VT, MVT::Other);
9164	FoldingSetNodeID ID;
9165	AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_LOAD, VTList: VTs, OpList: Ops);
9166	ID.AddInteger(I: VT.getRawBits());
9167	ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedLoadSDNode>(
9168	IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO));
9169	ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9170
9171	void *IP = nullptr;
9172	if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
9173	cast<VPStridedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
9174	return SDValue(E, 0);
9175	}
9176
9177	auto *N =
9178	newSDNode<VPStridedLoadSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs, Args&: AM,
9179	Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO);
9180	createOperands(Node: N, Vals: Ops);
9181	CSEMap.InsertNode(N, InsertPos: IP);
9182	InsertNode(N);
9183	SDValue V(N, 0);
9184	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9185	return V;
9186	}
9187
9188	SDValue SelectionDAG::getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain,
9189	SDValue Ptr, SDValue Stride,
9190	SDValue Mask, SDValue EVL,
9191	MachineMemOperand *MMO,
9192	bool IsExpanding) {
9193	SDValue Undef = getUNDEF(VT: Ptr.getValueType());
9194	return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, DL, Chain, Ptr,
9195	Offset: Undef, Stride, Mask, EVL, MemVT: VT, MMO, IsExpanding);
9196	}
9197
9198	SDValue SelectionDAG::getExtStridedLoadVP(
9199	ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain,
9200	SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT,
9201	MachineMemOperand *MMO, bool IsExpanding) {
9202	SDValue Undef = getUNDEF(VT: Ptr.getValueType());
9203	return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Offset: Undef,
9204	Stride, Mask, EVL, MemVT, MMO, IsExpanding);
9205	}
9206
9207	SDValue SelectionDAG::getStridedStoreVP(SDValue Chain, const SDLoc &DL,
9208	SDValue Val, SDValue Ptr,
9209	SDValue Offset, SDValue Stride,
9210	SDValue Mask, SDValue EVL, EVT MemVT,
9211	MachineMemOperand *MMO,
9212	ISD::MemIndexedMode AM,
9213	bool IsTruncating, bool IsCompressing) {
9214	assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9215	bool Indexed = AM != ISD::UNINDEXED;
9216	assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
9217	SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other)
9218	: getVTList(MVT::Other);
9219	SDValue Ops[] = {Chain, Val, Ptr, Offset, Stride, Mask, EVL};
9220	FoldingSetNodeID ID;
9221	AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTList: VTs, OpList: Ops);
9222	ID.AddInteger(I: MemVT.getRawBits());
9223	ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
9224	IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
9225	ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9226	void *IP = nullptr;
9227	if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
9228	cast<VPStridedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
9229	return SDValue(E, 0);
9230	}
9231	auto *N = newSDNode<VPStridedStoreSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(),
9232	Args&: VTs, Args&: AM, Args&: IsTruncating,
9233	Args&: IsCompressing, Args&: MemVT, Args&: MMO);
9234	createOperands(Node: N, Vals: Ops);
9235
9236	CSEMap.InsertNode(N, InsertPos: IP);
9237	InsertNode(N);
9238	SDValue V(N, 0);
9239	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9240	return V;
9241	}
9242
9243	SDValue SelectionDAG::getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL,
9244	SDValue Val, SDValue Ptr,
9245	SDValue Stride, SDValue Mask,
9246	SDValue EVL, EVT SVT,
9247	MachineMemOperand *MMO,
9248	bool IsCompressing) {
9249	EVT VT = Val.getValueType();
9250
9251	assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9252	if (VT == SVT)
9253	return getStridedStoreVP(Chain, DL, Val, Ptr, Offset: getUNDEF(VT: Ptr.getValueType()),
9254	Stride, Mask, EVL, MemVT: VT, MMO, AM: ISD::UNINDEXED,
9255	/*IsTruncating*/ false, IsCompressing);
9256
9257	assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
9258	"Should only be a truncating store, not extending!");
9259	assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
9260	assert(VT.isVector() == SVT.isVector() &&
9261	"Cannot use trunc store to convert to or from a vector!");
9262	assert((!VT.isVector() ||
9263	VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
9264	"Cannot use trunc store to change the number of vector elements!");
9265
9266	SDVTList VTs = getVTList(MVT::Other);
9267	SDValue Undef = getUNDEF(VT: Ptr.getValueType());
9268	SDValue Ops[] = {Chain, Val, Ptr, Undef, Stride, Mask, EVL};
9269	FoldingSetNodeID ID;
9270	AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTList: VTs, OpList: Ops);
9271	ID.AddInteger(I: SVT.getRawBits());
9272	ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
9273	IROrder: DL.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO));
9274	ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9275	void *IP = nullptr;
9276	if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
9277	cast<VPStridedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
9278	return SDValue(E, 0);
9279	}
9280	auto *N = newSDNode<VPStridedStoreSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(),
9281	Args&: VTs, Args: ISD::UNINDEXED, Args: true,
9282	Args&: IsCompressing, Args&: SVT, Args&: MMO);
9283	createOperands(Node: N, Vals: Ops);
9284
9285	CSEMap.InsertNode(N, InsertPos: IP);
9286	InsertNode(N);
9287	SDValue V(N, 0);
9288	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9289	return V;
9290	}
9291
9292	SDValue SelectionDAG::getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl,
9293	ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
9294	ISD::MemIndexType IndexType) {
9295	assert(Ops.size() == 6 && "Incompatible number of operands");
9296
9297	FoldingSetNodeID ID;
9298	AddNodeIDNode(ID, OpC: ISD::VP_GATHER, VTList: VTs, OpList: Ops);
9299	ID.AddInteger(I: VT.getRawBits());
9300	ID.AddInteger(I: getSyntheticNodeSubclassData<VPGatherSDNode>(
9301	IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType));
9302	ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9303	ID.AddInteger(I: MMO->getFlags());
9304	void *IP = nullptr;
9305	if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
9306	cast<VPGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
9307	return SDValue(E, 0);
9308	}
9309
9310	auto *N = newSDNode<VPGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
9311	Args&: VT, Args&: MMO, Args&: IndexType);
9312	createOperands(Node: N, Vals: Ops);
9313
9314	assert(N->getMask().getValueType().getVectorElementCount() ==
9315	N->getValueType(0).getVectorElementCount() &&
9316	"Vector width mismatch between mask and data");
9317	assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9318	N->getValueType(0).getVectorElementCount().isScalable() &&
9319	"Scalable flags of index and data do not match");
9320	assert(ElementCount::isKnownGE(
9321	N->getIndex().getValueType().getVectorElementCount(),
9322	N->getValueType(0).getVectorElementCount()) &&
9323	"Vector width mismatch between index and data");
9324	assert(isa<ConstantSDNode>(N->getScale()) &&
9325	N->getScale()->getAsAPIntVal().isPowerOf2() &&
9326	"Scale should be a constant power of 2");
9327
9328	CSEMap.InsertNode(N, InsertPos: IP);
9329	InsertNode(N);
9330	SDValue V(N, 0);
9331	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9332	return V;
9333	}
9334
9335	SDValue SelectionDAG::getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl,
9336	ArrayRef<SDValue> Ops,
9337	MachineMemOperand *MMO,
9338	ISD::MemIndexType IndexType) {
9339	assert(Ops.size() == 7 && "Incompatible number of operands");
9340
9341	FoldingSetNodeID ID;
9342	AddNodeIDNode(ID, OpC: ISD::VP_SCATTER, VTList: VTs, OpList: Ops);
9343	ID.AddInteger(I: VT.getRawBits());
9344	ID.AddInteger(I: getSyntheticNodeSubclassData<VPScatterSDNode>(
9345	IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType));
9346	ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9347	ID.AddInteger(I: MMO->getFlags());
9348	void *IP = nullptr;
9349	if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
9350	cast<VPScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
9351	return SDValue(E, 0);
9352	}
9353	auto *N = newSDNode<VPScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
9354	Args&: VT, Args&: MMO, Args&: IndexType);
9355	createOperands(Node: N, Vals: Ops);
9356
9357	assert(N->getMask().getValueType().getVectorElementCount() ==
9358	N->getValue().getValueType().getVectorElementCount() &&
9359	"Vector width mismatch between mask and data");
9360	assert(
9361	N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9362	N->getValue().getValueType().getVectorElementCount().isScalable() &&
9363	"Scalable flags of index and data do not match");
9364	assert(ElementCount::isKnownGE(
9365	N->getIndex().getValueType().getVectorElementCount(),
9366	N->getValue().getValueType().getVectorElementCount()) &&
9367	"Vector width mismatch between index and data");
9368	assert(isa<ConstantSDNode>(N->getScale()) &&
9369	N->getScale()->getAsAPIntVal().isPowerOf2() &&
9370	"Scale should be a constant power of 2");
9371
9372	CSEMap.InsertNode(N, InsertPos: IP);
9373	InsertNode(N);
9374	SDValue V(N, 0);
9375	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9376	return V;
9377	}
9378
9379	SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
9380	SDValue Base, SDValue Offset, SDValue Mask,
9381	SDValue PassThru, EVT MemVT,
9382	MachineMemOperand *MMO,
9383	ISD::MemIndexedMode AM,
9384	ISD::LoadExtType ExtTy, bool isExpanding) {
9385	bool Indexed = AM != ISD::UNINDEXED;
9386	assert((Indexed || Offset.isUndef()) &&
9387	"Unindexed masked load with an offset!");
9388	SDVTList VTs = Indexed ? getVTList(VT, Base.getValueType(), MVT::Other)
9389	: getVTList(VT, MVT::Other);
9390	SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru};
9391	FoldingSetNodeID ID;
9392	AddNodeIDNode(ID, OpC: ISD::MLOAD, VTList: VTs, OpList: Ops);
9393	ID.AddInteger(I: MemVT.getRawBits());
9394	ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedLoadSDNode>(
9395	IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO));
9396	ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9397	ID.AddInteger(I: MMO->getFlags());
9398	void *IP = nullptr;
9399	if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
9400	cast<MaskedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
9401	return SDValue(E, 0);
9402	}
9403	auto *N = newSDNode<MaskedLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
9404	Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO);
9405	createOperands(Node: N, Vals: Ops);
9406
9407	CSEMap.InsertNode(N, InsertPos: IP);
9408	InsertNode(N);
9409	SDValue V(N, 0);
9410	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9411	return V;
9412	}
9413
9414	SDValue SelectionDAG::getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl,
9415	SDValue Base, SDValue Offset,
9416	ISD::MemIndexedMode AM) {
9417	MaskedLoadSDNode *LD = cast<MaskedLoadSDNode>(Val&: OrigLoad);
9418	assert(LD->getOffset().isUndef() && "Masked load is already a indexed load!");
9419	return getMaskedLoad(VT: OrigLoad.getValueType(), dl, Chain: LD->getChain(), Base,
9420	Offset, Mask: LD->getMask(), PassThru: LD->getPassThru(),
9421	MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand(), AM,
9422	ExtTy: LD->getExtensionType(), isExpanding: LD->isExpandingLoad());
9423	}
9424
9425	SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
9426	SDValue Val, SDValue Base, SDValue Offset,
9427	SDValue Mask, EVT MemVT,
9428	MachineMemOperand *MMO,
9429	ISD::MemIndexedMode AM, bool IsTruncating,
9430	bool IsCompressing) {
9431	assert(Chain.getValueType() == MVT::Other &&
9432	"Invalid chain type");
9433	bool Indexed = AM != ISD::UNINDEXED;
9434	assert((Indexed || Offset.isUndef()) &&
9435	"Unindexed masked store with an offset!");
9436	SDVTList VTs = Indexed ? getVTList(Base.getValueType(), MVT::Other)
9437	: getVTList(MVT::Other);
9438	SDValue Ops[] = {Chain, Val, Base, Offset, Mask};
9439	FoldingSetNodeID ID;
9440	AddNodeIDNode(ID, OpC: ISD::MSTORE, VTList: VTs, OpList: Ops);
9441	ID.AddInteger(I: MemVT.getRawBits());
9442	ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedStoreSDNode>(
9443	IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
9444	ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9445	ID.AddInteger(I: MMO->getFlags());
9446	void *IP = nullptr;
9447	if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
9448	cast<MaskedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
9449	return SDValue(E, 0);
9450	}
9451	auto *N =
9452	newSDNode<MaskedStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
9453	Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO);
9454	createOperands(Node: N, Vals: Ops);
9455
9456	CSEMap.InsertNode(N, InsertPos: IP);
9457	InsertNode(N);
9458	SDValue V(N, 0);
9459	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9460	return V;
9461	}
9462
9463	SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl,
9464	SDValue Base, SDValue Offset,
9465	ISD::MemIndexedMode AM) {
9466	MaskedStoreSDNode *ST = cast<MaskedStoreSDNode>(Val&: OrigStore);
9467	assert(ST->getOffset().isUndef() &&
9468	"Masked store is already a indexed store!");
9469	return getMaskedStore(Chain: ST->getChain(), dl, Val: ST->getValue(), Base, Offset,
9470	Mask: ST->getMask(), MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(),
9471	AM, IsTruncating: ST->isTruncatingStore(), IsCompressing: ST->isCompressingStore());
9472	}
9473
9474	SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl,
9475	ArrayRef<SDValue> Ops,
9476	MachineMemOperand *MMO,
9477	ISD::MemIndexType IndexType,
9478	ISD::LoadExtType ExtTy) {
9479	assert(Ops.size() == 6 && "Incompatible number of operands");
9480
9481	FoldingSetNodeID ID;
9482	AddNodeIDNode(ID, OpC: ISD::MGATHER, VTList: VTs, OpList: Ops);
9483	ID.AddInteger(I: MemVT.getRawBits());
9484	ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedGatherSDNode>(
9485	IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy));
9486	ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9487	ID.AddInteger(I: MMO->getFlags());
9488	void *IP = nullptr;
9489	if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
9490	cast<MaskedGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
9491	return SDValue(E, 0);
9492	}
9493
9494	auto *N = newSDNode<MaskedGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
9495	Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy);
9496	createOperands(Node: N, Vals: Ops);
9497
9498	assert(N->getPassThru().getValueType() == N->getValueType(0) &&
9499	"Incompatible type of the PassThru value in MaskedGatherSDNode");
9500	assert(N->getMask().getValueType().getVectorElementCount() ==
9501	N->getValueType(0).getVectorElementCount() &&
9502	"Vector width mismatch between mask and data");
9503	assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9504	N->getValueType(0).getVectorElementCount().isScalable() &&
9505	"Scalable flags of index and data do not match");
9506	assert(ElementCount::isKnownGE(
9507	N->getIndex().getValueType().getVectorElementCount(),
9508	N->getValueType(0).getVectorElementCount()) &&
9509	"Vector width mismatch between index and data");
9510	assert(isa<ConstantSDNode>(N->getScale()) &&
9511	N->getScale()->getAsAPIntVal().isPowerOf2() &&
9512	"Scale should be a constant power of 2");
9513
9514	CSEMap.InsertNode(N, InsertPos: IP);
9515	InsertNode(N);
9516	SDValue V(N, 0);
9517	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9518	return V;
9519	}
9520
9521	SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl,
9522	ArrayRef<SDValue> Ops,
9523	MachineMemOperand *MMO,
9524	ISD::MemIndexType IndexType,
9525	bool IsTrunc) {
9526	assert(Ops.size() == 6 && "Incompatible number of operands");
9527
9528	FoldingSetNodeID ID;
9529	AddNodeIDNode(ID, OpC: ISD::MSCATTER, VTList: VTs, OpList: Ops);
9530	ID.AddInteger(I: MemVT.getRawBits());
9531	ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedScatterSDNode>(
9532	IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc));
9533	ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9534	ID.AddInteger(I: MMO->getFlags());
9535	void *IP = nullptr;
9536	if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
9537	cast<MaskedScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
9538	return SDValue(E, 0);
9539	}
9540
9541	auto *N = newSDNode<MaskedScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
9542	Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc);
9543	createOperands(Node: N, Vals: Ops);
9544
9545	assert(N->getMask().getValueType().getVectorElementCount() ==
9546	N->getValue().getValueType().getVectorElementCount() &&
9547	"Vector width mismatch between mask and data");
9548	assert(
9549	N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9550	N->getValue().getValueType().getVectorElementCount().isScalable() &&
9551	"Scalable flags of index and data do not match");
9552	assert(ElementCount::isKnownGE(
9553	N->getIndex().getValueType().getVectorElementCount(),
9554	N->getValue().getValueType().getVectorElementCount()) &&
9555	"Vector width mismatch between index and data");
9556	assert(isa<ConstantSDNode>(N->getScale()) &&
9557	N->getScale()->getAsAPIntVal().isPowerOf2() &&
9558	"Scale should be a constant power of 2");
9559
9560	CSEMap.InsertNode(N, InsertPos: IP);
9561	InsertNode(N);
9562	SDValue V(N, 0);
9563	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9564	return V;
9565	}
9566
9567	SDValue SelectionDAG::getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
9568	EVT MemVT, MachineMemOperand *MMO) {
9569	assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9570	SDVTList VTs = getVTList(MVT::Other);
9571	SDValue Ops[] = {Chain, Ptr};
9572	FoldingSetNodeID ID;
9573	AddNodeIDNode(ID, OpC: ISD::GET_FPENV_MEM, VTList: VTs, OpList: Ops);
9574	ID.AddInteger(I: MemVT.getRawBits());
9575	ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>(
9576	Opc: ISD::GET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO));
9577	ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9578	ID.AddInteger(I: MMO->getFlags());
9579	void *IP = nullptr;
9580	if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
9581	return SDValue(E, 0);
9582
9583	auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::GET_FPENV_MEM, Args: dl.getIROrder(),
9584	Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO);
9585	createOperands(Node: N, Vals: Ops);
9586
9587	CSEMap.InsertNode(N, InsertPos: IP);
9588	InsertNode(N);
9589	SDValue V(N, 0);
9590	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9591	return V;
9592	}
9593
9594	SDValue SelectionDAG::getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
9595	EVT MemVT, MachineMemOperand *MMO) {
9596	assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9597	SDVTList VTs = getVTList(MVT::Other);
9598	SDValue Ops[] = {Chain, Ptr};
9599	FoldingSetNodeID ID;
9600	AddNodeIDNode(ID, OpC: ISD::SET_FPENV_MEM, VTList: VTs, OpList: Ops);
9601	ID.AddInteger(I: MemVT.getRawBits());
9602	ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>(
9603	Opc: ISD::SET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO));
9604	ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9605	ID.AddInteger(I: MMO->getFlags());
9606	void *IP = nullptr;
9607	if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
9608	return SDValue(E, 0);
9609
9610	auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::SET_FPENV_MEM, Args: dl.getIROrder(),
9611	Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO);
9612	createOperands(Node: N, Vals: Ops);
9613
9614	CSEMap.InsertNode(N, InsertPos: IP);
9615	InsertNode(N);
9616	SDValue V(N, 0);
9617	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9618	return V;
9619	}
9620
9621	SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) {
9622	// select undef, T, F --> T (if T is a constant), otherwise F
9623	// select, ?, undef, F --> F
9624	// select, ?, T, undef --> T
9625	if (Cond.isUndef())
9626	return isConstantValueOfAnyType(N: T) ? T : F;
9627	if (T.isUndef())
9628	return F;
9629	if (F.isUndef())
9630	return T;
9631
9632	// select true, T, F --> T
9633	// select false, T, F --> F
9634	if (auto *CondC = dyn_cast<ConstantSDNode>(Val&: Cond))
9635	return CondC->isZero() ? F : T;
9636
9637	// TODO: This should simplify VSELECT with non-zero constant condition using
9638	// something like this (but check boolean contents to be complete?):
9639	if (ConstantSDNode *CondC = isConstOrConstSplat(N: Cond, /*AllowUndefs*/ false,
9640	/*AllowTruncation*/ true))
9641	if (CondC->isZero())
9642	return F;
9643
9644	// select ?, T, T --> T
9645	if (T == F)
9646	return T;
9647
9648	return SDValue();
9649	}
9650
9651	SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) {
9652	// shift undef, Y --> 0 (can always assume that the undef value is 0)
9653	if (X.isUndef())
9654	return getConstant(Val: 0, DL: SDLoc(X.getNode()), VT: X.getValueType());
9655	// shift X, undef --> undef (because it may shift by the bitwidth)
9656	if (Y.isUndef())
9657	return getUNDEF(VT: X.getValueType());
9658
9659	// shift 0, Y --> 0
9660	// shift X, 0 --> X
9661	if (isNullOrNullSplat(V: X) || isNullOrNullSplat(V: Y))
9662	return X;
9663
9664	// shift X, C >= bitwidth(X) --> undef
9665	// All vector elements must be too big (or undef) to avoid partial undefs.
9666	auto isShiftTooBig = [X](ConstantSDNode *Val) {
9667	return !Val || Val->getAPIntValue().uge(RHS: X.getScalarValueSizeInBits());
9668	};
9669	if (ISD::matchUnaryPredicate(Op: Y, Match: isShiftTooBig, AllowUndefs: true))
9670	return getUNDEF(VT: X.getValueType());
9671
9672	// shift i1/vXi1 X, Y --> X (any non-zero shift amount is undefined).
9673	if (X.getValueType().getScalarType() == MVT::i1)
9674	return X;
9675
9676	return SDValue();
9677	}
9678
9679	SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y,
9680	SDNodeFlags Flags) {
9681	// If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand
9682	// (an undef operand can be chosen to be Nan/Inf), then the result of this
9683	// operation is poison. That result can be relaxed to undef.
9684	ConstantFPSDNode *XC = isConstOrConstSplatFP(N: X, /* AllowUndefs */ true);
9685	ConstantFPSDNode *YC = isConstOrConstSplatFP(N: Y, /* AllowUndefs */ true);
9686	bool HasNan = (XC && XC->getValueAPF().isNaN()) ||
9687	(YC && YC->getValueAPF().isNaN());
9688	bool HasInf = (XC && XC->getValueAPF().isInfinity()) ||
9689	(YC && YC->getValueAPF().isInfinity());
9690
9691	if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef()))
9692	return getUNDEF(VT: X.getValueType());
9693
9694	if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef()))
9695	return getUNDEF(VT: X.getValueType());
9696
9697	if (!YC)
9698	return SDValue();
9699
9700	// X + -0.0 --> X
9701	if (Opcode == ISD::FADD)
9702	if (YC->getValueAPF().isNegZero())
9703	return X;
9704
9705	// X - +0.0 --> X
9706	if (Opcode == ISD::FSUB)
9707	if (YC->getValueAPF().isPosZero())
9708	return X;
9709
9710	// X * 1.0 --> X
9711	// X / 1.0 --> X
9712	if (Opcode == ISD::FMUL || Opcode == ISD::FDIV)
9713	if (YC->getValueAPF().isExactlyValue(V: 1.0))
9714	return X;
9715
9716	// X * 0.0 --> 0.0
9717	if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros())
9718	if (YC->getValueAPF().isZero())
9719	return getConstantFP(Val: 0.0, DL: SDLoc(Y), VT: Y.getValueType());
9720
9721	return SDValue();
9722	}
9723
9724	SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain,
9725	SDValue Ptr, SDValue SV, unsigned Align) {
9726	SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) };
9727	return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
9728	}
9729
9730	SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9731	ArrayRef<SDUse> Ops) {
9732	switch (Ops.size()) {
9733	case 0: return getNode(Opcode, DL, VT);
9734	case 1: return getNode(Opcode, DL, VT, N1: static_cast<const SDValue>(Ops[0]));
9735	case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1]);
9736	case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2]);
9737	default: break;
9738	}
9739
9740	// Copy from an SDUse array into an SDValue array for use with
9741	// the regular getNode logic.
9742	SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end());
9743	return getNode(Opcode, DL, VT, Ops: NewOps);
9744	}
9745
9746	SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9747	ArrayRef<SDValue> Ops) {
9748	SDNodeFlags Flags;
9749	if (Inserter)
9750	Flags = Inserter->getFlags();
9751	return getNode(Opcode, DL, VT, Ops, Flags);
9752	}
9753
9754	SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9755	ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
9756	unsigned NumOps = Ops.size();
9757	switch (NumOps) {
9758	case 0: return getNode(Opcode, DL, VT);
9759	case 1: return getNode(Opcode, DL, VT, N1: Ops[0], Flags);
9760	case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], Flags);
9761	case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2], Flags);
9762	default: break;
9763	}
9764
9765	#ifndef NDEBUG
9766	for (const auto &Op : Ops)
9767	assert(Op.getOpcode() != ISD::DELETED_NODE &&
9768	"Operand is DELETED_NODE!");
9769	#endif
9770
9771	switch (Opcode) {
9772	default: break;
9773	case ISD::BUILD_VECTOR:
9774	// Attempt to simplify BUILD_VECTOR.
9775	if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
9776	return V;
9777	break;
9778	case ISD::CONCAT_VECTORS:
9779	if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
9780	return V;
9781	break;
9782	case ISD::SELECT_CC:
9783	assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
9784	assert(Ops[0].getValueType() == Ops[1].getValueType() &&
9785	"LHS and RHS of condition must have same type!");
9786	assert(Ops[2].getValueType() == Ops[3].getValueType() &&
9787	"True and False arms of SelectCC must have same type!");
9788	assert(Ops[2].getValueType() == VT &&
9789	"select_cc node must be of same type as true and false value!");
9790	assert((!Ops[0].getValueType().isVector() ||
9791	Ops[0].getValueType().getVectorElementCount() ==
9792	VT.getVectorElementCount()) &&
9793	"Expected select_cc with vector result to have the same sized "
9794	"comparison type!");
9795	break;
9796	case ISD::BR_CC:
9797	assert(NumOps == 5 && "BR_CC takes 5 operands!");
9798	assert(Ops[2].getValueType() == Ops[3].getValueType() &&
9799	"LHS/RHS of comparison should match types!");
9800	break;
9801	case ISD::VP_ADD:
9802	case ISD::VP_SUB:
9803	// If it is VP_ADD/VP_SUB mask operation then turn it to VP_XOR
9804	if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
9805	Opcode = ISD::VP_XOR;
9806	break;
9807	case ISD::VP_MUL:
9808	// If it is VP_MUL mask operation then turn it to VP_AND
9809	if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
9810	Opcode = ISD::VP_AND;
9811	break;
9812	case ISD::VP_REDUCE_MUL:
9813	// If it is VP_REDUCE_MUL mask operation then turn it to VP_REDUCE_AND
9814	if (VT == MVT::i1)
9815	Opcode = ISD::VP_REDUCE_AND;
9816	break;
9817	case ISD::VP_REDUCE_ADD:
9818	// If it is VP_REDUCE_ADD mask operation then turn it to VP_REDUCE_XOR
9819	if (VT == MVT::i1)
9820	Opcode = ISD::VP_REDUCE_XOR;
9821	break;
9822	case ISD::VP_REDUCE_SMAX:
9823	case ISD::VP_REDUCE_UMIN:
9824	// If it is VP_REDUCE_SMAX/VP_REDUCE_UMIN mask operation then turn it to
9825	// VP_REDUCE_AND.
9826	if (VT == MVT::i1)
9827	Opcode = ISD::VP_REDUCE_AND;
9828	break;
9829	case ISD::VP_REDUCE_SMIN:
9830	case ISD::VP_REDUCE_UMAX:
9831	// If it is VP_REDUCE_SMIN/VP_REDUCE_UMAX mask operation then turn it to
9832	// VP_REDUCE_OR.
9833	if (VT == MVT::i1)
9834	Opcode = ISD::VP_REDUCE_OR;
9835	break;
9836	}
9837
9838	// Memoize nodes.
9839	SDNode *N;
9840	SDVTList VTs = getVTList(VT);
9841
9842	if (VT != MVT::Glue) {
9843	FoldingSetNodeID ID;
9844	AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
9845	void *IP = nullptr;
9846
9847	if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
9848	return SDValue(E, 0);
9849
9850	N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
9851	createOperands(Node: N, Vals: Ops);
9852
9853	CSEMap.InsertNode(N, InsertPos: IP);
9854	} else {
9855	N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
9856	createOperands(Node: N, Vals: Ops);
9857	}
9858
9859	N->setFlags(Flags);
9860	InsertNode(N);
9861	SDValue V(N, 0);
9862	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9863	return V;
9864	}
9865
9866	SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
9867	ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
9868	return getNode(Opcode, DL, VTList: getVTList(VTs: ResultTys), Ops);
9869	}
9870
9871	SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
9872	ArrayRef<SDValue> Ops) {
9873	SDNodeFlags Flags;
9874	if (Inserter)
9875	Flags = Inserter->getFlags();
9876	return getNode(Opcode, DL, VTList, Ops, Flags);
9877	}
9878
9879	SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
9880	ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
9881	if (VTList.NumVTs == 1)
9882	return getNode(Opcode, DL, VT: VTList.VTs[0], Ops, Flags);
9883
9884	#ifndef NDEBUG
9885	for (const auto &Op : Ops)
9886	assert(Op.getOpcode() != ISD::DELETED_NODE &&
9887	"Operand is DELETED_NODE!");
9888	#endif
9889
9890	switch (Opcode) {
9891	case ISD::SADDO:
9892	case ISD::UADDO:
9893	case ISD::SSUBO:
9894	case ISD::USUBO: {
9895	assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
9896	"Invalid add/sub overflow op!");
9897	assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
9898	Ops[0].getValueType() == Ops[1].getValueType() &&
9899	Ops[0].getValueType() == VTList.VTs[0] &&
9900	"Binary operator types must match!");
9901	SDValue N1 = Ops[0], N2 = Ops[1];
9902	canonicalizeCommutativeBinop(Opcode, N1, N2);
9903
9904	// (X +- 0) -> X with zero-overflow.
9905	ConstantSDNode *N2CV = isConstOrConstSplat(N: N2, /*AllowUndefs*/ false,
9906	/*AllowTruncation*/ true);
9907	if (N2CV && N2CV->isZero()) {
9908	SDValue ZeroOverFlow = getConstant(Val: 0, DL, VT: VTList.VTs[1]);
9909	return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {N1, ZeroOverFlow}, Flags);
9910	}
9911
9912	if (VTList.VTs[0].isVector() &&
9913	VTList.VTs[0].getVectorElementType() == MVT::i1 &&
9914	VTList.VTs[1].getVectorElementType() == MVT::i1) {
9915	SDValue F1 = getFreeze(V: N1);
9916	SDValue F2 = getFreeze(V: N2);
9917	// {vXi1,vXi1} (u/s)addo(vXi1 x, vXi1y) -> {xor(x,y),and(x,y)}
9918	if (Opcode == ISD::UADDO || Opcode == ISD::SADDO)
9919	return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
9920	Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2),
9921	getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: F1, N2: F2)},
9922	Flags);
9923	// {vXi1,vXi1} (u/s)subo(vXi1 x, vXi1y) -> {xor(x,y),and(~x,y)}
9924	if (Opcode == ISD::USUBO || Opcode == ISD::SSUBO) {
9925	SDValue NotF1 = getNOT(DL, Val: F1, VT: VTList.VTs[0]);
9926	return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
9927	Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2),
9928	getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: NotF1, N2: F2)},
9929	Flags);
9930	}
9931	}
9932	break;
9933	}
9934	case ISD::SADDO_CARRY:
9935	case ISD::UADDO_CARRY:
9936	case ISD::SSUBO_CARRY:
9937	case ISD::USUBO_CARRY:
9938	assert(VTList.NumVTs == 2 && Ops.size() == 3 &&
9939	"Invalid add/sub overflow op!");
9940	assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
9941	Ops[0].getValueType() == Ops[1].getValueType() &&
9942	Ops[0].getValueType() == VTList.VTs[0] &&
9943	Ops[2].getValueType() == VTList.VTs[1] &&
9944	"Binary operator types must match!");
9945	break;
9946	case ISD::SMUL_LOHI:
9947	case ISD::UMUL_LOHI: {
9948	assert(VTList.NumVTs == 2 && Ops.size() == 2 && "Invalid mul lo/hi op!");
9949	assert(VTList.VTs[0].isInteger() && VTList.VTs[0] == VTList.VTs[1] &&
9950	VTList.VTs[0] == Ops[0].getValueType() &&
9951	VTList.VTs[0] == Ops[1].getValueType() &&
9952	"Binary operator types must match!");
9953	// Constant fold.
9954	ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(Val: Ops[0]);
9955	ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: Ops[1]);
9956	if (LHS && RHS) {
9957	unsigned Width = VTList.VTs[0].getScalarSizeInBits();
9958	unsigned OutWidth = Width * 2;
9959	APInt Val = LHS->getAPIntValue();
9960	APInt Mul = RHS->getAPIntValue();
9961	if (Opcode == ISD::SMUL_LOHI) {
9962	Val = Val.sext(width: OutWidth);
9963	Mul = Mul.sext(width: OutWidth);
9964	} else {
9965	Val = Val.zext(width: OutWidth);
9966	Mul = Mul.zext(width: OutWidth);
9967	}
9968	Val *= Mul;
9969
9970	SDValue Hi =
9971	getConstant(Val: Val.extractBits(numBits: Width, bitPosition: Width), DL, VT: VTList.VTs[0]);
9972	SDValue Lo = getConstant(Val: Val.trunc(width: Width), DL, VT: VTList.VTs[0]);
9973	return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {Lo, Hi}, Flags);
9974	}
9975	break;
9976	}
9977	case ISD::FFREXP: {
9978	assert(VTList.NumVTs == 2 && Ops.size() == 1 && "Invalid ffrexp op!");
9979	assert(VTList.VTs[0].isFloatingPoint() && VTList.VTs[1].isInteger() &&
9980	VTList.VTs[0] == Ops[0].getValueType() && "frexp type mismatch");
9981
9982	if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val: Ops[0])) {
9983	int FrexpExp;
9984	APFloat FrexpMant =
9985	frexp(X: C->getValueAPF(), Exp&: FrexpExp, RM: APFloat::rmNearestTiesToEven);
9986	SDValue Result0 = getConstantFP(V: FrexpMant, DL, VT: VTList.VTs[0]);
9987	SDValue Result1 =
9988	getConstant(Val: FrexpMant.isFinite() ? FrexpExp : 0, DL, VT: VTList.VTs[1]);
9989	return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {Result0, Result1}, Flags);
9990	}
9991
9992	break;
9993	}
9994	case ISD::STRICT_FP_EXTEND:
9995	assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
9996	"Invalid STRICT_FP_EXTEND!");
9997	assert(VTList.VTs[0].isFloatingPoint() &&
9998	Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!");
9999	assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
10000	"STRICT_FP_EXTEND result type should be vector iff the operand "
10001	"type is vector!");
10002	assert((!VTList.VTs[0].isVector() ||
10003	VTList.VTs[0].getVectorElementCount() ==
10004	Ops[1].getValueType().getVectorElementCount()) &&
10005	"Vector element count mismatch!");
10006	assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) &&
10007	"Invalid fpext node, dst <= src!");
10008	break;
10009	case ISD::STRICT_FP_ROUND:
10010	assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!");
10011	assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
10012	"STRICT_FP_ROUND result type should be vector iff the operand "
10013	"type is vector!");
10014	assert((!VTList.VTs[0].isVector() ||
10015	VTList.VTs[0].getVectorElementCount() ==
10016	Ops[1].getValueType().getVectorElementCount()) &&
10017	"Vector element count mismatch!");
10018	assert(VTList.VTs[0].isFloatingPoint() &&
10019	Ops[1].getValueType().isFloatingPoint() &&
10020	VTList.VTs[0].bitsLT(Ops[1].getValueType()) &&
10021	isa<ConstantSDNode>(Ops[2]) &&
10022	(Ops[2]->getAsZExtVal() == 0 || Ops[2]->getAsZExtVal() == 1) &&
10023	"Invalid STRICT_FP_ROUND!");
10024	break;
10025	#if 0
10026	// FIXME: figure out how to safely handle things like
10027	// int foo(int x) { return 1 << (x & 255); }
10028	// int bar() { return foo(256); }
10029	case ISD::SRA_PARTS:
10030	case ISD::SRL_PARTS:
10031	case ISD::SHL_PARTS:
10032	if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
10033	cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
10034	return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
10035	else if (N3.getOpcode() == ISD::AND)
10036	if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
10037	// If the and is only masking out bits that cannot effect the shift,
10038	// eliminate the and.
10039	unsigned NumBits = VT.getScalarSizeInBits()*2;
10040	if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
10041	return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
10042	}
10043	break;
10044	#endif
10045	}
10046
10047	// Memoize the node unless it returns a glue result.
10048	SDNode *N;
10049	if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
10050	FoldingSetNodeID ID;
10051	AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
10052	void *IP = nullptr;
10053	if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
10054	return SDValue(E, 0);
10055
10056	N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList);
10057	createOperands(Node: N, Vals: Ops);
10058	CSEMap.InsertNode(N, InsertPos: IP);
10059	} else {
10060	N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList);
10061	createOperands(Node: N, Vals: Ops);
10062	}
10063
10064	N->setFlags(Flags);
10065	InsertNode(N);
10066	SDValue V(N, 0);
10067	NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10068	return V;
10069	}
10070
10071	SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
10072	SDVTList VTList) {
10073	return getNode(Opcode, DL, VTList, Ops: std::nullopt);
10074	}
10075
10076	SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10077	SDValue N1) {
10078	SDValue Ops[] = { N1 };
10079	return getNode(Opcode, DL, VTList, Ops);
10080	}
10081
10082	SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10083	SDValue N1, SDValue N2) {
10084	SDValue Ops[] = { N1, N2 };
10085	return getNode(Opcode, DL, VTList, Ops);
10086	}
10087
10088	SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10089	SDValue N1, SDValue N2, SDValue N3) {
10090	SDValue Ops[] = { N1, N2, N3 };
10091	return getNode(Opcode, DL, VTList, Ops);
10092	}
10093
10094	SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10095	SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
10096	SDValue Ops[] = { N1, N2, N3, N4 };
10097	return getNode(Opcode, DL, VTList, Ops);
10098	}
10099
10100	SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10101	SDValue N1, SDValue N2, SDValue N3, SDValue N4,
10102	SDValue N5) {
10103	SDValue Ops[] = { N1, N2, N3, N4, N5 };
10104	return getNode(Opcode, DL, VTList, Ops);
10105	}
10106
10107	SDVTList SelectionDAG::getVTList(EVT VT) {
10108	return makeVTList(VTs: SDNode::getValueTypeList(VT), NumVTs: 1);
10109	}
10110
10111	SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
10112	FoldingSetNodeID ID;
10113	ID.AddInteger(I: 2U);
10114	ID.AddInteger(I: VT1.getRawBits());
10115	ID.AddInteger(I: VT2.getRawBits());
10116
10117	void *IP = nullptr;
10118	SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
10119	if (!Result) {
10120	EVT *Array = Allocator.Allocate<EVT>(Num: 2);
10121	Array[0] = VT1;
10122	Array[1] = VT2;
10123	Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
10124	VTListMap.InsertNode(N: Result, InsertPos: IP);
10125	}
10126	return Result->getSDVTList();
10127	}
10128
10129	SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
10130	FoldingSetNodeID ID;
10131	ID.AddInteger(I: 3U);
10132	ID.AddInteger(I: VT1.getRawBits());
10133	ID.AddInteger(I: VT2.getRawBits());
10134	ID.AddInteger(I: VT3.getRawBits());
10135
10136	void *IP = nullptr;
10137	SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
10138	if (!Result) {
10139	EVT *Array = Allocator.Allocate<EVT>(Num: 3);
10140	Array[0] = VT1;
10141	Array[1] = VT2;
10142	Array[2] = VT3;
10143	Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
10144	VTListMap.InsertNode(N: Result, InsertPos: IP);
10145	}
10146	return Result->getSDVTList();
10147	}
10148
10149	SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
10150	FoldingSetNodeID ID;
10151	ID.AddInteger(I: 4U);
10152	ID.AddInteger(I: VT1.getRawBits());
10153	ID.AddInteger(I: VT2.getRawBits());
10154	ID.AddInteger(I: VT3.getRawBits());
10155	ID.AddInteger(I: VT4.getRawBits());
10156
10157	void *IP = nullptr;
10158	SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
10159	if (!Result) {
10160	EVT *Array = Allocator.Allocate<EVT>(Num: 4);
10161	Array[0] = VT1;
10162	Array[1] = VT2;
10163	Array[2] = VT3;
10164	Array[3] = VT4;
10165	Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
10166	VTListMap.InsertNode(N: Result, InsertPos: IP);
10167	}
10168	return Result->getSDVTList();
10169	}
10170
10171	SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
10172	unsigned NumVTs = VTs.size();
10173	FoldingSetNodeID ID;
10174	ID.AddInteger(I: NumVTs);
10175	for (unsigned index = 0; index < NumVTs; index++) {
10176	ID.AddInteger(I: VTs[index].getRawBits());
10177	}
10178
10179	void *IP = nullptr;
10180	SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
10181	if (!Result) {
10182	EVT *Array = Allocator.Allocate<EVT>(Num: NumVTs);
10183	llvm::copy(Range&: VTs, Out: Array);
10184	Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
10185	VTListMap.InsertNode(N: Result, InsertPos: IP);
10186	}
10187	return Result->getSDVTList();
10188	}
10189
10190
10191	/// UpdateNodeOperands - *Mutate* the specified node in-place to have the
10192	/// specified operands. If the resultant node already exists in the DAG,
10193	/// this does not modify the specified node, instead it returns the node that
10194	/// already exists. If the resultant node does not exist in the DAG, the
10195	/// input node is returned. As a degenerate case, if you specify the same
10196	/// input operands as the node already has, the input node is returned.
10197	SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
10198	assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
10199
10200	// Check to see if there is no change.
10201	if (Op == N->getOperand(Num: 0)) return N;
10202
10203	// See if the modified node already exists.
10204	void *InsertPos = nullptr;
10205	if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
10206	return Existing;
10207
10208	// Nope it doesn't. Remove the node from its current place in the maps.
10209	if (InsertPos)
10210	if (!RemoveNodeFromCSEMaps(N))
10211	InsertPos = nullptr;
10212
10213	// Now we update the operands.
10214	N->OperandList[0].set(Op);
10215
10216	updateDivergence(N);
10217	// If this gets put into a CSE map, add it.
10218	if (InsertPos) CSEMap.InsertNode(N, InsertPos);
10219	return N;
10220	}
10221
10222	SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
10223	assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
10224
10225	// Check to see if there is no change.
10226	if (Op1 == N->getOperand(Num: 0) && Op2 == N->getOperand(Num: 1))
10227	return N; // No operands changed, just return the input node.
10228
10229	// See if the modified node already exists.
10230	void *InsertPos = nullptr;
10231	if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
10232	return Existing;
10233
10234	// Nope it doesn't. Remove the node from its current place in the maps.
10235	if (InsertPos)
10236	if (!RemoveNodeFromCSEMaps(N))
10237	InsertPos = nullptr;
10238
10239	// Now we update the operands.
10240	if (N->OperandList[0] != Op1)
10241	N->OperandList[0].set(Op1);
10242	if (N->OperandList[1] != Op2)
10243	N->OperandList[1].set(Op2);
10244
10245	updateDivergence(N);
10246	// If this gets put into a CSE map, add it.
10247	if (InsertPos) CSEMap.InsertNode(N, InsertPos);
10248	return N;
10249	}
10250
10251	SDNode *SelectionDAG::
10252	UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
10253	SDValue Ops[] = { Op1, Op2, Op3 };
10254	return UpdateNodeOperands(N, Ops);
10255	}
10256
10257	SDNode *SelectionDAG::
10258	UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
10259	SDValue Op3, SDValue Op4) {
10260	SDValue Ops[] = { Op1, Op2, Op3, Op4 };
10261	return UpdateNodeOperands(N, Ops);
10262	}
10263
10264	SDNode *SelectionDAG::
10265	UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
10266	SDValue Op3, SDValue Op4, SDValue Op5) {
10267	SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
10268	return UpdateNodeOperands(N, Ops);
10269	}
10270
10271	SDNode *SelectionDAG::
10272	UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
10273	unsigned NumOps = Ops.size();
10274	assert(N->getNumOperands() == NumOps &&
10275	"Update with wrong number of operands");
10276
10277	// If no operands changed just return the input node.
10278	if (std::equal(first1: Ops.begin(), last1: Ops.end(), first2: N->op_begin()))
10279	return N;
10280
10281	// See if the modified node already exists.
10282	void *InsertPos = nullptr;
10283	if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
10284	return Existing;
10285
10286	// Nope it doesn't. Remove the node from its current place in the maps.
10287	if (InsertPos)
10288	if (!RemoveNodeFromCSEMaps(N))
10289	InsertPos = nullptr;
10290
10291	// Now we update the operands.
10292	for (unsigned i = 0; i != NumOps; ++i)
10293	if (N->OperandList[i] != Ops[i])
10294	N->OperandList[i].set(Ops[i]);
10295
10296	updateDivergence(N);
10297	// If this gets put into a CSE map, add it.
10298	if (InsertPos) CSEMap.InsertNode(N, InsertPos);
10299	return N;
10300	}
10301
10302	/// DropOperands - Release the operands and set this node to have
10303	/// zero operands.
10304	void SDNode::DropOperands() {
10305	// Unlike the code in MorphNodeTo that does this, we don't need to
10306	// watch for dead nodes here.
10307	for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
10308	SDUse &Use = *I++;
10309	Use.set(SDValue());
10310	}
10311	}
10312
10313	void SelectionDAG::setNodeMemRefs(MachineSDNode *N,
10314	ArrayRef<MachineMemOperand *> NewMemRefs) {
10315	if (NewMemRefs.empty()) {
10316	N->clearMemRefs();
10317	return;
10318	}
10319
10320	// Check if we can avoid allocating by storing a single reference directly.
10321	if (NewMemRefs.size() == 1) {
10322	N->MemRefs = NewMemRefs[0];
10323	N->NumMemRefs = 1;
10324	return;
10325	}
10326
10327	MachineMemOperand **MemRefsBuffer =
10328	Allocator.template Allocate<MachineMemOperand *>(Num: NewMemRefs.size());
10329	llvm::copy(Range&: NewMemRefs, Out: MemRefsBuffer);
10330	N->MemRefs = MemRefsBuffer;
10331	N->NumMemRefs = static_cast<int>(NewMemRefs.size());
10332	}
10333
10334	/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
10335	/// machine opcode.
10336	///
10337	SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10338	EVT VT) {
10339	SDVTList VTs = getVTList(VT);
10340	return SelectNodeTo(N, MachineOpc, VTs, Ops: std::nullopt);
10341	}
10342
10343	SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10344	EVT VT, SDValue Op1) {
10345	SDVTList VTs = getVTList(VT);
10346	SDValue Ops[] = { Op1 };
10347	return SelectNodeTo(N, MachineOpc, VTs, Ops);
10348	}
10349
10350	SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10351	EVT VT, SDValue Op1,
10352	SDValue Op2) {
10353	SDVTList VTs = getVTList(VT);
10354	SDValue Ops[] = { Op1, Op2 };
10355	return SelectNodeTo(N, MachineOpc, VTs, Ops);
10356	}
10357
10358	SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10359	EVT VT, SDValue Op1,
10360	SDValue Op2, SDValue Op3) {
10361	SDVTList VTs = getVTList(VT);
10362	SDValue Ops[] = { Op1, Op2, Op3 };
10363	return SelectNodeTo(N, MachineOpc, VTs, Ops);
10364	}
10365
10366	SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10367	EVT VT, ArrayRef<SDValue> Ops) {
10368	SDVTList VTs = getVTList(VT);
10369	return SelectNodeTo(N, MachineOpc, VTs, Ops);
10370	}
10371
10372	SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10373	EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
10374	SDVTList VTs = getVTList(VT1, VT2);
10375	return SelectNodeTo(N, MachineOpc, VTs, Ops);
10376	}
10377
10378	SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10379	EVT VT1, EVT VT2) {
10380	SDVTList VTs = getVTList(VT1, VT2);
10381	return SelectNodeTo(N, MachineOpc, VTs, Ops: std::nullopt);
10382	}
10383
10384	SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10385	EVT VT1, EVT VT2, EVT VT3,
10386	ArrayRef<SDValue> Ops) {
10387	SDVTList VTs = getVTList(VT1, VT2, VT3);
10388	return SelectNodeTo(N, MachineOpc, VTs, Ops);
10389	}
10390
10391	SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10392	EVT VT1, EVT VT2,
10393	SDValue Op1, SDValue Op2) {
10394	SDVTList VTs = getVTList(VT1, VT2);
10395	SDValue Ops[] = { Op1, Op2 };
10396	return SelectNodeTo(N, MachineOpc, VTs, Ops);
10397	}
10398
10399	SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10400	SDVTList VTs,ArrayRef<SDValue> Ops) {
10401	SDNode *New = MorphNodeTo(N, Opc: ~MachineOpc, VTs, Ops);
10402	// Reset the NodeID to -1.
10403	New->setNodeId(-1);
10404	if (New != N) {
10405	ReplaceAllUsesWith(From: N, To: New);
10406	RemoveDeadNode(N);
10407	}
10408	return New;
10409	}
10410
10411	/// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
10412	/// the line number information on the merged node since it is not possible to
10413	/// preserve the information that operation is associated with multiple lines.
10414	/// This will make the debugger working better at -O0, were there is a higher
10415	/// probability having other instructions associated with that line.
10416	///
10417	/// For IROrder, we keep the smaller of the two
10418	SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
10419	DebugLoc NLoc = N->getDebugLoc();
10420	if (NLoc && OptLevel == CodeGenOptLevel::None && OLoc.getDebugLoc() != NLoc) {
10421	N->setDebugLoc(DebugLoc());
10422	}
10423	unsigned Order = std::min(a: N->getIROrder(), b: OLoc.getIROrder());
10424	N->setIROrder(Order);
10425	return N;
10426	}
10427
10428	/// MorphNodeTo - This *mutates* the specified node to have the specified
10429	/// return type, opcode, and operands.
10430	///
10431	/// Note that MorphNodeTo returns the resultant node. If there is already a
10432	/// node of the specified opcode and operands, it returns that node instead of
10433	/// the current one. Note that the SDLoc need not be the same.
10434	///
10435	/// Using MorphNodeTo is faster than creating a new node and swapping it in
10436	/// with ReplaceAllUsesWith both because it often avoids allocating a new
10437	/// node, and because it doesn't require CSE recalculation for any of
10438	/// the node's users.
10439	///
10440	/// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
10441	/// As a consequence it isn't appropriate to use from within the DAG combiner or
10442	/// the legalizer which maintain worklists that would need to be updated when
10443	/// deleting things.
10444	SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
10445	SDVTList VTs, ArrayRef<SDValue> Ops) {
10446	// If an identical node already exists, use it.
10447	void *IP = nullptr;
10448	if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
10449	FoldingSetNodeID ID;
10450	AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: Ops);
10451	if (SDNode *ON = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos&: IP))
10452	return UpdateSDLocOnMergeSDNode(N: ON, OLoc: SDLoc(N));
10453	}
10454
10455	if (!RemoveNodeFromCSEMaps(N))
10456	IP = nullptr;
10457
10458	// Start the morphing.
10459	N->NodeType = Opc;
10460	N->ValueList = VTs.VTs;
10461	N->NumValues = VTs.NumVTs;
10462
10463	// Clear the operands list, updating used nodes to remove this from their
10464	// use list. Keep track of any operands that become dead as a result.
10465	SmallPtrSet<SDNode*, 16> DeadNodeSet;
10466	for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
10467	SDUse &Use = *I++;
10468	SDNode *Used = Use.getNode();
10469	Use.set(SDValue());
10470	if (Used->use_empty())
10471	DeadNodeSet.insert(Ptr: Used);
10472	}
10473
10474	// For MachineNode, initialize the memory references information.
10475	if (MachineSDNode *MN = dyn_cast<MachineSDNode>(Val: N))
10476	MN->clearMemRefs();
10477
10478	// Swap for an appropriately sized array from the recycler.
10479	removeOperands(Node: N);
10480	createOperands(Node: N, Vals: Ops);
10481
10482	// Delete any nodes that are still dead after adding the uses for the
10483	// new operands.
10484	if (!DeadNodeSet.empty()) {
10485	SmallVector<SDNode *, 16> DeadNodes;
10486	for (SDNode *N : DeadNodeSet)
10487	if (N->use_empty())
10488	DeadNodes.push_back(Elt: N);
10489	RemoveDeadNodes(DeadNodes);
10490	}
10491
10492	if (IP)
10493	CSEMap.InsertNode(N, InsertPos: IP); // Memoize the new node.
10494	return N;
10495	}
10496
10497	SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) {
10498	unsigned OrigOpc = Node->getOpcode();
10499	unsigned NewOpc;
10500	switch (OrigOpc) {
10501	default:
10502	llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!");
10503	#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
10504	case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break;
10505	#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
10506	case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break;
10507	#include "llvm/IR/ConstrainedOps.def"
10508	}
10509
10510	assert(Node->getNumValues() == 2 && "Unexpected number of results!");
10511
10512	// We're taking this node out of the chain, so we need to re-link things.
10513	SDValue InputChain = Node->getOperand(Num: 0);
10514	SDValue OutputChain = SDValue(Node, 1);
10515	ReplaceAllUsesOfValueWith(From: OutputChain, To: InputChain);
10516
10517	SmallVector<SDValue, 3> Ops;
10518	for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i)
10519	Ops.push_back(Elt: Node->getOperand(Num: i));
10520
10521	SDVTList VTs = getVTList(VT: Node->getValueType(ResNo: 0));
10522	SDNode *Res = MorphNodeTo(N: Node, Opc: NewOpc, VTs, Ops);
10523
10524	// MorphNodeTo can operate in two ways: if an existing node with the
10525	// specified operands exists, it can just return it. Otherwise, it
10526	// updates the node in place to have the requested operands.
10527	if (Res == Node) {
10528	// If we updated the node in place, reset the node ID. To the isel,
10529	// this should be just like a newly allocated machine node.
10530	Res->setNodeId(-1);
10531	} else {
10532	ReplaceAllUsesWith(From: Node, To: Res);
10533	RemoveDeadNode(N: Node);
10534	}
10535
10536	return Res;
10537	}
10538
10539	/// getMachineNode - These are used for target selectors to create a new node
10540	/// with specified return type(s), MachineInstr opcode, and operands.
10541	///
10542	/// Note that getMachineNode returns the resultant node. If there is already a
10543	/// node of the specified opcode and operands, it returns that node instead of
10544	/// the current one.
10545	MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10546	EVT VT) {
10547	SDVTList VTs = getVTList(VT);
10548	return getMachineNode(Opcode, dl, VTs, Ops: std::nullopt);
10549	}
10550
10551	MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10552	EVT VT, SDValue Op1) {
10553	SDVTList VTs = getVTList(VT);
10554	SDValue Ops[] = { Op1 };
10555	return getMachineNode(Opcode, dl, VTs, Ops);
10556	}
10557
10558	MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10559	EVT VT, SDValue Op1, SDValue Op2) {
10560	SDVTList VTs = getVTList(VT);
10561	SDValue Ops[] = { Op1, Op2 };
10562	return getMachineNode(Opcode, dl, VTs, Ops);
10563	}
10564
10565	MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10566	EVT VT, SDValue Op1, SDValue Op2,
10567	SDValue Op3) {
10568	SDVTList VTs = getVTList(VT);
10569	SDValue Ops[] = { Op1, Op2, Op3 };
10570	return getMachineNode(Opcode, dl, VTs, Ops);
10571	}
10572
10573	MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10574	EVT VT, ArrayRef<SDValue> Ops) {
10575	SDVTList VTs = getVTList(VT);
10576	return getMachineNode(Opcode, dl, VTs, Ops);
10577	}
10578
10579	MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10580	EVT VT1, EVT VT2, SDValue Op1,
10581	SDValue Op2) {
10582	SDVTList VTs = getVTList(VT1, VT2);
10583	SDValue Ops[] = { Op1, Op2 };
10584	return getMachineNode(Opcode, dl, VTs, Ops);
10585	}
10586
10587	MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10588	EVT VT1, EVT VT2, SDValue Op1,
10589	SDValue Op2, SDValue Op3) {
10590	SDVTList VTs = getVTList(VT1, VT2);
10591	SDValue Ops[] = { Op1, Op2, Op3 };
10592	return getMachineNode(Opcode, dl, VTs, Ops);
10593	}
10594
10595	MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10596	EVT VT1, EVT VT2,
10597	ArrayRef<SDValue> Ops) {
10598	SDVTList VTs = getVTList(VT1, VT2);
10599	return getMachineNode(Opcode, dl, VTs, Ops);
10600	}
10601
10602	MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10603	EVT VT1, EVT VT2, EVT VT3,
10604	SDValue Op1, SDValue Op2) {
10605	SDVTList VTs = getVTList(VT1, VT2, VT3);
10606	SDValue Ops[] = { Op1, Op2 };
10607	return getMachineNode(Opcode, dl, VTs, Ops);
10608	}
10609
10610	MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10611	EVT VT1, EVT VT2, EVT VT3,
10612	SDValue Op1, SDValue Op2,
10613	SDValue Op3) {
10614	SDVTList VTs = getVTList(VT1, VT2, VT3);
10615	SDValue Ops[] = { Op1, Op2, Op3 };
10616	return getMachineNode(Opcode, dl, VTs, Ops);
10617	}
10618
10619	MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10620	EVT VT1, EVT VT2, EVT VT3,
10621	ArrayRef<SDValue> Ops) {
10622	SDVTList VTs = getVTList(VT1, VT2, VT3);
10623	return getMachineNode(Opcode, dl, VTs, Ops);
10624	}
10625
10626	MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10627	ArrayRef<EVT> ResultTys,
10628	ArrayRef<SDValue> Ops) {
10629	SDVTList VTs = getVTList(VTs: ResultTys);
10630	return getMachineNode(Opcode, dl, VTs, Ops);
10631	}
10632
10633	MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL,
10634	SDVTList VTs,
10635	ArrayRef<SDValue> Ops) {
10636	bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
10637	MachineSDNode *N;
10638	void *IP = nullptr;
10639
10640	if (DoCSE) {
10641	FoldingSetNodeID ID;
10642	AddNodeIDNode(ID, OpC: ~Opcode, VTList: VTs, OpList: Ops);
10643	IP = nullptr;
10644	if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
10645	return cast<MachineSDNode>(Val: UpdateSDLocOnMergeSDNode(N: E, OLoc: DL));
10646	}
10647	}
10648
10649	// Allocate a new MachineSDNode.
10650	N = newSDNode<MachineSDNode>(Args: ~Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
10651	createOperands(Node: N, Vals: Ops);
10652
10653	if (DoCSE)
10654	CSEMap.InsertNode(N, InsertPos: IP);
10655
10656	InsertNode(N);
10657	NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating new machine node: ", G: this);
10658	return N;
10659	}
10660
10661	/// getTargetExtractSubreg - A convenience function for creating
10662	/// TargetOpcode::EXTRACT_SUBREG nodes.
10663	SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
10664	SDValue Operand) {
10665	SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
10666	SDNode *Subreg = getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL,
10667	VT, Op1: Operand, Op2: SRIdxVal);
10668	return SDValue(Subreg, 0);
10669	}
10670
10671	/// getTargetInsertSubreg - A convenience function for creating
10672	/// TargetOpcode::INSERT_SUBREG nodes.
10673	SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
10674	SDValue Operand, SDValue Subreg) {
10675	SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
10676	SDNode *Result = getMachineNode(Opcode: TargetOpcode::INSERT_SUBREG, dl: DL,
10677	VT, Op1: Operand, Op2: Subreg, Op3: SRIdxVal);
10678	return SDValue(Result, 0);
10679	}
10680
10681	/// getNodeIfExists - Get the specified node if it's already available, or
10682	/// else return NULL.
10683	SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
10684	ArrayRef<SDValue> Ops) {
10685	SDNodeFlags Flags;
10686	if (Inserter)
10687	Flags = Inserter->getFlags();
10688	return getNodeIfExists(Opcode, VTList, Ops, Flags);
10689	}
10690
10691	SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
10692	ArrayRef<SDValue> Ops,
10693	const SDNodeFlags Flags) {
10694	if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
10695	FoldingSetNodeID ID;
10696	AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
10697	void *IP = nullptr;
10698	if (SDNode *E = FindNodeOrInsertPos(ID, DL: SDLoc(), InsertPos&: IP)) {
10699	E->intersectFlagsWith(Flags);
10700	return E;
10701	}
10702	}
10703	return nullptr;
10704	}
10705
10706	/// doesNodeExist - Check if a node exists without modifying its flags.
10707	bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList,
10708	ArrayRef<SDValue> Ops) {
10709	if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
10710	FoldingSetNodeID ID;
10711	AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
10712	void *IP = nullptr;
10713	if (FindNodeOrInsertPos(ID, DL: SDLoc(), InsertPos&: IP))
10714	return true;
10715	}
10716	return false;
10717	}
10718
10719	/// getDbgValue - Creates a SDDbgValue node.
10720	///
10721	/// SDNode
10722	SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr,
10723	SDNode *N, unsigned R, bool IsIndirect,
10724	const DebugLoc &DL, unsigned O) {
10725	assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10726	"Expected inlined-at fields to agree");
10727	return new (DbgInfo->getAlloc())
10728	SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromNode(Node: N, ResNo: R),
10729	{}, IsIndirect, DL, O,
10730	/*IsVariadic=*/false);
10731	}
10732
10733	/// Constant
10734	SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var,
10735	DIExpression *Expr,
10736	const Value *C,
10737	const DebugLoc &DL, unsigned O) {
10738	assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10739	"Expected inlined-at fields to agree");
10740	return new (DbgInfo->getAlloc())
10741	SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromConst(Const: C), {},
10742	/*IsIndirect=*/false, DL, O,
10743	/*IsVariadic=*/false);
10744	}
10745
10746	/// FrameIndex
10747	SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
10748	DIExpression *Expr, unsigned FI,
10749	bool IsIndirect,
10750	const DebugLoc &DL,
10751	unsigned O) {
10752	assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10753	"Expected inlined-at fields to agree");
10754	return getFrameIndexDbgValue(Var, Expr, FI, Dependencies: {}, IsIndirect, DL, O);
10755	}
10756
10757	/// FrameIndex with dependencies
10758	SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
10759	DIExpression *Expr, unsigned FI,
10760	ArrayRef<SDNode *> Dependencies,
10761	bool IsIndirect,
10762	const DebugLoc &DL,
10763	unsigned O) {
10764	assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10765	"Expected inlined-at fields to agree");
10766	return new (DbgInfo->getAlloc())
10767	SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromFrameIdx(FrameIdx: FI),
10768	Dependencies, IsIndirect, DL, O,
10769	/*IsVariadic=*/false);
10770	}
10771
10772	/// VReg
10773	SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var, DIExpression *Expr,
10774	unsigned VReg, bool IsIndirect,
10775	const DebugLoc &DL, unsigned O) {
10776	assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10777	"Expected inlined-at fields to agree");
10778	return new (DbgInfo->getAlloc())
10779	SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromVReg(VReg),
10780	{}, IsIndirect, DL, O,
10781	/*IsVariadic=*/false);
10782	}
10783
10784	SDDbgValue *SelectionDAG::getDbgValueList(DIVariable *Var, DIExpression *Expr,
10785	ArrayRef<SDDbgOperand> Locs,
10786	ArrayRef<SDNode *> Dependencies,
10787	bool IsIndirect, const DebugLoc &DL,
10788	unsigned O, bool IsVariadic) {
10789	assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10790	"Expected inlined-at fields to agree");
10791	return new (DbgInfo->getAlloc())
10792	SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect,
10793	DL, O, IsVariadic);
10794	}
10795
10796	void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
10797	unsigned OffsetInBits, unsigned SizeInBits,
10798	bool InvalidateDbg) {
10799	SDNode *FromNode = From.getNode();
10800	SDNode *ToNode = To.getNode();
10801	assert(FromNode && ToNode && "Can't modify dbg values");
10802
10803	// PR35338
10804	// TODO: assert(From != To && "Redundant dbg value transfer");
10805	// TODO: assert(FromNode != ToNode && "Intranode dbg value transfer");
10806	if (From == To || FromNode == ToNode)
10807	return;
10808
10809	if (!FromNode->getHasDebugValue())
10810	return;
10811
10812	SDDbgOperand FromLocOp =
10813	SDDbgOperand::fromNode(Node: From.getNode(), ResNo: From.getResNo());
10814	SDDbgOperand ToLocOp = SDDbgOperand::fromNode(Node: To.getNode(), ResNo: To.getResNo());
10815
10816	SmallVector<SDDbgValue *, 2> ClonedDVs;
10817	for (SDDbgValue *Dbg : GetDbgValues(SD: FromNode)) {
10818	if (Dbg->isInvalidated())
10819	continue;
10820
10821	// TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value");
10822
10823	// Create a new location ops vector that is equal to the old vector, but
10824	// with each instance of FromLocOp replaced with ToLocOp.
10825	bool Changed = false;
10826	auto NewLocOps = Dbg->copyLocationOps();
10827	std::replace_if(
10828	first: NewLocOps.begin(), last: NewLocOps.end(),
10829	pred: [&Changed, FromLocOp](const SDDbgOperand &Op) {
10830	bool Match = Op == FromLocOp;
10831	Changed |= Match;
10832	return Match;
10833	},
10834	new_value: ToLocOp);
10835	// Ignore this SDDbgValue if we didn't find a matching location.
10836	if (!Changed)
10837	continue;
10838
10839	DIVariable *Var = Dbg->getVariable();
10840	auto *Expr = Dbg->getExpression();
10841	// If a fragment is requested, update the expression.
10842	if (SizeInBits) {
10843	// When splitting a larger (e.g., sign-extended) value whose
10844	// lower bits are described with an SDDbgValue, do not attempt
10845	// to transfer the SDDbgValue to the upper bits.
10846	if (auto FI = Expr->getFragmentInfo())
10847	if (OffsetInBits + SizeInBits > FI->SizeInBits)
10848	continue;
10849	auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits,
10850	SizeInBits);
10851	if (!Fragment)
10852	continue;
10853	Expr = *Fragment;
10854	}
10855
10856	auto AdditionalDependencies = Dbg->getAdditionalDependencies();
10857	// Clone the SDDbgValue and move it to To.
10858	SDDbgValue *Clone = getDbgValueList(
10859	Var, Expr, Locs: NewLocOps, Dependencies: AdditionalDependencies, IsIndirect: Dbg->isIndirect(),
10860	DL: Dbg->getDebugLoc(), O: std::max(a: ToNode->getIROrder(), b: Dbg->getOrder()),
10861	IsVariadic: Dbg->isVariadic());
10862	ClonedDVs.push_back(Elt: Clone);
10863
10864	if (InvalidateDbg) {
10865	// Invalidate value and indicate the SDDbgValue should not be emitted.
10866	Dbg->setIsInvalidated();
10867	Dbg->setIsEmitted();
10868	}
10869	}
10870
10871	for (SDDbgValue *Dbg : ClonedDVs) {
10872	assert(is_contained(Dbg->getSDNodes(), ToNode) &&
10873	"Transferred DbgValues should depend on the new SDNode");
10874	AddDbgValue(DB: Dbg, isParameter: false);
10875	}
10876	}
10877
10878	void SelectionDAG::salvageDebugInfo(SDNode &N) {
10879	if (!N.getHasDebugValue())
10880	return;
10881
10882	SmallVector<SDDbgValue *, 2> ClonedDVs;
10883	for (auto *DV : GetDbgValues(SD: &N)) {
10884	if (DV->isInvalidated())
10885	continue;
10886	switch (N.getOpcode()) {
10887	default:
10888	break;
10889	case ISD::ADD: {
10890	SDValue N0 = N.getOperand(Num: 0);
10891	SDValue N1 = N.getOperand(Num: 1);
10892	if (!isa<ConstantSDNode>(Val: N0)) {
10893	bool RHSConstant = isa<ConstantSDNode>(Val: N1);
10894	uint64_t Offset;
10895	if (RHSConstant)
10896	Offset = N.getConstantOperandVal(Num: 1);
10897	// We are not allowed to turn indirect debug values variadic, so
10898	// don't salvage those.
10899	if (!RHSConstant && DV->isIndirect())
10900	continue;
10901
10902	// Rewrite an ADD constant node into a DIExpression. Since we are
10903	// performing arithmetic to compute the variable's *value* in the
10904	// DIExpression, we need to mark the expression with a
10905	// DW_OP_stack_value.
10906	auto *DIExpr = DV->getExpression();
10907	auto NewLocOps = DV->copyLocationOps();
10908	bool Changed = false;
10909	size_t OrigLocOpsSize = NewLocOps.size();
10910	for (size_t i = 0; i < OrigLocOpsSize; ++i) {
10911	// We're not given a ResNo to compare against because the whole
10912	// node is going away. We know that any ISD::ADD only has one
10913	// result, so we can assume any node match is using the result.
10914	if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
10915	NewLocOps[i].getSDNode() != &N)
10916	continue;
10917	NewLocOps[i] = SDDbgOperand::fromNode(Node: N0.getNode(), ResNo: N0.getResNo());
10918	if (RHSConstant) {
10919	SmallVector<uint64_t, 3> ExprOps;
10920	DIExpression::appendOffset(Ops&: ExprOps, Offset);
10921	DIExpr = DIExpression::appendOpsToArg(Expr: DIExpr, Ops: ExprOps, ArgNo: i, StackValue: true);
10922	} else {
10923	// Convert to a variadic expression (if not already).
10924	// convertToVariadicExpression() returns a const pointer, so we use
10925	// a temporary const variable here.
10926	const auto *TmpDIExpr =
10927	DIExpression::convertToVariadicExpression(Expr: DIExpr);
10928	SmallVector<uint64_t, 3> ExprOps;
10929	ExprOps.push_back(Elt: dwarf::DW_OP_LLVM_arg);
10930	ExprOps.push_back(Elt: NewLocOps.size());
10931	ExprOps.push_back(Elt: dwarf::DW_OP_plus);
10932	SDDbgOperand RHS =
10933	SDDbgOperand::fromNode(Node: N1.getNode(), ResNo: N1.getResNo());
10934	NewLocOps.push_back(Elt: RHS);
10935	DIExpr = DIExpression::appendOpsToArg(Expr: TmpDIExpr, Ops: ExprOps, ArgNo: i, StackValue: true);
10936	}
10937	Changed = true;
10938	}
10939	(void)Changed;
10940	assert(Changed && "Salvage target doesn't use N");
10941
10942	bool IsVariadic =
10943	DV->isVariadic() || OrigLocOpsSize != NewLocOps.size();
10944
10945	auto AdditionalDependencies = DV->getAdditionalDependencies();
10946	SDDbgValue *Clone = getDbgValueList(
10947	Var: DV->getVariable(), Expr: DIExpr, Locs: NewLocOps, Dependencies: AdditionalDependencies,
10948	IsIndirect: DV->isIndirect(), DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic);
10949	ClonedDVs.push_back(Elt: Clone);
10950	DV->setIsInvalidated();
10951	DV->setIsEmitted();
10952	LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting";
10953	N0.getNode()->dumprFull(this);
10954	dbgs() << " into " << *DIExpr << '\n');
10955	}
10956	break;
10957	}
10958	case ISD::TRUNCATE: {
10959	SDValue N0 = N.getOperand(Num: 0);
10960	TypeSize FromSize = N0.getValueSizeInBits();
10961	TypeSize ToSize = N.getValueSizeInBits(ResNo: 0);
10962
10963	DIExpression *DbgExpression = DV->getExpression();
10964	auto ExtOps = DIExpression::getExtOps(FromSize, ToSize, Signed: false);
10965	auto NewLocOps = DV->copyLocationOps();
10966	bool Changed = false;
10967	for (size_t i = 0; i < NewLocOps.size(); ++i) {
10968	if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
10969	NewLocOps[i].getSDNode() != &N)
10970	continue;
10971
10972	NewLocOps[i] = SDDbgOperand::fromNode(Node: N0.getNode(), ResNo: N0.getResNo());
10973	DbgExpression = DIExpression::appendOpsToArg(Expr: DbgExpression, Ops: ExtOps, ArgNo: i);
10974	Changed = true;
10975	}
10976	assert(Changed && "Salvage target doesn't use N");
10977	(void)Changed;
10978
10979	SDDbgValue *Clone =
10980	getDbgValueList(Var: DV->getVariable(), Expr: DbgExpression, Locs: NewLocOps,
10981	Dependencies: DV->getAdditionalDependencies(), IsIndirect: DV->isIndirect(),
10982	DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic: DV->isVariadic());
10983
10984	ClonedDVs.push_back(Elt: Clone);
10985	DV->setIsInvalidated();
10986	DV->setIsEmitted();
10987	LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting"; N0.getNode()->dumprFull(this);
10988	dbgs() << " into " << *DbgExpression << '\n');
10989	break;
10990	}
10991	}
10992	}
10993
10994	for (SDDbgValue *Dbg : ClonedDVs) {
10995	assert(!Dbg->getSDNodes().empty() &&
10996	"Salvaged DbgValue should depend on a new SDNode");
10997	AddDbgValue(DB: Dbg, isParameter: false);
10998	}
10999	}
11000
11001	/// Creates a SDDbgLabel node.
11002	SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label,
11003	const DebugLoc &DL, unsigned O) {
11004	assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&
11005	"Expected inlined-at fields to agree");
11006	return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O);
11007	}
11008
11009	namespace {
11010
11011	/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
11012	/// pointed to by a use iterator is deleted, increment the use iterator
11013	/// so that it doesn't dangle.
11014	///
11015	class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
11016	SDNode::use_iterator &UI;
11017	SDNode::use_iterator &UE;
11018
11019	void NodeDeleted(SDNode *N, SDNode *E) override {
11020	// Increment the iterator as needed.
11021	while (UI != UE && N == *UI)
11022	++UI;
11023	}
11024
11025	public:
11026	RAUWUpdateListener(SelectionDAG &d,
11027	SDNode::use_iterator &ui,
11028	SDNode::use_iterator &ue)
11029	: SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
11030	};
11031
11032	} // end anonymous namespace
11033
11034	/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
11035	/// This can cause recursive merging of nodes in the DAG.
11036	///
11037	/// This version assumes From has a single result value.
11038	///
11039	void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
11040	SDNode *From = FromN.getNode();
11041	assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
11042	"Cannot replace with this method!");
11043	assert(From != To.getNode() && "Cannot replace uses of with self");
11044
11045	// Preserve Debug Values
11046	transferDbgValues(From: FromN, To);
11047	// Preserve extra info.
11048	copyExtraInfo(From, To: To.getNode());
11049
11050	// Iterate over all the existing uses of From. New uses will be added
11051	// to the beginning of the use list, which we avoid visiting.
11052	// This specifically avoids visiting uses of From that arise while the
11053	// replacement is happening, because any such uses would be the result
11054	// of CSE: If an existing node looks like From after one of its operands
11055	// is replaced by To, we don't want to replace of all its users with To
11056	// too. See PR3018 for more info.
11057	SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
11058	RAUWUpdateListener Listener(*this, UI, UE);
11059	while (UI != UE) {
11060	SDNode *User = *UI;
11061
11062	// This node is about to morph, remove its old self from the CSE maps.
11063	RemoveNodeFromCSEMaps(N: User);
11064
11065	// A user can appear in a use list multiple times, and when this
11066	// happens the uses are usually next to each other in the list.
11067	// To help reduce the number of CSE recomputations, process all
11068	// the uses of this user that we can find this way.
11069	do {
11070	SDUse &Use = UI.getUse();
11071	++UI;
11072	Use.set(To);
11073	if (To->isDivergent() != From->isDivergent())
11074	updateDivergence(N: User);
11075	} while (UI != UE && *UI == User);
11076	// Now that we have modified User, add it back to the CSE maps. If it
11077	// already exists there, recursively merge the results together.
11078	AddModifiedNodeToCSEMaps(N: User);
11079	}
11080
11081	// If we just RAUW'd the root, take note.
11082	if (FromN == getRoot())
11083	setRoot(To);
11084	}
11085
11086	/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
11087	/// This can cause recursive merging of nodes in the DAG.
11088	///
11089	/// This version assumes that for each value of From, there is a
11090	/// corresponding value in To in the same position with the same type.
11091	///
11092	void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
11093	#ifndef NDEBUG
11094	for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
11095	assert((!From->hasAnyUseOfValue(i) ||
11096	From->getValueType(i) == To->getValueType(i)) &&
11097	"Cannot use this version of ReplaceAllUsesWith!");
11098	#endif
11099
11100	// Handle the trivial case.
11101	if (From == To)
11102	return;
11103
11104	// Preserve Debug Info. Only do this if there's a use.
11105	for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
11106	if (From->hasAnyUseOfValue(Value: i)) {
11107	assert((i < To->getNumValues()) && "Invalid To location");
11108	transferDbgValues(From: SDValue(From, i), To: SDValue(To, i));
11109	}
11110	// Preserve extra info.
11111	copyExtraInfo(From, To);
11112
11113	// Iterate over just the existing users of From. See the comments in
11114	// the ReplaceAllUsesWith above.
11115	SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
11116	RAUWUpdateListener Listener(*this, UI, UE);
11117	while (UI != UE) {
11118	SDNode *User = *UI;
11119
11120	// This node is about to morph, remove its old self from the CSE maps.
11121	RemoveNodeFromCSEMaps(N: User);
11122
11123	// A user can appear in a use list multiple times, and when this
11124	// happens the uses are usually next to each other in the list.
11125	// To help reduce the number of CSE recomputations, process all
11126	// the uses of this user that we can find this way.
11127	do {
11128	SDUse &Use = UI.getUse();
11129	++UI;
11130	Use.setNode(To);
11131	if (To->isDivergent() != From->isDivergent())
11132	updateDivergence(N: User);
11133	} while (UI != UE && *UI == User);
11134
11135	// Now that we have modified User, add it back to the CSE maps. If it
11136	// already exists there, recursively merge the results together.
11137	AddModifiedNodeToCSEMaps(N: User);
11138	}
11139
11140	// If we just RAUW'd the root, take note.
11141	if (From == getRoot().getNode())
11142	setRoot(SDValue(To, getRoot().getResNo()));
11143	}
11144
11145	/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
11146	/// This can cause recursive merging of nodes in the DAG.
11147	///
11148	/// This version can replace From with any result values. To must match the
11149	/// number and types of values returned by From.
11150	void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
11151	if (From->getNumValues() == 1) // Handle the simple case efficiently.
11152	return ReplaceAllUsesWith(FromN: SDValue(From, 0), To: To[0]);
11153
11154	for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) {
11155	// Preserve Debug Info.
11156	transferDbgValues(From: SDValue(From, i), To: To[i]);
11157	// Preserve extra info.
11158	copyExtraInfo(From, To: To[i].getNode());
11159	}
11160
11161	// Iterate over just the existing users of From. See the comments in
11162	// the ReplaceAllUsesWith above.
11163	SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
11164	RAUWUpdateListener Listener(*this, UI, UE);
11165	while (UI != UE) {
11166	SDNode *User = *UI;
11167
11168	// This node is about to morph, remove its old self from the CSE maps.
11169	RemoveNodeFromCSEMaps(N: User);
11170
11171	// A user can appear in a use list multiple times, and when this happens the
11172	// uses are usually next to each other in the list. To help reduce the
11173	// number of CSE and divergence recomputations, process all the uses of this
11174	// user that we can find this way.
11175	bool To_IsDivergent = false;
11176	do {
11177	SDUse &Use = UI.getUse();
11178	const SDValue &ToOp = To[Use.getResNo()];
11179	++UI;
11180	Use.set(ToOp);
11181	To_IsDivergent |= ToOp->isDivergent();
11182	} while (UI != UE && *UI == User);
11183
11184	if (To_IsDivergent != From->isDivergent())
11185	updateDivergence(N: User);
11186
11187	// Now that we have modified User, add it back to the CSE maps. If it
11188	// already exists there, recursively merge the results together.
11189	AddModifiedNodeToCSEMaps(N: User);
11190	}
11191
11192	// If we just RAUW'd the root, take note.
11193	if (From == getRoot().getNode())
11194	setRoot(SDValue(To[getRoot().getResNo()]));
11195	}
11196
11197	/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
11198	/// uses of other values produced by From.getNode() alone. The Deleted
11199	/// vector is handled the same way as for ReplaceAllUsesWith.
11200	void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
11201	// Handle the really simple, really trivial case efficiently.
11202	if (From == To) return;
11203
11204	// Handle the simple, trivial, case efficiently.
11205	if (From.getNode()->getNumValues() == 1) {
11206	ReplaceAllUsesWith(FromN: From, To);
11207	return;
11208	}
11209
11210	// Preserve Debug Info.
11211	transferDbgValues(From, To);
11212	copyExtraInfo(From: From.getNode(), To: To.getNode());
11213
11214	// Iterate over just the existing users of From. See the comments in
11215	// the ReplaceAllUsesWith above.
11216	SDNode::use_iterator UI = From.getNode()->use_begin(),
11217	UE = From.getNode()->use_end();
11218	RAUWUpdateListener Listener(*this, UI, UE);
11219	while (UI != UE) {
11220	SDNode *User = *UI;
11221	bool UserRemovedFromCSEMaps = false;
11222
11223	// A user can appear in a use list multiple times, and when this
11224	// happens the uses are usually next to each other in the list.
11225	// To help reduce the number of CSE recomputations, process all
11226	// the uses of this user that we can find this way.
11227	do {
11228	SDUse &Use = UI.getUse();
11229
11230	// Skip uses of different values from the same node.
11231	if (Use.getResNo() != From.getResNo()) {
11232	++UI;
11233	continue;
11234	}
11235
11236	// If this node hasn't been modified yet, it's still in the CSE maps,
11237	// so remove its old self from the CSE maps.
11238	if (!UserRemovedFromCSEMaps) {
11239	RemoveNodeFromCSEMaps(N: User);
11240	UserRemovedFromCSEMaps = true;
11241	}
11242
11243	++UI;
11244	Use.set(To);
11245	if (To->isDivergent() != From->isDivergent())
11246	updateDivergence(N: User);
11247	} while (UI != UE && *UI == User);
11248	// We are iterating over all uses of the From node, so if a use
11249	// doesn't use the specific value, no changes are made.
11250	if (!UserRemovedFromCSEMaps)
11251	continue;
11252
11253	// Now that we have modified User, add it back to the CSE maps. If it
11254	// already exists there, recursively merge the results together.
11255	AddModifiedNodeToCSEMaps(N: User);
11256	}
11257
11258	// If we just RAUW'd the root, take note.
11259	if (From == getRoot())
11260	setRoot(To);
11261	}
11262
11263	namespace {
11264
11265	/// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
11266	/// to record information about a use.
11267	struct UseMemo {
11268	SDNode *User;
11269	unsigned Index;
11270	SDUse *Use;
11271	};
11272
11273	/// operator< - Sort Memos by User.
11274	bool operator<(const UseMemo &L, const UseMemo &R) {
11275	return (intptr_t)L.User < (intptr_t)R.User;
11276	}
11277
11278	/// RAUOVWUpdateListener - Helper for ReplaceAllUsesOfValuesWith - When the node
11279	/// pointed to by a UseMemo is deleted, set the User to nullptr to indicate that
11280	/// the node already has been taken care of recursively.
11281	class RAUOVWUpdateListener : public SelectionDAG::DAGUpdateListener {
11282	SmallVector<UseMemo, 4> &Uses;
11283
11284	void NodeDeleted(SDNode *N, SDNode *E) override {
11285	for (UseMemo &Memo : Uses)
11286	if (Memo.User == N)
11287	Memo.User = nullptr;
11288	}
11289
11290	public:
11291	RAUOVWUpdateListener(SelectionDAG &d, SmallVector<UseMemo, 4> &uses)
11292	: SelectionDAG::DAGUpdateListener(d), Uses(uses) {}
11293	};
11294
11295	} // end anonymous namespace
11296
11297	bool SelectionDAG::calculateDivergence(SDNode *N) {
11298	if (TLI->isSDNodeAlwaysUniform(N)) {
11299	assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) &&
11300	"Conflicting divergence information!");
11301	return false;
11302	}
11303	if (TLI->isSDNodeSourceOfDivergence(N, FLI, UA))
11304	return true;
11305	for (const auto &Op : N->ops()) {
11306	if (Op.Val.getValueType() != MVT::Other && Op.getNode()->isDivergent())
11307	return true;
11308	}
11309	return false;
11310	}
11311
11312	void SelectionDAG::updateDivergence(SDNode *N) {
11313	SmallVector<SDNode *, 16> Worklist(1, N);
11314	do {
11315	N = Worklist.pop_back_val();
11316	bool IsDivergent = calculateDivergence(N);
11317	if (N->SDNodeBits.IsDivergent != IsDivergent) {
11318	N->SDNodeBits.IsDivergent = IsDivergent;
11319	llvm::append_range(C&: Worklist, R: N->uses());
11320	}
11321	} while (!Worklist.empty());
11322	}
11323
11324	void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
11325	DenseMap<SDNode *, unsigned> Degree;
11326	Order.reserve(n: AllNodes.size());
11327	for (auto &N : allnodes()) {
11328	unsigned NOps = N.getNumOperands();
11329	Degree[&N] = NOps;
11330	if (0 == NOps)
11331	Order.push_back(x: &N);
11332	}
11333	for (size_t I = 0; I != Order.size(); ++I) {
11334	SDNode *N = Order[I];
11335	for (auto *U : N->uses()) {
11336	unsigned &UnsortedOps = Degree[U];
11337	if (0 == --UnsortedOps)
11338	Order.push_back(x: U);
11339	}
11340	}
11341	}
11342
11343	#ifndef NDEBUG
11344	void SelectionDAG::VerifyDAGDivergence() {
11345	std::vector<SDNode *> TopoOrder;
11346	CreateTopologicalOrder(Order&: TopoOrder);
11347	for (auto *N : TopoOrder) {
11348	assert(calculateDivergence(N) == N->isDivergent() &&
11349	"Divergence bit inconsistency detected");
11350	}
11351	}
11352	#endif
11353
11354	/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
11355	/// uses of other values produced by From.getNode() alone. The same value
11356	/// may appear in both the From and To list. The Deleted vector is
11357	/// handled the same way as for ReplaceAllUsesWith.
11358	void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
11359	const SDValue *To,
11360	unsigned Num){
11361	// Handle the simple, trivial case efficiently.
11362	if (Num == 1)
11363	return ReplaceAllUsesOfValueWith(From: *From, To: *To);
11364
11365	transferDbgValues(From: *From, To: *To);
11366	copyExtraInfo(From: From->getNode(), To: To->getNode());
11367
11368	// Read up all the uses and make records of them. This helps
11369	// processing new uses that are introduced during the
11370	// replacement process.
11371	SmallVector<UseMemo, 4> Uses;
11372	for (unsigned i = 0; i != Num; ++i) {
11373	unsigned FromResNo = From[i].getResNo();
11374	SDNode *FromNode = From[i].getNode();
11375	for (SDNode::use_iterator UI = FromNode->use_begin(),
11376	E = FromNode->use_end(); UI != E; ++UI) {
11377	SDUse &Use = UI.getUse();
11378	if (Use.getResNo() == FromResNo) {
11379	UseMemo Memo = { .User: *UI, .Index: i, .Use: &Use };
11380	Uses.push_back(Elt: Memo);
11381	}
11382	}
11383	}
11384
11385	// Sort the uses, so that all the uses from a given User are together.
11386	llvm::sort(C&: Uses);
11387	RAUOVWUpdateListener Listener(*this, Uses);
11388
11389	for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
11390	UseIndex != UseIndexEnd; ) {
11391	// We know that this user uses some value of From. If it is the right
11392	// value, update it.
11393	SDNode *User = Uses[UseIndex].User;
11394	// If the node has been deleted by recursive CSE updates when updating
11395	// another node, then just skip this entry.
11396	if (User == nullptr) {
11397	++UseIndex;
11398	continue;
11399	}
11400
11401	// This node is about to morph, remove its old self from the CSE maps.
11402	RemoveNodeFromCSEMaps(N: User);
11403
11404	// The Uses array is sorted, so all the uses for a given User
11405	// are next to each other in the list.
11406	// To help reduce the number of CSE recomputations, process all
11407	// the uses of this user that we can find this way.
11408	do {
11409	unsigned i = Uses[UseIndex].Index;
11410	SDUse &Use = *Uses[UseIndex].Use;
11411	++UseIndex;
11412
11413	Use.set(To[i]);
11414	} while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
11415
11416	// Now that we have modified User, add it back to the CSE maps. If it
11417	// already exists there, recursively merge the results together.
11418	AddModifiedNodeToCSEMaps(N: User);
11419	}
11420	}
11421
11422	/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
11423	/// based on their topological order. It returns the maximum id and a vector
11424	/// of the SDNodes* in assigned order by reference.
11425	unsigned SelectionDAG::AssignTopologicalOrder() {
11426	unsigned DAGSize = 0;
11427
11428	// SortedPos tracks the progress of the algorithm. Nodes before it are
11429	// sorted, nodes after it are unsorted. When the algorithm completes
11430	// it is at the end of the list.
11431	allnodes_iterator SortedPos = allnodes_begin();
11432
11433	// Visit all the nodes. Move nodes with no operands to the front of
11434	// the list immediately. Annotate nodes that do have operands with their
11435	// operand count. Before we do this, the Node Id fields of the nodes
11436	// may contain arbitrary values. After, the Node Id fields for nodes
11437	// before SortedPos will contain the topological sort index, and the
11438	// Node Id fields for nodes At SortedPos and after will contain the
11439	// count of outstanding operands.
11440	for (SDNode &N : llvm::make_early_inc_range(Range: allnodes())) {
11441	checkForCycles(N: &N, DAG: this);
11442	unsigned Degree = N.getNumOperands();
11443	if (Degree == 0) {
11444	// A node with no uses, add it to the result array immediately.
11445	N.setNodeId(DAGSize++);
11446	allnodes_iterator Q(&N);
11447	if (Q != SortedPos)
11448	SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT&: Q));
11449	assert(SortedPos != AllNodes.end() && "Overran node list");
11450	++SortedPos;
11451	} else {
11452	// Temporarily use the Node Id as scratch space for the degree count.
11453	N.setNodeId(Degree);
11454	}
11455	}
11456
11457	// Visit all the nodes. As we iterate, move nodes into sorted order,
11458	// such that by the time the end is reached all nodes will be sorted.
11459	for (SDNode &Node : allnodes()) {
11460	SDNode *N = &Node;
11461	checkForCycles(N, DAG: this);
11462	// N is in sorted position, so all its uses have one less operand
11463	// that needs to be sorted.
11464	for (SDNode *P : N->uses()) {
11465	unsigned Degree = P->getNodeId();
11466	assert(Degree != 0 && "Invalid node degree");
11467	--Degree;
11468	if (Degree == 0) {
11469	// All of P's operands are sorted, so P may sorted now.
11470	P->setNodeId(DAGSize++);
11471	if (P->getIterator() != SortedPos)
11472	SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT: P));
11473	assert(SortedPos != AllNodes.end() && "Overran node list");
11474	++SortedPos;
11475	} else {
11476	// Update P's outstanding operand count.
11477	P->setNodeId(Degree);
11478	}
11479	}
11480	if (Node.getIterator() == SortedPos) {
11481	#ifndef NDEBUG
11482	allnodes_iterator I(N);
11483	SDNode *S = &*++I;
11484	dbgs() << "Overran sorted position:\n";
11485	S->dumprFull(G: this); dbgs() << "\n";
11486	dbgs() << "Checking if this is due to cycles\n";
11487	checkForCycles(DAG: this, force: true);
11488	#endif
11489	llvm_unreachable(nullptr);
11490	}
11491	}
11492
11493	assert(SortedPos == AllNodes.end() &&
11494	"Topological sort incomplete!");
11495	assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
11496	"First node in topological sort is not the entry token!");
11497	assert(AllNodes.front().getNodeId() == 0 &&
11498	"First node in topological sort has non-zero id!");
11499	assert(AllNodes.front().getNumOperands() == 0 &&
11500	"First node in topological sort has operands!");
11501	assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
11502	"Last node in topologic sort has unexpected id!");
11503	assert(AllNodes.back().use_empty() &&
11504	"Last node in topologic sort has users!");
11505	assert(DAGSize == allnodes_size() && "Node count mismatch!");
11506	return DAGSize;
11507	}
11508
11509	/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
11510	/// value is produced by SD.
11511	void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) {
11512	for (SDNode *SD : DB->getSDNodes()) {
11513	if (!SD)
11514	continue;
11515	assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
11516	SD->setHasDebugValue(true);
11517	}
11518	DbgInfo->add(V: DB, isParameter);
11519	}
11520
11521	void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(L: DB); }
11522
11523	SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain,
11524	SDValue NewMemOpChain) {
11525	assert(isa<MemSDNode>(NewMemOpChain) && "Expected a memop node");
11526	assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT");
11527	// The new memory operation must have the same position as the old load in
11528	// terms of memory dependency. Create a TokenFactor for the old load and new
11529	// memory operation and update uses of the old load's output chain to use that
11530	// TokenFactor.
11531	if (OldChain == NewMemOpChain || OldChain.use_empty())
11532	return NewMemOpChain;
11533
11534	SDValue TokenFactor = getNode(ISD::TokenFactor, SDLoc(OldChain), MVT::Other,
11535	OldChain, NewMemOpChain);
11536	ReplaceAllUsesOfValueWith(From: OldChain, To: TokenFactor);
11537	UpdateNodeOperands(N: TokenFactor.getNode(), Op1: OldChain, Op2: NewMemOpChain);
11538	return TokenFactor;
11539	}
11540
11541	SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,
11542	SDValue NewMemOp) {
11543	assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node");
11544	SDValue OldChain = SDValue(OldLoad, 1);
11545	SDValue NewMemOpChain = NewMemOp.getValue(R: 1);
11546	return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain);
11547	}
11548
11549	SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op,
11550	Function **OutFunction) {
11551	assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol");
11552
11553	auto *Symbol = cast<ExternalSymbolSDNode>(Val&: Op)->getSymbol();
11554	auto *Module = MF->getFunction().getParent();
11555	auto *Function = Module->getFunction(Name: Symbol);
11556
11557	if (OutFunction != nullptr)
11558	*OutFunction = Function;
11559
11560	if (Function != nullptr) {
11561	auto PtrTy = TLI->getPointerTy(DL: getDataLayout(), AS: Function->getAddressSpace());
11562	return getGlobalAddress(GV: Function, DL: SDLoc(Op), VT: PtrTy);
11563	}
11564
11565	std::string ErrorStr;
11566	raw_string_ostream ErrorFormatter(ErrorStr);
11567	ErrorFormatter << "Undefined external symbol ";
11568	ErrorFormatter << '"' << Symbol << '"';
11569	report_fatal_error(reason: Twine(ErrorFormatter.str()));
11570	}
11571
11572	//===----------------------------------------------------------------------===//
11573	// SDNode Class
11574	//===----------------------------------------------------------------------===//
11575
11576	bool llvm::isNullConstant(SDValue V) {
11577	ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
11578	return Const != nullptr && Const->isZero();
11579	}
11580
11581	bool llvm::isNullFPConstant(SDValue V) {
11582	ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(Val&: V);
11583	return Const != nullptr && Const->isZero() && !Const->isNegative();
11584	}
11585
11586	bool llvm::isAllOnesConstant(SDValue V) {
11587	ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
11588	return Const != nullptr && Const->isAllOnes();
11589	}
11590
11591	bool llvm::isOneConstant(SDValue V) {
11592	ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
11593	return Const != nullptr && Const->isOne();
11594	}
11595
11596	bool llvm::isMinSignedConstant(SDValue V) {
11597	ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
11598	return Const != nullptr && Const->isMinSignedValue();
11599	}
11600
11601	bool llvm::isNeutralConstant(unsigned Opcode, SDNodeFlags Flags, SDValue V,
11602	unsigned OperandNo) {
11603	// NOTE: The cases should match with IR's ConstantExpr::getBinOpIdentity().
11604	// TODO: Target-specific opcodes could be added.
11605	if (auto *ConstV = isConstOrConstSplat(N: V, /*AllowUndefs*/ false,
11606	/*AllowTruncation*/ true)) {
11607	APInt Const = ConstV->getAPIntValue().trunc(width: V.getScalarValueSizeInBits());
11608	switch (Opcode) {
11609	case ISD::ADD:
11610	case ISD::OR:
11611	case ISD::XOR:
11612	case ISD::UMAX:
11613	return Const.isZero();
11614	case ISD::MUL:
11615	return Const.isOne();
11616	case ISD::AND:
11617	case ISD::UMIN:
11618	return Const.isAllOnes();
11619	case ISD::SMAX:
11620	return Const.isMinSignedValue();
11621	case ISD::SMIN:
11622	return Const.isMaxSignedValue();
11623	case ISD::SUB:
11624	case ISD::SHL:
11625	case ISD::SRA:
11626	case ISD::SRL:
11627	return OperandNo == 1 && Const.isZero();
11628	case ISD::UDIV:
11629	case ISD::SDIV:
11630	return OperandNo == 1 && Const.isOne();
11631	}
11632	} else if (auto *ConstFP = isConstOrConstSplatFP(N: V)) {
11633	switch (Opcode) {
11634	case ISD::FADD:
11635	return ConstFP->isZero() &&
11636	(Flags.hasNoSignedZeros() || ConstFP->isNegative());
11637	case ISD::FSUB:
11638	return OperandNo == 1 && ConstFP->isZero() &&
11639	(Flags.hasNoSignedZeros() || !ConstFP->isNegative());
11640	case ISD::FMUL:
11641	return ConstFP->isExactlyValue(V: 1.0);
11642	case ISD::FDIV:
11643	return OperandNo == 1 && ConstFP->isExactlyValue(V: 1.0);
11644	case ISD::FMINNUM:
11645	case ISD::FMAXNUM: {
11646	// Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
11647	EVT VT = V.getValueType();
11648	const fltSemantics &Semantics = SelectionDAG::EVTToAPFloatSemantics(VT);
11649	APFloat NeutralAF = !Flags.hasNoNaNs()
11650	? APFloat::getQNaN(Sem: Semantics)
11651	: !Flags.hasNoInfs()
11652	? APFloat::getInf(Sem: Semantics)
11653	: APFloat::getLargest(Sem: Semantics);
11654	if (Opcode == ISD::FMAXNUM)
11655	NeutralAF.changeSign();
11656
11657	return ConstFP->isExactlyValue(V: NeutralAF);
11658	}
11659	}
11660	}
11661	return false;
11662	}
11663
11664	SDValue llvm::peekThroughBitcasts(SDValue V) {
11665	while (V.getOpcode() == ISD::BITCAST)
11666	V = V.getOperand(i: 0);
11667	return V;
11668	}
11669
11670	SDValue llvm::peekThroughOneUseBitcasts(SDValue V) {
11671	while (V.getOpcode() == ISD::BITCAST && V.getOperand(i: 0).hasOneUse())
11672	V = V.getOperand(i: 0);
11673	return V;
11674	}
11675
11676	SDValue llvm::peekThroughExtractSubvectors(SDValue V) {
11677	while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR)
11678	V = V.getOperand(i: 0);
11679	return V;
11680	}
11681
11682	SDValue llvm::peekThroughTruncates(SDValue V) {
11683	while (V.getOpcode() == ISD::TRUNCATE)
11684	V = V.getOperand(i: 0);
11685	return V;
11686	}
11687
11688	bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) {
11689	if (V.getOpcode() != ISD::XOR)
11690	return false;
11691	V = peekThroughBitcasts(V: V.getOperand(i: 1));
11692	unsigned NumBits = V.getScalarValueSizeInBits();
11693	ConstantSDNode *C =
11694	isConstOrConstSplat(N: V, AllowUndefs, /*AllowTruncation*/ true);
11695	return C && (C->getAPIntValue().countr_one() >= NumBits);
11696	}
11697
11698	ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs,
11699	bool AllowTruncation) {
11700	EVT VT = N.getValueType();
11701	APInt DemandedElts = VT.isFixedLengthVector()
11702	? APInt::getAllOnes(numBits: VT.getVectorMinNumElements())
11703	: APInt(1, 1);
11704	return isConstOrConstSplat(N, DemandedElts, AllowUndefs, AllowTruncation);
11705	}
11706
11707	ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
11708	bool AllowUndefs,
11709	bool AllowTruncation) {
11710	if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N))
11711	return CN;
11712
11713	// SplatVectors can truncate their operands. Ignore that case here unless
11714	// AllowTruncation is set.
11715	if (N->getOpcode() == ISD::SPLAT_VECTOR) {
11716	EVT VecEltVT = N->getValueType(ResNo: 0).getVectorElementType();
11717	if (auto *CN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0))) {
11718	EVT CVT = CN->getValueType(ResNo: 0);
11719	assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension");
11720	if (AllowTruncation || CVT == VecEltVT)
11721	return CN;
11722	}
11723	}
11724
11725	if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) {
11726	BitVector UndefElements;
11727	ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, UndefElements: &UndefElements);
11728
11729	// BuildVectors can truncate their operands. Ignore that case here unless
11730	// AllowTruncation is set.
11731	// TODO: Look into whether we should allow UndefElements in non-DemandedElts
11732	if (CN && (UndefElements.none() || AllowUndefs)) {
11733	EVT CVT = CN->getValueType(ResNo: 0);
11734	EVT NSVT = N.getValueType().getScalarType();
11735	assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension");
11736	if (AllowTruncation || (CVT == NSVT))
11737	return CN;
11738	}
11739	}
11740
11741	return nullptr;
11742	}
11743
11744	ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) {
11745	EVT VT = N.getValueType();
11746	APInt DemandedElts = VT.isFixedLengthVector()
11747	? APInt::getAllOnes(numBits: VT.getVectorMinNumElements())
11748	: APInt(1, 1);
11749	return isConstOrConstSplatFP(N, DemandedElts, AllowUndefs);
11750	}
11751
11752	ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N,
11753	const APInt &DemandedElts,
11754	bool AllowUndefs) {
11755	if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val&: N))
11756	return CN;
11757
11758	if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) {
11759	BitVector UndefElements;
11760	ConstantFPSDNode *CN =
11761	BV->getConstantFPSplatNode(DemandedElts, UndefElements: &UndefElements);
11762	// TODO: Look into whether we should allow UndefElements in non-DemandedElts
11763	if (CN && (UndefElements.none() || AllowUndefs))
11764	return CN;
11765	}
11766
11767	if (N.getOpcode() == ISD::SPLAT_VECTOR)
11768	if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val: N.getOperand(i: 0)))
11769	return CN;
11770
11771	return nullptr;
11772	}
11773
11774	bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {
11775	// TODO: may want to use peekThroughBitcast() here.
11776	ConstantSDNode *C =
11777	isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
11778	return C && C->isZero();
11779	}
11780
11781	bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) {
11782	ConstantSDNode *C =
11783	isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation*/ true);
11784	return C && C->isOne();
11785	}
11786
11787	bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) {
11788	N = peekThroughBitcasts(V: N);
11789	unsigned BitWidth = N.getScalarValueSizeInBits();
11790	ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
11791	return C && C->isAllOnes() && C->getValueSizeInBits(ResNo: 0) == BitWidth;
11792	}
11793
11794	HandleSDNode::~HandleSDNode() {
11795	DropOperands();
11796	}
11797
11798	GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order,
11799	const DebugLoc &DL,
11800	const GlobalValue *GA, EVT VT,
11801	int64_t o, unsigned TF)
11802	: SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) {
11803	TheGlobal = GA;
11804	}
11805
11806	AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl,
11807	EVT VT, unsigned SrcAS,
11808	unsigned DestAS)
11809	: SDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT)),
11810	SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
11811
11812	MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
11813	SDVTList VTs, EVT memvt, MachineMemOperand *mmo)
11814	: SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
11815	MemSDNodeBits.IsVolatile = MMO->isVolatile();
11816	MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal();
11817	MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable();
11818	MemSDNodeBits.IsInvariant = MMO->isInvariant();
11819
11820	// We check here that the size of the memory operand fits within the size of
11821	// the MMO. This is because the MMO might indicate only a possible address
11822	// range instead of specifying the affected memory addresses precisely.
11823	assert(
11824	(!MMO->getType().isValid() ||
11825	TypeSize::isKnownLE(memvt.getStoreSize(), MMO->getSize().getValue())) &&
11826	"Size mismatch!");
11827	}
11828
11829	/// Profile - Gather unique data for the node.
11830	///
11831	void SDNode::Profile(FoldingSetNodeID &ID) const {
11832	AddNodeIDNode(ID, N: this);
11833	}
11834
11835	namespace {
11836
11837	struct EVTArray {
11838	std::vector<EVT> VTs;
11839
11840	EVTArray() {
11841	VTs.reserve(n: MVT::VALUETYPE_SIZE);
11842	for (unsigned i = 0; i < MVT::VALUETYPE_SIZE; ++i)
11843	VTs.push_back(x: MVT((MVT::SimpleValueType)i));
11844	}
11845	};
11846
11847	} // end anonymous namespace
11848
11849	/// getValueTypeList - Return a pointer to the specified value type.
11850	///
11851	const EVT *SDNode::getValueTypeList(EVT VT) {
11852	static std::set<EVT, EVT::compareRawBits> EVTs;
11853	static EVTArray SimpleVTArray;
11854	static sys::SmartMutex<true> VTMutex;
11855
11856	if (VT.isExtended()) {
11857	sys::SmartScopedLock<true> Lock(VTMutex);
11858	return &(*EVTs.insert(x: VT).first);
11859	}
11860	assert(VT.getSimpleVT() < MVT::VALUETYPE_SIZE && "Value type out of range!");
11861	return &SimpleVTArray.VTs[VT.getSimpleVT().SimpleTy];
11862	}
11863
11864	/// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
11865	/// indicated value. This method ignores uses of other values defined by this
11866	/// operation.
11867	bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const {
11868	assert(Value < getNumValues() && "Bad value!");
11869
11870	// TODO: Only iterate over uses of a given value of the node
11871	for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) {
11872	if (UI.getUse().getResNo() == Value) {
11873	if (NUses == 0)
11874	return false;
11875	--NUses;
11876	}
11877	}
11878
11879	// Found exactly the right number of uses?
11880	return NUses == 0;
11881	}
11882
11883	/// hasAnyUseOfValue - Return true if there are any use of the indicated
11884	/// value. This method ignores uses of other values defined by this operation.
11885	bool SDNode::hasAnyUseOfValue(unsigned Value) const {
11886	assert(Value < getNumValues() && "Bad value!");
11887
11888	for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI)
11889	if (UI.getUse().getResNo() == Value)
11890	return true;
11891
11892	return false;
11893	}
11894
11895	/// isOnlyUserOf - Return true if this node is the only use of N.
11896	bool SDNode::isOnlyUserOf(const SDNode *N) const {
11897	bool Seen = false;
11898	for (const SDNode *User : N->uses()) {
11899	if (User == this)
11900	Seen = true;
11901	else
11902	return false;
11903	}
11904
11905	return Seen;
11906	}
11907
11908	/// Return true if the only users of N are contained in Nodes.
11909	bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) {
11910	bool Seen = false;
11911	for (const SDNode *User : N->uses()) {
11912	if (llvm::is_contained(Range&: Nodes, Element: User))
11913	Seen = true;
11914	else
11915	return false;
11916	}
11917
11918	return Seen;
11919	}
11920
11921	/// isOperand - Return true if this node is an operand of N.
11922	bool SDValue::isOperandOf(const SDNode *N) const {
11923	return is_contained(Range: N->op_values(), Element: *this);
11924	}
11925
11926	bool SDNode::isOperandOf(const SDNode *N) const {
11927	return any_of(Range: N->op_values(),
11928	P: [this](SDValue Op) { return this == Op.getNode(); });
11929	}
11930
11931	/// reachesChainWithoutSideEffects - Return true if this operand (which must
11932	/// be a chain) reaches the specified operand without crossing any
11933	/// side-effecting instructions on any chain path. In practice, this looks
11934	/// through token factors and non-volatile loads. In order to remain efficient,
11935	/// this only looks a couple of nodes in, it does not do an exhaustive search.
11936	///
11937	/// Note that we only need to examine chains when we're searching for
11938	/// side-effects; SelectionDAG requires that all side-effects are represented
11939	/// by chains, even if another operand would force a specific ordering. This
11940	/// constraint is necessary to allow transformations like splitting loads.
11941	bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
11942	unsigned Depth) const {
11943	if (*this == Dest) return true;
11944
11945	// Don't search too deeply, we just want to be able to see through
11946	// TokenFactor's etc.
11947	if (Depth == 0) return false;
11948
11949	// If this is a token factor, all inputs to the TF happen in parallel.
11950	if (getOpcode() == ISD::TokenFactor) {
11951	// First, try a shallow search.
11952	if (is_contained(Range: (*this)->ops(), Element: Dest)) {
11953	// We found the chain we want as an operand of this TokenFactor.
11954	// Essentially, we reach the chain without side-effects if we could
11955	// serialize the TokenFactor into a simple chain of operations with
11956	// Dest as the last operation. This is automatically true if the
11957	// chain has one use: there are no other ordering constraints.
11958	// If the chain has more than one use, we give up: some other
11959	// use of Dest might force a side-effect between Dest and the current
11960	// node.
11961	if (Dest.hasOneUse())
11962	return true;
11963	}
11964	// Next, try a deep search: check whether every operand of the TokenFactor
11965	// reaches Dest.
11966	return llvm::all_of(Range: (*this)->ops(), P: [=](SDValue Op) {
11967	return Op.reachesChainWithoutSideEffects(Dest, Depth: Depth - 1);
11968	});
11969	}
11970
11971	// Loads don't have side effects, look through them.
11972	if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Val: *this)) {
11973	if (Ld->isUnordered())
11974	return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth: Depth-1);
11975	}
11976	return false;
11977	}
11978
11979	bool SDNode::hasPredecessor(const SDNode *N) const {
11980	SmallPtrSet<const SDNode *, 32> Visited;
11981	SmallVector<const SDNode *, 16> Worklist;
11982	Worklist.push_back(Elt: this);
11983	return hasPredecessorHelper(N, Visited, Worklist);
11984	}
11985
11986	void SDNode::intersectFlagsWith(const SDNodeFlags Flags) {
11987	this->Flags.intersectWith(Flags);
11988	}
11989
11990	SDValue
11991	SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
11992	ArrayRef<ISD::NodeType> CandidateBinOps,
11993	bool AllowPartials) {
11994	// The pattern must end in an extract from index 0.
11995	if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
11996	!isNullConstant(V: Extract->getOperand(Num: 1)))
11997	return SDValue();
11998
11999	// Match against one of the candidate binary ops.
12000	SDValue Op = Extract->getOperand(Num: 0);
12001	if (llvm::none_of(Range&: CandidateBinOps, P: [Op](ISD::NodeType BinOp) {
12002	return Op.getOpcode() == unsigned(BinOp);
12003	}))
12004	return SDValue();
12005
12006	// Floating-point reductions may require relaxed constraints on the final step
12007	// of the reduction because they may reorder intermediate operations.
12008	unsigned CandidateBinOp = Op.getOpcode();
12009	if (Op.getValueType().isFloatingPoint()) {
12010	SDNodeFlags Flags = Op->getFlags();
12011	switch (CandidateBinOp) {
12012	case ISD::FADD:
12013	if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation())
12014	return SDValue();
12015	break;
12016	default:
12017	llvm_unreachable("Unhandled FP opcode for binop reduction");
12018	}
12019	}
12020
12021	// Matching failed - attempt to see if we did enough stages that a partial
12022	// reduction from a subvector is possible.
12023	auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) {
12024	if (!AllowPartials || !Op)
12025	return SDValue();
12026	EVT OpVT = Op.getValueType();
12027	EVT OpSVT = OpVT.getScalarType();
12028	EVT SubVT = EVT::getVectorVT(Context&: *getContext(), VT: OpSVT, NumElements: NumSubElts);
12029	if (!TLI->isExtractSubvectorCheap(ResVT: SubVT, SrcVT: OpVT, Index: 0))
12030	return SDValue();
12031	BinOp = (ISD::NodeType)CandidateBinOp;
12032	return getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(Op), VT: SubVT, N1: Op,
12033	N2: getVectorIdxConstant(Val: 0, DL: SDLoc(Op)));
12034	};
12035
12036	// At each stage, we're looking for something that looks like:
12037	// %s = shufflevector <8 x i32> %op, <8 x i32> undef,
12038	// <8 x i32> <i32 2, i32 3, i32 undef, i32 undef,
12039	// i32 undef, i32 undef, i32 undef, i32 undef>
12040	// %a = binop <8 x i32> %op, %s
12041	// Where the mask changes according to the stage. E.g. for a 3-stage pyramid,
12042	// we expect something like:
12043	// <4,5,6,7,u,u,u,u>
12044	// <2,3,u,u,u,u,u,u>
12045	// <1,u,u,u,u,u,u,u>
12046	// While a partial reduction match would be:
12047	// <2,3,u,u,u,u,u,u>
12048	// <1,u,u,u,u,u,u,u>
12049	unsigned Stages = Log2_32(Value: Op.getValueType().getVectorNumElements());
12050	SDValue PrevOp;
12051	for (unsigned i = 0; i < Stages; ++i) {
12052	unsigned MaskEnd = (1 << i);
12053
12054	if (Op.getOpcode() != CandidateBinOp)
12055	return PartialReduction(PrevOp, MaskEnd);
12056
12057	SDValue Op0 = Op.getOperand(i: 0);
12058	SDValue Op1 = Op.getOperand(i: 1);
12059
12060	ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op0);
12061	if (Shuffle) {
12062	Op = Op1;
12063	} else {
12064	Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op1);
12065	Op = Op0;
12066	}
12067
12068	// The first operand of the shuffle should be the same as the other operand
12069	// of the binop.
12070	if (!Shuffle || Shuffle->getOperand(Num: 0) != Op)
12071	return PartialReduction(PrevOp, MaskEnd);
12072
12073	// Verify the shuffle has the expected (at this stage of the pyramid) mask.
12074	for (int Index = 0; Index < (int)MaskEnd; ++Index)
12075	if (Shuffle->getMaskElt(Idx: Index) != (int)(MaskEnd + Index))
12076	return PartialReduction(PrevOp, MaskEnd);
12077
12078	PrevOp = Op;
12079	}
12080
12081	// Handle subvector reductions, which tend to appear after the shuffle
12082	// reduction stages.
12083	while (Op.getOpcode() == CandidateBinOp) {
12084	unsigned NumElts = Op.getValueType().getVectorNumElements();
12085	SDValue Op0 = Op.getOperand(i: 0);
12086	SDValue Op1 = Op.getOperand(i: 1);
12087	if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
12088	Op1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
12089	Op0.getOperand(i: 0) != Op1.getOperand(i: 0))
12090	break;
12091	SDValue Src = Op0.getOperand(i: 0);
12092	unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
12093	if (NumSrcElts != (2 * NumElts))
12094	break;
12095	if (!(Op0.getConstantOperandAPInt(i: 1) == 0 &&
12096	Op1.getConstantOperandAPInt(i: 1) == NumElts) &&
12097	!(Op1.getConstantOperandAPInt(i: 1) == 0 &&
12098	Op0.getConstantOperandAPInt(i: 1) == NumElts))
12099	break;
12100	Op = Src;
12101	}
12102
12103	BinOp = (ISD::NodeType)CandidateBinOp;
12104	return Op;
12105	}
12106
12107	SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
12108	EVT VT = N->getValueType(ResNo: 0);
12109	EVT EltVT = VT.getVectorElementType();
12110	unsigned NE = VT.getVectorNumElements();
12111
12112	SDLoc dl(N);
12113
12114	// If ResNE is 0, fully unroll the vector op.
12115	if (ResNE == 0)
12116	ResNE = NE;
12117	else if (NE > ResNE)
12118	NE = ResNE;
12119
12120	if (N->getNumValues() == 2) {
12121	SmallVector<SDValue, 8> Scalars0, Scalars1;
12122	SmallVector<SDValue, 4> Operands(N->getNumOperands());
12123	EVT VT1 = N->getValueType(ResNo: 1);
12124	EVT EltVT1 = VT1.getVectorElementType();
12125
12126	unsigned i;
12127	for (i = 0; i != NE; ++i) {
12128	for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
12129	SDValue Operand = N->getOperand(Num: j);
12130	EVT OperandVT = Operand.getValueType();
12131
12132	// A vector operand; extract a single element.
12133	EVT OperandEltVT = OperandVT.getVectorElementType();
12134	Operands[j] = getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: OperandEltVT,
12135	N1: Operand, N2: getVectorIdxConstant(Val: i, DL: dl));
12136	}
12137
12138	SDValue EltOp = getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {EltVT, EltVT1}, Ops: Operands);
12139	Scalars0.push_back(Elt: EltOp);
12140	Scalars1.push_back(Elt: EltOp.getValue(R: 1));
12141	}
12142
12143	SDValue Vec0 = getBuildVector(VT, DL: dl, Ops: Scalars0);
12144	SDValue Vec1 = getBuildVector(VT: VT1, DL: dl, Ops: Scalars1);
12145	return getMergeValues(Ops: {Vec0, Vec1}, dl);
12146	}
12147
12148	assert(N->getNumValues() == 1 &&
12149	"Can't unroll a vector with multiple results!");
12150
12151	SmallVector<SDValue, 8> Scalars;
12152	SmallVector<SDValue, 4> Operands(N->getNumOperands());
12153
12154	unsigned i;
12155	for (i= 0; i != NE; ++i) {
12156	for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
12157	SDValue Operand = N->getOperand(Num: j);
12158	EVT OperandVT = Operand.getValueType();
12159	if (OperandVT.isVector()) {
12160	// A vector operand; extract a single element.
12161	EVT OperandEltVT = OperandVT.getVectorElementType();
12162	Operands[j] = getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: OperandEltVT,
12163	N1: Operand, N2: getVectorIdxConstant(Val: i, DL: dl));
12164	} else {
12165	// A scalar operand; just use it as is.
12166	Operands[j] = Operand;
12167	}
12168	}
12169
12170	switch (N->getOpcode()) {
12171	default: {
12172	Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, Ops: Operands,
12173	Flags: N->getFlags()));
12174	break;
12175	}
12176	case ISD::VSELECT:
12177	Scalars.push_back(Elt: getNode(Opcode: ISD::SELECT, DL: dl, VT: EltVT, Ops: Operands));
12178	break;
12179	case ISD::SHL:
12180	case ISD::SRA:
12181	case ISD::SRL:
12182	case ISD::ROTL:
12183	case ISD::ROTR:
12184	Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, N1: Operands[0],
12185	N2: getShiftAmountOperand(LHSTy: Operands[0].getValueType(),
12186	Op: Operands[1])));
12187	break;
12188	case ISD::SIGN_EXTEND_INREG: {
12189	EVT ExtVT = cast<VTSDNode>(Val&: Operands[1])->getVT().getVectorElementType();
12190	Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT,
12191	N1: Operands[0],
12192	N2: getValueType(VT: ExtVT)));
12193	}
12194	}
12195	}
12196
12197	for (; i < ResNE; ++i)
12198	Scalars.push_back(Elt: getUNDEF(VT: EltVT));
12199
12200	EVT VecVT = EVT::getVectorVT(Context&: *getContext(), VT: EltVT, NumElements: ResNE);
12201	return getBuildVector(VT: VecVT, DL: dl, Ops: Scalars);
12202	}
12203
12204	std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
12205	SDNode *N, unsigned ResNE) {
12206	unsigned Opcode = N->getOpcode();
12207	assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO ||
12208	Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||
12209	Opcode == ISD::UMULO || Opcode == ISD::SMULO) &&
12210	"Expected an overflow opcode");
12211
12212	EVT ResVT = N->getValueType(ResNo: 0);
12213	EVT OvVT = N->getValueType(ResNo: 1);
12214	EVT ResEltVT = ResVT.getVectorElementType();
12215	EVT OvEltVT = OvVT.getVectorElementType();
12216	SDLoc dl(N);
12217
12218	// If ResNE is 0, fully unroll the vector op.
12219	unsigned NE = ResVT.getVectorNumElements();
12220	if (ResNE == 0)
12221	ResNE = NE;
12222	else if (NE > ResNE)
12223	NE = ResNE;
12224
12225	SmallVector<SDValue, 8> LHSScalars;
12226	SmallVector<SDValue, 8> RHSScalars;
12227	ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: LHSScalars, Start: 0, Count: NE);
12228	ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: RHSScalars, Start: 0, Count: NE);
12229
12230	EVT SVT = TLI->getSetCCResultType(DL: getDataLayout(), Context&: *getContext(), VT: ResEltVT);
12231	SDVTList VTs = getVTList(VT1: ResEltVT, VT2: SVT);
12232	SmallVector<SDValue, 8> ResScalars;
12233	SmallVector<SDValue, 8> OvScalars;
12234	for (unsigned i = 0; i < NE; ++i) {
12235	SDValue Res = getNode(Opcode, DL: dl, VTList: VTs, N1: LHSScalars[i], N2: RHSScalars[i]);
12236	SDValue Ov =
12237	getSelect(DL: dl, VT: OvEltVT, Cond: Res.getValue(R: 1),
12238	LHS: getBoolConstant(V: true, DL: dl, VT: OvEltVT, OpVT: ResVT),
12239	RHS: getConstant(Val: 0, DL: dl, VT: OvEltVT));
12240
12241	ResScalars.push_back(Elt: Res);
12242	OvScalars.push_back(Elt: Ov);
12243	}
12244
12245	ResScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: ResEltVT));
12246	OvScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: OvEltVT));
12247
12248	EVT NewResVT = EVT::getVectorVT(Context&: *getContext(), VT: ResEltVT, NumElements: ResNE);
12249	EVT NewOvVT = EVT::getVectorVT(Context&: *getContext(), VT: OvEltVT, NumElements: ResNE);
12250	return std::make_pair(x: getBuildVector(VT: NewResVT, DL: dl, Ops: ResScalars),
12251	y: getBuildVector(VT: NewOvVT, DL: dl, Ops: OvScalars));
12252	}
12253
12254	bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
12255	LoadSDNode *Base,
12256	unsigned Bytes,
12257	int Dist) const {
12258	if (LD->isVolatile() || Base->isVolatile())
12259	return false;
12260	// TODO: probably too restrictive for atomics, revisit
12261	if (!LD->isSimple())
12262	return false;
12263	if (LD->isIndexed() || Base->isIndexed())
12264	return false;
12265	if (LD->getChain() != Base->getChain())
12266	return false;
12267	EVT VT = LD->getMemoryVT();
12268	if (VT.getSizeInBits() / 8 != Bytes)
12269	return false;
12270
12271	auto BaseLocDecomp = BaseIndexOffset::match(N: Base, DAG: *this);
12272	auto LocDecomp = BaseIndexOffset::match(N: LD, DAG: *this);
12273
12274	int64_t Offset = 0;
12275	if (BaseLocDecomp.equalBaseIndex(Other: LocDecomp, DAG: *this, Off&: Offset))
12276	return (Dist * (int64_t)Bytes == Offset);
12277	return false;
12278	}
12279
12280	/// InferPtrAlignment - Infer alignment of a load / store address. Return
12281	/// std::nullopt if it cannot be inferred.
12282	MaybeAlign SelectionDAG::InferPtrAlign(SDValue Ptr) const {
12283	// If this is a GlobalAddress + cst, return the alignment.
12284	const GlobalValue *GV = nullptr;
12285	int64_t GVOffset = 0;
12286	if (TLI->isGAPlusOffset(N: Ptr.getNode(), GA&: GV, Offset&: GVOffset)) {
12287	unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
12288	KnownBits Known(PtrWidth);
12289	llvm::computeKnownBits(V: GV, Known, DL: getDataLayout());
12290	unsigned AlignBits = Known.countMinTrailingZeros();
12291	if (AlignBits)
12292	return commonAlignment(A: Align(1ull << std::min(a: 31U, b: AlignBits)), Offset: GVOffset);
12293	}
12294
12295	// If this is a direct reference to a stack slot, use information about the
12296	// stack slot's alignment.
12297	int FrameIdx = INT_MIN;
12298	int64_t FrameOffset = 0;
12299	if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr)) {
12300	FrameIdx = FI->getIndex();
12301	} else if (isBaseWithConstantOffset(Op: Ptr) &&
12302	isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))) {
12303	// Handle FI+Cst
12304	FrameIdx = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex();
12305	FrameOffset = Ptr.getConstantOperandVal(i: 1);
12306	}
12307
12308	if (FrameIdx != INT_MIN) {
12309	const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
12310	return commonAlignment(A: MFI.getObjectAlign(ObjectIdx: FrameIdx), Offset: FrameOffset);
12311	}
12312
12313	return std::nullopt;
12314	}
12315
12316	/// Split the scalar node with EXTRACT_ELEMENT using the provided
12317	/// VTs and return the low/high part.
12318	std::pair<SDValue, SDValue> SelectionDAG::SplitScalar(const SDValue &N,
12319	const SDLoc &DL,
12320	const EVT &LoVT,
12321	const EVT &HiVT) {
12322	assert(!LoVT.isVector() && !HiVT.isVector() && !N.getValueType().isVector() &&
12323	"Split node must be a scalar type");
12324	SDValue Lo =
12325	getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: LoVT, N1: N, N2: getIntPtrConstant(Val: 0, DL));
12326	SDValue Hi =
12327	getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: HiVT, N1: N, N2: getIntPtrConstant(Val: 1, DL));
12328	return std::make_pair(x&: Lo, y&: Hi);
12329	}
12330
12331	/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
12332	/// which is split (or expanded) into two not necessarily identical pieces.
12333	std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
12334	// Currently all types are split in half.
12335	EVT LoVT, HiVT;
12336	if (!VT.isVector())
12337	LoVT = HiVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT);
12338	else
12339	LoVT = HiVT = VT.getHalfNumVectorElementsVT(Context&: *getContext());
12340
12341	return std::make_pair(x&: LoVT, y&: HiVT);
12342	}
12343
12344	/// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a
12345	/// type, dependent on an enveloping VT that has been split into two identical
12346	/// pieces. Sets the HiIsEmpty flag when hi type has zero storage size.
12347	std::pair<EVT, EVT>
12348	SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT,
12349	bool *HiIsEmpty) const {
12350	EVT EltTp = VT.getVectorElementType();
12351	// Examples:
12352	// custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty)
12353	// custom VL=9 with enveloping VL=8/8 yields 8/1
12354	// custom VL=10 with enveloping VL=8/8 yields 8/2
12355	// etc.
12356	ElementCount VTNumElts = VT.getVectorElementCount();
12357	ElementCount EnvNumElts = EnvVT.getVectorElementCount();
12358	assert(VTNumElts.isScalable() == EnvNumElts.isScalable() &&
12359	"Mixing fixed width and scalable vectors when enveloping a type");
12360	EVT LoVT, HiVT;
12361	if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) {
12362	LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts);
12363	HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts - EnvNumElts);
12364	*HiIsEmpty = false;
12365	} else {
12366	// Flag that hi type has zero storage size, but return split envelop type
12367	// (this would be easier if vector types with zero elements were allowed).
12368	LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts);
12369	HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts);
12370	*HiIsEmpty = true;
12371	}
12372	return std::make_pair(x&: LoVT, y&: HiVT);
12373	}
12374
12375	/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
12376	/// low/high part.
12377	std::pair<SDValue, SDValue>
12378	SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
12379	const EVT &HiVT) {
12380	assert(LoVT.isScalableVector() == HiVT.isScalableVector() &&
12381	LoVT.isScalableVector() == N.getValueType().isScalableVector() &&
12382	"Splitting vector with an invalid mixture of fixed and scalable "
12383	"vector types");
12384	assert(LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <=
12385	N.getValueType().getVectorMinNumElements() &&
12386	"More vector elements requested than available!");
12387	SDValue Lo, Hi;
12388	Lo =
12389	getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LoVT, N1: N, N2: getVectorIdxConstant(Val: 0, DL));
12390	// For scalable vectors it is safe to use LoVT.getVectorMinNumElements()
12391	// (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales
12392	// IDX with the runtime scaling factor of the result vector type. For
12393	// fixed-width result vectors, that runtime scaling factor is 1.
12394	Hi = getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: N,
12395	N2: getVectorIdxConstant(Val: LoVT.getVectorMinNumElements(), DL));
12396	return std::make_pair(x&: Lo, y&: Hi);
12397	}
12398
12399	std::pair<SDValue, SDValue> SelectionDAG::SplitEVL(SDValue N, EVT VecVT,
12400	const SDLoc &DL) {
12401	// Split the vector length parameter.
12402	// %evl -> umin(%evl, %halfnumelts) and usubsat(%evl - %halfnumelts).
12403	EVT VT = N.getValueType();
12404	assert(VecVT.getVectorElementCount().isKnownEven() &&
12405	"Expecting the mask to be an evenly-sized vector");
12406	unsigned HalfMinNumElts = VecVT.getVectorMinNumElements() / 2;
12407	SDValue HalfNumElts =
12408	VecVT.isFixedLengthVector()
12409	? getConstant(Val: HalfMinNumElts, DL, VT)
12410	: getVScale(DL, VT, MulImm: APInt(VT.getScalarSizeInBits(), HalfMinNumElts));
12411	SDValue Lo = getNode(Opcode: ISD::UMIN, DL, VT, N1: N, N2: HalfNumElts);
12412	SDValue Hi = getNode(Opcode: ISD::USUBSAT, DL, VT, N1: N, N2: HalfNumElts);
12413	return std::make_pair(x&: Lo, y&: Hi);
12414	}
12415
12416	/// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
12417	SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) {
12418	EVT VT = N.getValueType();
12419	EVT WideVT = EVT::getVectorVT(Context&: *getContext(), VT: VT.getVectorElementType(),
12420	NumElements: NextPowerOf2(A: VT.getVectorNumElements()));
12421	return getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideVT, N1: getUNDEF(VT: WideVT), N2: N,
12422	N3: getVectorIdxConstant(Val: 0, DL));
12423	}
12424
12425	void SelectionDAG::ExtractVectorElements(SDValue Op,
12426	SmallVectorImpl<SDValue> &Args,
12427	unsigned Start, unsigned Count,
12428	EVT EltVT) {
12429	EVT VT = Op.getValueType();
12430	if (Count == 0)
12431	Count = VT.getVectorNumElements();
12432	if (EltVT == EVT())
12433	EltVT = VT.getVectorElementType();
12434	SDLoc SL(Op);
12435	for (unsigned i = Start, e = Start + Count; i != e; ++i) {
12436	Args.push_back(Elt: getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Op,
12437	N2: getVectorIdxConstant(Val: i, DL: SL)));
12438	}
12439	}
12440
12441	// getAddressSpace - Return the address space this GlobalAddress belongs to.
12442	unsigned GlobalAddressSDNode::getAddressSpace() const {
12443	return getGlobal()->getType()->getAddressSpace();
12444	}
12445
12446	Type *ConstantPoolSDNode::getType() const {
12447	if (isMachineConstantPoolEntry())
12448	return Val.MachineCPVal->getType();
12449	return Val.ConstVal->getType();
12450	}
12451
12452	bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
12453	unsigned &SplatBitSize,
12454	bool &HasAnyUndefs,
12455	unsigned MinSplatBits,
12456	bool IsBigEndian) const {
12457	EVT VT = getValueType(ResNo: 0);
12458	assert(VT.isVector() && "Expected a vector type");
12459	unsigned VecWidth = VT.getSizeInBits();
12460	if (MinSplatBits > VecWidth)
12461	return false;
12462
12463	// FIXME: The widths are based on this node's type, but build vectors can
12464	// truncate their operands.
12465	SplatValue = APInt(VecWidth, 0);
12466	SplatUndef = APInt(VecWidth, 0);
12467
12468	// Get the bits. Bits with undefined values (when the corresponding element
12469	// of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
12470	// in SplatValue. If any of the values are not constant, give up and return
12471	// false.
12472	unsigned int NumOps = getNumOperands();
12473	assert(NumOps > 0 && "isConstantSplat has 0-size build vector");
12474	unsigned EltWidth = VT.getScalarSizeInBits();
12475
12476	for (unsigned j = 0; j < NumOps; ++j) {
12477	unsigned i = IsBigEndian ? NumOps - 1 - j : j;
12478	SDValue OpVal = getOperand(Num: i);
12479	unsigned BitPos = j * EltWidth;
12480
12481	if (OpVal.isUndef())
12482	SplatUndef.setBits(loBit: BitPos, hiBit: BitPos + EltWidth);
12483	else if (auto *CN = dyn_cast<ConstantSDNode>(Val&: OpVal))
12484	SplatValue.insertBits(SubBits: CN->getAPIntValue().zextOrTrunc(width: EltWidth), bitPosition: BitPos);
12485	else if (auto *CN = dyn_cast<ConstantFPSDNode>(Val&: OpVal))
12486	SplatValue.insertBits(SubBits: CN->getValueAPF().bitcastToAPInt(), bitPosition: BitPos);
12487	else
12488	return false;
12489	}
12490
12491	// The build_vector is all constants or undefs. Find the smallest element
12492	// size that splats the vector.
12493	HasAnyUndefs = (SplatUndef != 0);
12494
12495	// FIXME: This does not work for vectors with elements less than 8 bits.
12496	while (VecWidth > 8) {
12497	// If we can't split in half, stop here.
12498	if (VecWidth & 1)
12499	break;
12500
12501	unsigned HalfSize = VecWidth / 2;
12502	APInt HighValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: HalfSize);
12503	APInt LowValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: 0);
12504	APInt HighUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: HalfSize);
12505	APInt LowUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: 0);
12506
12507	// If the two halves do not match (ignoring undef bits), stop here.
12508	if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
12509	MinSplatBits > HalfSize)
12510	break;
12511
12512	SplatValue = HighValue | LowValue;
12513	SplatUndef = HighUndef & LowUndef;
12514
12515	VecWidth = HalfSize;
12516	}
12517
12518	// FIXME: The loop above only tries to split in halves. But if the input
12519	// vector for example is <3 x i16> it wouldn't be able to detect a
12520	// SplatBitSize of 16. No idea if that is a design flaw currently limiting
12521	// optimizations. I guess that back in the days when this helper was created
12522	// vectors normally was power-of-2 sized.
12523
12524	SplatBitSize = VecWidth;
12525	return true;
12526	}
12527
12528	SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts,
12529	BitVector *UndefElements) const {
12530	unsigned NumOps = getNumOperands();
12531	if (UndefElements) {
12532	UndefElements->clear();
12533	UndefElements->resize(N: NumOps);
12534	}
12535	assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
12536	if (!DemandedElts)
12537	return SDValue();
12538	SDValue Splatted;
12539	for (unsigned i = 0; i != NumOps; ++i) {
12540	if (!DemandedElts[i])
12541	continue;
12542	SDValue Op = getOperand(Num: i);
12543	if (Op.isUndef()) {
12544	if (UndefElements)
12545	(*UndefElements)[i] = true;
12546	} else if (!Splatted) {
12547	Splatted = Op;
12548	} else if (Splatted != Op) {
12549	return SDValue();
12550	}
12551	}
12552
12553	if (!Splatted) {
12554	unsigned FirstDemandedIdx = DemandedElts.countr_zero();
12555	assert(getOperand(FirstDemandedIdx).isUndef() &&
12556	"Can only have a splat without a constant for all undefs.");
12557	return getOperand(Num: FirstDemandedIdx);
12558	}
12559
12560	return Splatted;
12561	}
12562
12563	SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {
12564	APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands());
12565	return getSplatValue(DemandedElts, UndefElements);
12566	}
12567
12568	bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts,
12569	SmallVectorImpl<SDValue> &Sequence,
12570	BitVector *UndefElements) const {
12571	unsigned NumOps = getNumOperands();
12572	Sequence.clear();
12573	if (UndefElements) {
12574	UndefElements->clear();
12575	UndefElements->resize(N: NumOps);
12576	}
12577	assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
12578	if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(Value: NumOps))
12579	return false;
12580
12581	// Set the undefs even if we don't find a sequence (like getSplatValue).
12582	if (UndefElements)
12583	for (unsigned I = 0; I != NumOps; ++I)
12584	if (DemandedElts[I] && getOperand(Num: I).isUndef())
12585	(*UndefElements)[I] = true;
12586
12587	// Iteratively widen the sequence length looking for repetitions.
12588	for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) {
12589	Sequence.append(NumInputs: SeqLen, Elt: SDValue());
12590	for (unsigned I = 0; I != NumOps; ++I) {
12591	if (!DemandedElts[I])
12592	continue;
12593	SDValue &SeqOp = Sequence[I % SeqLen];
12594	SDValue Op = getOperand(Num: I);
12595	if (Op.isUndef()) {
12596	if (!SeqOp)
12597	SeqOp = Op;
12598	continue;
12599	}
12600	if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) {
12601	Sequence.clear();
12602	break;
12603	}
12604	SeqOp = Op;
12605	}
12606	if (!Sequence.empty())
12607	return true;
12608	}
12609
12610	assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern");
12611	return false;
12612	}
12613
12614	bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
12615	BitVector *UndefElements) const {
12616	APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands());
12617	return getRepeatedSequence(DemandedElts, Sequence, UndefElements);
12618	}
12619
12620	ConstantSDNode *
12621	BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts,
12622	BitVector *UndefElements) const {
12623	return dyn_cast_or_null<ConstantSDNode>(
12624	Val: getSplatValue(DemandedElts, UndefElements));
12625	}
12626
12627	ConstantSDNode *
12628	BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {
12629	return dyn_cast_or_null<ConstantSDNode>(Val: getSplatValue(UndefElements));
12630	}
12631
12632	ConstantFPSDNode *
12633	BuildVectorSDNode::getConstantFPSplatNode(const APInt &DemandedElts,
12634	BitVector *UndefElements) const {
12635	return dyn_cast_or_null<ConstantFPSDNode>(
12636	Val: getSplatValue(DemandedElts, UndefElements));
12637	}
12638
12639	ConstantFPSDNode *
12640	BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {
12641	return dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements));
12642	}
12643
12644	int32_t
12645	BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
12646	uint32_t BitWidth) const {
12647	if (ConstantFPSDNode *CN =
12648	dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements))) {
12649	bool IsExact;
12650	APSInt IntVal(BitWidth);
12651	const APFloat &APF = CN->getValueAPF();
12652	if (APF.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &IsExact) !=
12653	APFloat::opOK ||
12654	!IsExact)
12655	return -1;
12656
12657	return IntVal.exactLogBase2();
12658	}
12659	return -1;
12660	}
12661
12662	bool BuildVectorSDNode::getConstantRawBits(
12663	bool IsLittleEndian, unsigned DstEltSizeInBits,
12664	SmallVectorImpl<APInt> &RawBitElements, BitVector &UndefElements) const {
12665	// Early-out if this contains anything but Undef/Constant/ConstantFP.
12666	if (!isConstant())
12667	return false;
12668
12669	unsigned NumSrcOps = getNumOperands();
12670	unsigned SrcEltSizeInBits = getValueType(ResNo: 0).getScalarSizeInBits();
12671	assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
12672	"Invalid bitcast scale");
12673
12674	// Extract raw src bits.
12675	SmallVector<APInt> SrcBitElements(NumSrcOps,
12676	APInt::getZero(numBits: SrcEltSizeInBits));
12677	BitVector SrcUndeElements(NumSrcOps, false);
12678
12679	for (unsigned I = 0; I != NumSrcOps; ++I) {
12680	SDValue Op = getOperand(Num: I);
12681	if (Op.isUndef()) {
12682	SrcUndeElements.set(I);
12683	continue;
12684	}
12685	auto *CInt = dyn_cast<ConstantSDNode>(Val&: Op);
12686	auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op);
12687	assert((CInt || CFP) && "Unknown constant");
12688	SrcBitElements[I] = CInt ? CInt->getAPIntValue().trunc(width: SrcEltSizeInBits)
12689	: CFP->getValueAPF().bitcastToAPInt();
12690	}
12691
12692	// Recast to dst width.
12693	recastRawBits(IsLittleEndian, DstEltSizeInBits, DstBitElements&: RawBitElements,
12694	SrcBitElements, DstUndefElements&: UndefElements, SrcUndefElements: SrcUndeElements);
12695	return true;
12696	}
12697
12698	void BuildVectorSDNode::recastRawBits(bool IsLittleEndian,
12699	unsigned DstEltSizeInBits,
12700	SmallVectorImpl<APInt> &DstBitElements,
12701	ArrayRef<APInt> SrcBitElements,
12702	BitVector &DstUndefElements,
12703	const BitVector &SrcUndefElements) {
12704	unsigned NumSrcOps = SrcBitElements.size();
12705	unsigned SrcEltSizeInBits = SrcBitElements[0].getBitWidth();
12706	assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
12707	"Invalid bitcast scale");
12708	assert(NumSrcOps == SrcUndefElements.size() &&
12709	"Vector size mismatch");
12710
12711	unsigned NumDstOps = (NumSrcOps * SrcEltSizeInBits) / DstEltSizeInBits;
12712	DstUndefElements.clear();
12713	DstUndefElements.resize(N: NumDstOps, t: false);
12714	DstBitElements.assign(NumElts: NumDstOps, Elt: APInt::getZero(numBits: DstEltSizeInBits));
12715
12716	// Concatenate src elements constant bits together into dst element.
12717	if (SrcEltSizeInBits <= DstEltSizeInBits) {
12718	unsigned Scale = DstEltSizeInBits / SrcEltSizeInBits;
12719	for (unsigned I = 0; I != NumDstOps; ++I) {
12720	DstUndefElements.set(I);
12721	APInt &DstBits = DstBitElements[I];
12722	for (unsigned J = 0; J != Scale; ++J) {
12723	unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
12724	if (SrcUndefElements[Idx])
12725	continue;
12726	DstUndefElements.reset(Idx: I);
12727	const APInt &SrcBits = SrcBitElements[Idx];
12728	assert(SrcBits.getBitWidth() == SrcEltSizeInBits &&
12729	"Illegal constant bitwidths");
12730	DstBits.insertBits(SubBits: SrcBits, bitPosition: J * SrcEltSizeInBits);
12731	}
12732	}
12733	return;
12734	}
12735
12736	// Split src element constant bits into dst elements.
12737	unsigned Scale = SrcEltSizeInBits / DstEltSizeInBits;
12738	for (unsigned I = 0; I != NumSrcOps; ++I) {
12739	if (SrcUndefElements[I]) {
12740	DstUndefElements.set(I: I * Scale, E: (I + 1) * Scale);
12741	continue;
12742	}
12743	const APInt &SrcBits = SrcBitElements[I];
12744	for (unsigned J = 0; J != Scale; ++J) {
12745	unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
12746	APInt &DstBits = DstBitElements[Idx];
12747	DstBits = SrcBits.extractBits(numBits: DstEltSizeInBits, bitPosition: J * DstEltSizeInBits);
12748	}
12749	}
12750	}
12751
12752	bool BuildVectorSDNode::isConstant() const {
12753	for (const SDValue &Op : op_values()) {
12754	unsigned Opc = Op.getOpcode();
12755	if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP)
12756	return false;
12757	}
12758	return true;
12759	}
12760
12761	std::optional<std::pair<APInt, APInt>>
12762	BuildVectorSDNode::isConstantSequence() const {
12763	unsigned NumOps = getNumOperands();
12764	if (NumOps < 2)
12765	return std::nullopt;
12766
12767	if (!isa<ConstantSDNode>(Val: getOperand(Num: 0)) ||
12768	!isa<ConstantSDNode>(Val: getOperand(Num: 1)))
12769	return std::nullopt;
12770
12771	unsigned EltSize = getValueType(ResNo: 0).getScalarSizeInBits();
12772	APInt Start = getConstantOperandAPInt(Num: 0).trunc(width: EltSize);
12773	APInt Stride = getConstantOperandAPInt(Num: 1).trunc(width: EltSize) - Start;
12774
12775	if (Stride.isZero())
12776	return std::nullopt;
12777
12778	for (unsigned i = 2; i < NumOps; ++i) {
12779	if (!isa<ConstantSDNode>(Val: getOperand(Num: i)))
12780	return std::nullopt;
12781
12782	APInt Val = getConstantOperandAPInt(Num: i).trunc(width: EltSize);
12783	if (Val != (Start + (Stride * i)))
12784	return std::nullopt;
12785	}
12786
12787	return std::make_pair(x&: Start, y&: Stride);
12788	}
12789
12790	bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) {
12791	// Find the first non-undef value in the shuffle mask.
12792	unsigned i, e;
12793	for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i)
12794	/* search */;
12795
12796	// If all elements are undefined, this shuffle can be considered a splat
12797	// (although it should eventually get simplified away completely).
12798	if (i == e)
12799	return true;
12800
12801	// Make sure all remaining elements are either undef or the same as the first
12802	// non-undef value.
12803	for (int Idx = Mask[i]; i != e; ++i)
12804	if (Mask[i] >= 0 && Mask[i] != Idx)
12805	return false;
12806	return true;
12807	}
12808
12809	// Returns the SDNode if it is a constant integer BuildVector
12810	// or constant integer.
12811	SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) const {
12812	if (isa<ConstantSDNode>(Val: N))
12813	return N.getNode();
12814	if (ISD::isBuildVectorOfConstantSDNodes(N: N.getNode()))
12815	return N.getNode();
12816	// Treat a GlobalAddress supporting constant offset folding as a
12817	// constant integer.
12818	if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val&: N))
12819	if (GA->getOpcode() == ISD::GlobalAddress &&
12820	TLI->isOffsetFoldingLegal(GA))
12821	return GA;
12822	if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
12823	isa<ConstantSDNode>(Val: N.getOperand(i: 0)))
12824	return N.getNode();
12825	return nullptr;
12826	}
12827
12828	// Returns the SDNode if it is a constant float BuildVector
12829	// or constant float.
12830	SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const {
12831	if (isa<ConstantFPSDNode>(Val: N))
12832	return N.getNode();
12833
12834	if (ISD::isBuildVectorOfConstantFPSDNodes(N: N.getNode()))
12835	return N.getNode();
12836
12837	if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
12838	isa<ConstantFPSDNode>(Val: N.getOperand(i: 0)))
12839	return N.getNode();
12840
12841	return nullptr;
12842	}
12843
12844	void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
12845	assert(!Node->OperandList && "Node already has operands");
12846	assert(SDNode::getMaxNumOperands() >= Vals.size() &&
12847	"too many operands to fit into SDNode");
12848	SDUse *Ops = OperandRecycler.allocate(
12849	Cap: ArrayRecycler<SDUse>::Capacity::get(N: Vals.size()), Allocator&: OperandAllocator);
12850
12851	bool IsDivergent = false;
12852	for (unsigned I = 0; I != Vals.size(); ++I) {
12853	Ops[I].setUser(Node);
12854	Ops[I].setInitial(Vals[I]);
12855	if (Ops[I].Val.getValueType() != MVT::Other) // Skip Chain. It does not carry divergence.
12856	IsDivergent |= Ops[I].getNode()->isDivergent();
12857	}
12858	Node->NumOperands = Vals.size();
12859	Node->OperandList = Ops;
12860	if (!TLI->isSDNodeAlwaysUniform(N: Node)) {
12861	IsDivergent |= TLI->isSDNodeSourceOfDivergence(N: Node, FLI, UA);
12862	Node->SDNodeBits.IsDivergent = IsDivergent;
12863	}
12864	checkForCycles(N: Node);
12865	}
12866
12867	SDValue SelectionDAG::getTokenFactor(const SDLoc &DL,
12868	SmallVectorImpl<SDValue> &Vals) {
12869	size_t Limit = SDNode::getMaxNumOperands();
12870	while (Vals.size() > Limit) {
12871	unsigned SliceIdx = Vals.size() - Limit;
12872	auto ExtractedTFs = ArrayRef<SDValue>(Vals).slice(N: SliceIdx, M: Limit);
12873	SDValue NewTF = getNode(ISD::TokenFactor, DL, MVT::Other, ExtractedTFs);
12874	Vals.erase(CS: Vals.begin() + SliceIdx, CE: Vals.end());
12875	Vals.emplace_back(Args&: NewTF);
12876	}
12877	return getNode(ISD::TokenFactor, DL, MVT::Other, Vals);
12878	}
12879
12880	SDValue SelectionDAG::getNeutralElement(unsigned Opcode, const SDLoc &DL,
12881	EVT VT, SDNodeFlags Flags) {
12882	switch (Opcode) {
12883	default:
12884	return SDValue();
12885	case ISD::ADD:
12886	case ISD::OR:
12887	case ISD::XOR:
12888	case ISD::UMAX:
12889	return getConstant(Val: 0, DL, VT);
12890	case ISD::MUL:
12891	return getConstant(Val: 1, DL, VT);
12892	case ISD::AND:
12893	case ISD::UMIN:
12894	return getAllOnesConstant(DL, VT);
12895	case ISD::SMAX:
12896	return getConstant(Val: APInt::getSignedMinValue(numBits: VT.getSizeInBits()), DL, VT);
12897	case ISD::SMIN:
12898	return getConstant(Val: APInt::getSignedMaxValue(numBits: VT.getSizeInBits()), DL, VT);
12899	case ISD::FADD:
12900	return getConstantFP(Val: -0.0, DL, VT);
12901	case ISD::FMUL:
12902	return getConstantFP(Val: 1.0, DL, VT);
12903	case ISD::FMINNUM:
12904	case ISD::FMAXNUM: {
12905	// Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
12906	const fltSemantics &Semantics = EVTToAPFloatSemantics(VT);
12907	APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Sem: Semantics) :
12908	!Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics) :
12909	APFloat::getLargest(Sem: Semantics);
12910	if (Opcode == ISD::FMAXNUM)
12911	NeutralAF.changeSign();
12912
12913	return getConstantFP(V: NeutralAF, DL, VT);
12914	}
12915	case ISD::FMINIMUM:
12916	case ISD::FMAXIMUM: {
12917	// Neutral element for fminimum is Inf or FLT_MAX, depending on FMF.
12918	const fltSemantics &Semantics = EVTToAPFloatSemantics(VT);
12919	APFloat NeutralAF = !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics)
12920	: APFloat::getLargest(Sem: Semantics);
12921	if (Opcode == ISD::FMAXIMUM)
12922	NeutralAF.changeSign();
12923
12924	return getConstantFP(V: NeutralAF, DL, VT);
12925	}
12926
12927	}
12928	}
12929
12930	/// Helper used to make a call to a library function that has one argument of
12931	/// pointer type.
12932	///
12933	/// Such functions include 'fegetmode', 'fesetenv' and some others, which are
12934	/// used to get or set floating-point state. They have one argument of pointer
12935	/// type, which points to the memory region containing bits of the
12936	/// floating-point state. The value returned by such function is ignored in the
12937	/// created call.
12938	///
12939	/// \param LibFunc Reference to library function (value of RTLIB::Libcall).
12940	/// \param Ptr Pointer used to save/load state.
12941	/// \param InChain Ingoing token chain.
12942	/// \returns Outgoing chain token.
12943	SDValue SelectionDAG::makeStateFunctionCall(unsigned LibFunc, SDValue Ptr,
12944	SDValue InChain,
12945	const SDLoc &DLoc) {
12946	assert(InChain.getValueType() == MVT::Other && "Expected token chain");
12947	TargetLowering::ArgListTy Args;
12948	TargetLowering::ArgListEntry Entry;
12949	Entry.Node = Ptr;
12950	Entry.Ty = Ptr.getValueType().getTypeForEVT(Context&: *getContext());
12951	Args.push_back(x: Entry);
12952	RTLIB::Libcall LC = static_cast<RTLIB::Libcall>(LibFunc);
12953	SDValue Callee = getExternalSymbol(Sym: TLI->getLibcallName(Call: LC),
12954	VT: TLI->getPointerTy(DL: getDataLayout()));
12955	TargetLowering::CallLoweringInfo CLI(*this);
12956	CLI.setDebugLoc(DLoc).setChain(InChain).setLibCallee(
12957	CC: TLI->getLibcallCallingConv(Call: LC), ResultType: Type::getVoidTy(C&: *getContext()), Target: Callee,
12958	ArgsList: std::move(Args));
12959	return TLI->LowerCallTo(CLI).second;
12960	}
12961
12962	void SelectionDAG::copyExtraInfo(SDNode *From, SDNode *To) {
12963	assert(From && To && "Invalid SDNode; empty source SDValue?");
12964	auto I = SDEI.find(Val: From);
12965	if (I == SDEI.end())
12966	return;
12967
12968	// Use of operator[] on the DenseMap may cause an insertion, which invalidates
12969	// the iterator, hence the need to make a copy to prevent a use-after-free.
12970	NodeExtraInfo NEI = I->second;
12971	if (LLVM_LIKELY(!NEI.PCSections) && LLVM_LIKELY(!NEI.MMRA)) {
12972	// No deep copy required for the types of extra info set.
12973	//
12974	// FIXME: Investigate if other types of extra info also need deep copy. This
12975	// depends on the types of nodes they can be attached to: if some extra info
12976	// is only ever attached to nodes where a replacement To node is always the
12977	// node where later use and propagation of the extra info has the intended
12978	// semantics, no deep copy is required.
12979	SDEI[To] = std::move(NEI);
12980	return;
12981	}
12982
12983	// We need to copy NodeExtraInfo to all _new_ nodes that are being introduced
12984	// through the replacement of From with To. Otherwise, replacements of a node
12985	// (From) with more complex nodes (To and its operands) may result in lost
12986	// extra info where the root node (To) is insignificant in further propagating
12987	// and using extra info when further lowering to MIR.
12988	//
12989	// In the first step pre-populate the visited set with the nodes reachable
12990	// from the old From node. This avoids copying NodeExtraInfo to parts of the
12991	// DAG that is not new and should be left untouched.
12992	SmallVector<const SDNode *> Leafs{From}; // Leafs reachable with VisitFrom.
12993	DenseSet<const SDNode *> FromReach; // The set of nodes reachable from From.
12994	auto VisitFrom = [&](auto &&Self, const SDNode *N, int MaxDepth) {
12995	if (MaxDepth == 0) {
12996	// Remember this node in case we need to increase MaxDepth and continue
12997	// populating FromReach from this node.
12998	Leafs.emplace_back(Args&: N);
12999	return;
13000	}
13001	if (!FromReach.insert(V: N).second)
13002	return;
13003	for (const SDValue &Op : N->op_values())
13004	Self(Self, Op.getNode(), MaxDepth - 1);
13005	};
13006
13007	// Copy extra info to To and all its transitive operands (that are new).
13008	SmallPtrSet<const SDNode *, 8> Visited;
13009	auto DeepCopyTo = [&](auto &&Self, const SDNode *N) {
13010	if (FromReach.contains(V: N))
13011	return true;
13012	if (!Visited.insert(Ptr: N).second)
13013	return true;
13014	if (getEntryNode().getNode() == N)
13015	return false;
13016	for (const SDValue &Op : N->op_values()) {
13017	if (!Self(Self, Op.getNode()))
13018	return false;
13019	}
13020	// Copy only if entry node was not reached.
13021	SDEI[N] = NEI;
13022	return true;
13023	};
13024
13025	// We first try with a lower MaxDepth, assuming that the path to common
13026	// operands between From and To is relatively short. This significantly
13027	// improves performance in the common case. The initial MaxDepth is big
13028	// enough to avoid retry in the common case; the last MaxDepth is large
13029	// enough to avoid having to use the fallback below (and protects from
13030	// potential stack exhaustion from recursion).
13031	for (int PrevDepth = 0, MaxDepth = 16; MaxDepth <= 1024;
13032	PrevDepth = MaxDepth, MaxDepth *= 2, Visited.clear()) {
13033	// StartFrom is the previous (or initial) set of leafs reachable at the
13034	// previous maximum depth.
13035	SmallVector<const SDNode *> StartFrom;
13036	std::swap(LHS&: StartFrom, RHS&: Leafs);
13037	for (const SDNode *N : StartFrom)
13038	VisitFrom(VisitFrom, N, MaxDepth - PrevDepth);
13039	if (LLVM_LIKELY(DeepCopyTo(DeepCopyTo, To)))
13040	return;
13041	// This should happen very rarely (reached the entry node).
13042	LLVM_DEBUG(dbgs() << __func__ << ": MaxDepth=" << MaxDepth << " too low\n");
13043	assert(!Leafs.empty());
13044	}
13045
13046	// This should not happen - but if it did, that means the subgraph reachable
13047	// from From has depth greater or equal to maximum MaxDepth, and VisitFrom()
13048	// could not visit all reachable common operands. Consequently, we were able
13049	// to reach the entry node.
13050	errs() << "warning: incomplete propagation of SelectionDAG::NodeExtraInfo\n";
13051	assert(false && "From subgraph too complex - increase max. MaxDepth?");
13052	// Best-effort fallback if assertions disabled.
13053	SDEI[To] = std::move(NEI);
13054	}
13055
13056	#ifndef NDEBUG
13057	static void checkForCyclesHelper(const SDNode *N,
13058	SmallPtrSetImpl<const SDNode*> &Visited,
13059	SmallPtrSetImpl<const SDNode*> &Checked,
13060	const llvm::SelectionDAG *DAG) {
13061	// If this node has already been checked, don't check it again.
13062	if (Checked.count(Ptr: N))
13063	return;
13064
13065	// If a node has already been visited on this depth-first walk, reject it as
13066	// a cycle.
13067	if (!Visited.insert(Ptr: N).second) {
13068	errs() << "Detected cycle in SelectionDAG\n";
13069	dbgs() << "Offending node:\n";
13070	N->dumprFull(G: DAG); dbgs() << "\n";
13071	abort();
13072	}
13073
13074	for (const SDValue &Op : N->op_values())
13075	checkForCyclesHelper(N: Op.getNode(), Visited, Checked, DAG);
13076
13077	Checked.insert(Ptr: N);
13078	Visited.erase(Ptr: N);
13079	}
13080	#endif
13081
13082	void llvm::checkForCycles(const llvm::SDNode *N,
13083	const llvm::SelectionDAG *DAG,
13084	bool force) {
13085	#ifndef NDEBUG
13086	bool check = force;
13087	#ifdef EXPENSIVE_CHECKS
13088	check = true;
13089	#endif // EXPENSIVE_CHECKS
13090	if (check) {
13091	assert(N && "Checking nonexistent SDNode");
13092	SmallPtrSet<const SDNode*, 32> visited;
13093	SmallPtrSet<const SDNode*, 32> checked;
13094	checkForCyclesHelper(N, Visited&: visited, Checked&: checked, DAG);
13095	}
13096	#endif // !NDEBUG
13097	}
13098
13099	void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
13100	checkForCycles(N: DAG->getRoot().getNode(), DAG, force);
13101	}
13102