1	//===- ValueTracking.cpp - Walk computations to compute properties --------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file contains routines that help analyze properties that chains of
10	// computations have.
11	//
12	//===----------------------------------------------------------------------===//
13
14	#include "llvm/Analysis/ValueTracking.h"
15	#include "llvm/ADT/APFloat.h"
16	#include "llvm/ADT/APInt.h"
17	#include "llvm/ADT/ArrayRef.h"
18	#include "llvm/ADT/STLExtras.h"
19	#include "llvm/ADT/ScopeExit.h"
20	#include "llvm/ADT/SmallPtrSet.h"
21	#include "llvm/ADT/SmallSet.h"
22	#include "llvm/ADT/SmallVector.h"
23	#include "llvm/ADT/StringRef.h"
24	#include "llvm/ADT/iterator_range.h"
25	#include "llvm/Analysis/AliasAnalysis.h"
26	#include "llvm/Analysis/AssumeBundleQueries.h"
27	#include "llvm/Analysis/AssumptionCache.h"
28	#include "llvm/Analysis/ConstantFolding.h"
29	#include "llvm/Analysis/DomConditionCache.h"
30	#include "llvm/Analysis/GuardUtils.h"
31	#include "llvm/Analysis/InstructionSimplify.h"
32	#include "llvm/Analysis/Loads.h"
33	#include "llvm/Analysis/LoopInfo.h"
34	#include "llvm/Analysis/OptimizationRemarkEmitter.h"
35	#include "llvm/Analysis/TargetLibraryInfo.h"
36	#include "llvm/Analysis/VectorUtils.h"
37	#include "llvm/Analysis/WithCache.h"
38	#include "llvm/IR/Argument.h"
39	#include "llvm/IR/Attributes.h"
40	#include "llvm/IR/BasicBlock.h"
41	#include "llvm/IR/Constant.h"
42	#include "llvm/IR/ConstantRange.h"
43	#include "llvm/IR/Constants.h"
44	#include "llvm/IR/DerivedTypes.h"
45	#include "llvm/IR/DiagnosticInfo.h"
46	#include "llvm/IR/Dominators.h"
47	#include "llvm/IR/EHPersonalities.h"
48	#include "llvm/IR/Function.h"
49	#include "llvm/IR/GetElementPtrTypeIterator.h"
50	#include "llvm/IR/GlobalAlias.h"
51	#include "llvm/IR/GlobalValue.h"
52	#include "llvm/IR/GlobalVariable.h"
53	#include "llvm/IR/InstrTypes.h"
54	#include "llvm/IR/Instruction.h"
55	#include "llvm/IR/Instructions.h"
56	#include "llvm/IR/IntrinsicInst.h"
57	#include "llvm/IR/Intrinsics.h"
58	#include "llvm/IR/IntrinsicsAArch64.h"
59	#include "llvm/IR/IntrinsicsAMDGPU.h"
60	#include "llvm/IR/IntrinsicsRISCV.h"
61	#include "llvm/IR/IntrinsicsX86.h"
62	#include "llvm/IR/LLVMContext.h"
63	#include "llvm/IR/Metadata.h"
64	#include "llvm/IR/Module.h"
65	#include "llvm/IR/Operator.h"
66	#include "llvm/IR/PatternMatch.h"
67	#include "llvm/IR/Type.h"
68	#include "llvm/IR/User.h"
69	#include "llvm/IR/Value.h"
70	#include "llvm/Support/Casting.h"
71	#include "llvm/Support/CommandLine.h"
72	#include "llvm/Support/Compiler.h"
73	#include "llvm/Support/ErrorHandling.h"
74	#include "llvm/Support/KnownBits.h"
75	#include "llvm/Support/MathExtras.h"
76	#include <algorithm>
77	#include <cassert>
78	#include <cstdint>
79	#include <optional>
80	#include <utility>
81
82	using namespace llvm;
83	using namespace llvm::PatternMatch;
84
85	// Controls the number of uses of the value searched for possible
86	// dominating comparisons.
87	static cl::opt<unsigned> DomConditionsMaxUses("dom-conditions-max-uses",
88	cl::Hidden, cl::init(Val: 20));
89
90
91	/// Returns the bitwidth of the given scalar or pointer type. For vector types,
92	/// returns the element type's bitwidth.
93	static unsigned getBitWidth(Type *Ty, const DataLayout &DL) {
94	if (unsigned BitWidth = Ty->getScalarSizeInBits())
95	return BitWidth;
96
97	return DL.getPointerTypeSizeInBits(Ty);
98	}
99
100	// Given the provided Value and, potentially, a context instruction, return
101	// the preferred context instruction (if any).
102	static const Instruction *safeCxtI(const Value *V, const Instruction *CxtI) {
103	// If we've been provided with a context instruction, then use that (provided
104	// it has been inserted).
105	if (CxtI && CxtI->getParent())
106	return CxtI;
107
108	// If the value is really an already-inserted instruction, then use that.
109	CxtI = dyn_cast<Instruction>(Val: V);
110	if (CxtI && CxtI->getParent())
111	return CxtI;
112
113	return nullptr;
114	}
115
116	static const Instruction *safeCxtI(const Value *V1, const Value *V2, const Instruction *CxtI) {
117	// If we've been provided with a context instruction, then use that (provided
118	// it has been inserted).
119	if (CxtI && CxtI->getParent())
120	return CxtI;
121
122	// If the value is really an already-inserted instruction, then use that.
123	CxtI = dyn_cast<Instruction>(Val: V1);
124	if (CxtI && CxtI->getParent())
125	return CxtI;
126
127	CxtI = dyn_cast<Instruction>(Val: V2);
128	if (CxtI && CxtI->getParent())
129	return CxtI;
130
131	return nullptr;
132	}
133
134	static bool getShuffleDemandedElts(const ShuffleVectorInst *Shuf,
135	const APInt &DemandedElts,
136	APInt &DemandedLHS, APInt &DemandedRHS) {
137	if (isa<ScalableVectorType>(Val: Shuf->getType())) {
138	assert(DemandedElts == APInt(1,1));
139	DemandedLHS = DemandedRHS = DemandedElts;
140	return true;
141	}
142
143	int NumElts =
144	cast<FixedVectorType>(Val: Shuf->getOperand(i_nocapture: 0)->getType())->getNumElements();
145	return llvm::getShuffleDemandedElts(SrcWidth: NumElts, Mask: Shuf->getShuffleMask(),
146	DemandedElts, DemandedLHS, DemandedRHS);
147	}
148
149	static void computeKnownBits(const Value *V, const APInt &DemandedElts,
150	KnownBits &Known, unsigned Depth,
151	const SimplifyQuery &Q);
152
153	void llvm::computeKnownBits(const Value *V, KnownBits &Known, unsigned Depth,
154	const SimplifyQuery &Q) {
155	// Since the number of lanes in a scalable vector is unknown at compile time,
156	// we track one bit which is implicitly broadcast to all lanes. This means
157	// that all lanes in a scalable vector are considered demanded.
158	auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
159	APInt DemandedElts =
160	FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
161	::computeKnownBits(V, DemandedElts, Known, Depth, Q);
162	}
163
164	void llvm::computeKnownBits(const Value *V, KnownBits &Known,
165	const DataLayout &DL, unsigned Depth,
166	AssumptionCache *AC, const Instruction *CxtI,
167	const DominatorTree *DT, bool UseInstrInfo) {
168	computeKnownBits(
169	V, Known, Depth,
170	Q: SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo));
171	}
172
173	KnownBits llvm::computeKnownBits(const Value *V, const DataLayout &DL,
174	unsigned Depth, AssumptionCache *AC,
175	const Instruction *CxtI,
176	const DominatorTree *DT, bool UseInstrInfo) {
177	return computeKnownBits(
178	V, Depth, Q: SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo));
179	}
180
181	KnownBits llvm::computeKnownBits(const Value *V, const APInt &DemandedElts,
182	const DataLayout &DL, unsigned Depth,
183	AssumptionCache *AC, const Instruction *CxtI,
184	const DominatorTree *DT, bool UseInstrInfo) {
185	return computeKnownBits(
186	V, DemandedElts, Depth,
187	Q: SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo));
188	}
189
190	static bool haveNoCommonBitsSetSpecialCases(const Value *LHS, const Value *RHS,
191	const SimplifyQuery &SQ) {
192	// Look for an inverted mask: (X & ~M) op (Y & M).
193	{
194	Value *M;
195	if (match(V: LHS, P: m_c_And(L: m_Not(V: m_Value(V&: M)), R: m_Value())) &&
196	match(V: RHS, P: m_c_And(L: m_Specific(V: M), R: m_Value())) &&
197	isGuaranteedNotToBeUndef(V: M, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT))
198	return true;
199	}
200
201	// X op (Y & ~X)
202	if (match(V: RHS, P: m_c_And(L: m_Not(V: m_Specific(V: LHS)), R: m_Value())) &&
203	isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT))
204	return true;
205
206	// X op ((X & Y) ^ Y) -- this is the canonical form of the previous pattern
207	// for constant Y.
208	Value *Y;
209	if (match(V: RHS,
210	P: m_c_Xor(L: m_c_And(L: m_Specific(V: LHS), R: m_Value(V&: Y)), R: m_Deferred(V: Y))) &&
211	isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT) &&
212	isGuaranteedNotToBeUndef(V: Y, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT))
213	return true;
214
215	// Peek through extends to find a 'not' of the other side:
216	// (ext Y) op ext(~Y)
217	if (match(V: LHS, P: m_ZExtOrSExt(Op: m_Value(V&: Y))) &&
218	match(V: RHS, P: m_ZExtOrSExt(Op: m_Not(V: m_Specific(V: Y)))) &&
219	isGuaranteedNotToBeUndef(V: Y, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT))
220	return true;
221
222	// Look for: (A & B) op ~(A | B)
223	{
224	Value *A, *B;
225	if (match(V: LHS, P: m_And(L: m_Value(V&: A), R: m_Value(V&: B))) &&
226	match(V: RHS, P: m_Not(V: m_c_Or(L: m_Specific(V: A), R: m_Specific(V: B)))) &&
227	isGuaranteedNotToBeUndef(V: A, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT) &&
228	isGuaranteedNotToBeUndef(V: B, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT))
229	return true;
230	}
231
232	return false;
233	}
234
235	bool llvm::haveNoCommonBitsSet(const WithCache<const Value *> &LHSCache,
236	const WithCache<const Value *> &RHSCache,
237	const SimplifyQuery &SQ) {
238	const Value *LHS = LHSCache.getValue();
239	const Value *RHS = RHSCache.getValue();
240
241	assert(LHS->getType() == RHS->getType() &&
242	"LHS and RHS should have the same type");
243	assert(LHS->getType()->isIntOrIntVectorTy() &&
244	"LHS and RHS should be integers");
245
246	if (haveNoCommonBitsSetSpecialCases(LHS, RHS, SQ) ||
247	haveNoCommonBitsSetSpecialCases(LHS: RHS, RHS: LHS, SQ))
248	return true;
249
250	return KnownBits::haveNoCommonBitsSet(LHS: LHSCache.getKnownBits(Q: SQ),
251	RHS: RHSCache.getKnownBits(Q: SQ));
252	}
253
254	bool llvm::isOnlyUsedInZeroEqualityComparison(const Instruction *I) {
255	return !I->user_empty() && all_of(Range: I->users(), P: [](const User *U) {
256	ICmpInst::Predicate P;
257	return match(V: U, P: m_ICmp(Pred&: P, L: m_Value(), R: m_Zero())) && ICmpInst::isEquality(P);
258	});
259	}
260
261	static bool isKnownToBeAPowerOfTwo(const Value *V, bool OrZero, unsigned Depth,
262	const SimplifyQuery &Q);
263
264	bool llvm::isKnownToBeAPowerOfTwo(const Value *V, const DataLayout &DL,
265	bool OrZero, unsigned Depth,
266	AssumptionCache *AC, const Instruction *CxtI,
267	const DominatorTree *DT, bool UseInstrInfo) {
268	return ::isKnownToBeAPowerOfTwo(
269	V, OrZero, Depth,
270	Q: SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo));
271	}
272
273	static bool isKnownNonZero(const Value *V, const APInt &DemandedElts,
274	unsigned Depth, const SimplifyQuery &Q);
275
276	static bool isKnownNonZero(const Value *V, unsigned Depth,
277	const SimplifyQuery &Q);
278
279	bool llvm::isKnownNonZero(const Value *V, const DataLayout &DL, unsigned Depth,
280	AssumptionCache *AC, const Instruction *CxtI,
281	const DominatorTree *DT, bool UseInstrInfo) {
282	return ::isKnownNonZero(
283	V, Depth, Q: SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo));
284	}
285
286	bool llvm::isKnownNonNegative(const Value *V, const SimplifyQuery &SQ,
287	unsigned Depth) {
288	return computeKnownBits(V, Depth, Q: SQ).isNonNegative();
289	}
290
291	bool llvm::isKnownPositive(const Value *V, const SimplifyQuery &SQ,
292	unsigned Depth) {
293	if (auto *CI = dyn_cast<ConstantInt>(Val: V))
294	return CI->getValue().isStrictlyPositive();
295
296	// TODO: We'd doing two recursive queries here. We should factor this such
297	// that only a single query is needed.
298	return isKnownNonNegative(V, SQ, Depth) && ::isKnownNonZero(V, Depth, Q: SQ);
299	}
300
301	bool llvm::isKnownNegative(const Value *V, const SimplifyQuery &SQ,
302	unsigned Depth) {
303	return computeKnownBits(V, Depth, Q: SQ).isNegative();
304	}
305
306	static bool isKnownNonEqual(const Value *V1, const Value *V2, unsigned Depth,
307	const SimplifyQuery &Q);
308
309	bool llvm::isKnownNonEqual(const Value *V1, const Value *V2,
310	const DataLayout &DL, AssumptionCache *AC,
311	const Instruction *CxtI, const DominatorTree *DT,
312	bool UseInstrInfo) {
313	return ::isKnownNonEqual(
314	V1, V2, Depth: 0,
315	Q: SimplifyQuery(DL, DT, AC, safeCxtI(V1: V2, V2: V1, CxtI), UseInstrInfo));
316	}
317
318	bool llvm::MaskedValueIsZero(const Value *V, const APInt &Mask,
319	const SimplifyQuery &SQ, unsigned Depth) {
320	KnownBits Known(Mask.getBitWidth());
321	computeKnownBits(V, Known, Depth, Q: SQ);
322	return Mask.isSubsetOf(RHS: Known.Zero);
323	}
324
325	static unsigned ComputeNumSignBits(const Value *V, const APInt &DemandedElts,
326	unsigned Depth, const SimplifyQuery &Q);
327
328	static unsigned ComputeNumSignBits(const Value *V, unsigned Depth,
329	const SimplifyQuery &Q) {
330	auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
331	APInt DemandedElts =
332	FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
333	return ComputeNumSignBits(V, DemandedElts, Depth, Q);
334	}
335
336	unsigned llvm::ComputeNumSignBits(const Value *V, const DataLayout &DL,
337	unsigned Depth, AssumptionCache *AC,
338	const Instruction *CxtI,
339	const DominatorTree *DT, bool UseInstrInfo) {
340	return ::ComputeNumSignBits(
341	V, Depth, Q: SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo));
342	}
343
344	unsigned llvm::ComputeMaxSignificantBits(const Value *V, const DataLayout &DL,
345	unsigned Depth, AssumptionCache *AC,
346	const Instruction *CxtI,
347	const DominatorTree *DT) {
348	unsigned SignBits = ComputeNumSignBits(V, DL, Depth, AC, CxtI, DT);
349	return V->getType()->getScalarSizeInBits() - SignBits + 1;
350	}
351
352	static void computeKnownBitsAddSub(bool Add, const Value *Op0, const Value *Op1,
353	bool NSW, const APInt &DemandedElts,
354	KnownBits &KnownOut, KnownBits &Known2,
355	unsigned Depth, const SimplifyQuery &Q) {
356	computeKnownBits(V: Op1, DemandedElts, Known&: KnownOut, Depth: Depth + 1, Q);
357
358	// If one operand is unknown and we have no nowrap information,
359	// the result will be unknown independently of the second operand.
360	if (KnownOut.isUnknown() && !NSW)
361	return;
362
363	computeKnownBits(V: Op0, DemandedElts, Known&: Known2, Depth: Depth + 1, Q);
364	KnownOut = KnownBits::computeForAddSub(Add, NSW, LHS: Known2, RHS: KnownOut);
365	}
366
367	static void computeKnownBitsMul(const Value *Op0, const Value *Op1, bool NSW,
368	const APInt &DemandedElts, KnownBits &Known,
369	KnownBits &Known2, unsigned Depth,
370	const SimplifyQuery &Q) {
371	computeKnownBits(V: Op1, DemandedElts, Known, Depth: Depth + 1, Q);
372	computeKnownBits(V: Op0, DemandedElts, Known&: Known2, Depth: Depth + 1, Q);
373
374	bool isKnownNegative = false;
375	bool isKnownNonNegative = false;
376	// If the multiplication is known not to overflow, compute the sign bit.
377	if (NSW) {
378	if (Op0 == Op1) {
379	// The product of a number with itself is non-negative.
380	isKnownNonNegative = true;
381	} else {
382	bool isKnownNonNegativeOp1 = Known.isNonNegative();
383	bool isKnownNonNegativeOp0 = Known2.isNonNegative();
384	bool isKnownNegativeOp1 = Known.isNegative();
385	bool isKnownNegativeOp0 = Known2.isNegative();
386	// The product of two numbers with the same sign is non-negative.
387	isKnownNonNegative = (isKnownNegativeOp1 && isKnownNegativeOp0) ||
388	(isKnownNonNegativeOp1 && isKnownNonNegativeOp0);
389	// The product of a negative number and a non-negative number is either
390	// negative or zero.
391	if (!isKnownNonNegative)
392	isKnownNegative =
393	(isKnownNegativeOp1 && isKnownNonNegativeOp0 &&
394	Known2.isNonZero()) ||
395	(isKnownNegativeOp0 && isKnownNonNegativeOp1 && Known.isNonZero());
396	}
397	}
398
399	bool SelfMultiply = Op0 == Op1;
400	if (SelfMultiply)
401	SelfMultiply &=
402	isGuaranteedNotToBeUndef(V: Op0, AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT, Depth: Depth + 1);
403	Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
404
405	// Only make use of no-wrap flags if we failed to compute the sign bit
406	// directly. This matters if the multiplication always overflows, in
407	// which case we prefer to follow the result of the direct computation,
408	// though as the program is invoking undefined behaviour we can choose
409	// whatever we like here.
410	if (isKnownNonNegative && !Known.isNegative())
411	Known.makeNonNegative();
412	else if (isKnownNegative && !Known.isNonNegative())
413	Known.makeNegative();
414	}
415
416	void llvm::computeKnownBitsFromRangeMetadata(const MDNode &Ranges,
417	KnownBits &Known) {
418	unsigned BitWidth = Known.getBitWidth();
419	unsigned NumRanges = Ranges.getNumOperands() / 2;
420	assert(NumRanges >= 1);
421
422	Known.Zero.setAllBits();
423	Known.One.setAllBits();
424
425	for (unsigned i = 0; i < NumRanges; ++i) {
426	ConstantInt *Lower =
427	mdconst::extract<ConstantInt>(MD: Ranges.getOperand(I: 2 * i + 0));
428	ConstantInt *Upper =
429	mdconst::extract<ConstantInt>(MD: Ranges.getOperand(I: 2 * i + 1));
430	ConstantRange Range(Lower->getValue(), Upper->getValue());
431
432	// The first CommonPrefixBits of all values in Range are equal.
433	unsigned CommonPrefixBits =
434	(Range.getUnsignedMax() ^ Range.getUnsignedMin()).countl_zero();
435	APInt Mask = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: CommonPrefixBits);
436	APInt UnsignedMax = Range.getUnsignedMax().zextOrTrunc(width: BitWidth);
437	Known.One &= UnsignedMax & Mask;
438	Known.Zero &= ~UnsignedMax & Mask;
439	}
440	}
441
442	static bool isEphemeralValueOf(const Instruction *I, const Value *E) {
443	SmallVector<const Value *, 16> WorkSet(1, I);
444	SmallPtrSet<const Value *, 32> Visited;
445	SmallPtrSet<const Value *, 16> EphValues;
446
447	// The instruction defining an assumption's condition itself is always
448	// considered ephemeral to that assumption (even if it has other
449	// non-ephemeral users). See r246696's test case for an example.
450	if (is_contained(Range: I->operands(), Element: E))
451	return true;
452
453	while (!WorkSet.empty()) {
454	const Value *V = WorkSet.pop_back_val();
455	if (!Visited.insert(Ptr: V).second)
456	continue;
457
458	// If all uses of this value are ephemeral, then so is this value.
459	if (llvm::all_of(Range: V->users(), P: [&](const User *U) {
460	return EphValues.count(Ptr: U);
461	})) {
462	if (V == E)
463	return true;
464
465	if (V == I || (isa<Instruction>(Val: V) &&
466	!cast<Instruction>(Val: V)->mayHaveSideEffects() &&
467	!cast<Instruction>(Val: V)->isTerminator())) {
468	EphValues.insert(Ptr: V);
469	if (const User *U = dyn_cast<User>(Val: V))
470	append_range(C&: WorkSet, R: U->operands());
471	}
472	}
473	}
474
475	return false;
476	}
477
478	// Is this an intrinsic that cannot be speculated but also cannot trap?
479	bool llvm::isAssumeLikeIntrinsic(const Instruction *I) {
480	if (const IntrinsicInst *CI = dyn_cast<IntrinsicInst>(Val: I))
481	return CI->isAssumeLikeIntrinsic();
482
483	return false;
484	}
485
486	bool llvm::isValidAssumeForContext(const Instruction *Inv,
487	const Instruction *CxtI,
488	const DominatorTree *DT,
489	bool AllowEphemerals) {
490	// There are two restrictions on the use of an assume:
491	// 1. The assume must dominate the context (or the control flow must
492	// reach the assume whenever it reaches the context).
493	// 2. The context must not be in the assume's set of ephemeral values
494	// (otherwise we will use the assume to prove that the condition
495	// feeding the assume is trivially true, thus causing the removal of
496	// the assume).
497
498	if (Inv->getParent() == CxtI->getParent()) {
499	// If Inv and CtxI are in the same block, check if the assume (Inv) is first
500	// in the BB.
501	if (Inv->comesBefore(Other: CxtI))
502	return true;
503
504	// Don't let an assume affect itself - this would cause the problems
505	// `isEphemeralValueOf` is trying to prevent, and it would also make
506	// the loop below go out of bounds.
507	if (!AllowEphemerals && Inv == CxtI)
508	return false;
509
510	// The context comes first, but they're both in the same block.
511	// Make sure there is nothing in between that might interrupt
512	// the control flow, not even CxtI itself.
513	// We limit the scan distance between the assume and its context instruction
514	// to avoid a compile-time explosion. This limit is chosen arbitrarily, so
515	// it can be adjusted if needed (could be turned into a cl::opt).
516	auto Range = make_range(x: CxtI->getIterator(), y: Inv->getIterator());
517	if (!isGuaranteedToTransferExecutionToSuccessor(Range, ScanLimit: 15))
518	return false;
519
520	return AllowEphemerals || !isEphemeralValueOf(I: Inv, E: CxtI);
521	}
522
523	// Inv and CxtI are in different blocks.
524	if (DT) {
525	if (DT->dominates(Def: Inv, User: CxtI))
526	return true;
527	} else if (Inv->getParent() == CxtI->getParent()->getSinglePredecessor()) {
528	// We don't have a DT, but this trivially dominates.
529	return true;
530	}
531
532	return false;
533	}
534
535	// TODO: cmpExcludesZero misses many cases where `RHS` is non-constant but
536	// we still have enough information about `RHS` to conclude non-zero. For
537	// example Pred=EQ, RHS=isKnownNonZero. cmpExcludesZero is called in loops
538	// so the extra compile time may not be worth it, but possibly a second API
539	// should be created for use outside of loops.
540	static bool cmpExcludesZero(CmpInst::Predicate Pred, const Value *RHS) {
541	// v u> y implies v != 0.
542	if (Pred == ICmpInst::ICMP_UGT)
543	return true;
544
545	// Special-case v != 0 to also handle v != null.
546	if (Pred == ICmpInst::ICMP_NE)
547	return match(V: RHS, P: m_Zero());
548
549	// All other predicates - rely on generic ConstantRange handling.
550	const APInt *C;
551	auto Zero = APInt::getZero(numBits: RHS->getType()->getScalarSizeInBits());
552	if (match(V: RHS, P: m_APInt(Res&: C))) {
553	ConstantRange TrueValues = ConstantRange::makeExactICmpRegion(Pred, Other: *C);
554	return !TrueValues.contains(Val: Zero);
555	}
556
557	auto *VC = dyn_cast<ConstantDataVector>(Val: RHS);
558	if (VC == nullptr)
559	return false;
560
561	for (unsigned ElemIdx = 0, NElem = VC->getNumElements(); ElemIdx < NElem;
562	++ElemIdx) {
563	ConstantRange TrueValues = ConstantRange::makeExactICmpRegion(
564	Pred, Other: VC->getElementAsAPInt(i: ElemIdx));
565	if (TrueValues.contains(Val: Zero))
566	return false;
567	}
568	return true;
569	}
570
571	static bool isKnownNonZeroFromAssume(const Value *V, const SimplifyQuery &Q) {
572	// Use of assumptions is context-sensitive. If we don't have a context, we
573	// cannot use them!
574	if (!Q.AC || !Q.CxtI)
575	return false;
576
577	for (AssumptionCache::ResultElem &Elem : Q.AC->assumptionsFor(V)) {
578	if (!Elem.Assume)
579	continue;
580
581	AssumeInst *I = cast<AssumeInst>(Val&: Elem.Assume);
582	assert(I->getFunction() == Q.CxtI->getFunction() &&
583	"Got assumption for the wrong function!");
584
585	if (Elem.Index != AssumptionCache::ExprResultIdx) {
586	if (!V->getType()->isPointerTy())
587	continue;
588	if (RetainedKnowledge RK = getKnowledgeFromBundle(
589	Assume&: *I, BOI: I->bundle_op_info_begin()[Elem.Index])) {
590	if (RK.WasOn == V &&
591	(RK.AttrKind == Attribute::NonNull ||
592	(RK.AttrKind == Attribute::Dereferenceable &&
593	!NullPointerIsDefined(F: Q.CxtI->getFunction(),
594	AS: V->getType()->getPointerAddressSpace()))) &&
595	isValidAssumeForContext(Inv: I, CxtI: Q.CxtI, DT: Q.DT))
596	return true;
597	}
598	continue;
599	}
600
601	// Warning: This loop can end up being somewhat performance sensitive.
602	// We're running this loop for once for each value queried resulting in a
603	// runtime of ~O(#assumes * #values).
604
605	Value *RHS;
606	CmpInst::Predicate Pred;
607	auto m_V = m_CombineOr(L: m_Specific(V), R: m_PtrToInt(Op: m_Specific(V)));
608	if (!match(V: I->getArgOperand(i: 0), P: m_c_ICmp(Pred, L: m_V, R: m_Value(V&: RHS))))
609	return false;
610
611	if (cmpExcludesZero(Pred, RHS) && isValidAssumeForContext(Inv: I, CxtI: Q.CxtI, DT: Q.DT))
612	return true;
613	}
614
615	return false;
616	}
617
618	static void computeKnownBitsFromCmp(const Value *V, CmpInst::Predicate Pred,
619	Value *LHS, Value *RHS, KnownBits &Known,
620	const SimplifyQuery &Q) {
621	if (RHS->getType()->isPointerTy()) {
622	// Handle comparison of pointer to null explicitly, as it will not be
623	// covered by the m_APInt() logic below.
624	if (LHS == V && match(V: RHS, P: m_Zero())) {
625	switch (Pred) {
626	case ICmpInst::ICMP_EQ:
627	Known.setAllZero();
628	break;
629	case ICmpInst::ICMP_SGE:
630	case ICmpInst::ICMP_SGT:
631	Known.makeNonNegative();
632	break;
633	case ICmpInst::ICMP_SLT:
634	Known.makeNegative();
635	break;
636	default:
637	break;
638	}
639	}
640	return;
641	}
642
643	unsigned BitWidth = Known.getBitWidth();
644	auto m_V =
645	m_CombineOr(L: m_Specific(V), R: m_PtrToIntSameSize(DL: Q.DL, Op: m_Specific(V)));
646
647	const APInt *Mask, *C;
648	uint64_t ShAmt;
649	switch (Pred) {
650	case ICmpInst::ICMP_EQ:
651	// assume(V = C)
652	if (match(V: LHS, P: m_V) && match(V: RHS, P: m_APInt(Res&: C))) {
653	Known = Known.unionWith(RHS: KnownBits::makeConstant(C: *C));
654	// assume(V & Mask = C)
655	} else if (match(V: LHS, P: m_And(L: m_V, R: m_APInt(Res&: Mask))) &&
656	match(V: RHS, P: m_APInt(Res&: C))) {
657	// For one bits in Mask, we can propagate bits from C to V.
658	Known.Zero |= ~*C & *Mask;
659	Known.One |= *C & *Mask;
660	// assume(V | Mask = C)
661	} else if (match(V: LHS, P: m_Or(L: m_V, R: m_APInt(Res&: Mask))) && match(V: RHS, P: m_APInt(Res&: C))) {
662	// For zero bits in Mask, we can propagate bits from C to V.
663	Known.Zero |= ~*C & ~*Mask;
664	Known.One |= *C & ~*Mask;
665	// assume(V ^ Mask = C)
666	} else if (match(V: LHS, P: m_Xor(L: m_V, R: m_APInt(Res&: Mask))) &&
667	match(V: RHS, P: m_APInt(Res&: C))) {
668	// Equivalent to assume(V == Mask ^ C)
669	Known = Known.unionWith(RHS: KnownBits::makeConstant(C: *C ^ *Mask));
670	// assume(V << ShAmt = C)
671	} else if (match(V: LHS, P: m_Shl(L: m_V, R: m_ConstantInt(V&: ShAmt))) &&
672	match(V: RHS, P: m_APInt(Res&: C)) && ShAmt < BitWidth) {
673	// For those bits in C that are known, we can propagate them to known
674	// bits in V shifted to the right by ShAmt.
675	KnownBits RHSKnown = KnownBits::makeConstant(C: *C);
676	RHSKnown.Zero.lshrInPlace(ShiftAmt: ShAmt);
677	RHSKnown.One.lshrInPlace(ShiftAmt: ShAmt);
678	Known = Known.unionWith(RHS: RHSKnown);
679	// assume(V >> ShAmt = C)
680	} else if (match(V: LHS, P: m_Shr(L: m_V, R: m_ConstantInt(V&: ShAmt))) &&
681	match(V: RHS, P: m_APInt(Res&: C)) && ShAmt < BitWidth) {
682	KnownBits RHSKnown = KnownBits::makeConstant(C: *C);
683	// For those bits in RHS that are known, we can propagate them to known
684	// bits in V shifted to the right by C.
685	Known.Zero |= RHSKnown.Zero << ShAmt;
686	Known.One |= RHSKnown.One << ShAmt;
687	}
688	break;
689	case ICmpInst::ICMP_NE: {
690	// assume (V & B != 0) where B is a power of 2
691	const APInt *BPow2;
692	if (match(V: LHS, P: m_And(L: m_V, R: m_Power2(V&: BPow2))) && match(V: RHS, P: m_Zero()))
693	Known.One |= *BPow2;
694	break;
695	}
696	default:
697	const APInt *Offset = nullptr;
698	if (match(V: LHS, P: m_CombineOr(L: m_V, R: m_Add(L: m_V, R: m_APInt(Res&: Offset)))) &&
699	match(V: RHS, P: m_APInt(Res&: C))) {
700	ConstantRange LHSRange = ConstantRange::makeAllowedICmpRegion(Pred, Other: *C);
701	if (Offset)
702	LHSRange = LHSRange.sub(Other: *Offset);
703	Known = Known.unionWith(RHS: LHSRange.toKnownBits());
704	}
705	break;
706	}
707	}
708
709	static void computeKnownBitsFromCond(const Value *V, Value *Cond,
710	KnownBits &Known, unsigned Depth,
711	const SimplifyQuery &SQ, bool Invert) {
712	Value *A, *B;
713	if (Depth < MaxAnalysisRecursionDepth &&
714	(Invert ? match(V: Cond, P: m_LogicalOr(L: m_Value(V&: A), R: m_Value(V&: B)))
715	: match(V: Cond, P: m_LogicalAnd(L: m_Value(V&: A), R: m_Value(V&: B))))) {
716	computeKnownBitsFromCond(V, Cond: A, Known, Depth: Depth + 1, SQ, Invert);
717	computeKnownBitsFromCond(V, Cond: B, Known, Depth: Depth + 1, SQ, Invert);
718	}
719
720	if (auto *Cmp = dyn_cast<ICmpInst>(Val: Cond))
721	computeKnownBitsFromCmp(
722	V, Pred: Invert ? Cmp->getInversePredicate() : Cmp->getPredicate(),
723	LHS: Cmp->getOperand(i_nocapture: 0), RHS: Cmp->getOperand(i_nocapture: 1), Known, Q: SQ);
724	}
725
726	void llvm::computeKnownBitsFromContext(const Value *V, KnownBits &Known,
727	unsigned Depth, const SimplifyQuery &Q) {
728	if (!Q.CxtI)
729	return;
730
731	if (Q.DC && Q.DT) {
732	// Handle dominating conditions.
733	for (BranchInst *BI : Q.DC->conditionsFor(V)) {
734	BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
735	if (Q.DT->dominates(BBE: Edge0, BB: Q.CxtI->getParent()))
736	computeKnownBitsFromCond(V, Cond: BI->getCondition(), Known, Depth, SQ: Q,
737	/*Invert*/ false);
738
739	BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
740	if (Q.DT->dominates(BBE: Edge1, BB: Q.CxtI->getParent()))
741	computeKnownBitsFromCond(V, Cond: BI->getCondition(), Known, Depth, SQ: Q,
742	/*Invert*/ true);
743	}
744
745	if (Known.hasConflict())
746	Known.resetAll();
747	}
748
749	if (!Q.AC)
750	return;
751
752	unsigned BitWidth = Known.getBitWidth();
753
754	// Note that the patterns below need to be kept in sync with the code
755	// in AssumptionCache::updateAffectedValues.
756
757	for (AssumptionCache::ResultElem &Elem : Q.AC->assumptionsFor(V)) {
758	if (!Elem.Assume)
759	continue;
760
761	AssumeInst *I = cast<AssumeInst>(Val&: Elem.Assume);
762	assert(I->getParent()->getParent() == Q.CxtI->getParent()->getParent() &&
763	"Got assumption for the wrong function!");
764
765	if (Elem.Index != AssumptionCache::ExprResultIdx) {
766	if (!V->getType()->isPointerTy())
767	continue;
768	if (RetainedKnowledge RK = getKnowledgeFromBundle(
769	Assume&: *I, BOI: I->bundle_op_info_begin()[Elem.Index])) {
770	if (RK.WasOn == V && RK.AttrKind == Attribute::Alignment &&
771	isPowerOf2_64(Value: RK.ArgValue) &&
772	isValidAssumeForContext(Inv: I, CxtI: Q.CxtI, DT: Q.DT))
773	Known.Zero.setLowBits(Log2_64(Value: RK.ArgValue));
774	}
775	continue;
776	}
777
778	// Warning: This loop can end up being somewhat performance sensitive.
779	// We're running this loop for once for each value queried resulting in a
780	// runtime of ~O(#assumes * #values).
781
782	Value *Arg = I->getArgOperand(i: 0);
783
784	if (Arg == V && isValidAssumeForContext(Inv: I, CxtI: Q.CxtI, DT: Q.DT)) {
785	assert(BitWidth == 1 && "assume operand is not i1?");
786	(void)BitWidth;
787	Known.setAllOnes();
788	return;
789	}
790	if (match(V: Arg, P: m_Not(V: m_Specific(V))) &&
791	isValidAssumeForContext(Inv: I, CxtI: Q.CxtI, DT: Q.DT)) {
792	assert(BitWidth == 1 && "assume operand is not i1?");
793	(void)BitWidth;
794	Known.setAllZero();
795	return;
796	}
797
798	// The remaining tests are all recursive, so bail out if we hit the limit.
799	if (Depth == MaxAnalysisRecursionDepth)
800	continue;
801
802	ICmpInst *Cmp = dyn_cast<ICmpInst>(Val: Arg);
803	if (!Cmp)
804	continue;
805
806	if (!isValidAssumeForContext(Inv: I, CxtI: Q.CxtI, DT: Q.DT))
807	continue;
808
809	computeKnownBitsFromCmp(V, Pred: Cmp->getPredicate(), LHS: Cmp->getOperand(i_nocapture: 0),
810	RHS: Cmp->getOperand(i_nocapture: 1), Known, Q);
811	}
812
813	// Conflicting assumption: Undefined behavior will occur on this execution
814	// path.
815	if (Known.hasConflict())
816	Known.resetAll();
817	}
818
819	/// Compute known bits from a shift operator, including those with a
820	/// non-constant shift amount. Known is the output of this function. Known2 is a
821	/// pre-allocated temporary with the same bit width as Known and on return
822	/// contains the known bit of the shift value source. KF is an
823	/// operator-specific function that, given the known-bits and a shift amount,
824	/// compute the implied known-bits of the shift operator's result respectively
825	/// for that shift amount. The results from calling KF are conservatively
826	/// combined for all permitted shift amounts.
827	static void computeKnownBitsFromShiftOperator(
828	const Operator *I, const APInt &DemandedElts, KnownBits &Known,
829	KnownBits &Known2, unsigned Depth, const SimplifyQuery &Q,
830	function_ref<KnownBits(const KnownBits &, const KnownBits &, bool)> KF) {
831	computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Depth: Depth + 1, Q);
832	computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Depth: Depth + 1, Q);
833	// To limit compile-time impact, only query isKnownNonZero() if we know at
834	// least something about the shift amount.
835	bool ShAmtNonZero =
836	Known.isNonZero() ||
837	(Known.getMaxValue().ult(RHS: Known.getBitWidth()) &&
838	isKnownNonZero(V: I->getOperand(i: 1), DemandedElts, Depth: Depth + 1, Q));
839	Known = KF(Known2, Known, ShAmtNonZero);
840	}
841
842	static KnownBits
843	getKnownBitsFromAndXorOr(const Operator *I, const APInt &DemandedElts,
844	const KnownBits &KnownLHS, const KnownBits &KnownRHS,
845	unsigned Depth, const SimplifyQuery &Q) {
846	unsigned BitWidth = KnownLHS.getBitWidth();
847	KnownBits KnownOut(BitWidth);
848	bool IsAnd = false;
849	bool HasKnownOne = !KnownLHS.One.isZero() || !KnownRHS.One.isZero();
850	Value *X = nullptr, *Y = nullptr;
851
852	switch (I->getOpcode()) {
853	case Instruction::And:
854	KnownOut = KnownLHS & KnownRHS;
855	IsAnd = true;
856	// and(x, -x) is common idioms that will clear all but lowest set
857	// bit. If we have a single known bit in x, we can clear all bits
858	// above it.
859	// TODO: instcombine often reassociates independent `and` which can hide
860	// this pattern. Try to match and(x, and(-x, y)) / and(and(x, y), -x).
861	if (HasKnownOne && match(V: I, P: m_c_And(L: m_Value(V&: X), R: m_Neg(V: m_Deferred(V: X))))) {
862	// -(-x) == x so using whichever (LHS/RHS) gets us a better result.
863	if (KnownLHS.countMaxTrailingZeros() <= KnownRHS.countMaxTrailingZeros())
864	KnownOut = KnownLHS.blsi();
865	else
866	KnownOut = KnownRHS.blsi();
867	}
868	break;
869	case Instruction::Or:
870	KnownOut = KnownLHS | KnownRHS;
871	break;
872	case Instruction::Xor:
873	KnownOut = KnownLHS ^ KnownRHS;
874	// xor(x, x-1) is common idioms that will clear all but lowest set
875	// bit. If we have a single known bit in x, we can clear all bits
876	// above it.
877	// TODO: xor(x, x-1) is often rewritting as xor(x, x-C) where C !=
878	// -1 but for the purpose of demanded bits (xor(x, x-C) &
879	// Demanded) == (xor(x, x-1) & Demanded). Extend the xor pattern
880	// to use arbitrary C if xor(x, x-C) as the same as xor(x, x-1).
881	if (HasKnownOne &&
882	match(V: I, P: m_c_Xor(L: m_Value(V&: X), R: m_c_Add(L: m_Deferred(V: X), R: m_AllOnes())))) {
883	const KnownBits &XBits = I->getOperand(i: 0) == X ? KnownLHS : KnownRHS;
884	KnownOut = XBits.blsmsk();
885	}
886	break;
887	default:
888	llvm_unreachable("Invalid Op used in 'analyzeKnownBitsFromAndXorOr'");
889	}
890
891	// and(x, add (x, -1)) is a common idiom that always clears the low bit;
892	// xor/or(x, add (x, -1)) is an idiom that will always set the low bit.
893	// here we handle the more general case of adding any odd number by
894	// matching the form and/xor/or(x, add(x, y)) where y is odd.
895	// TODO: This could be generalized to clearing any bit set in y where the
896	// following bit is known to be unset in y.
897	if (!KnownOut.Zero[0] && !KnownOut.One[0] &&
898	(match(V: I, P: m_c_BinOp(L: m_Value(V&: X), R: m_c_Add(L: m_Deferred(V: X), R: m_Value(V&: Y)))) ||
899	match(V: I, P: m_c_BinOp(L: m_Value(V&: X), R: m_Sub(L: m_Deferred(V: X), R: m_Value(V&: Y)))) ||
900	match(V: I, P: m_c_BinOp(L: m_Value(V&: X), R: m_Sub(L: m_Value(V&: Y), R: m_Deferred(V: X)))))) {
901	KnownBits KnownY(BitWidth);
902	computeKnownBits(V: Y, DemandedElts, Known&: KnownY, Depth: Depth + 1, Q);
903	if (KnownY.countMinTrailingOnes() > 0) {
904	if (IsAnd)
905	KnownOut.Zero.setBit(0);
906	else
907	KnownOut.One.setBit(0);
908	}
909	}
910	return KnownOut;
911	}
912
913	// Public so this can be used in `SimplifyDemandedUseBits`.
914	KnownBits llvm::analyzeKnownBitsFromAndXorOr(const Operator *I,
915	const KnownBits &KnownLHS,
916	const KnownBits &KnownRHS,
917	unsigned Depth,
918	const SimplifyQuery &SQ) {
919	auto *FVTy = dyn_cast<FixedVectorType>(Val: I->getType());
920	APInt DemandedElts =
921	FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
922
923	return getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS, KnownRHS, Depth,
924	Q: SQ);
925	}
926
927	ConstantRange llvm::getVScaleRange(const Function *F, unsigned BitWidth) {
928	Attribute Attr = F->getFnAttribute(Attribute::VScaleRange);
929	// Without vscale_range, we only know that vscale is non-zero.
930	if (!Attr.isValid())
931	return ConstantRange(APInt(BitWidth, 1), APInt::getZero(numBits: BitWidth));
932
933	unsigned AttrMin = Attr.getVScaleRangeMin();
934	// Minimum is larger than vscale width, result is always poison.
935	if ((unsigned)llvm::bit_width(Value: AttrMin) > BitWidth)
936	return ConstantRange::getEmpty(BitWidth);
937
938	APInt Min(BitWidth, AttrMin);
939	std::optional<unsigned> AttrMax = Attr.getVScaleRangeMax();
940	if (!AttrMax || (unsigned)llvm::bit_width(Value: *AttrMax) > BitWidth)
941	return ConstantRange(Min, APInt::getZero(numBits: BitWidth));
942
943	return ConstantRange(Min, APInt(BitWidth, *AttrMax) + 1);
944	}
945
946	static void computeKnownBitsFromOperator(const Operator *I,
947	const APInt &DemandedElts,
948	KnownBits &Known, unsigned Depth,
949	const SimplifyQuery &Q) {
950	unsigned BitWidth = Known.getBitWidth();
951
952	KnownBits Known2(BitWidth);
953	switch (I->getOpcode()) {
954	default: break;
955	case Instruction::Load:
956	if (MDNode *MD =
957	Q.IIQ.getMetadata(I: cast<LoadInst>(Val: I), KindID: LLVMContext::MD_range))
958	computeKnownBitsFromRangeMetadata(Ranges: *MD, Known);
959	break;
960	case Instruction::And:
961	computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Depth: Depth + 1, Q);
962	computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Depth: Depth + 1, Q);
963
964	Known = getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS: Known2, KnownRHS: Known, Depth, Q);
965	break;
966	case Instruction::Or:
967	computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Depth: Depth + 1, Q);
968	computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Depth: Depth + 1, Q);
969
970	Known = getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS: Known2, KnownRHS: Known, Depth, Q);
971	break;
972	case Instruction::Xor:
973	computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Depth: Depth + 1, Q);
974	computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Depth: Depth + 1, Q);
975
976	Known = getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS: Known2, KnownRHS: Known, Depth, Q);
977	break;
978	case Instruction::Mul: {
979	bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
980	computeKnownBitsMul(Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1), NSW, DemandedElts,
981	Known, Known2, Depth, Q);
982	break;
983	}
984	case Instruction::UDiv: {
985	computeKnownBits(V: I->getOperand(i: 0), Known, Depth: Depth + 1, Q);
986	computeKnownBits(V: I->getOperand(i: 1), Known&: Known2, Depth: Depth + 1, Q);
987	Known =
988	KnownBits::udiv(LHS: Known, RHS: Known2, Exact: Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)));
989	break;
990	}
991	case Instruction::SDiv: {
992	computeKnownBits(V: I->getOperand(i: 0), Known, Depth: Depth + 1, Q);
993	computeKnownBits(V: I->getOperand(i: 1), Known&: Known2, Depth: Depth + 1, Q);
994	Known =
995	KnownBits::sdiv(LHS: Known, RHS: Known2, Exact: Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)));
996	break;
997	}
998	case Instruction::Select: {
999	computeKnownBits(V: I->getOperand(i: 2), Known, Depth: Depth + 1, Q);
1000	computeKnownBits(V: I->getOperand(i: 1), Known&: Known2, Depth: Depth + 1, Q);
1001
1002	// Only known if known in both the LHS and RHS.
1003	Known = Known.intersectWith(RHS: Known2);
1004	break;
1005	}
1006	case Instruction::FPTrunc:
1007	case Instruction::FPExt:
1008	case Instruction::FPToUI:
1009	case Instruction::FPToSI:
1010	case Instruction::SIToFP:
1011	case Instruction::UIToFP:
1012	break; // Can't work with floating point.
1013	case Instruction::PtrToInt:
1014	case Instruction::IntToPtr:
1015	// Fall through and handle them the same as zext/trunc.
1016	[[fallthrough]];
1017	case Instruction::ZExt:
1018	case Instruction::Trunc: {
1019	Type *SrcTy = I->getOperand(i: 0)->getType();
1020
1021	unsigned SrcBitWidth;
1022	// Note that we handle pointer operands here because of inttoptr/ptrtoint
1023	// which fall through here.
1024	Type *ScalarTy = SrcTy->getScalarType();
1025	SrcBitWidth = ScalarTy->isPointerTy() ?
1026	Q.DL.getPointerTypeSizeInBits(ScalarTy) :
1027	Q.DL.getTypeSizeInBits(Ty: ScalarTy);
1028
1029	assert(SrcBitWidth && "SrcBitWidth can't be zero");
1030	Known = Known.anyextOrTrunc(BitWidth: SrcBitWidth);
1031	computeKnownBits(V: I->getOperand(i: 0), Known, Depth: Depth + 1, Q);
1032	if (auto *Inst = dyn_cast<PossiblyNonNegInst>(Val: I);
1033	Inst && Inst->hasNonNeg() && !Known.isNegative())
1034	Known.makeNonNegative();
1035	Known = Known.zextOrTrunc(BitWidth);
1036	break;
1037	}
1038	case Instruction::BitCast: {
1039	Type *SrcTy = I->getOperand(i: 0)->getType();
1040	if (SrcTy->isIntOrPtrTy() &&
1041	// TODO: For now, not handling conversions like:
1042	// (bitcast i64 %x to <2 x i32>)
1043	!I->getType()->isVectorTy()) {
1044	computeKnownBits(V: I->getOperand(i: 0), Known, Depth: Depth + 1, Q);
1045	break;
1046	}
1047
1048	// Handle cast from vector integer type to scalar or vector integer.
1049	auto *SrcVecTy = dyn_cast<FixedVectorType>(Val: SrcTy);
1050	if (!SrcVecTy || !SrcVecTy->getElementType()->isIntegerTy() ||
1051	!I->getType()->isIntOrIntVectorTy() ||
1052	isa<ScalableVectorType>(Val: I->getType()))
1053	break;
1054
1055	// Look through a cast from narrow vector elements to wider type.
1056	// Examples: v4i32 -> v2i64, v3i8 -> v24
1057	unsigned SubBitWidth = SrcVecTy->getScalarSizeInBits();
1058	if (BitWidth % SubBitWidth == 0) {
1059	// Known bits are automatically intersected across demanded elements of a
1060	// vector. So for example, if a bit is computed as known zero, it must be
1061	// zero across all demanded elements of the vector.
1062	//
1063	// For this bitcast, each demanded element of the output is sub-divided
1064	// across a set of smaller vector elements in the source vector. To get
1065	// the known bits for an entire element of the output, compute the known
1066	// bits for each sub-element sequentially. This is done by shifting the
1067	// one-set-bit demanded elements parameter across the sub-elements for
1068	// consecutive calls to computeKnownBits. We are using the demanded
1069	// elements parameter as a mask operator.
1070	//
1071	// The known bits of each sub-element are then inserted into place
1072	// (dependent on endian) to form the full result of known bits.
1073	unsigned NumElts = DemandedElts.getBitWidth();
1074	unsigned SubScale = BitWidth / SubBitWidth;
1075	APInt SubDemandedElts = APInt::getZero(numBits: NumElts * SubScale);
1076	for (unsigned i = 0; i != NumElts; ++i) {
1077	if (DemandedElts[i])
1078	SubDemandedElts.setBit(i * SubScale);
1079	}
1080
1081	KnownBits KnownSrc(SubBitWidth);
1082	for (unsigned i = 0; i != SubScale; ++i) {
1083	computeKnownBits(V: I->getOperand(i: 0), DemandedElts: SubDemandedElts.shl(shiftAmt: i), Known&: KnownSrc,
1084	Depth: Depth + 1, Q);
1085	unsigned ShiftElt = Q.DL.isLittleEndian() ? i : SubScale - 1 - i;
1086	Known.insertBits(SubBits: KnownSrc, BitPosition: ShiftElt * SubBitWidth);
1087	}
1088	}
1089	break;
1090	}
1091	case Instruction::SExt: {
1092	// Compute the bits in the result that are not present in the input.
1093	unsigned SrcBitWidth = I->getOperand(i: 0)->getType()->getScalarSizeInBits();
1094
1095	Known = Known.trunc(BitWidth: SrcBitWidth);
1096	computeKnownBits(V: I->getOperand(i: 0), Known, Depth: Depth + 1, Q);
1097	// If the sign bit of the input is known set or clear, then we know the
1098	// top bits of the result.
1099	Known = Known.sext(BitWidth);
1100	break;
1101	}
1102	case Instruction::Shl: {
1103	bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1104	bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1105	auto KF = [NUW, NSW](const KnownBits &KnownVal, const KnownBits &KnownAmt,
1106	bool ShAmtNonZero) {
1107	return KnownBits::shl(LHS: KnownVal, RHS: KnownAmt, NUW, NSW, ShAmtNonZero);
1108	};
1109	computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Depth, Q,
1110	KF);
1111	// Trailing zeros of a right-shifted constant never decrease.
1112	const APInt *C;
1113	if (match(V: I->getOperand(i: 0), P: m_APInt(Res&: C)))
1114	Known.Zero.setLowBits(C->countr_zero());
1115	break;
1116	}
1117	case Instruction::LShr: {
1118	auto KF = [](const KnownBits &KnownVal, const KnownBits &KnownAmt,
1119	bool ShAmtNonZero) {
1120	return KnownBits::lshr(LHS: KnownVal, RHS: KnownAmt, ShAmtNonZero);
1121	};
1122	computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Depth, Q,
1123	KF);
1124	// Leading zeros of a left-shifted constant never decrease.
1125	const APInt *C;
1126	if (match(V: I->getOperand(i: 0), P: m_APInt(Res&: C)))
1127	Known.Zero.setHighBits(C->countl_zero());
1128	break;
1129	}
1130	case Instruction::AShr: {
1131	auto KF = [](const KnownBits &KnownVal, const KnownBits &KnownAmt,
1132	bool ShAmtNonZero) {
1133	return KnownBits::ashr(LHS: KnownVal, RHS: KnownAmt, ShAmtNonZero);
1134	};
1135	computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Depth, Q,
1136	KF);
1137	break;
1138	}
1139	case Instruction::Sub: {
1140	bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1141	computeKnownBitsAddSub(Add: false, Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1), NSW,
1142	DemandedElts, KnownOut&: Known, Known2, Depth, Q);
1143	break;
1144	}
1145	case Instruction::Add: {
1146	bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1147	computeKnownBitsAddSub(Add: true, Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1), NSW,
1148	DemandedElts, KnownOut&: Known, Known2, Depth, Q);
1149	break;
1150	}
1151	case Instruction::SRem:
1152	computeKnownBits(V: I->getOperand(i: 0), Known, Depth: Depth + 1, Q);
1153	computeKnownBits(V: I->getOperand(i: 1), Known&: Known2, Depth: Depth + 1, Q);
1154	Known = KnownBits::srem(LHS: Known, RHS: Known2);
1155	break;
1156
1157	case Instruction::URem:
1158	computeKnownBits(V: I->getOperand(i: 0), Known, Depth: Depth + 1, Q);
1159	computeKnownBits(V: I->getOperand(i: 1), Known&: Known2, Depth: Depth + 1, Q);
1160	Known = KnownBits::urem(LHS: Known, RHS: Known2);
1161	break;
1162	case Instruction::Alloca:
1163	Known.Zero.setLowBits(Log2(A: cast<AllocaInst>(Val: I)->getAlign()));
1164	break;
1165	case Instruction::GetElementPtr: {
1166	// Analyze all of the subscripts of this getelementptr instruction
1167	// to determine if we can prove known low zero bits.
1168	computeKnownBits(V: I->getOperand(i: 0), Known, Depth: Depth + 1, Q);
1169	// Accumulate the constant indices in a separate variable
1170	// to minimize the number of calls to computeForAddSub.
1171	APInt AccConstIndices(BitWidth, 0, /*IsSigned*/ true);
1172
1173	gep_type_iterator GTI = gep_type_begin(GEP: I);
1174	for (unsigned i = 1, e = I->getNumOperands(); i != e; ++i, ++GTI) {
1175	// TrailZ can only become smaller, short-circuit if we hit zero.
1176	if (Known.isUnknown())
1177	break;
1178
1179	Value *Index = I->getOperand(i);
1180
1181	// Handle case when index is zero.
1182	Constant *CIndex = dyn_cast<Constant>(Val: Index);
1183	if (CIndex && CIndex->isZeroValue())
1184	continue;
1185
1186	if (StructType *STy = GTI.getStructTypeOrNull()) {
1187	// Handle struct member offset arithmetic.
1188
1189	assert(CIndex &&
1190	"Access to structure field must be known at compile time");
1191
1192	if (CIndex->getType()->isVectorTy())
1193	Index = CIndex->getSplatValue();
1194
1195	unsigned Idx = cast<ConstantInt>(Val: Index)->getZExtValue();
1196	const StructLayout *SL = Q.DL.getStructLayout(Ty: STy);
1197	uint64_t Offset = SL->getElementOffset(Idx);
1198	AccConstIndices += Offset;
1199	continue;
1200	}
1201
1202	// Handle array index arithmetic.
1203	Type *IndexedTy = GTI.getIndexedType();
1204	if (!IndexedTy->isSized()) {
1205	Known.resetAll();
1206	break;
1207	}
1208
1209	unsigned IndexBitWidth = Index->getType()->getScalarSizeInBits();
1210	KnownBits IndexBits(IndexBitWidth);
1211	computeKnownBits(V: Index, Known&: IndexBits, Depth: Depth + 1, Q);
1212	TypeSize IndexTypeSize = GTI.getSequentialElementStride(DL: Q.DL);
1213	uint64_t TypeSizeInBytes = IndexTypeSize.getKnownMinValue();
1214	KnownBits ScalingFactor(IndexBitWidth);
1215	// Multiply by current sizeof type.
1216	// &A[i] == A + i * sizeof(*A[i]).
1217	if (IndexTypeSize.isScalable()) {
1218	// For scalable types the only thing we know about sizeof is
1219	// that this is a multiple of the minimum size.
1220	ScalingFactor.Zero.setLowBits(llvm::countr_zero(Val: TypeSizeInBytes));
1221	} else if (IndexBits.isConstant()) {
1222	APInt IndexConst = IndexBits.getConstant();
1223	APInt ScalingFactor(IndexBitWidth, TypeSizeInBytes);
1224	IndexConst *= ScalingFactor;
1225	AccConstIndices += IndexConst.sextOrTrunc(width: BitWidth);
1226	continue;
1227	} else {
1228	ScalingFactor =
1229	KnownBits::makeConstant(C: APInt(IndexBitWidth, TypeSizeInBytes));
1230	}
1231	IndexBits = KnownBits::mul(LHS: IndexBits, RHS: ScalingFactor);
1232
1233	// If the offsets have a different width from the pointer, according
1234	// to the language reference we need to sign-extend or truncate them
1235	// to the width of the pointer.
1236	IndexBits = IndexBits.sextOrTrunc(BitWidth);
1237
1238	// Note that inbounds does *not* guarantee nsw for the addition, as only
1239	// the offset is signed, while the base address is unsigned.
1240	Known = KnownBits::computeForAddSub(
1241	/*Add=*/true, /*NSW=*/false, LHS: Known, RHS: IndexBits);
1242	}
1243	if (!Known.isUnknown() && !AccConstIndices.isZero()) {
1244	KnownBits Index = KnownBits::makeConstant(C: AccConstIndices);
1245	Known = KnownBits::computeForAddSub(
1246	/*Add=*/true, /*NSW=*/false, LHS: Known, RHS: Index);
1247	}
1248	break;
1249	}
1250	case Instruction::PHI: {
1251	const PHINode *P = cast<PHINode>(Val: I);
1252	BinaryOperator *BO = nullptr;
1253	Value *R = nullptr, *L = nullptr;
1254	if (matchSimpleRecurrence(P, BO, Start&: R, Step&: L)) {
1255	// Handle the case of a simple two-predecessor recurrence PHI.
1256	// There's a lot more that could theoretically be done here, but
1257	// this is sufficient to catch some interesting cases.
1258	unsigned Opcode = BO->getOpcode();
1259
1260	// If this is a shift recurrence, we know the bits being shifted in.
1261	// We can combine that with information about the start value of the
1262	// recurrence to conclude facts about the result.
1263	if ((Opcode == Instruction::LShr || Opcode == Instruction::AShr ||
1264	Opcode == Instruction::Shl) &&
1265	BO->getOperand(i_nocapture: 0) == I) {
1266
1267	// We have matched a recurrence of the form:
1268	// %iv = [R, %entry], [%iv.next, %backedge]
1269	// %iv.next = shift_op %iv, L
1270
1271	// Recurse with the phi context to avoid concern about whether facts
1272	// inferred hold at original context instruction. TODO: It may be
1273	// correct to use the original context. IF warranted, explore and
1274	// add sufficient tests to cover.
1275	SimplifyQuery RecQ = Q;
1276	RecQ.CxtI = P;
1277	computeKnownBits(V: R, DemandedElts, Known&: Known2, Depth: Depth + 1, Q: RecQ);
1278	switch (Opcode) {
1279	case Instruction::Shl:
1280	// A shl recurrence will only increase the tailing zeros
1281	Known.Zero.setLowBits(Known2.countMinTrailingZeros());
1282	break;
1283	case Instruction::LShr:
1284	// A lshr recurrence will preserve the leading zeros of the
1285	// start value
1286	Known.Zero.setHighBits(Known2.countMinLeadingZeros());
1287	break;
1288	case Instruction::AShr:
1289	// An ashr recurrence will extend the initial sign bit
1290	Known.Zero.setHighBits(Known2.countMinLeadingZeros());
1291	Known.One.setHighBits(Known2.countMinLeadingOnes());
1292	break;
1293	};
1294	}
1295
1296	// Check for operations that have the property that if
1297	// both their operands have low zero bits, the result
1298	// will have low zero bits.
1299	if (Opcode == Instruction::Add ||
1300	Opcode == Instruction::Sub ||
1301	Opcode == Instruction::And ||
1302	Opcode == Instruction::Or ||
1303	Opcode == Instruction::Mul) {
1304	// Change the context instruction to the "edge" that flows into the
1305	// phi. This is important because that is where the value is actually
1306	// "evaluated" even though it is used later somewhere else. (see also
1307	// D69571).
1308	SimplifyQuery RecQ = Q;
1309
1310	unsigned OpNum = P->getOperand(i_nocapture: 0) == R ? 0 : 1;
1311	Instruction *RInst = P->getIncomingBlock(i: OpNum)->getTerminator();
1312	Instruction *LInst = P->getIncomingBlock(i: 1-OpNum)->getTerminator();
1313
1314	// Ok, we have a PHI of the form L op= R. Check for low
1315	// zero bits.
1316	RecQ.CxtI = RInst;
1317	computeKnownBits(V: R, Known&: Known2, Depth: Depth + 1, Q: RecQ);
1318
1319	// We need to take the minimum number of known bits
1320	KnownBits Known3(BitWidth);
1321	RecQ.CxtI = LInst;
1322	computeKnownBits(V: L, Known&: Known3, Depth: Depth + 1, Q: RecQ);
1323
1324	Known.Zero.setLowBits(std::min(a: Known2.countMinTrailingZeros(),
1325	b: Known3.countMinTrailingZeros()));
1326
1327	auto *OverflowOp = dyn_cast<OverflowingBinaryOperator>(Val: BO);
1328	if (OverflowOp && Q.IIQ.hasNoSignedWrap(Op: OverflowOp)) {
1329	// If initial value of recurrence is nonnegative, and we are adding
1330	// a nonnegative number with nsw, the result can only be nonnegative
1331	// or poison value regardless of the number of times we execute the
1332	// add in phi recurrence. If initial value is negative and we are
1333	// adding a negative number with nsw, the result can only be
1334	// negative or poison value. Similar arguments apply to sub and mul.
1335	//
1336	// (add non-negative, non-negative) --> non-negative
1337	// (add negative, negative) --> negative
1338	if (Opcode == Instruction::Add) {
1339	if (Known2.isNonNegative() && Known3.isNonNegative())
1340	Known.makeNonNegative();
1341	else if (Known2.isNegative() && Known3.isNegative())
1342	Known.makeNegative();
1343	}
1344
1345	// (sub nsw non-negative, negative) --> non-negative
1346	// (sub nsw negative, non-negative) --> negative
1347	else if (Opcode == Instruction::Sub && BO->getOperand(i_nocapture: 0) == I) {
1348	if (Known2.isNonNegative() && Known3.isNegative())
1349	Known.makeNonNegative();
1350	else if (Known2.isNegative() && Known3.isNonNegative())
1351	Known.makeNegative();
1352	}
1353
1354	// (mul nsw non-negative, non-negative) --> non-negative
1355	else if (Opcode == Instruction::Mul && Known2.isNonNegative() &&
1356	Known3.isNonNegative())
1357	Known.makeNonNegative();
1358	}
1359
1360	break;
1361	}
1362	}
1363
1364	// Unreachable blocks may have zero-operand PHI nodes.
1365	if (P->getNumIncomingValues() == 0)
1366	break;
1367
1368	// Otherwise take the unions of the known bit sets of the operands,
1369	// taking conservative care to avoid excessive recursion.
1370	if (Depth < MaxAnalysisRecursionDepth - 1 && Known.isUnknown()) {
1371	// Skip if every incoming value references to ourself.
1372	if (isa_and_nonnull<UndefValue>(Val: P->hasConstantValue()))
1373	break;
1374
1375	Known.Zero.setAllBits();
1376	Known.One.setAllBits();
1377	for (unsigned u = 0, e = P->getNumIncomingValues(); u < e; ++u) {
1378	Value *IncValue = P->getIncomingValue(i: u);
1379	// Skip direct self references.
1380	if (IncValue == P) continue;
1381
1382	// Change the context instruction to the "edge" that flows into the
1383	// phi. This is important because that is where the value is actually
1384	// "evaluated" even though it is used later somewhere else. (see also
1385	// D69571).
1386	SimplifyQuery RecQ = Q;
1387	RecQ.CxtI = P->getIncomingBlock(i: u)->getTerminator();
1388
1389	Known2 = KnownBits(BitWidth);
1390
1391	// Recurse, but cap the recursion to one level, because we don't
1392	// want to waste time spinning around in loops.
1393	// TODO: See if we can base recursion limiter on number of incoming phi
1394	// edges so we don't overly clamp analysis.
1395	computeKnownBits(V: IncValue, Known&: Known2, Depth: MaxAnalysisRecursionDepth - 1, Q: RecQ);
1396
1397	// See if we can further use a conditional branch into the phi
1398	// to help us determine the range of the value.
1399	if (!Known2.isConstant()) {
1400	ICmpInst::Predicate Pred;
1401	const APInt *RHSC;
1402	BasicBlock *TrueSucc, *FalseSucc;
1403	// TODO: Use RHS Value and compute range from its known bits.
1404	if (match(V: RecQ.CxtI,
1405	P: m_Br(C: m_c_ICmp(Pred, L: m_Specific(V: IncValue), R: m_APInt(Res&: RHSC)),
1406	T: m_BasicBlock(V&: TrueSucc), F: m_BasicBlock(V&: FalseSucc)))) {
1407	// Check for cases of duplicate successors.
1408	if ((TrueSucc == P->getParent()) != (FalseSucc == P->getParent())) {
1409	// If we're using the false successor, invert the predicate.
1410	if (FalseSucc == P->getParent())
1411	Pred = CmpInst::getInversePredicate(pred: Pred);
1412	// Get the knownbits implied by the incoming phi condition.
1413	auto CR = ConstantRange::makeExactICmpRegion(Pred, Other: *RHSC);
1414	KnownBits KnownUnion = Known2.unionWith(RHS: CR.toKnownBits());
1415	// We can have conflicts here if we are analyzing deadcode (its
1416	// impossible for us reach this BB based the icmp).
1417	if (KnownUnion.hasConflict()) {
1418	// No reason to continue analyzing in a known dead region, so
1419	// just resetAll and break. This will cause us to also exit the
1420	// outer loop.
1421	Known.resetAll();
1422	break;
1423	}
1424	Known2 = KnownUnion;
1425	}
1426	}
1427	}
1428
1429	Known = Known.intersectWith(RHS: Known2);
1430	// If all bits have been ruled out, there's no need to check
1431	// more operands.
1432	if (Known.isUnknown())
1433	break;
1434	}
1435	}
1436	break;
1437	}
1438	case Instruction::Call:
1439	case Instruction::Invoke:
1440	// If range metadata is attached to this call, set known bits from that,
1441	// and then intersect with known bits based on other properties of the
1442	// function.
1443	if (MDNode *MD =
1444	Q.IIQ.getMetadata(I: cast<Instruction>(Val: I), KindID: LLVMContext::MD_range))
1445	computeKnownBitsFromRangeMetadata(Ranges: *MD, Known);
1446	if (const Value *RV = cast<CallBase>(Val: I)->getReturnedArgOperand()) {
1447	if (RV->getType() == I->getType()) {
1448	computeKnownBits(V: RV, Known&: Known2, Depth: Depth + 1, Q);
1449	Known = Known.unionWith(RHS: Known2);
1450	}
1451	}
1452	if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
1453	switch (II->getIntrinsicID()) {
1454	default: break;
1455	case Intrinsic::abs: {
1456	computeKnownBits(V: I->getOperand(i: 0), Known&: Known2, Depth: Depth + 1, Q);
1457	bool IntMinIsPoison = match(V: II->getArgOperand(i: 1), P: m_One());
1458	Known = Known2.abs(IntMinIsPoison);
1459	break;
1460	}
1461	case Intrinsic::bitreverse:
1462	computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Depth: Depth + 1, Q);
1463	Known.Zero |= Known2.Zero.reverseBits();
1464	Known.One |= Known2.One.reverseBits();
1465	break;
1466	case Intrinsic::bswap:
1467	computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Depth: Depth + 1, Q);
1468	Known.Zero |= Known2.Zero.byteSwap();
1469	Known.One |= Known2.One.byteSwap();
1470	break;
1471	case Intrinsic::ctlz: {
1472	computeKnownBits(V: I->getOperand(i: 0), Known&: Known2, Depth: Depth + 1, Q);
1473	// If we have a known 1, its position is our upper bound.
1474	unsigned PossibleLZ = Known2.countMaxLeadingZeros();
1475	// If this call is poison for 0 input, the result will be less than 2^n.
1476	if (II->getArgOperand(i: 1) == ConstantInt::getTrue(Context&: II->getContext()))
1477	PossibleLZ = std::min(a: PossibleLZ, b: BitWidth - 1);
1478	unsigned LowBits = llvm::bit_width(Value: PossibleLZ);
1479	Known.Zero.setBitsFrom(LowBits);
1480	break;
1481	}
1482	case Intrinsic::cttz: {
1483	computeKnownBits(V: I->getOperand(i: 0), Known&: Known2, Depth: Depth + 1, Q);
1484	// If we have a known 1, its position is our upper bound.
1485	unsigned PossibleTZ = Known2.countMaxTrailingZeros();
1486	// If this call is poison for 0 input, the result will be less than 2^n.
1487	if (II->getArgOperand(i: 1) == ConstantInt::getTrue(Context&: II->getContext()))
1488	PossibleTZ = std::min(a: PossibleTZ, b: BitWidth - 1);
1489	unsigned LowBits = llvm::bit_width(Value: PossibleTZ);
1490	Known.Zero.setBitsFrom(LowBits);
1491	break;
1492	}
1493	case Intrinsic::ctpop: {
1494	computeKnownBits(V: I->getOperand(i: 0), Known&: Known2, Depth: Depth + 1, Q);
1495	// We can bound the space the count needs. Also, bits known to be zero
1496	// can't contribute to the population.
1497	unsigned BitsPossiblySet = Known2.countMaxPopulation();
1498	unsigned LowBits = llvm::bit_width(Value: BitsPossiblySet);
1499	Known.Zero.setBitsFrom(LowBits);
1500	// TODO: we could bound KnownOne using the lower bound on the number
1501	// of bits which might be set provided by popcnt KnownOne2.
1502	break;
1503	}
1504	case Intrinsic::fshr:
1505	case Intrinsic::fshl: {
1506	const APInt *SA;
1507	if (!match(V: I->getOperand(i: 2), P: m_APInt(Res&: SA)))
1508	break;
1509
1510	// Normalize to funnel shift left.
1511	uint64_t ShiftAmt = SA->urem(RHS: BitWidth);
1512	if (II->getIntrinsicID() == Intrinsic::fshr)
1513	ShiftAmt = BitWidth - ShiftAmt;
1514
1515	KnownBits Known3(BitWidth);
1516	computeKnownBits(V: I->getOperand(i: 0), Known&: Known2, Depth: Depth + 1, Q);
1517	computeKnownBits(V: I->getOperand(i: 1), Known&: Known3, Depth: Depth + 1, Q);
1518
1519	Known.Zero =
1520	Known2.Zero.shl(shiftAmt: ShiftAmt) | Known3.Zero.lshr(shiftAmt: BitWidth - ShiftAmt);
1521	Known.One =
1522	Known2.One.shl(shiftAmt: ShiftAmt) | Known3.One.lshr(shiftAmt: BitWidth - ShiftAmt);
1523	break;
1524	}
1525	case Intrinsic::uadd_sat:
1526	computeKnownBits(V: I->getOperand(i: 0), Known, Depth: Depth + 1, Q);
1527	computeKnownBits(V: I->getOperand(i: 1), Known&: Known2, Depth: Depth + 1, Q);
1528	Known = KnownBits::uadd_sat(LHS: Known, RHS: Known2);
1529	break;
1530	case Intrinsic::usub_sat:
1531	computeKnownBits(V: I->getOperand(i: 0), Known, Depth: Depth + 1, Q);
1532	computeKnownBits(V: I->getOperand(i: 1), Known&: Known2, Depth: Depth + 1, Q);
1533	Known = KnownBits::usub_sat(LHS: Known, RHS: Known2);
1534	break;
1535	case Intrinsic::sadd_sat:
1536	computeKnownBits(V: I->getOperand(i: 0), Known, Depth: Depth + 1, Q);
1537	computeKnownBits(V: I->getOperand(i: 1), Known&: Known2, Depth: Depth + 1, Q);
1538	Known = KnownBits::sadd_sat(LHS: Known, RHS: Known2);
1539	break;
1540	case Intrinsic::ssub_sat:
1541	computeKnownBits(V: I->getOperand(i: 0), Known, Depth: Depth + 1, Q);
1542	computeKnownBits(V: I->getOperand(i: 1), Known&: Known2, Depth: Depth + 1, Q);
1543	Known = KnownBits::ssub_sat(LHS: Known, RHS: Known2);
1544	break;
1545	case Intrinsic::umin:
1546	computeKnownBits(V: I->getOperand(i: 0), Known, Depth: Depth + 1, Q);
1547	computeKnownBits(V: I->getOperand(i: 1), Known&: Known2, Depth: Depth + 1, Q);
1548	Known = KnownBits::umin(LHS: Known, RHS: Known2);
1549	break;
1550	case Intrinsic::umax:
1551	computeKnownBits(V: I->getOperand(i: 0), Known, Depth: Depth + 1, Q);
1552	computeKnownBits(V: I->getOperand(i: 1), Known&: Known2, Depth: Depth + 1, Q);
1553	Known = KnownBits::umax(LHS: Known, RHS: Known2);
1554	break;
1555	case Intrinsic::smin:
1556	computeKnownBits(V: I->getOperand(i: 0), Known, Depth: Depth + 1, Q);
1557	computeKnownBits(V: I->getOperand(i: 1), Known&: Known2, Depth: Depth + 1, Q);
1558	Known = KnownBits::smin(LHS: Known, RHS: Known2);
1559	break;
1560	case Intrinsic::smax:
1561	computeKnownBits(V: I->getOperand(i: 0), Known, Depth: Depth + 1, Q);
1562	computeKnownBits(V: I->getOperand(i: 1), Known&: Known2, Depth: Depth + 1, Q);
1563	Known = KnownBits::smax(LHS: Known, RHS: Known2);
1564	break;
1565	case Intrinsic::ptrmask: {
1566	computeKnownBits(V: I->getOperand(i: 0), Known, Depth: Depth + 1, Q);
1567
1568	const Value *Mask = I->getOperand(i: 1);
1569	Known2 = KnownBits(Mask->getType()->getScalarSizeInBits());
1570	computeKnownBits(V: Mask, Known&: Known2, Depth: Depth + 1, Q);
1571	// TODO: 1-extend would be more precise.
1572	Known &= Known2.anyextOrTrunc(BitWidth);
1573	break;
1574	}
1575	case Intrinsic::x86_sse42_crc32_64_64:
1576	Known.Zero.setBitsFrom(32);
1577	break;
1578	case Intrinsic::riscv_vsetvli:
1579	case Intrinsic::riscv_vsetvlimax:
1580	// Assume that VL output is <= 65536.
1581	// TODO: Take SEW and LMUL into account.
1582	if (BitWidth > 17)
1583	Known.Zero.setBitsFrom(17);
1584	break;
1585	case Intrinsic::vscale: {
1586	if (!II->getParent() || !II->getFunction())
1587	break;
1588
1589	Known = getVScaleRange(F: II->getFunction(), BitWidth).toKnownBits();
1590	break;
1591	}
1592	}
1593	}
1594	break;
1595	case Instruction::ShuffleVector: {
1596	auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: I);
1597	// FIXME: Do we need to handle ConstantExpr involving shufflevectors?
1598	if (!Shuf) {
1599	Known.resetAll();
1600	return;
1601	}
1602	// For undef elements, we don't know anything about the common state of
1603	// the shuffle result.
1604	APInt DemandedLHS, DemandedRHS;
1605	if (!getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS)) {
1606	Known.resetAll();
1607	return;
1608	}
1609	Known.One.setAllBits();
1610	Known.Zero.setAllBits();
1611	if (!!DemandedLHS) {
1612	const Value *LHS = Shuf->getOperand(i_nocapture: 0);
1613	computeKnownBits(V: LHS, DemandedElts: DemandedLHS, Known, Depth: Depth + 1, Q);
1614	// If we don't know any bits, early out.
1615	if (Known.isUnknown())
1616	break;
1617	}
1618	if (!!DemandedRHS) {
1619	const Value *RHS = Shuf->getOperand(i_nocapture: 1);
1620	computeKnownBits(V: RHS, DemandedElts: DemandedRHS, Known&: Known2, Depth: Depth + 1, Q);
1621	Known = Known.intersectWith(RHS: Known2);
1622	}
1623	break;
1624	}
1625	case Instruction::InsertElement: {
1626	if (isa<ScalableVectorType>(Val: I->getType())) {
1627	Known.resetAll();
1628	return;
1629	}
1630	const Value *Vec = I->getOperand(i: 0);
1631	const Value *Elt = I->getOperand(i: 1);
1632	auto *CIdx = dyn_cast<ConstantInt>(Val: I->getOperand(i: 2));
1633	// Early out if the index is non-constant or out-of-range.
1634	unsigned NumElts = DemandedElts.getBitWidth();
1635	if (!CIdx || CIdx->getValue().uge(RHS: NumElts)) {
1636	Known.resetAll();
1637	return;
1638	}
1639	Known.One.setAllBits();
1640	Known.Zero.setAllBits();
1641	unsigned EltIdx = CIdx->getZExtValue();
1642	// Do we demand the inserted element?
1643	if (DemandedElts[EltIdx]) {
1644	computeKnownBits(V: Elt, Known, Depth: Depth + 1, Q);
1645	// If we don't know any bits, early out.
1646	if (Known.isUnknown())
1647	break;
1648	}
1649	// We don't need the base vector element that has been inserted.
1650	APInt DemandedVecElts = DemandedElts;
1651	DemandedVecElts.clearBit(BitPosition: EltIdx);
1652	if (!!DemandedVecElts) {
1653	computeKnownBits(V: Vec, DemandedElts: DemandedVecElts, Known&: Known2, Depth: Depth + 1, Q);
1654	Known = Known.intersectWith(RHS: Known2);
1655	}
1656	break;
1657	}
1658	case Instruction::ExtractElement: {
1659	// Look through extract element. If the index is non-constant or
1660	// out-of-range demand all elements, otherwise just the extracted element.
1661	const Value *Vec = I->getOperand(i: 0);
1662	const Value *Idx = I->getOperand(i: 1);
1663	auto *CIdx = dyn_cast<ConstantInt>(Val: Idx);
1664	if (isa<ScalableVectorType>(Val: Vec->getType())) {
1665	// FIXME: there's probably *something* we can do with scalable vectors
1666	Known.resetAll();
1667	break;
1668	}
1669	unsigned NumElts = cast<FixedVectorType>(Val: Vec->getType())->getNumElements();
1670	APInt DemandedVecElts = APInt::getAllOnes(numBits: NumElts);
1671	if (CIdx && CIdx->getValue().ult(RHS: NumElts))
1672	DemandedVecElts = APInt::getOneBitSet(numBits: NumElts, BitNo: CIdx->getZExtValue());
1673	computeKnownBits(V: Vec, DemandedElts: DemandedVecElts, Known, Depth: Depth + 1, Q);
1674	break;
1675	}
1676	case Instruction::ExtractValue:
1677	if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I->getOperand(i: 0))) {
1678	const ExtractValueInst *EVI = cast<ExtractValueInst>(Val: I);
1679	if (EVI->getNumIndices() != 1) break;
1680	if (EVI->getIndices()[0] == 0) {
1681	switch (II->getIntrinsicID()) {
1682	default: break;
1683	case Intrinsic::uadd_with_overflow:
1684	case Intrinsic::sadd_with_overflow:
1685	computeKnownBitsAddSub(Add: true, Op0: II->getArgOperand(i: 0),
1686	Op1: II->getArgOperand(i: 1), NSW: false, DemandedElts,
1687	KnownOut&: Known, Known2, Depth, Q);
1688	break;
1689	case Intrinsic::usub_with_overflow:
1690	case Intrinsic::ssub_with_overflow:
1691	computeKnownBitsAddSub(Add: false, Op0: II->getArgOperand(i: 0),
1692	Op1: II->getArgOperand(i: 1), NSW: false, DemandedElts,
1693	KnownOut&: Known, Known2, Depth, Q);
1694	break;
1695	case Intrinsic::umul_with_overflow:
1696	case Intrinsic::smul_with_overflow:
1697	computeKnownBitsMul(Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1), NSW: false,
1698	DemandedElts, Known, Known2, Depth, Q);
1699	break;
1700	}
1701	}
1702	}
1703	break;
1704	case Instruction::Freeze:
1705	if (isGuaranteedNotToBePoison(V: I->getOperand(i: 0), AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT,
1706	Depth: Depth + 1))
1707	computeKnownBits(V: I->getOperand(i: 0), Known, Depth: Depth + 1, Q);
1708	break;
1709	}
1710	}
1711
1712	/// Determine which bits of V are known to be either zero or one and return
1713	/// them.
1714	KnownBits llvm::computeKnownBits(const Value *V, const APInt &DemandedElts,
1715	unsigned Depth, const SimplifyQuery &Q) {
1716	KnownBits Known(getBitWidth(Ty: V->getType(), DL: Q.DL));
1717	::computeKnownBits(V, DemandedElts, Known, Depth, Q);
1718	return Known;
1719	}
1720
1721	/// Determine which bits of V are known to be either zero or one and return
1722	/// them.
1723	KnownBits llvm::computeKnownBits(const Value *V, unsigned Depth,
1724	const SimplifyQuery &Q) {
1725	KnownBits Known(getBitWidth(Ty: V->getType(), DL: Q.DL));
1726	computeKnownBits(V, Known, Depth, Q);
1727	return Known;
1728	}
1729
1730	/// Determine which bits of V are known to be either zero or one and return
1731	/// them in the Known bit set.
1732	///
1733	/// NOTE: we cannot consider 'undef' to be "IsZero" here. The problem is that
1734	/// we cannot optimize based on the assumption that it is zero without changing
1735	/// it to be an explicit zero. If we don't change it to zero, other code could
1736	/// optimized based on the contradictory assumption that it is non-zero.
1737	/// Because instcombine aggressively folds operations with undef args anyway,
1738	/// this won't lose us code quality.
1739	///
1740	/// This function is defined on values with integer type, values with pointer
1741	/// type, and vectors of integers. In the case
1742	/// where V is a vector, known zero, and known one values are the
1743	/// same width as the vector element, and the bit is set only if it is true
1744	/// for all of the demanded elements in the vector specified by DemandedElts.
1745	void computeKnownBits(const Value *V, const APInt &DemandedElts,
1746	KnownBits &Known, unsigned Depth,
1747	const SimplifyQuery &Q) {
1748	if (!DemandedElts) {
1749	// No demanded elts, better to assume we don't know anything.
1750	Known.resetAll();
1751	return;
1752	}
1753
1754	assert(V && "No Value?");
1755	assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
1756
1757	#ifndef NDEBUG
1758	Type *Ty = V->getType();
1759	unsigned BitWidth = Known.getBitWidth();
1760
1761	assert((Ty->isIntOrIntVectorTy(BitWidth) || Ty->isPtrOrPtrVectorTy()) &&
1762	"Not integer or pointer type!");
1763
1764	if (auto *FVTy = dyn_cast<FixedVectorType>(Val: Ty)) {
1765	assert(
1766	FVTy->getNumElements() == DemandedElts.getBitWidth() &&
1767	"DemandedElt width should equal the fixed vector number of elements");
1768	} else {
1769	assert(DemandedElts == APInt(1, 1) &&
1770	"DemandedElt width should be 1 for scalars or scalable vectors");
1771	}
1772
1773	Type *ScalarTy = Ty->getScalarType();
1774	if (ScalarTy->isPointerTy()) {
1775	assert(BitWidth == Q.DL.getPointerTypeSizeInBits(ScalarTy) &&
1776	"V and Known should have same BitWidth");
1777	} else {
1778	assert(BitWidth == Q.DL.getTypeSizeInBits(ScalarTy) &&
1779	"V and Known should have same BitWidth");
1780	}
1781	#endif
1782
1783	const APInt *C;
1784	if (match(V, P: m_APInt(Res&: C))) {
1785	// We know all of the bits for a scalar constant or a splat vector constant!
1786	Known = KnownBits::makeConstant(C: *C);
1787	return;
1788	}
1789	// Null and aggregate-zero are all-zeros.
1790	if (isa<ConstantPointerNull>(Val: V) || isa<ConstantAggregateZero>(Val: V)) {
1791	Known.setAllZero();
1792	return;
1793	}
1794	// Handle a constant vector by taking the intersection of the known bits of
1795	// each element.
1796	if (const ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(Val: V)) {
1797	assert(!isa<ScalableVectorType>(V->getType()));
1798	// We know that CDV must be a vector of integers. Take the intersection of
1799	// each element.
1800	Known.Zero.setAllBits(); Known.One.setAllBits();
1801	for (unsigned i = 0, e = CDV->getNumElements(); i != e; ++i) {
1802	if (!DemandedElts[i])
1803	continue;
1804	APInt Elt = CDV->getElementAsAPInt(i);
1805	Known.Zero &= ~Elt;
1806	Known.One &= Elt;
1807	}
1808	if (Known.hasConflict())
1809	Known.resetAll();
1810	return;
1811	}
1812
1813	if (const auto *CV = dyn_cast<ConstantVector>(Val: V)) {
1814	assert(!isa<ScalableVectorType>(V->getType()));
1815	// We know that CV must be a vector of integers. Take the intersection of
1816	// each element.
1817	Known.Zero.setAllBits(); Known.One.setAllBits();
1818	for (unsigned i = 0, e = CV->getNumOperands(); i != e; ++i) {
1819	if (!DemandedElts[i])
1820	continue;
1821	Constant *Element = CV->getAggregateElement(Elt: i);
1822	if (isa<PoisonValue>(Val: Element))
1823	continue;
1824	auto *ElementCI = dyn_cast_or_null<ConstantInt>(Val: Element);
1825	if (!ElementCI) {
1826	Known.resetAll();
1827	return;
1828	}
1829	const APInt &Elt = ElementCI->getValue();
1830	Known.Zero &= ~Elt;
1831	Known.One &= Elt;
1832	}
1833	if (Known.hasConflict())
1834	Known.resetAll();
1835	return;
1836	}
1837
1838	// Start out not knowing anything.
1839	Known.resetAll();
1840
1841	// We can't imply anything about undefs.
1842	if (isa<UndefValue>(Val: V))
1843	return;
1844
1845	// There's no point in looking through other users of ConstantData for
1846	// assumptions. Confirm that we've handled them all.
1847	assert(!isa<ConstantData>(V) && "Unhandled constant data!");
1848
1849	// All recursive calls that increase depth must come after this.
1850	if (Depth == MaxAnalysisRecursionDepth)
1851	return;
1852
1853	// A weak GlobalAlias is totally unknown. A non-weak GlobalAlias has
1854	// the bits of its aliasee.
1855	if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(Val: V)) {
1856	if (!GA->isInterposable())
1857	computeKnownBits(V: GA->getAliasee(), Known, Depth: Depth + 1, Q);
1858	return;
1859	}
1860
1861	if (const Operator *I = dyn_cast<Operator>(Val: V))
1862	computeKnownBitsFromOperator(I, DemandedElts, Known, Depth, Q);
1863	else if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: V)) {
1864	if (std::optional<ConstantRange> CR = GV->getAbsoluteSymbolRange())
1865	Known = CR->toKnownBits();
1866	}
1867
1868	// Aligned pointers have trailing zeros - refine Known.Zero set
1869	if (isa<PointerType>(Val: V->getType())) {
1870	Align Alignment = V->getPointerAlignment(DL: Q.DL);
1871	Known.Zero.setLowBits(Log2(A: Alignment));
1872	}
1873
1874	// computeKnownBitsFromContext strictly refines Known.
1875	// Therefore, we run them after computeKnownBitsFromOperator.
1876
1877	// Check whether we can determine known bits from context such as assumes.
1878	computeKnownBitsFromContext(V, Known, Depth, Q);
1879
1880	assert((Known.Zero & Known.One) == 0 && "Bits known to be one AND zero?");
1881	}
1882
1883	/// Try to detect a recurrence that the value of the induction variable is
1884	/// always a power of two (or zero).
1885	static bool isPowerOfTwoRecurrence(const PHINode *PN, bool OrZero,
1886	unsigned Depth, SimplifyQuery &Q) {
1887	BinaryOperator *BO = nullptr;
1888	Value *Start = nullptr, *Step = nullptr;
1889	if (!matchSimpleRecurrence(P: PN, BO, Start, Step))
1890	return false;
1891
1892	// Initial value must be a power of two.
1893	for (const Use &U : PN->operands()) {
1894	if (U.get() == Start) {
1895	// Initial value comes from a different BB, need to adjust context
1896	// instruction for analysis.
1897	Q.CxtI = PN->getIncomingBlock(U)->getTerminator();
1898	if (!isKnownToBeAPowerOfTwo(V: Start, OrZero, Depth, Q))
1899	return false;
1900	}
1901	}
1902
1903	// Except for Mul, the induction variable must be on the left side of the
1904	// increment expression, otherwise its value can be arbitrary.
1905	if (BO->getOpcode() != Instruction::Mul && BO->getOperand(i_nocapture: 1) != Step)
1906	return false;
1907
1908	Q.CxtI = BO->getParent()->getTerminator();
1909	switch (BO->getOpcode()) {
1910	case Instruction::Mul:
1911	// Power of two is closed under multiplication.
1912	return (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: BO) ||
1913	Q.IIQ.hasNoSignedWrap(Op: BO)) &&
1914	isKnownToBeAPowerOfTwo(V: Step, OrZero, Depth, Q);
1915	case Instruction::SDiv:
1916	// Start value must not be signmask for signed division, so simply being a
1917	// power of two is not sufficient, and it has to be a constant.
1918	if (!match(V: Start, P: m_Power2()) || match(V: Start, P: m_SignMask()))
1919	return false;
1920	[[fallthrough]];
1921	case Instruction::UDiv:
1922	// Divisor must be a power of two.
1923	// If OrZero is false, cannot guarantee induction variable is non-zero after
1924	// division, same for Shr, unless it is exact division.
1925	return (OrZero || Q.IIQ.isExact(Op: BO)) &&
1926	isKnownToBeAPowerOfTwo(V: Step, OrZero: false, Depth, Q);
1927	case Instruction::Shl:
1928	return OrZero || Q.IIQ.hasNoUnsignedWrap(Op: BO) || Q.IIQ.hasNoSignedWrap(Op: BO);
1929	case Instruction::AShr:
1930	if (!match(V: Start, P: m_Power2()) || match(V: Start, P: m_SignMask()))
1931	return false;
1932	[[fallthrough]];
1933	case Instruction::LShr:
1934	return OrZero || Q.IIQ.isExact(Op: BO);
1935	default:
1936	return false;
1937	}
1938	}
1939
1940	/// Return true if the given value is known to have exactly one
1941	/// bit set when defined. For vectors return true if every element is known to
1942	/// be a power of two when defined. Supports values with integer or pointer
1943	/// types and vectors of integers.
1944	bool isKnownToBeAPowerOfTwo(const Value *V, bool OrZero, unsigned Depth,
1945	const SimplifyQuery &Q) {
1946	assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
1947
1948	if (isa<Constant>(Val: V))
1949	return OrZero ? match(V, P: m_Power2OrZero()) : match(V, P: m_Power2());
1950
1951	// i1 is by definition a power of 2 or zero.
1952	if (OrZero && V->getType()->getScalarSizeInBits() == 1)
1953	return true;
1954
1955	auto *I = dyn_cast<Instruction>(Val: V);
1956	if (!I)
1957	return false;
1958
1959	if (Q.CxtI && match(V, P: m_VScale())) {
1960	const Function *F = Q.CxtI->getFunction();
1961	// The vscale_range indicates vscale is a power-of-two.
1962	return F->hasFnAttribute(Attribute::VScaleRange);
1963	}
1964
1965	// 1 << X is clearly a power of two if the one is not shifted off the end. If
1966	// it is shifted off the end then the result is undefined.
1967	if (match(V: I, P: m_Shl(L: m_One(), R: m_Value())))
1968	return true;
1969
1970	// (signmask) >>l X is clearly a power of two if the one is not shifted off
1971	// the bottom. If it is shifted off the bottom then the result is undefined.
1972	if (match(V: I, P: m_LShr(L: m_SignMask(), R: m_Value())))
1973	return true;
1974
1975	// The remaining tests are all recursive, so bail out if we hit the limit.
1976	if (Depth++ == MaxAnalysisRecursionDepth)
1977	return false;
1978
1979	switch (I->getOpcode()) {
1980	case Instruction::ZExt:
1981	return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Depth, Q);
1982	case Instruction::Trunc:
1983	return OrZero && isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Depth, Q);
1984	case Instruction::Shl:
1985	if (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: I) || Q.IIQ.hasNoSignedWrap(Op: I))
1986	return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Depth, Q);
1987	return false;
1988	case Instruction::LShr:
1989	if (OrZero || Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)))
1990	return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Depth, Q);
1991	return false;
1992	case Instruction::UDiv:
1993	if (Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)))
1994	return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Depth, Q);
1995	return false;
1996	case Instruction::Mul:
1997	return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), OrZero, Depth, Q) &&
1998	isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Depth, Q) &&
1999	(OrZero || isKnownNonZero(V: I, Depth, Q));
2000	case Instruction::And:
2001	// A power of two and'd with anything is a power of two or zero.
2002	if (OrZero &&
2003	(isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), /*OrZero*/ true, Depth, Q) ||
2004	isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), /*OrZero*/ true, Depth, Q)))
2005	return true;
2006	// X & (-X) is always a power of two or zero.
2007	if (match(V: I->getOperand(i: 0), P: m_Neg(V: m_Specific(V: I->getOperand(i: 1)))) ||
2008	match(V: I->getOperand(i: 1), P: m_Neg(V: m_Specific(V: I->getOperand(i: 0)))))
2009	return OrZero || isKnownNonZero(V: I->getOperand(i: 0), Depth, Q);
2010	return false;
2011	case Instruction::Add: {
2012	// Adding a power-of-two or zero to the same power-of-two or zero yields
2013	// either the original power-of-two, a larger power-of-two or zero.
2014	const OverflowingBinaryOperator *VOBO = cast<OverflowingBinaryOperator>(Val: V);
2015	if (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: VOBO) ||
2016	Q.IIQ.hasNoSignedWrap(Op: VOBO)) {
2017	if (match(V: I->getOperand(i: 0),
2018	P: m_c_And(L: m_Specific(V: I->getOperand(i: 1)), R: m_Value())) &&
2019	isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), OrZero, Depth, Q))
2020	return true;
2021	if (match(V: I->getOperand(i: 1),
2022	P: m_c_And(L: m_Specific(V: I->getOperand(i: 0)), R: m_Value())) &&
2023	isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Depth, Q))
2024	return true;
2025
2026	unsigned BitWidth = V->getType()->getScalarSizeInBits();
2027	KnownBits LHSBits(BitWidth);
2028	computeKnownBits(V: I->getOperand(i: 0), Known&: LHSBits, Depth, Q);
2029
2030	KnownBits RHSBits(BitWidth);
2031	computeKnownBits(V: I->getOperand(i: 1), Known&: RHSBits, Depth, Q);
2032	// If i8 V is a power of two or zero:
2033	// ZeroBits: 1 1 1 0 1 1 1 1
2034	// ~ZeroBits: 0 0 0 1 0 0 0 0
2035	if ((~(LHSBits.Zero & RHSBits.Zero)).isPowerOf2())
2036	// If OrZero isn't set, we cannot give back a zero result.
2037	// Make sure either the LHS or RHS has a bit set.
2038	if (OrZero || RHSBits.One.getBoolValue() || LHSBits.One.getBoolValue())
2039	return true;
2040	}
2041	return false;
2042	}
2043	case Instruction::Select:
2044	return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), OrZero, Depth, Q) &&
2045	isKnownToBeAPowerOfTwo(V: I->getOperand(i: 2), OrZero, Depth, Q);
2046	case Instruction::PHI: {
2047	// A PHI node is power of two if all incoming values are power of two, or if
2048	// it is an induction variable where in each step its value is a power of
2049	// two.
2050	auto *PN = cast<PHINode>(Val: I);
2051	SimplifyQuery RecQ = Q;
2052
2053	// Check if it is an induction variable and always power of two.
2054	if (isPowerOfTwoRecurrence(PN, OrZero, Depth, Q&: RecQ))
2055	return true;
2056
2057	// Recursively check all incoming values. Limit recursion to 2 levels, so
2058	// that search complexity is limited to number of operands^2.
2059	unsigned NewDepth = std::max(a: Depth, b: MaxAnalysisRecursionDepth - 1);
2060	return llvm::all_of(Range: PN->operands(), P: [&](const Use &U) {
2061	// Value is power of 2 if it is coming from PHI node itself by induction.
2062	if (U.get() == PN)
2063	return true;
2064
2065	// Change the context instruction to the incoming block where it is
2066	// evaluated.
2067	RecQ.CxtI = PN->getIncomingBlock(U)->getTerminator();
2068	return isKnownToBeAPowerOfTwo(V: U.get(), OrZero, Depth: NewDepth, Q: RecQ);
2069	});
2070	}
2071	case Instruction::Invoke:
2072	case Instruction::Call: {
2073	if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
2074	switch (II->getIntrinsicID()) {
2075	case Intrinsic::umax:
2076	case Intrinsic::smax:
2077	case Intrinsic::umin:
2078	case Intrinsic::smin:
2079	return isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 1), OrZero, Depth, Q) &&
2080	isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 0), OrZero, Depth, Q);
2081	// bswap/bitreverse just move around bits, but don't change any 1s/0s
2082	// thus dont change pow2/non-pow2 status.
2083	case Intrinsic::bitreverse:
2084	case Intrinsic::bswap:
2085	return isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 0), OrZero, Depth, Q);
2086	case Intrinsic::fshr:
2087	case Intrinsic::fshl:
2088	// If Op0 == Op1, this is a rotate. is_pow2(rotate(x, y)) == is_pow2(x)
2089	if (II->getArgOperand(i: 0) == II->getArgOperand(i: 1))
2090	return isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 0), OrZero, Depth, Q);
2091	break;
2092	default:
2093	break;
2094	}
2095	}
2096	return false;
2097	}
2098	default:
2099	return false;
2100	}
2101	}
2102
2103	/// Test whether a GEP's result is known to be non-null.
2104	///
2105	/// Uses properties inherent in a GEP to try to determine whether it is known
2106	/// to be non-null.
2107	///
2108	/// Currently this routine does not support vector GEPs.
2109	static bool isGEPKnownNonNull(const GEPOperator *GEP, unsigned Depth,
2110	const SimplifyQuery &Q) {
2111	const Function *F = nullptr;
2112	if (const Instruction *I = dyn_cast<Instruction>(Val: GEP))
2113	F = I->getFunction();
2114
2115	if (!GEP->isInBounds() ||
2116	NullPointerIsDefined(F, AS: GEP->getPointerAddressSpace()))
2117	return false;
2118
2119	// FIXME: Support vector-GEPs.
2120	assert(GEP->getType()->isPointerTy() && "We only support plain pointer GEP");
2121
2122	// If the base pointer is non-null, we cannot walk to a null address with an
2123	// inbounds GEP in address space zero.
2124	if (isKnownNonZero(V: GEP->getPointerOperand(), Depth, Q))
2125	return true;
2126
2127	// Walk the GEP operands and see if any operand introduces a non-zero offset.
2128	// If so, then the GEP cannot produce a null pointer, as doing so would
2129	// inherently violate the inbounds contract within address space zero.
2130	for (gep_type_iterator GTI = gep_type_begin(GEP), GTE = gep_type_end(GEP);
2131	GTI != GTE; ++GTI) {
2132	// Struct types are easy -- they must always be indexed by a constant.
2133	if (StructType *STy = GTI.getStructTypeOrNull()) {
2134	ConstantInt *OpC = cast<ConstantInt>(Val: GTI.getOperand());
2135	unsigned ElementIdx = OpC->getZExtValue();
2136	const StructLayout *SL = Q.DL.getStructLayout(Ty: STy);
2137	uint64_t ElementOffset = SL->getElementOffset(Idx: ElementIdx);
2138	if (ElementOffset > 0)
2139	return true;
2140	continue;
2141	}
2142
2143	// If we have a zero-sized type, the index doesn't matter. Keep looping.
2144	if (GTI.getSequentialElementStride(DL: Q.DL).isZero())
2145	continue;
2146
2147	// Fast path the constant operand case both for efficiency and so we don't
2148	// increment Depth when just zipping down an all-constant GEP.
2149	if (ConstantInt *OpC = dyn_cast<ConstantInt>(Val: GTI.getOperand())) {
2150	if (!OpC->isZero())
2151	return true;
2152	continue;
2153	}
2154
2155	// We post-increment Depth here because while isKnownNonZero increments it
2156	// as well, when we pop back up that increment won't persist. We don't want
2157	// to recurse 10k times just because we have 10k GEP operands. We don't
2158	// bail completely out because we want to handle constant GEPs regardless
2159	// of depth.
2160	if (Depth++ >= MaxAnalysisRecursionDepth)
2161	continue;
2162
2163	if (isKnownNonZero(V: GTI.getOperand(), Depth, Q))
2164	return true;
2165	}
2166
2167	return false;
2168	}
2169
2170	static bool isKnownNonNullFromDominatingCondition(const Value *V,
2171	const Instruction *CtxI,
2172	const DominatorTree *DT) {
2173	assert(!isa<Constant>(V) && "Called for constant?");
2174
2175	if (!CtxI || !DT)
2176	return false;
2177
2178	unsigned NumUsesExplored = 0;
2179	for (const auto *U : V->users()) {
2180	// Avoid massive lists
2181	if (NumUsesExplored >= DomConditionsMaxUses)
2182	break;
2183	NumUsesExplored++;
2184
2185	// If the value is used as an argument to a call or invoke, then argument
2186	// attributes may provide an answer about null-ness.
2187	if (const auto *CB = dyn_cast<CallBase>(Val: U))
2188	if (auto *CalledFunc = CB->getCalledFunction())
2189	for (const Argument &Arg : CalledFunc->args())
2190	if (CB->getArgOperand(i: Arg.getArgNo()) == V &&
2191	Arg.hasNonNullAttr(/* AllowUndefOrPoison */ false) &&
2192	DT->dominates(Def: CB, User: CtxI))
2193	return true;
2194
2195	// If the value is used as a load/store, then the pointer must be non null.
2196	if (V == getLoadStorePointerOperand(V: U)) {
2197	const Instruction *I = cast<Instruction>(Val: U);
2198	if (!NullPointerIsDefined(F: I->getFunction(),
2199	AS: V->getType()->getPointerAddressSpace()) &&
2200	DT->dominates(Def: I, User: CtxI))
2201	return true;
2202	}
2203
2204	if ((match(V: U, P: m_IDiv(L: m_Value(), R: m_Specific(V))) ||
2205	match(V: U, P: m_IRem(L: m_Value(), R: m_Specific(V)))) &&
2206	isValidAssumeForContext(Inv: cast<Instruction>(Val: U), CxtI: CtxI, DT))
2207	return true;
2208
2209	// Consider only compare instructions uniquely controlling a branch
2210	Value *RHS;
2211	CmpInst::Predicate Pred;
2212	if (!match(V: U, P: m_c_ICmp(Pred, L: m_Specific(V), R: m_Value(V&: RHS))))
2213	continue;
2214
2215	bool NonNullIfTrue;
2216	if (cmpExcludesZero(Pred, RHS))
2217	NonNullIfTrue = true;
2218	else if (cmpExcludesZero(Pred: CmpInst::getInversePredicate(pred: Pred), RHS))
2219	NonNullIfTrue = false;
2220	else
2221	continue;
2222
2223	SmallVector<const User *, 4> WorkList;
2224	SmallPtrSet<const User *, 4> Visited;
2225	for (const auto *CmpU : U->users()) {
2226	assert(WorkList.empty() && "Should be!");
2227	if (Visited.insert(Ptr: CmpU).second)
2228	WorkList.push_back(Elt: CmpU);
2229
2230	while (!WorkList.empty()) {
2231	auto *Curr = WorkList.pop_back_val();
2232
2233	// If a user is an AND, add all its users to the work list. We only
2234	// propagate "pred != null" condition through AND because it is only
2235	// correct to assume that all conditions of AND are met in true branch.
2236	// TODO: Support similar logic of OR and EQ predicate?
2237	if (NonNullIfTrue)
2238	if (match(V: Curr, P: m_LogicalAnd(L: m_Value(), R: m_Value()))) {
2239	for (const auto *CurrU : Curr->users())
2240	if (Visited.insert(Ptr: CurrU).second)
2241	WorkList.push_back(Elt: CurrU);
2242	continue;
2243	}
2244
2245	if (const BranchInst *BI = dyn_cast<BranchInst>(Val: Curr)) {
2246	assert(BI->isConditional() && "uses a comparison!");
2247
2248	BasicBlock *NonNullSuccessor =
2249	BI->getSuccessor(i: NonNullIfTrue ? 0 : 1);
2250	BasicBlockEdge Edge(BI->getParent(), NonNullSuccessor);
2251	if (Edge.isSingleEdge() && DT->dominates(BBE: Edge, BB: CtxI->getParent()))
2252	return true;
2253	} else if (NonNullIfTrue && isGuard(U: Curr) &&
2254	DT->dominates(Def: cast<Instruction>(Val: Curr), User: CtxI)) {
2255	return true;
2256	}
2257	}
2258	}
2259	}
2260
2261	return false;
2262	}
2263
2264	/// Does the 'Range' metadata (which must be a valid MD_range operand list)
2265	/// ensure that the value it's attached to is never Value? 'RangeType' is
2266	/// is the type of the value described by the range.
2267	static bool rangeMetadataExcludesValue(const MDNode* Ranges, const APInt& Value) {
2268	const unsigned NumRanges = Ranges->getNumOperands() / 2;
2269	assert(NumRanges >= 1);
2270	for (unsigned i = 0; i < NumRanges; ++i) {
2271	ConstantInt *Lower =
2272	mdconst::extract<ConstantInt>(MD: Ranges->getOperand(I: 2 * i + 0));
2273	ConstantInt *Upper =
2274	mdconst::extract<ConstantInt>(MD: Ranges->getOperand(I: 2 * i + 1));
2275	ConstantRange Range(Lower->getValue(), Upper->getValue());
2276	if (Range.contains(Val: Value))
2277	return false;
2278	}
2279	return true;
2280	}
2281
2282	/// Try to detect a recurrence that monotonically increases/decreases from a
2283	/// non-zero starting value. These are common as induction variables.
2284	static bool isNonZeroRecurrence(const PHINode *PN) {
2285	BinaryOperator *BO = nullptr;
2286	Value *Start = nullptr, *Step = nullptr;
2287	const APInt *StartC, *StepC;
2288	if (!matchSimpleRecurrence(P: PN, BO, Start, Step) ||
2289	!match(V: Start, P: m_APInt(Res&: StartC)) || StartC->isZero())
2290	return false;
2291
2292	switch (BO->getOpcode()) {
2293	case Instruction::Add:
2294	// Starting from non-zero and stepping away from zero can never wrap back
2295	// to zero.
2296	return BO->hasNoUnsignedWrap() ||
2297	(BO->hasNoSignedWrap() && match(V: Step, P: m_APInt(Res&: StepC)) &&
2298	StartC->isNegative() == StepC->isNegative());
2299	case Instruction::Mul:
2300	return (BO->hasNoUnsignedWrap() || BO->hasNoSignedWrap()) &&
2301	match(V: Step, P: m_APInt(Res&: StepC)) && !StepC->isZero();
2302	case Instruction::Shl:
2303	return BO->hasNoUnsignedWrap() || BO->hasNoSignedWrap();
2304	case Instruction::AShr:
2305	case Instruction::LShr:
2306	return BO->isExact();
2307	default:
2308	return false;
2309	}
2310	}
2311
2312	static bool isNonZeroAdd(const APInt &DemandedElts, unsigned Depth,
2313	const SimplifyQuery &Q, unsigned BitWidth, Value *X,
2314	Value *Y, bool NSW) {
2315	KnownBits XKnown = computeKnownBits(V: X, DemandedElts, Depth, Q);
2316	KnownBits YKnown = computeKnownBits(V: Y, DemandedElts, Depth, Q);
2317
2318	// If X and Y are both non-negative (as signed values) then their sum is not
2319	// zero unless both X and Y are zero.
2320	if (XKnown.isNonNegative() && YKnown.isNonNegative())
2321	if (isKnownNonZero(V: Y, DemandedElts, Depth, Q) ||
2322	isKnownNonZero(V: X, DemandedElts, Depth, Q))
2323	return true;
2324
2325	// If X and Y are both negative (as signed values) then their sum is not
2326	// zero unless both X and Y equal INT_MIN.
2327	if (XKnown.isNegative() && YKnown.isNegative()) {
2328	APInt Mask = APInt::getSignedMaxValue(numBits: BitWidth);
2329	// The sign bit of X is set. If some other bit is set then X is not equal
2330	// to INT_MIN.
2331	if (XKnown.One.intersects(RHS: Mask))
2332	return true;
2333	// The sign bit of Y is set. If some other bit is set then Y is not equal
2334	// to INT_MIN.
2335	if (YKnown.One.intersects(RHS: Mask))
2336	return true;
2337	}
2338
2339	// The sum of a non-negative number and a power of two is not zero.
2340	if (XKnown.isNonNegative() &&
2341	isKnownToBeAPowerOfTwo(V: Y, /*OrZero*/ false, Depth, Q))
2342	return true;
2343	if (YKnown.isNonNegative() &&
2344	isKnownToBeAPowerOfTwo(V: X, /*OrZero*/ false, Depth, Q))
2345	return true;
2346
2347	return KnownBits::computeForAddSub(/*Add*/ true, NSW, LHS: XKnown, RHS: YKnown)
2348	.isNonZero();
2349	}
2350
2351	static bool isNonZeroSub(const APInt &DemandedElts, unsigned Depth,
2352	const SimplifyQuery &Q, unsigned BitWidth, Value *X,
2353	Value *Y) {
2354	// TODO: Move this case into isKnownNonEqual().
2355	if (auto *C = dyn_cast<Constant>(Val: X))
2356	if (C->isNullValue() && isKnownNonZero(V: Y, DemandedElts, Depth, Q))
2357	return true;
2358
2359	return ::isKnownNonEqual(V1: X, V2: Y, Depth, Q);
2360	}
2361
2362	static bool isNonZeroShift(const Operator *I, const APInt &DemandedElts,
2363	unsigned Depth, const SimplifyQuery &Q,
2364	const KnownBits &KnownVal) {
2365	auto ShiftOp = [&](const APInt &Lhs, const APInt &Rhs) {
2366	switch (I->getOpcode()) {
2367	case Instruction::Shl:
2368	return Lhs.shl(ShiftAmt: Rhs);
2369	case Instruction::LShr:
2370	return Lhs.lshr(ShiftAmt: Rhs);
2371	case Instruction::AShr:
2372	return Lhs.ashr(ShiftAmt: Rhs);
2373	default:
2374	llvm_unreachable("Unknown Shift Opcode");
2375	}
2376	};
2377
2378	auto InvShiftOp = [&](const APInt &Lhs, const APInt &Rhs) {
2379	switch (I->getOpcode()) {
2380	case Instruction::Shl:
2381	return Lhs.lshr(ShiftAmt: Rhs);
2382	case Instruction::LShr:
2383	case Instruction::AShr:
2384	return Lhs.shl(ShiftAmt: Rhs);
2385	default:
2386	llvm_unreachable("Unknown Shift Opcode");
2387	}
2388	};
2389
2390	if (KnownVal.isUnknown())
2391	return false;
2392
2393	KnownBits KnownCnt =
2394	computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Depth, Q);
2395	APInt MaxShift = KnownCnt.getMaxValue();
2396	unsigned NumBits = KnownVal.getBitWidth();
2397	if (MaxShift.uge(RHS: NumBits))
2398	return false;
2399
2400	if (!ShiftOp(KnownVal.One, MaxShift).isZero())
2401	return true;
2402
2403	// If all of the bits shifted out are known to be zero, and Val is known
2404	// non-zero then at least one non-zero bit must remain.
2405	if (InvShiftOp(KnownVal.Zero, NumBits - MaxShift)
2406	.eq(RHS: InvShiftOp(APInt::getAllOnes(numBits: NumBits), NumBits - MaxShift)) &&
2407	isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Depth, Q))
2408	return true;
2409
2410	return false;
2411	}
2412
2413	static bool isKnownNonZeroFromOperator(const Operator *I,
2414	const APInt &DemandedElts,
2415	unsigned Depth, const SimplifyQuery &Q) {
2416	unsigned BitWidth = getBitWidth(Ty: I->getType()->getScalarType(), DL: Q.DL);
2417	switch (I->getOpcode()) {
2418	case Instruction::Alloca:
2419	// Alloca never returns null, malloc might.
2420	return I->getType()->getPointerAddressSpace() == 0;
2421	case Instruction::GetElementPtr:
2422	if (I->getType()->isPointerTy())
2423	return isGEPKnownNonNull(GEP: cast<GEPOperator>(Val: I), Depth, Q);
2424	break;
2425	case Instruction::BitCast: {
2426	// We need to be a bit careful here. We can only peek through the bitcast
2427	// if the scalar size of elements in the operand are smaller than and a
2428	// multiple of the size they are casting too. Take three cases:
2429	//
2430	// 1) Unsafe:
2431	// bitcast <2 x i16> %NonZero to <4 x i8>
2432	//
2433	// %NonZero can have 2 non-zero i16 elements, but isKnownNonZero on a
2434	// <4 x i8> requires that all 4 i8 elements be non-zero which isn't
2435	// guranteed (imagine just sign bit set in the 2 i16 elements).
2436	//
2437	// 2) Unsafe:
2438	// bitcast <4 x i3> %NonZero to <3 x i4>
2439	//
2440	// Even though the scalar size of the src (`i3`) is smaller than the
2441	// scalar size of the dst `i4`, because `i3` is not a multiple of `i4`
2442	// its possible for the `3 x i4` elements to be zero because there are
2443	// some elements in the destination that don't contain any full src
2444	// element.
2445	//
2446	// 3) Safe:
2447	// bitcast <4 x i8> %NonZero to <2 x i16>
2448	//
2449	// This is always safe as non-zero in the 4 i8 elements implies
2450	// non-zero in the combination of any two adjacent ones. Since i8 is a
2451	// multiple of i16, each i16 is guranteed to have 2 full i8 elements.
2452	// This all implies the 2 i16 elements are non-zero.
2453	Type *FromTy = I->getOperand(i: 0)->getType();
2454	if ((FromTy->isIntOrIntVectorTy() || FromTy->isPtrOrPtrVectorTy()) &&
2455	(BitWidth % getBitWidth(Ty: FromTy->getScalarType(), DL: Q.DL)) == 0)
2456	return isKnownNonZero(V: I->getOperand(i: 0), Depth, Q);
2457	} break;
2458	case Instruction::IntToPtr:
2459	// Note that we have to take special care to avoid looking through
2460	// truncating casts, e.g., int2ptr/ptr2int with appropriate sizes, as well
2461	// as casts that can alter the value, e.g., AddrSpaceCasts.
2462	if (!isa<ScalableVectorType>(Val: I->getType()) &&
2463	Q.DL.getTypeSizeInBits(Ty: I->getOperand(i: 0)->getType()).getFixedValue() <=
2464	Q.DL.getTypeSizeInBits(Ty: I->getType()).getFixedValue())
2465	return isKnownNonZero(V: I->getOperand(i: 0), Depth, Q);
2466	break;
2467	case Instruction::PtrToInt:
2468	// Similar to int2ptr above, we can look through ptr2int here if the cast
2469	// is a no-op or an extend and not a truncate.
2470	if (!isa<ScalableVectorType>(Val: I->getType()) &&
2471	Q.DL.getTypeSizeInBits(Ty: I->getOperand(i: 0)->getType()).getFixedValue() <=
2472	Q.DL.getTypeSizeInBits(Ty: I->getType()).getFixedValue())
2473	return isKnownNonZero(V: I->getOperand(i: 0), Depth, Q);
2474	break;
2475	case Instruction::Sub:
2476	return isNonZeroSub(DemandedElts, Depth, Q, BitWidth, X: I->getOperand(i: 0),
2477	Y: I->getOperand(i: 1));
2478	case Instruction::Or:
2479	// X | Y != 0 if X != 0 or Y != 0.
2480	return isKnownNonZero(V: I->getOperand(i: 1), DemandedElts, Depth, Q) ||
2481	isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Depth, Q);
2482	case Instruction::SExt:
2483	case Instruction::ZExt:
2484	// ext X != 0 if X != 0.
2485	return isKnownNonZero(V: I->getOperand(i: 0), Depth, Q);
2486
2487	case Instruction::Shl: {
2488	// shl nsw/nuw can't remove any non-zero bits.
2489	const OverflowingBinaryOperator *BO = cast<OverflowingBinaryOperator>(Val: I);
2490	if (Q.IIQ.hasNoUnsignedWrap(Op: BO) || Q.IIQ.hasNoSignedWrap(Op: BO))
2491	return isKnownNonZero(V: I->getOperand(i: 0), Depth, Q);
2492
2493	// shl X, Y != 0 if X is odd. Note that the value of the shift is undefined
2494	// if the lowest bit is shifted off the end.
2495	KnownBits Known(BitWidth);
2496	computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Depth, Q);
2497	if (Known.One[0])
2498	return true;
2499
2500	return isNonZeroShift(I, DemandedElts, Depth, Q, KnownVal: Known);
2501	}
2502	case Instruction::LShr:
2503	case Instruction::AShr: {
2504	// shr exact can only shift out zero bits.
2505	const PossiblyExactOperator *BO = cast<PossiblyExactOperator>(Val: I);
2506	if (BO->isExact())
2507	return isKnownNonZero(V: I->getOperand(i: 0), Depth, Q);
2508
2509	// shr X, Y != 0 if X is negative. Note that the value of the shift is not
2510	// defined if the sign bit is shifted off the end.
2511	KnownBits Known =
2512	computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Depth, Q);
2513	if (Known.isNegative())
2514	return true;
2515
2516	return isNonZeroShift(I, DemandedElts, Depth, Q, KnownVal: Known);
2517	}
2518	case Instruction::UDiv:
2519	case Instruction::SDiv: {
2520	// X / Y
2521	// div exact can only produce a zero if the dividend is zero.
2522	if (cast<PossiblyExactOperator>(Val: I)->isExact())
2523	return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Depth, Q);
2524
2525	std::optional<bool> XUgeY;
2526	KnownBits XKnown =
2527	computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Depth, Q);
2528	// If X is fully unknown we won't be able to figure anything out so don't
2529	// both computing knownbits for Y.
2530	if (XKnown.isUnknown())
2531	return false;
2532
2533	KnownBits YKnown =
2534	computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Depth, Q);
2535	if (I->getOpcode() == Instruction::SDiv) {
2536	// For signed division need to compare abs value of the operands.
2537	XKnown = XKnown.abs(/*IntMinIsPoison*/ false);
2538	YKnown = YKnown.abs(/*IntMinIsPoison*/ false);
2539	}
2540	// If X u>= Y then div is non zero (0/0 is UB).
2541	XUgeY = KnownBits::uge(LHS: XKnown, RHS: YKnown);
2542	// If X is total unknown or X u< Y we won't be able to prove non-zero
2543	// with compute known bits so just return early.
2544	return XUgeY && *XUgeY;
2545	}
2546	case Instruction::Add: {
2547	// X + Y.
2548
2549	// If Add has nuw wrap flag, then if either X or Y is non-zero the result is
2550	// non-zero.
2551	auto *BO = cast<OverflowingBinaryOperator>(Val: I);
2552	if (Q.IIQ.hasNoUnsignedWrap(Op: BO))
2553	return isKnownNonZero(V: I->getOperand(i: 1), DemandedElts, Depth, Q) ||
2554	isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Depth, Q);
2555
2556	return isNonZeroAdd(DemandedElts, Depth, Q, BitWidth, X: I->getOperand(i: 0),
2557	Y: I->getOperand(i: 1), NSW: Q.IIQ.hasNoSignedWrap(Op: BO));
2558	}
2559	case Instruction::Mul: {
2560	// If X and Y are non-zero then so is X * Y as long as the multiplication
2561	// does not overflow.
2562	const OverflowingBinaryOperator *BO = cast<OverflowingBinaryOperator>(Val: I);
2563	if (Q.IIQ.hasNoSignedWrap(Op: BO) || Q.IIQ.hasNoUnsignedWrap(Op: BO))
2564	return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Depth, Q) &&
2565	isKnownNonZero(V: I->getOperand(i: 1), DemandedElts, Depth, Q);
2566
2567	// If either X or Y is odd, then if the other is non-zero the result can't
2568	// be zero.
2569	KnownBits XKnown =
2570	computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Depth, Q);
2571	if (XKnown.One[0])
2572	return isKnownNonZero(V: I->getOperand(i: 1), DemandedElts, Depth, Q);
2573
2574	KnownBits YKnown =
2575	computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Depth, Q);
2576	if (YKnown.One[0])
2577	return XKnown.isNonZero() ||
2578	isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Depth, Q);
2579
2580	// If there exists any subset of X (sX) and subset of Y (sY) s.t sX * sY is
2581	// non-zero, then X * Y is non-zero. We can find sX and sY by just taking
2582	// the lowest known One of X and Y. If they are non-zero, the result
2583	// must be non-zero. We can check if LSB(X) * LSB(Y) != 0 by doing
2584	// X.CountLeadingZeros + Y.CountLeadingZeros < BitWidth.
2585	return (XKnown.countMaxTrailingZeros() + YKnown.countMaxTrailingZeros()) <
2586	BitWidth;
2587	}
2588	case Instruction::Select: {
2589	// (C ? X : Y) != 0 if X != 0 and Y != 0.
2590
2591	// First check if the arm is non-zero using `isKnownNonZero`. If that fails,
2592	// then see if the select condition implies the arm is non-zero. For example
2593	// (X != 0 ? X : Y), we know the true arm is non-zero as the `X` "return" is
2594	// dominated by `X != 0`.
2595	auto SelectArmIsNonZero = [&](bool IsTrueArm) {
2596	Value *Op;
2597	Op = IsTrueArm ? I->getOperand(i: 1) : I->getOperand(i: 2);
2598	// Op is trivially non-zero.
2599	if (isKnownNonZero(V: Op, DemandedElts, Depth, Q))
2600	return true;
2601
2602	// The condition of the select dominates the true/false arm. Check if the
2603	// condition implies that a given arm is non-zero.
2604	Value *X;
2605	CmpInst::Predicate Pred;
2606	if (!match(V: I->getOperand(i: 0), P: m_c_ICmp(Pred, L: m_Specific(V: Op), R: m_Value(V&: X))))
2607	return false;
2608
2609	if (!IsTrueArm)
2610	Pred = ICmpInst::getInversePredicate(pred: Pred);
2611
2612	return cmpExcludesZero(Pred, RHS: X);
2613	};
2614
2615	if (SelectArmIsNonZero(/* IsTrueArm */ true) &&
2616	SelectArmIsNonZero(/* IsTrueArm */ false))
2617	return true;
2618	break;
2619	}
2620	case Instruction::PHI: {
2621	auto *PN = cast<PHINode>(Val: I);
2622	if (Q.IIQ.UseInstrInfo && isNonZeroRecurrence(PN))
2623	return true;
2624
2625	// Check if all incoming values are non-zero using recursion.
2626	SimplifyQuery RecQ = Q;
2627	unsigned NewDepth = std::max(a: Depth, b: MaxAnalysisRecursionDepth - 1);
2628	return llvm::all_of(Range: PN->operands(), P: [&](const Use &U) {
2629	if (U.get() == PN)
2630	return true;
2631	RecQ.CxtI = PN->getIncomingBlock(U)->getTerminator();
2632	// Check if the branch on the phi excludes zero.
2633	ICmpInst::Predicate Pred;
2634	Value *X;
2635	BasicBlock *TrueSucc, *FalseSucc;
2636	if (match(V: RecQ.CxtI,
2637	P: m_Br(C: m_c_ICmp(Pred, L: m_Specific(V: U.get()), R: m_Value(V&: X)),
2638	T: m_BasicBlock(V&: TrueSucc), F: m_BasicBlock(V&: FalseSucc)))) {
2639	// Check for cases of duplicate successors.
2640	if ((TrueSucc == PN->getParent()) != (FalseSucc == PN->getParent())) {
2641	// If we're using the false successor, invert the predicate.
2642	if (FalseSucc == PN->getParent())
2643	Pred = CmpInst::getInversePredicate(pred: Pred);
2644	if (cmpExcludesZero(Pred, RHS: X))
2645	return true;
2646	}
2647	}
2648	// Finally recurse on the edge and check it directly.
2649	return isKnownNonZero(V: U.get(), DemandedElts, Depth: NewDepth, Q: RecQ);
2650	});
2651	}
2652	case Instruction::ExtractElement:
2653	if (const auto *EEI = dyn_cast<ExtractElementInst>(Val: I)) {
2654	const Value *Vec = EEI->getVectorOperand();
2655	const Value *Idx = EEI->getIndexOperand();
2656	auto *CIdx = dyn_cast<ConstantInt>(Val: Idx);
2657	if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType())) {
2658	unsigned NumElts = VecTy->getNumElements();
2659	APInt DemandedVecElts = APInt::getAllOnes(numBits: NumElts);
2660	if (CIdx && CIdx->getValue().ult(RHS: NumElts))
2661	DemandedVecElts = APInt::getOneBitSet(numBits: NumElts, BitNo: CIdx->getZExtValue());
2662	return isKnownNonZero(V: Vec, DemandedElts: DemandedVecElts, Depth, Q);
2663	}
2664	}
2665	break;
2666	case Instruction::Freeze:
2667	return isKnownNonZero(V: I->getOperand(i: 0), Depth, Q) &&
2668	isGuaranteedNotToBePoison(V: I->getOperand(i: 0), AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT,
2669	Depth);
2670	case Instruction::Load: {
2671	auto *LI = cast<LoadInst>(Val: I);
2672	// A Load tagged with nonnull or dereferenceable with null pointer undefined
2673	// is never null.
2674	if (auto *PtrT = dyn_cast<PointerType>(Val: I->getType()))
2675	if (Q.IIQ.getMetadata(I: LI, KindID: LLVMContext::MD_nonnull) ||
2676	(Q.IIQ.getMetadata(I: LI, KindID: LLVMContext::MD_dereferenceable) &&
2677	!NullPointerIsDefined(F: LI->getFunction(), AS: PtrT->getAddressSpace())))
2678	return true;
2679
2680	// No need to fall through to computeKnownBits as range metadata is already
2681	// handled in isKnownNonZero.
2682	return false;
2683	}
2684	case Instruction::Call:
2685	case Instruction::Invoke:
2686	if (I->getType()->isPointerTy()) {
2687	const auto *Call = cast<CallBase>(Val: I);
2688	if (Call->isReturnNonNull())
2689	return true;
2690	if (const auto *RP = getArgumentAliasingToReturnedPointer(Call, MustPreserveNullness: true))
2691	return isKnownNonZero(V: RP, Depth, Q);
2692	} else if (const Value *RV = cast<CallBase>(Val: I)->getReturnedArgOperand()) {
2693	if (RV->getType() == I->getType() && isKnownNonZero(V: RV, Depth, Q))
2694	return true;
2695	}
2696
2697	if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
2698	switch (II->getIntrinsicID()) {
2699	case Intrinsic::sshl_sat:
2700	case Intrinsic::ushl_sat:
2701	case Intrinsic::abs:
2702	case Intrinsic::bitreverse:
2703	case Intrinsic::bswap:
2704	case Intrinsic::ctpop:
2705	return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Depth, Q);
2706	case Intrinsic::ssub_sat:
2707	return isNonZeroSub(DemandedElts, Depth, Q, BitWidth,
2708	X: II->getArgOperand(i: 0), Y: II->getArgOperand(i: 1));
2709	case Intrinsic::sadd_sat:
2710	return isNonZeroAdd(DemandedElts, Depth, Q, BitWidth,
2711	X: II->getArgOperand(i: 0), Y: II->getArgOperand(i: 1),
2712	/*NSW*/ true);
2713	case Intrinsic::umax:
2714	case Intrinsic::uadd_sat:
2715	return isKnownNonZero(V: II->getArgOperand(i: 1), DemandedElts, Depth, Q) ||
2716	isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Depth, Q);
2717	case Intrinsic::smin:
2718	case Intrinsic::smax: {
2719	auto KnownOpImpliesNonZero = [&](const KnownBits &K) {
2720	return II->getIntrinsicID() == Intrinsic::smin
2721	? K.isNegative()
2722	: K.isStrictlyPositive();
2723	};
2724	KnownBits XKnown =
2725	computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Depth, Q);
2726	if (KnownOpImpliesNonZero(XKnown))
2727	return true;
2728	KnownBits YKnown =
2729	computeKnownBits(V: II->getArgOperand(i: 1), DemandedElts, Depth, Q);
2730	if (KnownOpImpliesNonZero(YKnown))
2731	return true;
2732
2733	if (XKnown.isNonZero() && YKnown.isNonZero())
2734	return true;
2735	}
2736	[[fallthrough]];
2737	case Intrinsic::umin:
2738	return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Depth, Q) &&
2739	isKnownNonZero(V: II->getArgOperand(i: 1), DemandedElts, Depth, Q);
2740	case Intrinsic::cttz:
2741	return computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Depth, Q)
2742	.Zero[0];
2743	case Intrinsic::ctlz:
2744	return computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Depth, Q)
2745	.isNonNegative();
2746	case Intrinsic::fshr:
2747	case Intrinsic::fshl:
2748	// If Op0 == Op1, this is a rotate. rotate(x, y) != 0 iff x != 0.
2749	if (II->getArgOperand(i: 0) == II->getArgOperand(i: 1))
2750	return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Depth, Q);
2751	break;
2752	case Intrinsic::vscale:
2753	return true;
2754	case Intrinsic::experimental_get_vector_length:
2755	return isKnownNonZero(V: I->getOperand(i: 0), Depth, Q);
2756	default:
2757	break;
2758	}
2759	break;
2760	}
2761
2762	return false;
2763	}
2764
2765	KnownBits Known(BitWidth);
2766	computeKnownBits(V: I, DemandedElts, Known, Depth, Q);
2767	return Known.One != 0;
2768	}
2769
2770	/// Return true if the given value is known to be non-zero when defined. For
2771	/// vectors, return true if every demanded element is known to be non-zero when
2772	/// defined. For pointers, if the context instruction and dominator tree are
2773	/// specified, perform context-sensitive analysis and return true if the
2774	/// pointer couldn't possibly be null at the specified instruction.
2775	/// Supports values with integer or pointer type and vectors of integers.
2776	bool isKnownNonZero(const Value *V, const APInt &DemandedElts, unsigned Depth,
2777	const SimplifyQuery &Q) {
2778
2779	#ifndef NDEBUG
2780	Type *Ty = V->getType();
2781	assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
2782
2783	if (auto *FVTy = dyn_cast<FixedVectorType>(Val: Ty)) {
2784	assert(
2785	FVTy->getNumElements() == DemandedElts.getBitWidth() &&
2786	"DemandedElt width should equal the fixed vector number of elements");
2787	} else {
2788	assert(DemandedElts == APInt(1, 1) &&
2789	"DemandedElt width should be 1 for scalars");
2790	}
2791	#endif
2792
2793	if (auto *C = dyn_cast<Constant>(Val: V)) {
2794	if (C->isNullValue())
2795	return false;
2796	if (isa<ConstantInt>(Val: C))
2797	// Must be non-zero due to null test above.
2798	return true;
2799
2800	// For constant vectors, check that all elements are undefined or known
2801	// non-zero to determine that the whole vector is known non-zero.
2802	if (auto *VecTy = dyn_cast<FixedVectorType>(Val: C->getType())) {
2803	for (unsigned i = 0, e = VecTy->getNumElements(); i != e; ++i) {
2804	if (!DemandedElts[i])
2805	continue;
2806	Constant *Elt = C->getAggregateElement(Elt: i);
2807	if (!Elt || Elt->isNullValue())
2808	return false;
2809	if (!isa<UndefValue>(Val: Elt) && !isa<ConstantInt>(Val: Elt))
2810	return false;
2811	}
2812	return true;
2813	}
2814
2815	// A global variable in address space 0 is non null unless extern weak
2816	// or an absolute symbol reference. Other address spaces may have null as a
2817	// valid address for a global, so we can't assume anything.
2818	if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: V)) {
2819	if (!GV->isAbsoluteSymbolRef() && !GV->hasExternalWeakLinkage() &&
2820	GV->getType()->getAddressSpace() == 0)
2821	return true;
2822	}
2823
2824	// For constant expressions, fall through to the Operator code below.
2825	if (!isa<ConstantExpr>(Val: V))
2826	return false;
2827	}
2828
2829	if (auto *I = dyn_cast<Instruction>(Val: V)) {
2830	if (MDNode *Ranges = Q.IIQ.getMetadata(I, KindID: LLVMContext::MD_range)) {
2831	// If the possible ranges don't contain zero, then the value is
2832	// definitely non-zero.
2833	if (auto *Ty = dyn_cast<IntegerType>(Val: V->getType())) {
2834	const APInt ZeroValue(Ty->getBitWidth(), 0);
2835	if (rangeMetadataExcludesValue(Ranges, Value: ZeroValue))
2836	return true;
2837	}
2838	}
2839	}
2840
2841	if (!isa<Constant>(Val: V) && isKnownNonZeroFromAssume(V, Q))
2842	return true;
2843
2844	// Some of the tests below are recursive, so bail out if we hit the limit.
2845	if (Depth++ >= MaxAnalysisRecursionDepth)
2846	return false;
2847
2848	// Check for pointer simplifications.
2849
2850	if (PointerType *PtrTy = dyn_cast<PointerType>(Val: V->getType())) {
2851	// A byval, inalloca may not be null in a non-default addres space. A
2852	// nonnull argument is assumed never 0.
2853	if (const Argument *A = dyn_cast<Argument>(Val: V)) {
2854	if (((A->hasPassPointeeByValueCopyAttr() &&
2855	!NullPointerIsDefined(F: A->getParent(), AS: PtrTy->getAddressSpace())) ||
2856	A->hasNonNullAttr()))
2857	return true;
2858	}
2859	}
2860
2861	if (const auto *I = dyn_cast<Operator>(Val: V))
2862	if (isKnownNonZeroFromOperator(I, DemandedElts, Depth, Q))
2863	return true;
2864
2865	if (!isa<Constant>(Val: V) &&
2866	isKnownNonNullFromDominatingCondition(V, CtxI: Q.CxtI, DT: Q.DT))
2867	return true;
2868
2869	return false;
2870	}
2871
2872	bool isKnownNonZero(const Value *V, unsigned Depth, const SimplifyQuery &Q) {
2873	auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
2874	APInt DemandedElts =
2875	FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
2876	return isKnownNonZero(V, DemandedElts, Depth, Q);
2877	}
2878
2879	/// If the pair of operators are the same invertible function, return the
2880	/// the operands of the function corresponding to each input. Otherwise,
2881	/// return std::nullopt. An invertible function is one that is 1-to-1 and maps
2882	/// every input value to exactly one output value. This is equivalent to
2883	/// saying that Op1 and Op2 are equal exactly when the specified pair of
2884	/// operands are equal, (except that Op1 and Op2 may be poison more often.)
2885	static std::optional<std::pair<Value*, Value*>>
2886	getInvertibleOperands(const Operator *Op1,
2887	const Operator *Op2) {
2888	if (Op1->getOpcode() != Op2->getOpcode())
2889	return std::nullopt;
2890
2891	auto getOperands = [&](unsigned OpNum) -> auto {
2892	return std::make_pair(x: Op1->getOperand(i: OpNum), y: Op2->getOperand(i: OpNum));
2893	};
2894
2895	switch (Op1->getOpcode()) {
2896	default:
2897	break;
2898	case Instruction::Add:
2899	case Instruction::Sub:
2900	if (Op1->getOperand(i: 0) == Op2->getOperand(i: 0))
2901	return getOperands(1);
2902	if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1))
2903	return getOperands(0);
2904	break;
2905	case Instruction::Mul: {
2906	// invertible if A * B == (A * B) mod 2^N where A, and B are integers
2907	// and N is the bitwdith. The nsw case is non-obvious, but proven by
2908	// alive2: https://alive2.llvm.org/ce/z/Z6D5qK
2909	auto *OBO1 = cast<OverflowingBinaryOperator>(Val: Op1);
2910	auto *OBO2 = cast<OverflowingBinaryOperator>(Val: Op2);
2911	if ((!OBO1->hasNoUnsignedWrap() || !OBO2->hasNoUnsignedWrap()) &&
2912	(!OBO1->hasNoSignedWrap() || !OBO2->hasNoSignedWrap()))
2913	break;
2914
2915	// Assume operand order has been canonicalized
2916	if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1) &&
2917	isa<ConstantInt>(Val: Op1->getOperand(i: 1)) &&
2918	!cast<ConstantInt>(Val: Op1->getOperand(i: 1))->isZero())
2919	return getOperands(0);
2920	break;
2921	}
2922	case Instruction::Shl: {
2923	// Same as multiplies, with the difference that we don't need to check
2924	// for a non-zero multiply. Shifts always multiply by non-zero.
2925	auto *OBO1 = cast<OverflowingBinaryOperator>(Val: Op1);
2926	auto *OBO2 = cast<OverflowingBinaryOperator>(Val: Op2);
2927	if ((!OBO1->hasNoUnsignedWrap() || !OBO2->hasNoUnsignedWrap()) &&
2928	(!OBO1->hasNoSignedWrap() || !OBO2->hasNoSignedWrap()))
2929	break;
2930
2931	if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1))
2932	return getOperands(0);
2933	break;
2934	}
2935	case Instruction::AShr:
2936	case Instruction::LShr: {
2937	auto *PEO1 = cast<PossiblyExactOperator>(Val: Op1);
2938	auto *PEO2 = cast<PossiblyExactOperator>(Val: Op2);
2939	if (!PEO1->isExact() || !PEO2->isExact())
2940	break;
2941
2942	if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1))
2943	return getOperands(0);
2944	break;
2945	}
2946	case Instruction::SExt:
2947	case Instruction::ZExt:
2948	if (Op1->getOperand(i: 0)->getType() == Op2->getOperand(i: 0)->getType())
2949	return getOperands(0);
2950	break;
2951	case Instruction::PHI: {
2952	const PHINode *PN1 = cast<PHINode>(Val: Op1);
2953	const PHINode *PN2 = cast<PHINode>(Val: Op2);
2954
2955	// If PN1 and PN2 are both recurrences, can we prove the entire recurrences
2956	// are a single invertible function of the start values? Note that repeated
2957	// application of an invertible function is also invertible
2958	BinaryOperator *BO1 = nullptr;
2959	Value *Start1 = nullptr, *Step1 = nullptr;
2960	BinaryOperator *BO2 = nullptr;
2961	Value *Start2 = nullptr, *Step2 = nullptr;
2962	if (PN1->getParent() != PN2->getParent() ||
2963	!matchSimpleRecurrence(P: PN1, BO&: BO1, Start&: Start1, Step&: Step1) ||
2964	!matchSimpleRecurrence(P: PN2, BO&: BO2, Start&: Start2, Step&: Step2))
2965	break;
2966
2967	auto Values = getInvertibleOperands(Op1: cast<Operator>(Val: BO1),
2968	Op2: cast<Operator>(Val: BO2));
2969	if (!Values)
2970	break;
2971
2972	// We have to be careful of mutually defined recurrences here. Ex:
2973	// * X_i = X_(i-1) OP Y_(i-1), and Y_i = X_(i-1) OP V
2974	// * X_i = Y_i = X_(i-1) OP Y_(i-1)
2975	// The invertibility of these is complicated, and not worth reasoning
2976	// about (yet?).
2977	if (Values->first != PN1 || Values->second != PN2)
2978	break;
2979
2980	return std::make_pair(x&: Start1, y&: Start2);
2981	}
2982	}
2983	return std::nullopt;
2984	}
2985
2986	/// Return true if V2 == V1 + X, where X is known non-zero.
2987	static bool isAddOfNonZero(const Value *V1, const Value *V2, unsigned Depth,
2988	const SimplifyQuery &Q) {
2989	const BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: V1);
2990	if (!BO || BO->getOpcode() != Instruction::Add)
2991	return false;
2992	Value *Op = nullptr;
2993	if (V2 == BO->getOperand(i_nocapture: 0))
2994	Op = BO->getOperand(i_nocapture: 1);
2995	else if (V2 == BO->getOperand(i_nocapture: 1))
2996	Op = BO->getOperand(i_nocapture: 0);
2997	else
2998	return false;
2999	return isKnownNonZero(V: Op, Depth: Depth + 1, Q);
3000	}
3001
3002	/// Return true if V2 == V1 * C, where V1 is known non-zero, C is not 0/1 and
3003	/// the multiplication is nuw or nsw.
3004	static bool isNonEqualMul(const Value *V1, const Value *V2, unsigned Depth,
3005	const SimplifyQuery &Q) {
3006	if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: V2)) {
3007	const APInt *C;
3008	return match(V: OBO, P: m_Mul(L: m_Specific(V: V1), R: m_APInt(Res&: C))) &&
3009	(OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) &&
3010	!C->isZero() && !C->isOne() && isKnownNonZero(V: V1, Depth: Depth + 1, Q);
3011	}
3012	return false;
3013	}
3014
3015	/// Return true if V2 == V1 << C, where V1 is known non-zero, C is not 0 and
3016	/// the shift is nuw or nsw.
3017	static bool isNonEqualShl(const Value *V1, const Value *V2, unsigned Depth,
3018	const SimplifyQuery &Q) {
3019	if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: V2)) {
3020	const APInt *C;
3021	return match(V: OBO, P: m_Shl(L: m_Specific(V: V1), R: m_APInt(Res&: C))) &&
3022	(OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) &&
3023	!C->isZero() && isKnownNonZero(V: V1, Depth: Depth + 1, Q);
3024	}
3025	return false;
3026	}
3027
3028	static bool isNonEqualPHIs(const PHINode *PN1, const PHINode *PN2,
3029	unsigned Depth, const SimplifyQuery &Q) {
3030	// Check two PHIs are in same block.
3031	if (PN1->getParent() != PN2->getParent())
3032	return false;
3033
3034	SmallPtrSet<const BasicBlock *, 8> VisitedBBs;
3035	bool UsedFullRecursion = false;
3036	for (const BasicBlock *IncomBB : PN1->blocks()) {
3037	if (!VisitedBBs.insert(Ptr: IncomBB).second)
3038	continue; // Don't reprocess blocks that we have dealt with already.
3039	const Value *IV1 = PN1->getIncomingValueForBlock(BB: IncomBB);
3040	const Value *IV2 = PN2->getIncomingValueForBlock(BB: IncomBB);
3041	const APInt *C1, *C2;
3042	if (match(V: IV1, P: m_APInt(Res&: C1)) && match(V: IV2, P: m_APInt(Res&: C2)) && *C1 != *C2)
3043	continue;
3044
3045	// Only one pair of phi operands is allowed for full recursion.
3046	if (UsedFullRecursion)
3047	return false;
3048
3049	SimplifyQuery RecQ = Q;
3050	RecQ.CxtI = IncomBB->getTerminator();
3051	if (!isKnownNonEqual(V1: IV1, V2: IV2, Depth: Depth + 1, Q: RecQ))
3052	return false;
3053	UsedFullRecursion = true;
3054	}
3055	return true;
3056	}
3057
3058	static bool isNonEqualSelect(const Value *V1, const Value *V2, unsigned Depth,
3059	const SimplifyQuery &Q) {
3060	const SelectInst *SI1 = dyn_cast<SelectInst>(Val: V1);
3061	if (!SI1)
3062	return false;
3063
3064	if (const SelectInst *SI2 = dyn_cast<SelectInst>(Val: V2)) {
3065	const Value *Cond1 = SI1->getCondition();
3066	const Value *Cond2 = SI2->getCondition();
3067	if (Cond1 == Cond2)
3068	return isKnownNonEqual(V1: SI1->getTrueValue(), V2: SI2->getTrueValue(),
3069	Depth: Depth + 1, Q) &&
3070	isKnownNonEqual(V1: SI1->getFalseValue(), V2: SI2->getFalseValue(),
3071	Depth: Depth + 1, Q);
3072	}
3073	return isKnownNonEqual(V1: SI1->getTrueValue(), V2, Depth: Depth + 1, Q) &&
3074	isKnownNonEqual(V1: SI1->getFalseValue(), V2, Depth: Depth + 1, Q);
3075	}
3076
3077	// Check to see if A is both a GEP and is the incoming value for a PHI in the
3078	// loop, and B is either a ptr or another GEP. If the PHI has 2 incoming values,
3079	// one of them being the recursive GEP A and the other a ptr at same base and at
3080	// the same/higher offset than B we are only incrementing the pointer further in
3081	// loop if offset of recursive GEP is greater than 0.
3082	static bool isNonEqualPointersWithRecursiveGEP(const Value *A, const Value *B,
3083	const SimplifyQuery &Q) {
3084	if (!A->getType()->isPointerTy() || !B->getType()->isPointerTy())
3085	return false;
3086
3087	auto *GEPA = dyn_cast<GEPOperator>(Val: A);
3088	if (!GEPA || GEPA->getNumIndices() != 1 || !isa<Constant>(Val: GEPA->idx_begin()))
3089	return false;
3090
3091	// Handle 2 incoming PHI values with one being a recursive GEP.
3092	auto *PN = dyn_cast<PHINode>(Val: GEPA->getPointerOperand());
3093	if (!PN || PN->getNumIncomingValues() != 2)
3094	return false;
3095
3096	// Search for the recursive GEP as an incoming operand, and record that as
3097	// Step.
3098	Value *Start = nullptr;
3099	Value *Step = const_cast<Value *>(A);
3100	if (PN->getIncomingValue(i: 0) == Step)
3101	Start = PN->getIncomingValue(i: 1);
3102	else if (PN->getIncomingValue(i: 1) == Step)
3103	Start = PN->getIncomingValue(i: 0);
3104	else
3105	return false;
3106
3107	// Other incoming node base should match the B base.
3108	// StartOffset >= OffsetB && StepOffset > 0?
3109	// StartOffset <= OffsetB && StepOffset < 0?
3110	// Is non-equal if above are true.
3111	// We use stripAndAccumulateInBoundsConstantOffsets to restrict the
3112	// optimisation to inbounds GEPs only.
3113	unsigned IndexWidth = Q.DL.getIndexTypeSizeInBits(Ty: Start->getType());
3114	APInt StartOffset(IndexWidth, 0);
3115	Start = Start->stripAndAccumulateInBoundsConstantOffsets(DL: Q.DL, Offset&: StartOffset);
3116	APInt StepOffset(IndexWidth, 0);
3117	Step = Step->stripAndAccumulateInBoundsConstantOffsets(DL: Q.DL, Offset&: StepOffset);
3118
3119	// Check if Base Pointer of Step matches the PHI.
3120	if (Step != PN)
3121	return false;
3122	APInt OffsetB(IndexWidth, 0);
3123	B = B->stripAndAccumulateInBoundsConstantOffsets(DL: Q.DL, Offset&: OffsetB);
3124	return Start == B &&
3125	((StartOffset.sge(RHS: OffsetB) && StepOffset.isStrictlyPositive()) ||
3126	(StartOffset.sle(RHS: OffsetB) && StepOffset.isNegative()));
3127	}
3128
3129	/// Return true if it is known that V1 != V2.
3130	static bool isKnownNonEqual(const Value *V1, const Value *V2, unsigned Depth,
3131	const SimplifyQuery &Q) {
3132	if (V1 == V2)
3133	return false;
3134	if (V1->getType() != V2->getType())
3135	// We can't look through casts yet.
3136	return false;
3137
3138	if (Depth >= MaxAnalysisRecursionDepth)
3139	return false;
3140
3141	// See if we can recurse through (exactly one of) our operands. This
3142	// requires our operation be 1-to-1 and map every input value to exactly
3143	// one output value. Such an operation is invertible.
3144	auto *O1 = dyn_cast<Operator>(Val: V1);
3145	auto *O2 = dyn_cast<Operator>(Val: V2);
3146	if (O1 && O2 && O1->getOpcode() == O2->getOpcode()) {
3147	if (auto Values = getInvertibleOperands(Op1: O1, Op2: O2))
3148	return isKnownNonEqual(V1: Values->first, V2: Values->second, Depth: Depth + 1, Q);
3149
3150	if (const PHINode *PN1 = dyn_cast<PHINode>(Val: V1)) {
3151	const PHINode *PN2 = cast<PHINode>(Val: V2);
3152	// FIXME: This is missing a generalization to handle the case where one is
3153	// a PHI and another one isn't.
3154	if (isNonEqualPHIs(PN1, PN2, Depth, Q))
3155	return true;
3156	};
3157	}
3158
3159	if (isAddOfNonZero(V1, V2, Depth, Q) || isAddOfNonZero(V1: V2, V2: V1, Depth, Q))
3160	return true;
3161
3162	if (isNonEqualMul(V1, V2, Depth, Q) || isNonEqualMul(V1: V2, V2: V1, Depth, Q))
3163	return true;
3164
3165	if (isNonEqualShl(V1, V2, Depth, Q) || isNonEqualShl(V1: V2, V2: V1, Depth, Q))
3166	return true;
3167
3168	if (V1->getType()->isIntOrIntVectorTy()) {
3169	// Are any known bits in V1 contradictory to known bits in V2? If V1
3170	// has a known zero where V2 has a known one, they must not be equal.
3171	KnownBits Known1 = computeKnownBits(V: V1, Depth, Q);
3172	if (!Known1.isUnknown()) {
3173	KnownBits Known2 = computeKnownBits(V: V2, Depth, Q);
3174	if (Known1.Zero.intersects(RHS: Known2.One) ||
3175	Known2.Zero.intersects(RHS: Known1.One))
3176	return true;
3177	}
3178	}
3179
3180	if (isNonEqualSelect(V1, V2, Depth, Q) || isNonEqualSelect(V1: V2, V2: V1, Depth, Q))
3181	return true;
3182
3183	if (isNonEqualPointersWithRecursiveGEP(A: V1, B: V2, Q) ||
3184	isNonEqualPointersWithRecursiveGEP(A: V2, B: V1, Q))
3185	return true;
3186
3187	Value *A, *B;
3188	// PtrToInts are NonEqual if their Ptrs are NonEqual.
3189	// Check PtrToInt type matches the pointer size.
3190	if (match(V: V1, P: m_PtrToIntSameSize(DL: Q.DL, Op: m_Value(V&: A))) &&
3191	match(V: V2, P: m_PtrToIntSameSize(DL: Q.DL, Op: m_Value(V&: B))))
3192	return isKnownNonEqual(V1: A, V2: B, Depth: Depth + 1, Q);
3193
3194	return false;
3195	}
3196
3197	// Match a signed min+max clamp pattern like smax(smin(In, CHigh), CLow).
3198	// Returns the input and lower/upper bounds.
3199	static bool isSignedMinMaxClamp(const Value *Select, const Value *&In,
3200	const APInt *&CLow, const APInt *&CHigh) {
3201	assert(isa<Operator>(Select) &&
3202	cast<Operator>(Select)->getOpcode() == Instruction::Select &&
3203	"Input should be a Select!");
3204
3205	const Value *LHS = nullptr, *RHS = nullptr;
3206	SelectPatternFlavor SPF = matchSelectPattern(V: Select, LHS, RHS).Flavor;
3207	if (SPF != SPF_SMAX && SPF != SPF_SMIN)
3208	return false;
3209
3210	if (!match(V: RHS, P: m_APInt(Res&: CLow)))
3211	return false;
3212
3213	const Value *LHS2 = nullptr, *RHS2 = nullptr;
3214	SelectPatternFlavor SPF2 = matchSelectPattern(V: LHS, LHS&: LHS2, RHS&: RHS2).Flavor;
3215	if (getInverseMinMaxFlavor(SPF) != SPF2)
3216	return false;
3217
3218	if (!match(V: RHS2, P: m_APInt(Res&: CHigh)))
3219	return false;
3220
3221	if (SPF == SPF_SMIN)
3222	std::swap(a&: CLow, b&: CHigh);
3223
3224	In = LHS2;
3225	return CLow->sle(RHS: *CHigh);
3226	}
3227
3228	static bool isSignedMinMaxIntrinsicClamp(const IntrinsicInst *II,
3229	const APInt *&CLow,
3230	const APInt *&CHigh) {
3231	assert((II->getIntrinsicID() == Intrinsic::smin ||
3232	II->getIntrinsicID() == Intrinsic::smax) && "Must be smin/smax");
3233
3234	Intrinsic::ID InverseID = getInverseMinMaxIntrinsic(MinMaxID: II->getIntrinsicID());
3235	auto *InnerII = dyn_cast<IntrinsicInst>(Val: II->getArgOperand(i: 0));
3236	if (!InnerII || InnerII->getIntrinsicID() != InverseID ||
3237	!match(V: II->getArgOperand(i: 1), P: m_APInt(Res&: CLow)) ||
3238	!match(V: InnerII->getArgOperand(i: 1), P: m_APInt(Res&: CHigh)))
3239	return false;
3240
3241	if (II->getIntrinsicID() == Intrinsic::smin)
3242	std::swap(a&: CLow, b&: CHigh);
3243	return CLow->sle(RHS: *CHigh);
3244	}
3245
3246	/// For vector constants, loop over the elements and find the constant with the
3247	/// minimum number of sign bits. Return 0 if the value is not a vector constant
3248	/// or if any element was not analyzed; otherwise, return the count for the
3249	/// element with the minimum number of sign bits.
3250	static unsigned computeNumSignBitsVectorConstant(const Value *V,
3251	const APInt &DemandedElts,
3252	unsigned TyBits) {
3253	const auto *CV = dyn_cast<Constant>(Val: V);
3254	if (!CV || !isa<FixedVectorType>(Val: CV->getType()))
3255	return 0;
3256
3257	unsigned MinSignBits = TyBits;
3258	unsigned NumElts = cast<FixedVectorType>(Val: CV->getType())->getNumElements();
3259	for (unsigned i = 0; i != NumElts; ++i) {
3260	if (!DemandedElts[i])
3261	continue;
3262	// If we find a non-ConstantInt, bail out.
3263	auto *Elt = dyn_cast_or_null<ConstantInt>(Val: CV->getAggregateElement(Elt: i));
3264	if (!Elt)
3265	return 0;
3266
3267	MinSignBits = std::min(a: MinSignBits, b: Elt->getValue().getNumSignBits());
3268	}
3269
3270	return MinSignBits;
3271	}
3272
3273	static unsigned ComputeNumSignBitsImpl(const Value *V,
3274	const APInt &DemandedElts,
3275	unsigned Depth, const SimplifyQuery &Q);
3276
3277	static unsigned ComputeNumSignBits(const Value *V, const APInt &DemandedElts,
3278	unsigned Depth, const SimplifyQuery &Q) {
3279	unsigned Result = ComputeNumSignBitsImpl(V, DemandedElts, Depth, Q);
3280	assert(Result > 0 && "At least one sign bit needs to be present!");
3281	return Result;
3282	}
3283
3284	/// Return the number of times the sign bit of the register is replicated into
3285	/// the other bits. We know that at least 1 bit is always equal to the sign bit
3286	/// (itself), but other cases can give us information. For example, immediately
3287	/// after an "ashr X, 2", we know that the top 3 bits are all equal to each
3288	/// other, so we return 3. For vectors, return the number of sign bits for the
3289	/// vector element with the minimum number of known sign bits of the demanded
3290	/// elements in the vector specified by DemandedElts.
3291	static unsigned ComputeNumSignBitsImpl(const Value *V,
3292	const APInt &DemandedElts,
3293	unsigned Depth, const SimplifyQuery &Q) {
3294	Type *Ty = V->getType();
3295	#ifndef NDEBUG
3296	assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
3297
3298	if (auto *FVTy = dyn_cast<FixedVectorType>(Val: Ty)) {
3299	assert(
3300	FVTy->getNumElements() == DemandedElts.getBitWidth() &&
3301	"DemandedElt width should equal the fixed vector number of elements");
3302	} else {
3303	assert(DemandedElts == APInt(1, 1) &&
3304	"DemandedElt width should be 1 for scalars");
3305	}
3306	#endif
3307
3308	// We return the minimum number of sign bits that are guaranteed to be present
3309	// in V, so for undef we have to conservatively return 1. We don't have the
3310	// same behavior for poison though -- that's a FIXME today.
3311
3312	Type *ScalarTy = Ty->getScalarType();
3313	unsigned TyBits = ScalarTy->isPointerTy() ?
3314	Q.DL.getPointerTypeSizeInBits(ScalarTy) :
3315	Q.DL.getTypeSizeInBits(Ty: ScalarTy);
3316
3317	unsigned Tmp, Tmp2;
3318	unsigned FirstAnswer = 1;
3319
3320	// Note that ConstantInt is handled by the general computeKnownBits case
3321	// below.
3322
3323	if (Depth == MaxAnalysisRecursionDepth)
3324	return 1;
3325
3326	if (auto *U = dyn_cast<Operator>(Val: V)) {
3327	switch (Operator::getOpcode(V)) {
3328	default: break;
3329	case Instruction::SExt:
3330	Tmp = TyBits - U->getOperand(i: 0)->getType()->getScalarSizeInBits();
3331	return ComputeNumSignBits(V: U->getOperand(i: 0), Depth: Depth + 1, Q) + Tmp;
3332
3333	case Instruction::SDiv: {
3334	const APInt *Denominator;
3335	// sdiv X, C -> adds log(C) sign bits.
3336	if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: Denominator))) {
3337
3338	// Ignore non-positive denominator.
3339	if (!Denominator->isStrictlyPositive())
3340	break;
3341
3342	// Calculate the incoming numerator bits.
3343	unsigned NumBits = ComputeNumSignBits(V: U->getOperand(i: 0), Depth: Depth + 1, Q);
3344
3345	// Add floor(log(C)) bits to the numerator bits.
3346	return std::min(a: TyBits, b: NumBits + Denominator->logBase2());
3347	}
3348	break;
3349	}
3350
3351	case Instruction::SRem: {
3352	Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), Depth: Depth + 1, Q);
3353
3354	const APInt *Denominator;
3355	// srem X, C -> we know that the result is within [-C+1,C) when C is a
3356	// positive constant. This let us put a lower bound on the number of sign
3357	// bits.
3358	if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: Denominator))) {
3359
3360	// Ignore non-positive denominator.
3361	if (Denominator->isStrictlyPositive()) {
3362	// Calculate the leading sign bit constraints by examining the
3363	// denominator. Given that the denominator is positive, there are two
3364	// cases:
3365	//
3366	// 1. The numerator is positive. The result range is [0,C) and
3367	// [0,C) u< (1 << ceilLogBase2(C)).
3368	//
3369	// 2. The numerator is negative. Then the result range is (-C,0] and
3370	// integers in (-C,0] are either 0 or >u (-1 << ceilLogBase2(C)).
3371	//
3372	// Thus a lower bound on the number of sign bits is `TyBits -
3373	// ceilLogBase2(C)`.
3374
3375	unsigned ResBits = TyBits - Denominator->ceilLogBase2();
3376	Tmp = std::max(a: Tmp, b: ResBits);
3377	}
3378	}
3379	return Tmp;
3380	}
3381
3382	case Instruction::AShr: {
3383	Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), Depth: Depth + 1, Q);
3384	// ashr X, C -> adds C sign bits. Vectors too.
3385	const APInt *ShAmt;
3386	if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: ShAmt))) {
3387	if (ShAmt->uge(RHS: TyBits))
3388	break; // Bad shift.
3389	unsigned ShAmtLimited = ShAmt->getZExtValue();
3390	Tmp += ShAmtLimited;
3391	if (Tmp > TyBits) Tmp = TyBits;
3392	}
3393	return Tmp;
3394	}
3395	case Instruction::Shl: {
3396	const APInt *ShAmt;
3397	if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: ShAmt))) {
3398	// shl destroys sign bits.
3399	Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), Depth: Depth + 1, Q);
3400	if (ShAmt->uge(RHS: TyBits) || // Bad shift.
3401	ShAmt->uge(RHS: Tmp)) break; // Shifted all sign bits out.
3402	Tmp2 = ShAmt->getZExtValue();
3403	return Tmp - Tmp2;
3404	}
3405	break;
3406	}
3407	case Instruction::And:
3408	case Instruction::Or:
3409	case Instruction::Xor: // NOT is handled here.
3410	// Logical binary ops preserve the number of sign bits at the worst.
3411	Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), Depth: Depth + 1, Q);
3412	if (Tmp != 1) {
3413	Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 1), Depth: Depth + 1, Q);
3414	FirstAnswer = std::min(a: Tmp, b: Tmp2);
3415	// We computed what we know about the sign bits as our first
3416	// answer. Now proceed to the generic code that uses
3417	// computeKnownBits, and pick whichever answer is better.
3418	}
3419	break;
3420
3421	case Instruction::Select: {
3422	// If we have a clamp pattern, we know that the number of sign bits will
3423	// be the minimum of the clamp min/max range.
3424	const Value *X;
3425	const APInt *CLow, *CHigh;
3426	if (isSignedMinMaxClamp(Select: U, In&: X, CLow, CHigh))
3427	return std::min(a: CLow->getNumSignBits(), b: CHigh->getNumSignBits());
3428
3429	Tmp = ComputeNumSignBits(V: U->getOperand(i: 1), Depth: Depth + 1, Q);
3430	if (Tmp == 1) break;
3431	Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 2), Depth: Depth + 1, Q);
3432	return std::min(a: Tmp, b: Tmp2);
3433	}
3434
3435	case Instruction::Add:
3436	// Add can have at most one carry bit. Thus we know that the output
3437	// is, at worst, one more bit than the inputs.
3438	Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), Depth: Depth + 1, Q);
3439	if (Tmp == 1) break;
3440
3441	// Special case decrementing a value (ADD X, -1):
3442	if (const auto *CRHS = dyn_cast<Constant>(Val: U->getOperand(i: 1)))
3443	if (CRHS->isAllOnesValue()) {
3444	KnownBits Known(TyBits);
3445	computeKnownBits(V: U->getOperand(i: 0), Known, Depth: Depth + 1, Q);
3446
3447	// If the input is known to be 0 or 1, the output is 0/-1, which is
3448	// all sign bits set.
3449	if ((Known.Zero | 1).isAllOnes())
3450	return TyBits;
3451
3452	// If we are subtracting one from a positive number, there is no carry
3453	// out of the result.
3454	if (Known.isNonNegative())
3455	return Tmp;
3456	}
3457
3458	Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 1), Depth: Depth + 1, Q);
3459	if (Tmp2 == 1) break;
3460	return std::min(a: Tmp, b: Tmp2) - 1;
3461
3462	case Instruction::Sub:
3463	Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 1), Depth: Depth + 1, Q);
3464	if (Tmp2 == 1) break;
3465
3466	// Handle NEG.
3467	if (const auto *CLHS = dyn_cast<Constant>(Val: U->getOperand(i: 0)))
3468	if (CLHS->isNullValue()) {
3469	KnownBits Known(TyBits);
3470	computeKnownBits(V: U->getOperand(i: 1), Known, Depth: Depth + 1, Q);
3471	// If the input is known to be 0 or 1, the output is 0/-1, which is
3472	// all sign bits set.
3473	if ((Known.Zero | 1).isAllOnes())
3474	return TyBits;
3475
3476	// If the input is known to be positive (the sign bit is known clear),
3477	// the output of the NEG has the same number of sign bits as the
3478	// input.
3479	if (Known.isNonNegative())
3480	return Tmp2;
3481
3482	// Otherwise, we treat this like a SUB.
3483	}
3484
3485	// Sub can have at most one carry bit. Thus we know that the output
3486	// is, at worst, one more bit than the inputs.
3487	Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), Depth: Depth + 1, Q);
3488	if (Tmp == 1) break;
3489	return std::min(a: Tmp, b: Tmp2) - 1;
3490
3491	case Instruction::Mul: {
3492	// The output of the Mul can be at most twice the valid bits in the
3493	// inputs.
3494	unsigned SignBitsOp0 = ComputeNumSignBits(V: U->getOperand(i: 0), Depth: Depth + 1, Q);
3495	if (SignBitsOp0 == 1) break;
3496	unsigned SignBitsOp1 = ComputeNumSignBits(V: U->getOperand(i: 1), Depth: Depth + 1, Q);
3497	if (SignBitsOp1 == 1) break;
3498	unsigned OutValidBits =
3499	(TyBits - SignBitsOp0 + 1) + (TyBits - SignBitsOp1 + 1);
3500	return OutValidBits > TyBits ? 1 : TyBits - OutValidBits + 1;
3501	}
3502
3503	case Instruction::PHI: {
3504	const PHINode *PN = cast<PHINode>(Val: U);
3505	unsigned NumIncomingValues = PN->getNumIncomingValues();
3506	// Don't analyze large in-degree PHIs.
3507	if (NumIncomingValues > 4) break;
3508	// Unreachable blocks may have zero-operand PHI nodes.
3509	if (NumIncomingValues == 0) break;
3510
3511	// Take the minimum of all incoming values. This can't infinitely loop
3512	// because of our depth threshold.
3513	SimplifyQuery RecQ = Q;
3514	Tmp = TyBits;
3515	for (unsigned i = 0, e = NumIncomingValues; i != e; ++i) {
3516	if (Tmp == 1) return Tmp;
3517	RecQ.CxtI = PN->getIncomingBlock(i)->getTerminator();
3518	Tmp = std::min(
3519	a: Tmp, b: ComputeNumSignBits(V: PN->getIncomingValue(i), Depth: Depth + 1, Q: RecQ));
3520	}
3521	return Tmp;
3522	}
3523
3524	case Instruction::Trunc: {
3525	// If the input contained enough sign bits that some remain after the
3526	// truncation, then we can make use of that. Otherwise we don't know
3527	// anything.
3528	Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), Depth: Depth + 1, Q);
3529	unsigned OperandTyBits = U->getOperand(i: 0)->getType()->getScalarSizeInBits();
3530	if (Tmp > (OperandTyBits - TyBits))
3531	return Tmp - (OperandTyBits - TyBits);
3532
3533	return 1;
3534	}
3535
3536	case Instruction::ExtractElement:
3537	// Look through extract element. At the moment we keep this simple and
3538	// skip tracking the specific element. But at least we might find
3539	// information valid for all elements of the vector (for example if vector
3540	// is sign extended, shifted, etc).
3541	return ComputeNumSignBits(V: U->getOperand(i: 0), Depth: Depth + 1, Q);
3542
3543	case Instruction::ShuffleVector: {
3544	// Collect the minimum number of sign bits that are shared by every vector
3545	// element referenced by the shuffle.
3546	auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: U);
3547	if (!Shuf) {
3548	// FIXME: Add support for shufflevector constant expressions.
3549	return 1;
3550	}
3551	APInt DemandedLHS, DemandedRHS;
3552	// For undef elements, we don't know anything about the common state of
3553	// the shuffle result.
3554	if (!getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS))
3555	return 1;
3556	Tmp = std::numeric_limits<unsigned>::max();
3557	if (!!DemandedLHS) {
3558	const Value *LHS = Shuf->getOperand(i_nocapture: 0);
3559	Tmp = ComputeNumSignBits(V: LHS, DemandedElts: DemandedLHS, Depth: Depth + 1, Q);
3560	}
3561	// If we don't know anything, early out and try computeKnownBits
3562	// fall-back.
3563	if (Tmp == 1)
3564	break;
3565	if (!!DemandedRHS) {
3566	const Value *RHS = Shuf->getOperand(i_nocapture: 1);
3567	Tmp2 = ComputeNumSignBits(V: RHS, DemandedElts: DemandedRHS, Depth: Depth + 1, Q);
3568	Tmp = std::min(a: Tmp, b: Tmp2);
3569	}
3570	// If we don't know anything, early out and try computeKnownBits
3571	// fall-back.
3572	if (Tmp == 1)
3573	break;
3574	assert(Tmp <= TyBits && "Failed to determine minimum sign bits");
3575	return Tmp;
3576	}
3577	case Instruction::Call: {
3578	if (const auto *II = dyn_cast<IntrinsicInst>(Val: U)) {
3579	switch (II->getIntrinsicID()) {
3580	default: break;
3581	case Intrinsic::abs:
3582	Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), Depth: Depth + 1, Q);
3583	if (Tmp == 1) break;
3584
3585	// Absolute value reduces number of sign bits by at most 1.
3586	return Tmp - 1;
3587	case Intrinsic::smin:
3588	case Intrinsic::smax: {
3589	const APInt *CLow, *CHigh;
3590	if (isSignedMinMaxIntrinsicClamp(II, CLow, CHigh))
3591	return std::min(a: CLow->getNumSignBits(), b: CHigh->getNumSignBits());
3592	}
3593	}
3594	}
3595	}
3596	}
3597	}
3598
3599	// Finally, if we can prove that the top bits of the result are 0's or 1's,
3600	// use this information.
3601
3602	// If we can examine all elements of a vector constant successfully, we're
3603	// done (we can't do any better than that). If not, keep trying.
3604	if (unsigned VecSignBits =
3605	computeNumSignBitsVectorConstant(V, DemandedElts, TyBits))
3606	return VecSignBits;
3607
3608	KnownBits Known(TyBits);
3609	computeKnownBits(V, DemandedElts, Known, Depth, Q);
3610
3611	// If we know that the sign bit is either zero or one, determine the number of
3612	// identical bits in the top of the input value.
3613	return std::max(a: FirstAnswer, b: Known.countMinSignBits());
3614	}
3615
3616	Intrinsic::ID llvm::getIntrinsicForCallSite(const CallBase &CB,
3617	const TargetLibraryInfo *TLI) {
3618	const Function *F = CB.getCalledFunction();
3619	if (!F)
3620	return Intrinsic::not_intrinsic;
3621
3622	if (F->isIntrinsic())
3623	return F->getIntrinsicID();
3624
3625	// We are going to infer semantics of a library function based on mapping it
3626	// to an LLVM intrinsic. Check that the library function is available from
3627	// this callbase and in this environment.
3628	LibFunc Func;
3629	if (F->hasLocalLinkage() || !TLI || !TLI->getLibFunc(CB, F&: Func) ||
3630	!CB.onlyReadsMemory())
3631	return Intrinsic::not_intrinsic;
3632
3633	switch (Func) {
3634	default:
3635	break;
3636	case LibFunc_sin:
3637	case LibFunc_sinf:
3638	case LibFunc_sinl:
3639	return Intrinsic::sin;
3640	case LibFunc_cos:
3641	case LibFunc_cosf:
3642	case LibFunc_cosl:
3643	return Intrinsic::cos;
3644	case LibFunc_exp:
3645	case LibFunc_expf:
3646	case LibFunc_expl:
3647	return Intrinsic::exp;
3648	case LibFunc_exp2:
3649	case LibFunc_exp2f:
3650	case LibFunc_exp2l:
3651	return Intrinsic::exp2;
3652	case LibFunc_log:
3653	case LibFunc_logf:
3654	case LibFunc_logl:
3655	return Intrinsic::log;
3656	case LibFunc_log10:
3657	case LibFunc_log10f:
3658	case LibFunc_log10l:
3659	return Intrinsic::log10;
3660	case LibFunc_log2:
3661	case LibFunc_log2f:
3662	case LibFunc_log2l:
3663	return Intrinsic::log2;
3664	case LibFunc_fabs:
3665	case LibFunc_fabsf:
3666	case LibFunc_fabsl:
3667	return Intrinsic::fabs;
3668	case LibFunc_fmin:
3669	case LibFunc_fminf:
3670	case LibFunc_fminl:
3671	return Intrinsic::minnum;
3672	case LibFunc_fmax:
3673	case LibFunc_fmaxf:
3674	case LibFunc_fmaxl:
3675	return Intrinsic::maxnum;
3676	case LibFunc_copysign:
3677	case LibFunc_copysignf:
3678	case LibFunc_copysignl:
3679	return Intrinsic::copysign;
3680	case LibFunc_floor:
3681	case LibFunc_floorf:
3682	case LibFunc_floorl:
3683	return Intrinsic::floor;
3684	case LibFunc_ceil:
3685	case LibFunc_ceilf:
3686	case LibFunc_ceill:
3687	return Intrinsic::ceil;
3688	case LibFunc_trunc:
3689	case LibFunc_truncf:
3690	case LibFunc_truncl:
3691	return Intrinsic::trunc;
3692	case LibFunc_rint:
3693	case LibFunc_rintf:
3694	case LibFunc_rintl:
3695	return Intrinsic::rint;
3696	case LibFunc_nearbyint:
3697	case LibFunc_nearbyintf:
3698	case LibFunc_nearbyintl:
3699	return Intrinsic::nearbyint;
3700	case LibFunc_round:
3701	case LibFunc_roundf:
3702	case LibFunc_roundl:
3703	return Intrinsic::round;
3704	case LibFunc_roundeven:
3705	case LibFunc_roundevenf:
3706	case LibFunc_roundevenl:
3707	return Intrinsic::roundeven;
3708	case LibFunc_pow:
3709	case LibFunc_powf:
3710	case LibFunc_powl:
3711	return Intrinsic::pow;
3712	case LibFunc_sqrt:
3713	case LibFunc_sqrtf:
3714	case LibFunc_sqrtl:
3715	return Intrinsic::sqrt;
3716	}
3717
3718	return Intrinsic::not_intrinsic;
3719	}
3720
3721	/// Return true if it's possible to assume IEEE treatment of input denormals in
3722	/// \p F for \p Val.
3723	static bool inputDenormalIsIEEE(const Function &F, const Type *Ty) {
3724	Ty = Ty->getScalarType();
3725	return F.getDenormalMode(FPType: Ty->getFltSemantics()).Input == DenormalMode::IEEE;
3726	}
3727
3728	static bool inputDenormalIsIEEEOrPosZero(const Function &F, const Type *Ty) {
3729	Ty = Ty->getScalarType();
3730	DenormalMode Mode = F.getDenormalMode(FPType: Ty->getFltSemantics());
3731	return Mode.Input == DenormalMode::IEEE ||
3732	Mode.Input == DenormalMode::PositiveZero;
3733	}
3734
3735	static bool outputDenormalIsIEEEOrPosZero(const Function &F, const Type *Ty) {
3736	Ty = Ty->getScalarType();
3737	DenormalMode Mode = F.getDenormalMode(FPType: Ty->getFltSemantics());
3738	return Mode.Output == DenormalMode::IEEE ||
3739	Mode.Output == DenormalMode::PositiveZero;
3740	}
3741
3742	bool KnownFPClass::isKnownNeverLogicalZero(const Function &F, Type *Ty) const {
3743	return isKnownNeverZero() &&
3744	(isKnownNeverSubnormal() || inputDenormalIsIEEE(F, Ty));
3745	}
3746
3747	bool KnownFPClass::isKnownNeverLogicalNegZero(const Function &F,
3748	Type *Ty) const {
3749	return isKnownNeverNegZero() &&
3750	(isKnownNeverNegSubnormal() || inputDenormalIsIEEEOrPosZero(F, Ty));
3751	}
3752
3753	bool KnownFPClass::isKnownNeverLogicalPosZero(const Function &F,
3754	Type *Ty) const {
3755	if (!isKnownNeverPosZero())
3756	return false;
3757
3758	// If we know there are no denormals, nothing can be flushed to zero.
3759	if (isKnownNeverSubnormal())
3760	return true;
3761
3762	DenormalMode Mode = F.getDenormalMode(FPType: Ty->getScalarType()->getFltSemantics());
3763	switch (Mode.Input) {
3764	case DenormalMode::IEEE:
3765	return true;
3766	case DenormalMode::PreserveSign:
3767	// Negative subnormal won't flush to +0
3768	return isKnownNeverPosSubnormal();
3769	case DenormalMode::PositiveZero:
3770	default:
3771	// Both positive and negative subnormal could flush to +0
3772	return false;
3773	}
3774
3775	llvm_unreachable("covered switch over denormal mode");
3776	}
3777
3778	void KnownFPClass::propagateDenormal(const KnownFPClass &Src, const Function &F,
3779	Type *Ty) {
3780	KnownFPClasses = Src.KnownFPClasses;
3781	// If we aren't assuming the source can't be a zero, we don't have to check if
3782	// a denormal input could be flushed.
3783	if (!Src.isKnownNeverPosZero() && !Src.isKnownNeverNegZero())
3784	return;
3785
3786	// If we know the input can't be a denormal, it can't be flushed to 0.
3787	if (Src.isKnownNeverSubnormal())
3788	return;
3789
3790	DenormalMode Mode = F.getDenormalMode(FPType: Ty->getScalarType()->getFltSemantics());
3791
3792	if (!Src.isKnownNeverPosSubnormal() && Mode != DenormalMode::getIEEE())
3793	KnownFPClasses |= fcPosZero;
3794
3795	if (!Src.isKnownNeverNegSubnormal() && Mode != DenormalMode::getIEEE()) {
3796	if (Mode != DenormalMode::getPositiveZero())
3797	KnownFPClasses |= fcNegZero;
3798
3799	if (Mode.Input == DenormalMode::PositiveZero ||
3800	Mode.Output == DenormalMode::PositiveZero ||
3801	Mode.Input == DenormalMode::Dynamic ||
3802	Mode.Output == DenormalMode::Dynamic)
3803	KnownFPClasses |= fcPosZero;
3804	}
3805	}
3806
3807	void KnownFPClass::propagateCanonicalizingSrc(const KnownFPClass &Src,
3808	const Function &F, Type *Ty) {
3809	propagateDenormal(Src, F, Ty);
3810	propagateNaN(Src, /*PreserveSign=*/true);
3811	}
3812
3813	/// Given an exploded icmp instruction, return true if the comparison only
3814	/// checks the sign bit. If it only checks the sign bit, set TrueIfSigned if
3815	/// the result of the comparison is true when the input value is signed.
3816	bool llvm::isSignBitCheck(ICmpInst::Predicate Pred, const APInt &RHS,
3817	bool &TrueIfSigned) {
3818	switch (Pred) {
3819	case ICmpInst::ICMP_SLT: // True if LHS s< 0
3820	TrueIfSigned = true;
3821	return RHS.isZero();
3822	case ICmpInst::ICMP_SLE: // True if LHS s<= -1
3823	TrueIfSigned = true;
3824	return RHS.isAllOnes();
3825	case ICmpInst::ICMP_SGT: // True if LHS s> -1
3826	TrueIfSigned = false;
3827	return RHS.isAllOnes();
3828	case ICmpInst::ICMP_SGE: // True if LHS s>= 0
3829	TrueIfSigned = false;
3830	return RHS.isZero();
3831	case ICmpInst::ICMP_UGT:
3832	// True if LHS u> RHS and RHS == sign-bit-mask - 1
3833	TrueIfSigned = true;
3834	return RHS.isMaxSignedValue();
3835	case ICmpInst::ICMP_UGE:
3836	// True if LHS u>= RHS and RHS == sign-bit-mask (2^7, 2^15, 2^31, etc)
3837	TrueIfSigned = true;
3838	return RHS.isMinSignedValue();
3839	case ICmpInst::ICMP_ULT:
3840	// True if LHS u< RHS and RHS == sign-bit-mask (2^7, 2^15, 2^31, etc)
3841	TrueIfSigned = false;
3842	return RHS.isMinSignedValue();
3843	case ICmpInst::ICMP_ULE:
3844	// True if LHS u<= RHS and RHS == sign-bit-mask - 1
3845	TrueIfSigned = false;
3846	return RHS.isMaxSignedValue();
3847	default:
3848	return false;
3849	}
3850	}
3851
3852	/// Returns a pair of values, which if passed to llvm.is.fpclass, returns the
3853	/// same result as an fcmp with the given operands.
3854	std::pair<Value *, FPClassTest> llvm::fcmpToClassTest(FCmpInst::Predicate Pred,
3855	const Function &F,
3856	Value *LHS, Value *RHS,
3857	bool LookThroughSrc) {
3858	const APFloat *ConstRHS;
3859	if (!match(V: RHS, P: m_APFloatAllowUndef(Res&: ConstRHS)))
3860	return {nullptr, fcAllFlags};
3861
3862	return fcmpToClassTest(Pred, F, LHS, ConstRHS, LookThroughSrc);
3863	}
3864
3865	std::pair<Value *, FPClassTest>
3866	llvm::fcmpToClassTest(FCmpInst::Predicate Pred, const Function &F, Value *LHS,
3867	const APFloat *ConstRHS, bool LookThroughSrc) {
3868
3869	auto [Src, ClassIfTrue, ClassIfFalse] =
3870	fcmpImpliesClass(Pred, F, LHS, RHS: *ConstRHS, LookThroughSrc);
3871	if (Src && ClassIfTrue == ~ClassIfFalse)
3872	return {Src, ClassIfTrue};
3873	return {nullptr, fcAllFlags};
3874	}
3875
3876	/// Return the return value for fcmpImpliesClass for a compare that produces an
3877	/// exact class test.
3878	static std::tuple<Value *, FPClassTest, FPClassTest> exactClass(Value *V,
3879	FPClassTest M) {
3880	return {V, M, ~M};
3881	}
3882
3883	std::tuple<Value *, FPClassTest, FPClassTest>
3884	llvm::fcmpImpliesClass(CmpInst::Predicate Pred, const Function &F, Value *LHS,
3885	FPClassTest RHSClass, bool LookThroughSrc) {
3886	assert(RHSClass != fcNone);
3887
3888	const FPClassTest OrigClass = RHSClass;
3889
3890	Value *Src = LHS;
3891	const bool IsNegativeRHS = (RHSClass & fcNegative) == RHSClass;
3892	const bool IsPositiveRHS = (RHSClass & fcPositive) == RHSClass;
3893	const bool IsNaN = (RHSClass & ~fcNan) == fcNone;
3894
3895	if (IsNaN) {
3896	// fcmp o__ x, nan -> false
3897	// fcmp u__ x, nan -> true
3898	return exactClass(V: Src, M: CmpInst::isOrdered(predicate: Pred) ? fcNone : fcAllFlags);
3899	}
3900
3901	// fcmp ord x, zero|normal|subnormal|inf -> ~fcNan
3902	if (Pred == FCmpInst::FCMP_ORD)
3903	return exactClass(V: Src, M: ~fcNan);
3904
3905	// fcmp uno x, zero|normal|subnormal|inf -> fcNan
3906	if (Pred == FCmpInst::FCMP_UNO)
3907	return exactClass(V: Src, M: fcNan);
3908
3909	if (Pred == FCmpInst::FCMP_TRUE)
3910	return exactClass(V: Src, M: fcAllFlags);
3911
3912	if (Pred == FCmpInst::FCMP_FALSE)
3913	return exactClass(V: Src, M: fcNone);
3914
3915	const bool IsFabs = LookThroughSrc && match(V: LHS, P: m_FAbs(Op0: m_Value(V&: Src)));
3916	if (IsFabs)
3917	RHSClass = llvm::inverse_fabs(Mask: RHSClass);
3918
3919	const bool IsZero = (OrigClass & fcZero) == OrigClass;
3920	if (IsZero) {
3921	assert(Pred != FCmpInst::FCMP_ORD && Pred != FCmpInst::FCMP_UNO);
3922	// Compares with fcNone are only exactly equal to fcZero if input denormals
3923	// are not flushed.
3924	// TODO: Handle DAZ by expanding masks to cover subnormal cases.
3925	if (!inputDenormalIsIEEE(F, Ty: LHS->getType()))
3926	return {nullptr, fcAllFlags, fcAllFlags};
3927
3928	switch (Pred) {
3929	case FCmpInst::FCMP_OEQ: // Match x == 0.0
3930	return exactClass(V: Src, M: fcZero);
3931	case FCmpInst::FCMP_UEQ: // Match isnan(x) || (x == 0.0)
3932	return exactClass(V: Src, M: fcZero | fcNan);
3933	case FCmpInst::FCMP_UNE: // Match (x != 0.0)
3934	return exactClass(V: Src, M: ~fcZero);
3935	case FCmpInst::FCMP_ONE: // Match !isnan(x) && x != 0.0
3936	return exactClass(V: Src, M: ~fcNan & ~fcZero);
3937	case FCmpInst::FCMP_ORD:
3938	// Canonical form of ord/uno is with a zero. We could also handle
3939	// non-canonical other non-NaN constants or LHS == RHS.
3940	return exactClass(V: Src, M: ~fcNan);
3941	case FCmpInst::FCMP_UNO:
3942	return exactClass(V: Src, M: fcNan);
3943	case FCmpInst::FCMP_OGT: // x > 0
3944	return exactClass(V: Src, M: fcPosSubnormal | fcPosNormal | fcPosInf);
3945	case FCmpInst::FCMP_UGT: // isnan(x) || x > 0
3946	return exactClass(V: Src, M: fcPosSubnormal | fcPosNormal | fcPosInf | fcNan);
3947	case FCmpInst::FCMP_OGE: // x >= 0
3948	return exactClass(V: Src, M: fcPositive | fcNegZero);
3949	case FCmpInst::FCMP_UGE: // isnan(x) || x >= 0
3950	return exactClass(V: Src, M: fcPositive | fcNegZero | fcNan);
3951	case FCmpInst::FCMP_OLT: // x < 0
3952	return exactClass(V: Src, M: fcNegSubnormal | fcNegNormal | fcNegInf);
3953	case FCmpInst::FCMP_ULT: // isnan(x) || x < 0
3954	return exactClass(V: Src, M: fcNegSubnormal | fcNegNormal | fcNegInf | fcNan);
3955	case FCmpInst::FCMP_OLE: // x <= 0
3956	return exactClass(V: Src, M: fcNegative | fcPosZero);
3957	case FCmpInst::FCMP_ULE: // isnan(x) || x <= 0
3958	return exactClass(V: Src, M: fcNegative | fcPosZero | fcNan);
3959	default:
3960	llvm_unreachable("all compare types are handled");
3961	}
3962
3963	return {nullptr, fcAllFlags, fcAllFlags};
3964	}
3965
3966	const bool IsDenormalRHS = (OrigClass & fcSubnormal) == OrigClass;
3967
3968	const bool IsInf = (OrigClass & fcInf) == OrigClass;
3969	if (IsInf) {
3970	FPClassTest Mask = fcAllFlags;
3971
3972	switch (Pred) {
3973	case FCmpInst::FCMP_OEQ:
3974	case FCmpInst::FCMP_UNE: {
3975	// Match __builtin_isinf patterns
3976	//
3977	// fcmp oeq x, +inf -> is_fpclass x, fcPosInf
3978	// fcmp oeq fabs(x), +inf -> is_fpclass x, fcInf
3979	// fcmp oeq x, -inf -> is_fpclass x, fcNegInf
3980	// fcmp oeq fabs(x), -inf -> is_fpclass x, 0 -> false
3981	//
3982	// fcmp une x, +inf -> is_fpclass x, ~fcPosInf
3983	// fcmp une fabs(x), +inf -> is_fpclass x, ~fcInf
3984	// fcmp une x, -inf -> is_fpclass x, ~fcNegInf
3985	// fcmp une fabs(x), -inf -> is_fpclass x, fcAllFlags -> true
3986	if (IsNegativeRHS) {
3987	Mask = fcNegInf;
3988	if (IsFabs)
3989	Mask = fcNone;
3990	} else {
3991	Mask = fcPosInf;
3992	if (IsFabs)
3993	Mask |= fcNegInf;
3994	}
3995	break;
3996	}
3997	case FCmpInst::FCMP_ONE:
3998	case FCmpInst::FCMP_UEQ: {
3999	// Match __builtin_isinf patterns
4000	// fcmp one x, -inf -> is_fpclass x, fcNegInf
4001	// fcmp one fabs(x), -inf -> is_fpclass x, ~fcNegInf & ~fcNan
4002	// fcmp one x, +inf -> is_fpclass x, ~fcNegInf & ~fcNan
4003	// fcmp one fabs(x), +inf -> is_fpclass x, ~fcInf & fcNan
4004	//
4005	// fcmp ueq x, +inf -> is_fpclass x, fcPosInf|fcNan
4006	// fcmp ueq (fabs x), +inf -> is_fpclass x, fcInf|fcNan
4007	// fcmp ueq x, -inf -> is_fpclass x, fcNegInf|fcNan
4008	// fcmp ueq fabs(x), -inf -> is_fpclass x, fcNan
4009	if (IsNegativeRHS) {
4010	Mask = ~fcNegInf & ~fcNan;
4011	if (IsFabs)
4012	Mask = ~fcNan;
4013	} else {
4014	Mask = ~fcPosInf & ~fcNan;
4015	if (IsFabs)
4016	Mask &= ~fcNegInf;
4017	}
4018
4019	break;
4020	}
4021	case FCmpInst::FCMP_OLT:
4022	case FCmpInst::FCMP_UGE: {
4023	if (IsNegativeRHS) {
4024	// No value is ordered and less than negative infinity.
4025	// All values are unordered with or at least negative infinity.
4026	// fcmp olt x, -inf -> false
4027	// fcmp uge x, -inf -> true
4028	Mask = fcNone;
4029	break;
4030	}
4031
4032	// fcmp olt fabs(x), +inf -> fcFinite
4033	// fcmp uge fabs(x), +inf -> ~fcFinite
4034	// fcmp olt x, +inf -> fcFinite|fcNegInf
4035	// fcmp uge x, +inf -> ~(fcFinite|fcNegInf)
4036	Mask = fcFinite;
4037	if (!IsFabs)
4038	Mask |= fcNegInf;
4039	break;
4040	}
4041	case FCmpInst::FCMP_OGE:
4042	case FCmpInst::FCMP_ULT: {
4043	if (IsNegativeRHS) {
4044	// fcmp oge x, -inf -> ~fcNan
4045	// fcmp oge fabs(x), -inf -> ~fcNan
4046	// fcmp ult x, -inf -> fcNan
4047	// fcmp ult fabs(x), -inf -> fcNan
4048	Mask = ~fcNan;
4049	break;
4050	}
4051
4052	// fcmp oge fabs(x), +inf -> fcInf
4053	// fcmp oge x, +inf -> fcPosInf
4054	// fcmp ult fabs(x), +inf -> ~fcInf
4055	// fcmp ult x, +inf -> ~fcPosInf
4056	Mask = fcPosInf;
4057	if (IsFabs)
4058	Mask |= fcNegInf;
4059	break;
4060	}
4061	case FCmpInst::FCMP_OGT:
4062	case FCmpInst::FCMP_ULE: {
4063	if (IsNegativeRHS) {
4064	// fcmp ogt x, -inf -> fcmp one x, -inf
4065	// fcmp ogt fabs(x), -inf -> fcmp ord x, x
4066	// fcmp ule x, -inf -> fcmp ueq x, -inf
4067	// fcmp ule fabs(x), -inf -> fcmp uno x, x
4068	Mask = IsFabs ? ~fcNan : ~(fcNegInf | fcNan);
4069	break;
4070	}
4071
4072	// No value is ordered and greater than infinity.
4073	Mask = fcNone;
4074	break;
4075	}
4076	case FCmpInst::FCMP_OLE:
4077	case FCmpInst::FCMP_UGT: {
4078	if (IsNegativeRHS) {
4079	Mask = IsFabs ? fcNone : fcNegInf;
4080	break;
4081	}
4082
4083	// fcmp ole x, +inf -> fcmp ord x, x
4084	// fcmp ole fabs(x), +inf -> fcmp ord x, x
4085	// fcmp ole x, -inf -> fcmp oeq x, -inf
4086	// fcmp ole fabs(x), -inf -> false
4087	Mask = ~fcNan;
4088	break;
4089	}
4090	default:
4091	llvm_unreachable("all compare types are handled");
4092	}
4093
4094	// Invert the comparison for the unordered cases.
4095	if (FCmpInst::isUnordered(predicate: Pred))
4096	Mask = ~Mask;
4097
4098	return exactClass(V: Src, M: Mask);
4099	}
4100
4101	if (Pred == FCmpInst::FCMP_OEQ)
4102	return {Src, RHSClass, fcAllFlags};
4103
4104	if (Pred == FCmpInst::FCMP_UEQ) {
4105	FPClassTest Class = RHSClass | fcNan;
4106	return {Src, Class, ~fcNan};
4107	}
4108
4109	if (Pred == FCmpInst::FCMP_ONE)
4110	return {Src, ~fcNan, RHSClass | fcNan};
4111
4112	if (Pred == FCmpInst::FCMP_UNE)
4113	return {Src, fcAllFlags, RHSClass};
4114
4115	assert((RHSClass == fcNone || RHSClass == fcPosNormal ||
4116	RHSClass == fcNegNormal || RHSClass == fcNormal ||
4117	RHSClass == fcPosSubnormal || RHSClass == fcNegSubnormal ||
4118	RHSClass == fcSubnormal) &&
4119	"should have been recognized as an exact class test");
4120
4121	if (IsNegativeRHS) {
4122	// TODO: Handle fneg(fabs)
4123	if (IsFabs) {
4124	// fabs(x) o> -k -> fcmp ord x, x
4125	// fabs(x) u> -k -> true
4126	// fabs(x) o< -k -> false
4127	// fabs(x) u< -k -> fcmp uno x, x
4128	switch (Pred) {
4129	case FCmpInst::FCMP_OGT:
4130	case FCmpInst::FCMP_OGE:
4131	return {Src, ~fcNan, fcNan};
4132	case FCmpInst::FCMP_UGT:
4133	case FCmpInst::FCMP_UGE:
4134	return {Src, fcAllFlags, fcNone};
4135	case FCmpInst::FCMP_OLT:
4136	case FCmpInst::FCMP_OLE:
4137	return {Src, fcNone, fcAllFlags};
4138	case FCmpInst::FCMP_ULT:
4139	case FCmpInst::FCMP_ULE:
4140	return {Src, fcNan, ~fcNan};
4141	default:
4142	break;
4143	}
4144
4145	return {nullptr, fcAllFlags, fcAllFlags};
4146	}
4147
4148	FPClassTest ClassesLE = fcNegInf | fcNegNormal;
4149	FPClassTest ClassesGE = fcPositive | fcNegZero | fcNegSubnormal;
4150
4151	if (IsDenormalRHS)
4152	ClassesLE |= fcNegSubnormal;
4153	else
4154	ClassesGE |= fcNegNormal;
4155
4156	switch (Pred) {
4157	case FCmpInst::FCMP_OGT:
4158	case FCmpInst::FCMP_OGE:
4159	return {Src, ClassesGE, ~ClassesGE | RHSClass};
4160	case FCmpInst::FCMP_UGT:
4161	case FCmpInst::FCMP_UGE:
4162	return {Src, ClassesGE | fcNan, ~(ClassesGE | fcNan) | RHSClass};
4163	case FCmpInst::FCMP_OLT:
4164	case FCmpInst::FCMP_OLE:
4165	return {Src, ClassesLE, ~ClassesLE | RHSClass};
4166	case FCmpInst::FCMP_ULT:
4167	case FCmpInst::FCMP_ULE:
4168	return {Src, ClassesLE | fcNan, ~(ClassesLE | fcNan) | RHSClass};
4169	default:
4170	break;
4171	}
4172	} else if (IsPositiveRHS) {
4173	FPClassTest ClassesGE = fcPosNormal | fcPosInf;
4174	FPClassTest ClassesLE = fcNegative | fcPosZero | fcPosSubnormal;
4175	if (IsDenormalRHS)
4176	ClassesGE |= fcPosSubnormal;
4177	else
4178	ClassesLE |= fcPosNormal;
4179
4180	if (IsFabs) {
4181	ClassesGE = llvm::inverse_fabs(Mask: ClassesGE);
4182	ClassesLE = llvm::inverse_fabs(Mask: ClassesLE);
4183	}
4184
4185	switch (Pred) {
4186	case FCmpInst::FCMP_OGT:
4187	case FCmpInst::FCMP_OGE:
4188	return {Src, ClassesGE, ~ClassesGE | RHSClass};
4189	case FCmpInst::FCMP_UGT:
4190	case FCmpInst::FCMP_UGE:
4191	return {Src, ClassesGE | fcNan, ~(ClassesGE | fcNan) | RHSClass};
4192	case FCmpInst::FCMP_OLT:
4193	case FCmpInst::FCMP_OLE:
4194	return {Src, ClassesLE, ~ClassesLE | RHSClass};
4195	case FCmpInst::FCMP_ULT:
4196	case FCmpInst::FCMP_ULE:
4197	return {Src, ClassesLE | fcNan, ~(ClassesLE | fcNan) | RHSClass};
4198	default:
4199	break;
4200	}
4201	}
4202
4203	return {nullptr, fcAllFlags, fcAllFlags};
4204	}
4205
4206	std::tuple<Value *, FPClassTest, FPClassTest>
4207	llvm::fcmpImpliesClass(CmpInst::Predicate Pred, const Function &F, Value *LHS,
4208	const APFloat &ConstRHS, bool LookThroughSrc) {
4209	// We can refine checks against smallest normal / largest denormal to an
4210	// exact class test.
4211	if (!ConstRHS.isNegative() && ConstRHS.isSmallestNormalized()) {
4212	Value *Src = LHS;
4213	const bool IsFabs = LookThroughSrc && match(V: LHS, P: m_FAbs(Op0: m_Value(V&: Src)));
4214
4215	FPClassTest Mask;
4216	// Match pattern that's used in __builtin_isnormal.
4217	switch (Pred) {
4218	case FCmpInst::FCMP_OLT:
4219	case FCmpInst::FCMP_UGE: {
4220	// fcmp olt x, smallest_normal -> fcNegInf|fcNegNormal|fcSubnormal|fcZero
4221	// fcmp olt fabs(x), smallest_normal -> fcSubnormal|fcZero
4222	// fcmp uge x, smallest_normal -> fcNan|fcPosNormal|fcPosInf
4223	// fcmp uge fabs(x), smallest_normal -> ~(fcSubnormal|fcZero)
4224	Mask = fcZero | fcSubnormal;
4225	if (!IsFabs)
4226	Mask |= fcNegNormal | fcNegInf;
4227
4228	break;
4229	}
4230	case FCmpInst::FCMP_OGE:
4231	case FCmpInst::FCMP_ULT: {
4232	// fcmp oge x, smallest_normal -> fcPosNormal | fcPosInf
4233	// fcmp oge fabs(x), smallest_normal -> fcInf | fcNormal
4234	// fcmp ult x, smallest_normal -> ~(fcPosNormal | fcPosInf)
4235	// fcmp ult fabs(x), smallest_normal -> ~(fcInf | fcNormal)
4236	Mask = fcPosInf | fcPosNormal;
4237	if (IsFabs)
4238	Mask |= fcNegInf | fcNegNormal;
4239	break;
4240	}
4241	default:
4242	return fcmpImpliesClass(Pred, F, LHS, RHSClass: ConstRHS.classify(),
4243	LookThroughSrc);
4244	}
4245
4246	// Invert the comparison for the unordered cases.
4247	if (FCmpInst::isUnordered(predicate: Pred))
4248	Mask = ~Mask;
4249
4250	return exactClass(V: Src, M: Mask);
4251	}
4252
4253	return fcmpImpliesClass(Pred, F, LHS, RHSClass: ConstRHS.classify(), LookThroughSrc);
4254	}
4255
4256	std::tuple<Value *, FPClassTest, FPClassTest>
4257	llvm::fcmpImpliesClass(CmpInst::Predicate Pred, const Function &F, Value *LHS,
4258	Value *RHS, bool LookThroughSrc) {
4259	const APFloat *ConstRHS;
4260	if (!match(V: RHS, P: m_APFloatAllowUndef(Res&: ConstRHS)))
4261	return {nullptr, fcAllFlags, fcAllFlags};
4262
4263	// TODO: Just call computeKnownFPClass for RHS to handle non-constants.
4264	return fcmpImpliesClass(Pred, F, LHS, ConstRHS: *ConstRHS, LookThroughSrc);
4265	}
4266
4267	static void computeKnownFPClassFromCond(const Value *V, Value *Cond,
4268	bool CondIsTrue,
4269	const Instruction *CxtI,
4270	KnownFPClass &KnownFromContext) {
4271	CmpInst::Predicate Pred;
4272	Value *LHS;
4273	uint64_t ClassVal = 0;
4274	const APFloat *CRHS;
4275	const APInt *RHS;
4276	if (match(V: Cond, P: m_FCmp(Pred, L: m_Value(V&: LHS), R: m_APFloat(Res&: CRHS)))) {
4277	auto [CmpVal, MaskIfTrue, MaskIfFalse] = fcmpImpliesClass(
4278	Pred, F: *CxtI->getParent()->getParent(), LHS, ConstRHS: *CRHS, LookThroughSrc: LHS != V);
4279	if (CmpVal == V)
4280	KnownFromContext.knownNot(RuleOut: ~(CondIsTrue ? MaskIfTrue : MaskIfFalse));
4281	} else if (match(Cond, m_Intrinsic<Intrinsic::is_fpclass>(
4282	m_Value(LHS), m_ConstantInt(ClassVal)))) {
4283	FPClassTest Mask = static_cast<FPClassTest>(ClassVal);
4284	KnownFromContext.knownNot(RuleOut: CondIsTrue ? ~Mask : Mask);
4285	} else if (match(V: Cond, P: m_ICmp(Pred, L: m_ElementWiseBitCast(Op: m_Value(V&: LHS)),
4286	R: m_APInt(Res&: RHS)))) {
4287	bool TrueIfSigned;
4288	if (!isSignBitCheck(Pred, RHS: *RHS, TrueIfSigned))
4289	return;
4290	if (TrueIfSigned == CondIsTrue)
4291	KnownFromContext.signBitMustBeOne();
4292	else
4293	KnownFromContext.signBitMustBeZero();
4294	}
4295	}
4296
4297	static KnownFPClass computeKnownFPClassFromContext(const Value *V,
4298	const SimplifyQuery &Q) {
4299	KnownFPClass KnownFromContext;
4300
4301	if (!Q.CxtI)
4302	return KnownFromContext;
4303
4304	if (Q.DC && Q.DT) {
4305	// Handle dominating conditions.
4306	for (BranchInst *BI : Q.DC->conditionsFor(V)) {
4307	Value *Cond = BI->getCondition();
4308
4309	BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
4310	if (Q.DT->dominates(BBE: Edge0, BB: Q.CxtI->getParent()))
4311	computeKnownFPClassFromCond(V, Cond, /*CondIsTrue=*/true, CxtI: Q.CxtI,
4312	KnownFromContext);
4313
4314	BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
4315	if (Q.DT->dominates(BBE: Edge1, BB: Q.CxtI->getParent()))
4316	computeKnownFPClassFromCond(V, Cond, /*CondIsTrue=*/false, CxtI: Q.CxtI,
4317	KnownFromContext);
4318	}
4319	}
4320
4321	if (!Q.AC)
4322	return KnownFromContext;
4323
4324	// Try to restrict the floating-point classes based on information from
4325	// assumptions.
4326	for (auto &AssumeVH : Q.AC->assumptionsFor(V)) {
4327	if (!AssumeVH)
4328	continue;
4329	CallInst *I = cast<CallInst>(Val&: AssumeVH);
4330
4331	assert(I->getFunction() == Q.CxtI->getParent()->getParent() &&
4332	"Got assumption for the wrong function!");
4333	assert(I->getCalledFunction()->getIntrinsicID() == Intrinsic::assume &&
4334	"must be an assume intrinsic");
4335
4336	if (!isValidAssumeForContext(Inv: I, CxtI: Q.CxtI, DT: Q.DT))
4337	continue;
4338
4339	computeKnownFPClassFromCond(V, Cond: I->getArgOperand(i: 0), /*CondIsTrue=*/true,
4340	CxtI: Q.CxtI, KnownFromContext);
4341	}
4342
4343	return KnownFromContext;
4344	}
4345
4346	void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
4347	FPClassTest InterestedClasses, KnownFPClass &Known,
4348	unsigned Depth, const SimplifyQuery &Q);
4349
4350	static void computeKnownFPClass(const Value *V, KnownFPClass &Known,
4351	FPClassTest InterestedClasses, unsigned Depth,
4352	const SimplifyQuery &Q) {
4353	auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
4354	APInt DemandedElts =
4355	FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
4356	computeKnownFPClass(V, DemandedElts, InterestedClasses, Known, Depth, Q);
4357	}
4358
4359	static void computeKnownFPClassForFPTrunc(const Operator *Op,
4360	const APInt &DemandedElts,
4361	FPClassTest InterestedClasses,
4362	KnownFPClass &Known, unsigned Depth,
4363	const SimplifyQuery &Q) {
4364	if ((InterestedClasses &
4365	(KnownFPClass::OrderedLessThanZeroMask | fcNan)) == fcNone)
4366	return;
4367
4368	KnownFPClass KnownSrc;
4369	computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses,
4370	Known&: KnownSrc, Depth: Depth + 1, Q);
4371
4372	// Sign should be preserved
4373	// TODO: Handle cannot be ordered greater than zero
4374	if (KnownSrc.cannotBeOrderedLessThanZero())
4375	Known.knownNot(RuleOut: KnownFPClass::OrderedLessThanZeroMask);
4376
4377	Known.propagateNaN(Src: KnownSrc, PreserveSign: true);
4378
4379	// Infinity needs a range check.
4380	}
4381
4382	void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
4383	FPClassTest InterestedClasses, KnownFPClass &Known,
4384	unsigned Depth, const SimplifyQuery &Q) {
4385	assert(Known.isUnknown() && "should not be called with known information");
4386
4387	if (!DemandedElts) {
4388	// No demanded elts, better to assume we don't know anything.
4389	Known.resetAll();
4390	return;
4391	}
4392
4393	assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
4394
4395	if (auto *CFP = dyn_cast_or_null<ConstantFP>(Val: V)) {
4396	Known.KnownFPClasses = CFP->getValueAPF().classify();
4397	Known.SignBit = CFP->isNegative();
4398	return;
4399	}
4400
4401	// Try to handle fixed width vector constants
4402	auto *VFVTy = dyn_cast<FixedVectorType>(Val: V->getType());
4403	const Constant *CV = dyn_cast<Constant>(Val: V);
4404	if (VFVTy && CV) {
4405	Known.KnownFPClasses = fcNone;
4406	bool SignBitAllZero = true;
4407	bool SignBitAllOne = true;
4408
4409	// For vectors, verify that each element is not NaN.
4410	unsigned NumElts = VFVTy->getNumElements();
4411	for (unsigned i = 0; i != NumElts; ++i) {
4412	Constant *Elt = CV->getAggregateElement(Elt: i);
4413	if (!Elt) {
4414	Known = KnownFPClass();
4415	return;
4416	}
4417	if (isa<UndefValue>(Val: Elt))
4418	continue;
4419	auto *CElt = dyn_cast<ConstantFP>(Val: Elt);
4420	if (!CElt) {
4421	Known = KnownFPClass();
4422	return;
4423	}
4424
4425	const APFloat &C = CElt->getValueAPF();
4426	Known.KnownFPClasses |= C.classify();
4427	if (C.isNegative())
4428	SignBitAllZero = false;
4429	else
4430	SignBitAllOne = false;
4431	}
4432	if (SignBitAllOne != SignBitAllZero)
4433	Known.SignBit = SignBitAllOne;
4434	return;
4435	}
4436
4437	FPClassTest KnownNotFromFlags = fcNone;
4438	if (const auto *CB = dyn_cast<CallBase>(Val: V))
4439	KnownNotFromFlags |= CB->getRetNoFPClass();
4440	else if (const auto *Arg = dyn_cast<Argument>(Val: V))
4441	KnownNotFromFlags |= Arg->getNoFPClass();
4442
4443	const Operator *Op = dyn_cast<Operator>(Val: V);
4444	if (const FPMathOperator *FPOp = dyn_cast_or_null<FPMathOperator>(Val: Op)) {
4445	if (FPOp->hasNoNaNs())
4446	KnownNotFromFlags |= fcNan;
4447	if (FPOp->hasNoInfs())
4448	KnownNotFromFlags |= fcInf;
4449	}
4450
4451	KnownFPClass AssumedClasses = computeKnownFPClassFromContext(V, Q);
4452	KnownNotFromFlags |= ~AssumedClasses.KnownFPClasses;
4453
4454	// We no longer need to find out about these bits from inputs if we can
4455	// assume this from flags/attributes.
4456	InterestedClasses &= ~KnownNotFromFlags;
4457
4458	auto ClearClassesFromFlags = make_scope_exit(F: [=, &Known] {
4459	Known.knownNot(RuleOut: KnownNotFromFlags);
4460	if (!Known.SignBit && AssumedClasses.SignBit) {
4461	if (*AssumedClasses.SignBit)
4462	Known.signBitMustBeOne();
4463	else
4464	Known.signBitMustBeZero();
4465	}
4466	});
4467
4468	if (!Op)
4469	return;
4470
4471	// All recursive calls that increase depth must come after this.
4472	if (Depth == MaxAnalysisRecursionDepth)
4473	return;
4474
4475	const unsigned Opc = Op->getOpcode();
4476	switch (Opc) {
4477	case Instruction::FNeg: {
4478	computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses,
4479	Known, Depth: Depth + 1, Q);
4480	Known.fneg();
4481	break;
4482	}
4483	case Instruction::Select: {
4484	Value *Cond = Op->getOperand(i: 0);
4485	Value *LHS = Op->getOperand(i: 1);
4486	Value *RHS = Op->getOperand(i: 2);
4487
4488	FPClassTest FilterLHS = fcAllFlags;
4489	FPClassTest FilterRHS = fcAllFlags;
4490
4491	Value *TestedValue = nullptr;
4492	FPClassTest MaskIfTrue = fcAllFlags;
4493	FPClassTest MaskIfFalse = fcAllFlags;
4494	uint64_t ClassVal = 0;
4495	const Function *F = cast<Instruction>(Val: Op)->getFunction();
4496	CmpInst::Predicate Pred;
4497	Value *CmpLHS, *CmpRHS;
4498	if (F && match(V: Cond, P: m_FCmp(Pred, L: m_Value(V&: CmpLHS), R: m_Value(V&: CmpRHS)))) {
4499	// If the select filters out a value based on the class, it no longer
4500	// participates in the class of the result
4501
4502	// TODO: In some degenerate cases we can infer something if we try again
4503	// without looking through sign operations.
4504	bool LookThroughFAbsFNeg = CmpLHS != LHS && CmpLHS != RHS;
4505	std::tie(args&: TestedValue, args&: MaskIfTrue, args&: MaskIfFalse) =
4506	fcmpImpliesClass(Pred, F: *F, LHS: CmpLHS, RHS: CmpRHS, LookThroughSrc: LookThroughFAbsFNeg);
4507	} else if (match(Cond,
4508	m_Intrinsic<Intrinsic::is_fpclass>(
4509	m_Value(TestedValue), m_ConstantInt(ClassVal)))) {
4510	FPClassTest TestedMask = static_cast<FPClassTest>(ClassVal);
4511	MaskIfTrue = TestedMask;
4512	MaskIfFalse = ~TestedMask;
4513	}
4514
4515	if (TestedValue == LHS) {
4516	// match !isnan(x) ? x : y
4517	FilterLHS = MaskIfTrue;
4518	} else if (TestedValue == RHS) { // && IsExactClass
4519	// match !isnan(x) ? y : x
4520	FilterRHS = MaskIfFalse;
4521	}
4522
4523	KnownFPClass Known2;
4524	computeKnownFPClass(V: LHS, DemandedElts, InterestedClasses: InterestedClasses & FilterLHS, Known,
4525	Depth: Depth + 1, Q);
4526	Known.KnownFPClasses &= FilterLHS;
4527
4528	computeKnownFPClass(V: RHS, DemandedElts, InterestedClasses: InterestedClasses & FilterRHS,
4529	Known&: Known2, Depth: Depth + 1, Q);
4530	Known2.KnownFPClasses &= FilterRHS;
4531
4532	Known |= Known2;
4533	break;
4534	}
4535	case Instruction::Call: {
4536	const CallInst *II = cast<CallInst>(Val: Op);
4537	const Intrinsic::ID IID = II->getIntrinsicID();
4538	switch (IID) {
4539	case Intrinsic::fabs: {
4540	if ((InterestedClasses & (fcNan | fcPositive)) != fcNone) {
4541	// If we only care about the sign bit we don't need to inspect the
4542	// operand.
4543	computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
4544	InterestedClasses, Known, Depth: Depth + 1, Q);
4545	}
4546
4547	Known.fabs();
4548	break;
4549	}
4550	case Intrinsic::copysign: {
4551	KnownFPClass KnownSign;
4552
4553	computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
4554	Known, Depth: Depth + 1, Q);
4555	computeKnownFPClass(V: II->getArgOperand(i: 1), DemandedElts, InterestedClasses,
4556	Known&: KnownSign, Depth: Depth + 1, Q);
4557	Known.copysign(Sign: KnownSign);
4558	break;
4559	}
4560	case Intrinsic::fma:
4561	case Intrinsic::fmuladd: {
4562	if ((InterestedClasses & fcNegative) == fcNone)
4563	break;
4564
4565	if (II->getArgOperand(i: 0) != II->getArgOperand(i: 1))
4566	break;
4567
4568	// The multiply cannot be -0 and therefore the add can't be -0
4569	Known.knownNot(RuleOut: fcNegZero);
4570
4571	// x * x + y is non-negative if y is non-negative.
4572	KnownFPClass KnownAddend;
4573	computeKnownFPClass(V: II->getArgOperand(i: 2), DemandedElts, InterestedClasses,
4574	Known&: KnownAddend, Depth: Depth + 1, Q);
4575
4576	if (KnownAddend.cannotBeOrderedLessThanZero())
4577	Known.knownNot(RuleOut: fcNegative);
4578	break;
4579	}
4580	case Intrinsic::sqrt:
4581	case Intrinsic::experimental_constrained_sqrt: {
4582	KnownFPClass KnownSrc;
4583	FPClassTest InterestedSrcs = InterestedClasses;
4584	if (InterestedClasses & fcNan)
4585	InterestedSrcs |= KnownFPClass::OrderedLessThanZeroMask;
4586
4587	computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
4588	Known&: KnownSrc, Depth: Depth + 1, Q);
4589
4590	if (KnownSrc.isKnownNeverPosInfinity())
4591	Known.knownNot(RuleOut: fcPosInf);
4592	if (KnownSrc.isKnownNever(Mask: fcSNan))
4593	Known.knownNot(RuleOut: fcSNan);
4594
4595	// Any negative value besides -0 returns a nan.
4596	if (KnownSrc.isKnownNeverNaN() && KnownSrc.cannotBeOrderedLessThanZero())
4597	Known.knownNot(RuleOut: fcNan);
4598
4599	// The only negative value that can be returned is -0 for -0 inputs.
4600	Known.knownNot(RuleOut: fcNegInf | fcNegSubnormal | fcNegNormal);
4601
4602	// If the input denormal mode could be PreserveSign, a negative
4603	// subnormal input could produce a negative zero output.
4604	const Function *F = II->getFunction();
4605	if (Q.IIQ.hasNoSignedZeros(Op: II) ||
4606	(F && KnownSrc.isKnownNeverLogicalNegZero(F: *F, Ty: II->getType()))) {
4607	Known.knownNot(RuleOut: fcNegZero);
4608	if (KnownSrc.isKnownNeverNaN())
4609	Known.signBitMustBeZero();
4610	}
4611
4612	break;
4613	}
4614	case Intrinsic::sin:
4615	case Intrinsic::cos: {
4616	// Return NaN on infinite inputs.
4617	KnownFPClass KnownSrc;
4618	computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
4619	Known&: KnownSrc, Depth: Depth + 1, Q);
4620	Known.knownNot(RuleOut: fcInf);
4621	if (KnownSrc.isKnownNeverNaN() && KnownSrc.isKnownNeverInfinity())
4622	Known.knownNot(RuleOut: fcNan);
4623	break;
4624	}
4625	case Intrinsic::maxnum:
4626	case Intrinsic::minnum:
4627	case Intrinsic::minimum:
4628	case Intrinsic::maximum: {
4629	KnownFPClass KnownLHS, KnownRHS;
4630	computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
4631	Known&: KnownLHS, Depth: Depth + 1, Q);
4632	computeKnownFPClass(V: II->getArgOperand(i: 1), DemandedElts, InterestedClasses,
4633	Known&: KnownRHS, Depth: Depth + 1, Q);
4634
4635	bool NeverNaN = KnownLHS.isKnownNeverNaN() || KnownRHS.isKnownNeverNaN();
4636	Known = KnownLHS | KnownRHS;
4637
4638	// If either operand is not NaN, the result is not NaN.
4639	if (NeverNaN && (IID == Intrinsic::minnum || IID == Intrinsic::maxnum))
4640	Known.knownNot(RuleOut: fcNan);
4641
4642	if (IID == Intrinsic::maxnum) {
4643	// If at least one operand is known to be positive, the result must be
4644	// positive.
4645	if ((KnownLHS.cannotBeOrderedLessThanZero() &&
4646	KnownLHS.isKnownNeverNaN()) ||
4647	(KnownRHS.cannotBeOrderedLessThanZero() &&
4648	KnownRHS.isKnownNeverNaN()))
4649	Known.knownNot(RuleOut: KnownFPClass::OrderedLessThanZeroMask);
4650	} else if (IID == Intrinsic::maximum) {
4651	// If at least one operand is known to be positive, the result must be
4652	// positive.
4653	if (KnownLHS.cannotBeOrderedLessThanZero() ||
4654	KnownRHS.cannotBeOrderedLessThanZero())
4655	Known.knownNot(RuleOut: KnownFPClass::OrderedLessThanZeroMask);
4656	} else if (IID == Intrinsic::minnum) {
4657	// If at least one operand is known to be negative, the result must be
4658	// negative.
4659	if ((KnownLHS.cannotBeOrderedGreaterThanZero() &&
4660	KnownLHS.isKnownNeverNaN()) ||
4661	(KnownRHS.cannotBeOrderedGreaterThanZero() &&
4662	KnownRHS.isKnownNeverNaN()))
4663	Known.knownNot(RuleOut: KnownFPClass::OrderedGreaterThanZeroMask);
4664	} else {
4665	// If at least one operand is known to be negative, the result must be
4666	// negative.
4667	if (KnownLHS.cannotBeOrderedGreaterThanZero() ||
4668	KnownRHS.cannotBeOrderedGreaterThanZero())
4669	Known.knownNot(RuleOut: KnownFPClass::OrderedGreaterThanZeroMask);
4670	}
4671
4672	// Fixup zero handling if denormals could be returned as a zero.
4673	//
4674	// As there's no spec for denormal flushing, be conservative with the
4675	// treatment of denormals that could be flushed to zero. For older
4676	// subtargets on AMDGPU the min/max instructions would not flush the
4677	// output and return the original value.
4678	//
4679	if ((Known.KnownFPClasses & fcZero) != fcNone &&
4680	!Known.isKnownNeverSubnormal()) {
4681	const Function *Parent = II->getFunction();
4682	if (!Parent)
4683	break;
4684
4685	DenormalMode Mode = Parent->getDenormalMode(
4686	FPType: II->getType()->getScalarType()->getFltSemantics());
4687	if (Mode != DenormalMode::getIEEE())
4688	Known.KnownFPClasses |= fcZero;
4689	}
4690
4691	if (Known.isKnownNeverNaN()) {
4692	if (KnownLHS.SignBit && KnownRHS.SignBit &&
4693	*KnownLHS.SignBit == *KnownRHS.SignBit) {
4694	if (*KnownLHS.SignBit)
4695	Known.signBitMustBeOne();
4696	else
4697	Known.signBitMustBeZero();
4698	} else if ((IID == Intrinsic::maximum || IID == Intrinsic::minimum) ||
4699	((KnownLHS.isKnownNeverNegZero() ||
4700	KnownRHS.isKnownNeverPosZero()) &&
4701	(KnownLHS.isKnownNeverPosZero() ||
4702	KnownRHS.isKnownNeverNegZero()))) {
4703	if ((IID == Intrinsic::maximum || IID == Intrinsic::maxnum) &&
4704	(KnownLHS.SignBit == false || KnownRHS.SignBit == false))
4705	Known.signBitMustBeZero();
4706	else if ((IID == Intrinsic::minimum || IID == Intrinsic::minnum) &&
4707	(KnownLHS.SignBit == true || KnownRHS.SignBit == true))
4708	Known.signBitMustBeOne();
4709	}
4710	}
4711	break;
4712	}
4713	case Intrinsic::canonicalize: {
4714	KnownFPClass KnownSrc;
4715	computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
4716	Known&: KnownSrc, Depth: Depth + 1, Q);
4717
4718	// This is essentially a stronger form of
4719	// propagateCanonicalizingSrc. Other "canonicalizing" operations don't
4720	// actually have an IR canonicalization guarantee.
4721
4722	// Canonicalize may flush denormals to zero, so we have to consider the
4723	// denormal mode to preserve known-not-0 knowledge.
4724	Known.KnownFPClasses = KnownSrc.KnownFPClasses | fcZero | fcQNan;
4725
4726	// Stronger version of propagateNaN
4727	// Canonicalize is guaranteed to quiet signaling nans.
4728	if (KnownSrc.isKnownNeverNaN())
4729	Known.knownNot(RuleOut: fcNan);
4730	else
4731	Known.knownNot(RuleOut: fcSNan);
4732
4733	const Function *F = II->getFunction();
4734	if (!F)
4735	break;
4736
4737	// If the parent function flushes denormals, the canonical output cannot
4738	// be a denormal.
4739	const fltSemantics &FPType =
4740	II->getType()->getScalarType()->getFltSemantics();
4741	DenormalMode DenormMode = F->getDenormalMode(FPType);
4742	if (DenormMode == DenormalMode::getIEEE()) {
4743	if (KnownSrc.isKnownNever(Mask: fcPosZero))
4744	Known.knownNot(RuleOut: fcPosZero);
4745	if (KnownSrc.isKnownNever(Mask: fcNegZero))
4746	Known.knownNot(RuleOut: fcNegZero);
4747	break;
4748	}
4749
4750	if (DenormMode.inputsAreZero() || DenormMode.outputsAreZero())
4751	Known.knownNot(RuleOut: fcSubnormal);
4752
4753	if (DenormMode.Input == DenormalMode::PositiveZero ||
4754	(DenormMode.Output == DenormalMode::PositiveZero &&
4755	DenormMode.Input == DenormalMode::IEEE))
4756	Known.knownNot(RuleOut: fcNegZero);
4757
4758	break;
4759	}
4760	case Intrinsic::trunc:
4761	case Intrinsic::floor:
4762	case Intrinsic::ceil:
4763	case Intrinsic::rint:
4764	case Intrinsic::nearbyint:
4765	case Intrinsic::round:
4766	case Intrinsic::roundeven: {
4767	KnownFPClass KnownSrc;
4768	FPClassTest InterestedSrcs = InterestedClasses;
4769	if (InterestedSrcs & fcPosFinite)
4770	InterestedSrcs |= fcPosFinite;
4771	if (InterestedSrcs & fcNegFinite)
4772	InterestedSrcs |= fcNegFinite;
4773	computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
4774	Known&: KnownSrc, Depth: Depth + 1, Q);
4775
4776	// Integer results cannot be subnormal.
4777	Known.knownNot(RuleOut: fcSubnormal);
4778
4779	Known.propagateNaN(Src: KnownSrc, PreserveSign: true);
4780
4781	// Pass through infinities, except PPC_FP128 is a special case for
4782	// intrinsics other than trunc.
4783	if (IID == Intrinsic::trunc || !V->getType()->isMultiUnitFPType()) {
4784	if (KnownSrc.isKnownNeverPosInfinity())
4785	Known.knownNot(RuleOut: fcPosInf);
4786	if (KnownSrc.isKnownNeverNegInfinity())
4787	Known.knownNot(RuleOut: fcNegInf);
4788	}
4789
4790	// Negative round ups to 0 produce -0
4791	if (KnownSrc.isKnownNever(Mask: fcPosFinite))
4792	Known.knownNot(RuleOut: fcPosFinite);
4793	if (KnownSrc.isKnownNever(Mask: fcNegFinite))
4794	Known.knownNot(RuleOut: fcNegFinite);
4795
4796	break;
4797	}
4798	case Intrinsic::exp:
4799	case Intrinsic::exp2:
4800	case Intrinsic::exp10: {
4801	Known.knownNot(RuleOut: fcNegative);
4802	if ((InterestedClasses & fcNan) == fcNone)
4803	break;
4804
4805	KnownFPClass KnownSrc;
4806	computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
4807	Known&: KnownSrc, Depth: Depth + 1, Q);
4808	if (KnownSrc.isKnownNeverNaN()) {
4809	Known.knownNot(RuleOut: fcNan);
4810	Known.signBitMustBeZero();
4811	}
4812
4813	break;
4814	}
4815	case Intrinsic::fptrunc_round: {
4816	computeKnownFPClassForFPTrunc(Op, DemandedElts, InterestedClasses, Known,
4817	Depth, Q);
4818	break;
4819	}
4820	case Intrinsic::log:
4821	case Intrinsic::log10:
4822	case Intrinsic::log2:
4823	case Intrinsic::experimental_constrained_log:
4824	case Intrinsic::experimental_constrained_log10:
4825	case Intrinsic::experimental_constrained_log2: {
4826	// log(+inf) -> +inf
4827	// log([+-]0.0) -> -inf
4828	// log(-inf) -> nan
4829	// log(-x) -> nan
4830	if ((InterestedClasses & (fcNan | fcInf)) == fcNone)
4831	break;
4832
4833	FPClassTest InterestedSrcs = InterestedClasses;
4834	if ((InterestedClasses & fcNegInf) != fcNone)
4835	InterestedSrcs |= fcZero | fcSubnormal;
4836	if ((InterestedClasses & fcNan) != fcNone)
4837	InterestedSrcs |= fcNan | (fcNegative & ~fcNan);
4838
4839	KnownFPClass KnownSrc;
4840	computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
4841	Known&: KnownSrc, Depth: Depth + 1, Q);
4842
4843	if (KnownSrc.isKnownNeverPosInfinity())
4844	Known.knownNot(RuleOut: fcPosInf);
4845
4846	if (KnownSrc.isKnownNeverNaN() && KnownSrc.cannotBeOrderedLessThanZero())
4847	Known.knownNot(RuleOut: fcNan);
4848
4849	const Function *F = II->getFunction();
4850	if (F && KnownSrc.isKnownNeverLogicalZero(F: *F, Ty: II->getType()))
4851	Known.knownNot(RuleOut: fcNegInf);
4852
4853	break;
4854	}
4855	case Intrinsic::powi: {
4856	if ((InterestedClasses & fcNegative) == fcNone)
4857	break;
4858
4859	const Value *Exp = II->getArgOperand(i: 1);
4860	Type *ExpTy = Exp->getType();
4861	unsigned BitWidth = ExpTy->getScalarType()->getIntegerBitWidth();
4862	KnownBits ExponentKnownBits(BitWidth);
4863	computeKnownBits(V: Exp, DemandedElts: isa<VectorType>(Val: ExpTy) ? DemandedElts : APInt(1, 1),
4864	Known&: ExponentKnownBits, Depth: Depth + 1, Q);
4865
4866	if (ExponentKnownBits.Zero[0]) { // Is even
4867	Known.knownNot(RuleOut: fcNegative);
4868	break;
4869	}
4870
4871	// Given that exp is an integer, here are the
4872	// ways that pow can return a negative value:
4873	//
4874	// pow(-x, exp) --> negative if exp is odd and x is negative.
4875	// pow(-0, exp) --> -inf if exp is negative odd.
4876	// pow(-0, exp) --> -0 if exp is positive odd.
4877	// pow(-inf, exp) --> -0 if exp is negative odd.
4878	// pow(-inf, exp) --> -inf if exp is positive odd.
4879	KnownFPClass KnownSrc;
4880	computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: fcNegative,
4881	Known&: KnownSrc, Depth: Depth + 1, Q);
4882	if (KnownSrc.isKnownNever(Mask: fcNegative))
4883	Known.knownNot(RuleOut: fcNegative);
4884	break;
4885	}
4886	case Intrinsic::ldexp: {
4887	KnownFPClass KnownSrc;
4888	computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
4889	Known&: KnownSrc, Depth: Depth + 1, Q);
4890	Known.propagateNaN(Src: KnownSrc, /*PropagateSign=*/PreserveSign: true);
4891
4892	// Sign is preserved, but underflows may produce zeroes.
4893	if (KnownSrc.isKnownNever(Mask: fcNegative))
4894	Known.knownNot(RuleOut: fcNegative);
4895	else if (KnownSrc.cannotBeOrderedLessThanZero())
4896	Known.knownNot(RuleOut: KnownFPClass::OrderedLessThanZeroMask);
4897
4898	if (KnownSrc.isKnownNever(Mask: fcPositive))
4899	Known.knownNot(RuleOut: fcPositive);
4900	else if (KnownSrc.cannotBeOrderedGreaterThanZero())
4901	Known.knownNot(RuleOut: KnownFPClass::OrderedGreaterThanZeroMask);
4902
4903	// Can refine inf/zero handling based on the exponent operand.
4904	const FPClassTest ExpInfoMask = fcZero | fcSubnormal | fcInf;
4905	if ((InterestedClasses & ExpInfoMask) == fcNone)
4906	break;
4907	if ((KnownSrc.KnownFPClasses & ExpInfoMask) == fcNone)
4908	break;
4909
4910	const fltSemantics &Flt =
4911	II->getType()->getScalarType()->getFltSemantics();
4912	unsigned Precision = APFloat::semanticsPrecision(Flt);
4913	const Value *ExpArg = II->getArgOperand(i: 1);
4914	ConstantRange ExpRange = computeConstantRange(
4915	V: ExpArg, ForSigned: true, UseInstrInfo: Q.IIQ.UseInstrInfo, AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT, Depth: Depth + 1);
4916
4917	const int MantissaBits = Precision - 1;
4918	if (ExpRange.getSignedMin().sge(RHS: static_cast<int64_t>(MantissaBits)))
4919	Known.knownNot(RuleOut: fcSubnormal);
4920
4921	const Function *F = II->getFunction();
4922	const APInt *ConstVal = ExpRange.getSingleElement();
4923	if (ConstVal && ConstVal->isZero()) {
4924	// ldexp(x, 0) -> x, so propagate everything.
4925	Known.propagateCanonicalizingSrc(Src: KnownSrc, F: *F, Ty: II->getType());
4926	} else if (ExpRange.isAllNegative()) {
4927	// If we know the power is <= 0, can't introduce inf
4928	if (KnownSrc.isKnownNeverPosInfinity())
4929	Known.knownNot(RuleOut: fcPosInf);
4930	if (KnownSrc.isKnownNeverNegInfinity())
4931	Known.knownNot(RuleOut: fcNegInf);
4932	} else if (ExpRange.isAllNonNegative()) {
4933	// If we know the power is >= 0, can't introduce subnormal or zero
4934	if (KnownSrc.isKnownNeverPosSubnormal())
4935	Known.knownNot(RuleOut: fcPosSubnormal);
4936	if (KnownSrc.isKnownNeverNegSubnormal())
4937	Known.knownNot(RuleOut: fcNegSubnormal);
4938	if (F && KnownSrc.isKnownNeverLogicalPosZero(F: *F, Ty: II->getType()))
4939	Known.knownNot(RuleOut: fcPosZero);
4940	if (F && KnownSrc.isKnownNeverLogicalNegZero(F: *F, Ty: II->getType()))
4941	Known.knownNot(RuleOut: fcNegZero);
4942	}
4943
4944	break;
4945	}
4946	case Intrinsic::arithmetic_fence: {
4947	computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
4948	Known, Depth: Depth + 1, Q);
4949	break;
4950	}
4951	case Intrinsic::experimental_constrained_sitofp:
4952	case Intrinsic::experimental_constrained_uitofp:
4953	// Cannot produce nan
4954	Known.knownNot(RuleOut: fcNan);
4955
4956	// sitofp and uitofp turn into +0.0 for zero.
4957	Known.knownNot(RuleOut: fcNegZero);
4958
4959	// Integers cannot be subnormal
4960	Known.knownNot(RuleOut: fcSubnormal);
4961
4962	if (IID == Intrinsic::experimental_constrained_uitofp)
4963	Known.signBitMustBeZero();
4964
4965	// TODO: Copy inf handling from instructions
4966	break;
4967	default:
4968	break;
4969	}
4970
4971	break;
4972	}
4973	case Instruction::FAdd:
4974	case Instruction::FSub: {
4975	KnownFPClass KnownLHS, KnownRHS;
4976	bool WantNegative =
4977	Op->getOpcode() == Instruction::FAdd &&
4978	(InterestedClasses & KnownFPClass::OrderedLessThanZeroMask) != fcNone;
4979	bool WantNaN = (InterestedClasses & fcNan) != fcNone;
4980	bool WantNegZero = (InterestedClasses & fcNegZero) != fcNone;
4981
4982	if (!WantNaN && !WantNegative && !WantNegZero)
4983	break;
4984
4985	FPClassTest InterestedSrcs = InterestedClasses;
4986	if (WantNegative)
4987	InterestedSrcs |= KnownFPClass::OrderedLessThanZeroMask;
4988	if (InterestedClasses & fcNan)
4989	InterestedSrcs |= fcInf;
4990	computeKnownFPClass(V: Op->getOperand(i: 1), DemandedElts, InterestedClasses: InterestedSrcs,
4991	Known&: KnownRHS, Depth: Depth + 1, Q);
4992
4993	if ((WantNaN && KnownRHS.isKnownNeverNaN()) ||
4994	(WantNegative && KnownRHS.cannotBeOrderedLessThanZero()) ||
4995	WantNegZero || Opc == Instruction::FSub) {
4996
4997	// RHS is canonically cheaper to compute. Skip inspecting the LHS if
4998	// there's no point.
4999	computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5000	Known&: KnownLHS, Depth: Depth + 1, Q);
5001	// Adding positive and negative infinity produces NaN.
5002	// TODO: Check sign of infinities.
5003	if (KnownLHS.isKnownNeverNaN() && KnownRHS.isKnownNeverNaN() &&
5004	(KnownLHS.isKnownNeverInfinity() || KnownRHS.isKnownNeverInfinity()))
5005	Known.knownNot(RuleOut: fcNan);
5006
5007	// FIXME: Context function should always be passed in separately
5008	const Function *F = cast<Instruction>(Val: Op)->getFunction();
5009
5010	if (Op->getOpcode() == Instruction::FAdd) {
5011	if (KnownLHS.cannotBeOrderedLessThanZero() &&
5012	KnownRHS.cannotBeOrderedLessThanZero())
5013	Known.knownNot(RuleOut: KnownFPClass::OrderedLessThanZeroMask);
5014	if (!F)
5015	break;
5016
5017	// (fadd x, 0.0) is guaranteed to return +0.0, not -0.0.
5018	if ((KnownLHS.isKnownNeverLogicalNegZero(F: *F, Ty: Op->getType()) ||
5019	KnownRHS.isKnownNeverLogicalNegZero(F: *F, Ty: Op->getType())) &&
5020	// Make sure output negative denormal can't flush to -0
5021	outputDenormalIsIEEEOrPosZero(F: *F, Ty: Op->getType()))
5022	Known.knownNot(RuleOut: fcNegZero);
5023	} else {
5024	if (!F)
5025	break;
5026
5027	// Only fsub -0, +0 can return -0
5028	if ((KnownLHS.isKnownNeverLogicalNegZero(F: *F, Ty: Op->getType()) ||
5029	KnownRHS.isKnownNeverLogicalPosZero(F: *F, Ty: Op->getType())) &&
5030	// Make sure output negative denormal can't flush to -0
5031	outputDenormalIsIEEEOrPosZero(F: *F, Ty: Op->getType()))
5032	Known.knownNot(RuleOut: fcNegZero);
5033	}
5034	}
5035
5036	break;
5037	}
5038	case Instruction::FMul: {
5039	// X * X is always non-negative or a NaN.
5040	if (Op->getOperand(i: 0) == Op->getOperand(i: 1))
5041	Known.knownNot(RuleOut: fcNegative);
5042
5043	if ((InterestedClasses & fcNan) != fcNan)
5044	break;
5045
5046	// fcSubnormal is only needed in case of DAZ.
5047	const FPClassTest NeedForNan = fcNan | fcInf | fcZero | fcSubnormal;
5048
5049	KnownFPClass KnownLHS, KnownRHS;
5050	computeKnownFPClass(V: Op->getOperand(i: 1), DemandedElts, InterestedClasses: NeedForNan, Known&: KnownRHS,
5051	Depth: Depth + 1, Q);
5052	if (!KnownRHS.isKnownNeverNaN())
5053	break;
5054
5055	computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses: NeedForNan, Known&: KnownLHS,
5056	Depth: Depth + 1, Q);
5057	if (!KnownLHS.isKnownNeverNaN())
5058	break;
5059
5060	if (KnownLHS.SignBit && KnownRHS.SignBit) {
5061	if (*KnownLHS.SignBit == *KnownRHS.SignBit)
5062	Known.signBitMustBeZero();
5063	else
5064	Known.signBitMustBeOne();
5065	}
5066
5067	// If 0 * +/-inf produces NaN.
5068	if (KnownLHS.isKnownNeverInfinity() && KnownRHS.isKnownNeverInfinity()) {
5069	Known.knownNot(RuleOut: fcNan);
5070	break;
5071	}
5072
5073	const Function *F = cast<Instruction>(Val: Op)->getFunction();
5074	if (!F)
5075	break;
5076
5077	if ((KnownRHS.isKnownNeverInfinity() ||
5078	KnownLHS.isKnownNeverLogicalZero(F: *F, Ty: Op->getType())) &&
5079	(KnownLHS.isKnownNeverInfinity() ||
5080	KnownRHS.isKnownNeverLogicalZero(F: *F, Ty: Op->getType())))
5081	Known.knownNot(RuleOut: fcNan);
5082
5083	break;
5084	}
5085	case Instruction::FDiv:
5086	case Instruction::FRem: {
5087	if (Op->getOperand(i: 0) == Op->getOperand(i: 1)) {
5088	// TODO: Could filter out snan if we inspect the operand
5089	if (Op->getOpcode() == Instruction::FDiv) {
5090	// X / X is always exactly 1.0 or a NaN.
5091	Known.KnownFPClasses = fcNan | fcPosNormal;
5092	} else {
5093	// X % X is always exactly [+-]0.0 or a NaN.
5094	Known.KnownFPClasses = fcNan | fcZero;
5095	}
5096
5097	break;
5098	}
5099
5100	const bool WantNan = (InterestedClasses & fcNan) != fcNone;
5101	const bool WantNegative = (InterestedClasses & fcNegative) != fcNone;
5102	const bool WantPositive =
5103	Opc == Instruction::FRem && (InterestedClasses & fcPositive) != fcNone;
5104	if (!WantNan && !WantNegative && !WantPositive)
5105	break;
5106
5107	KnownFPClass KnownLHS, KnownRHS;
5108
5109	computeKnownFPClass(V: Op->getOperand(i: 1), DemandedElts,
5110	InterestedClasses: fcNan | fcInf | fcZero | fcNegative, Known&: KnownRHS,
5111	Depth: Depth + 1, Q);
5112
5113	bool KnowSomethingUseful =
5114	KnownRHS.isKnownNeverNaN() || KnownRHS.isKnownNever(Mask: fcNegative);
5115
5116	if (KnowSomethingUseful || WantPositive) {
5117	const FPClassTest InterestedLHS =
5118	WantPositive ? fcAllFlags
5119	: fcNan | fcInf | fcZero | fcSubnormal | fcNegative;
5120
5121	computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts,
5122	InterestedClasses: InterestedClasses & InterestedLHS, Known&: KnownLHS,
5123	Depth: Depth + 1, Q);
5124	}
5125
5126	const Function *F = cast<Instruction>(Val: Op)->getFunction();
5127
5128	if (Op->getOpcode() == Instruction::FDiv) {
5129	// Only 0/0, Inf/Inf produce NaN.
5130	if (KnownLHS.isKnownNeverNaN() && KnownRHS.isKnownNeverNaN() &&
5131	(KnownLHS.isKnownNeverInfinity() ||
5132	KnownRHS.isKnownNeverInfinity()) &&
5133	((F && KnownLHS.isKnownNeverLogicalZero(F: *F, Ty: Op->getType())) ||
5134	(F && KnownRHS.isKnownNeverLogicalZero(F: *F, Ty: Op->getType())))) {
5135	Known.knownNot(RuleOut: fcNan);
5136	}
5137
5138	// X / -0.0 is -Inf (or NaN).
5139	// +X / +X is +X
5140	if (KnownLHS.isKnownNever(Mask: fcNegative) && KnownRHS.isKnownNever(Mask: fcNegative))
5141	Known.knownNot(RuleOut: fcNegative);
5142	} else {
5143	// Inf REM x and x REM 0 produce NaN.
5144	if (KnownLHS.isKnownNeverNaN() && KnownRHS.isKnownNeverNaN() &&
5145	KnownLHS.isKnownNeverInfinity() && F &&
5146	KnownRHS.isKnownNeverLogicalZero(F: *F, Ty: Op->getType())) {
5147	Known.knownNot(RuleOut: fcNan);
5148	}
5149
5150	// The sign for frem is the same as the first operand.
5151	if (KnownLHS.cannotBeOrderedLessThanZero())
5152	Known.knownNot(RuleOut: KnownFPClass::OrderedLessThanZeroMask);
5153	if (KnownLHS.cannotBeOrderedGreaterThanZero())
5154	Known.knownNot(RuleOut: KnownFPClass::OrderedGreaterThanZeroMask);
5155
5156	// See if we can be more aggressive about the sign of 0.
5157	if (KnownLHS.isKnownNever(Mask: fcNegative))
5158	Known.knownNot(RuleOut: fcNegative);
5159	if (KnownLHS.isKnownNever(Mask: fcPositive))
5160	Known.knownNot(RuleOut: fcPositive);
5161	}
5162
5163	break;
5164	}
5165	case Instruction::FPExt: {
5166	// Infinity, nan and zero propagate from source.
5167	computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses,
5168	Known, Depth: Depth + 1, Q);
5169
5170	const fltSemantics &DstTy =
5171	Op->getType()->getScalarType()->getFltSemantics();
5172	const fltSemantics &SrcTy =
5173	Op->getOperand(i: 0)->getType()->getScalarType()->getFltSemantics();
5174
5175	// All subnormal inputs should be in the normal range in the result type.
5176	if (APFloat::isRepresentableAsNormalIn(Src: SrcTy, Dst: DstTy))
5177	Known.knownNot(RuleOut: fcSubnormal);
5178
5179	// Sign bit of a nan isn't guaranteed.
5180	if (!Known.isKnownNeverNaN())
5181	Known.SignBit = std::nullopt;
5182	break;
5183	}
5184	case Instruction::FPTrunc: {
5185	computeKnownFPClassForFPTrunc(Op, DemandedElts, InterestedClasses, Known,
5186	Depth, Q);
5187	break;
5188	}
5189	case Instruction::SIToFP:
5190	case Instruction::UIToFP: {
5191	// Cannot produce nan
5192	Known.knownNot(RuleOut: fcNan);
5193
5194	// Integers cannot be subnormal
5195	Known.knownNot(RuleOut: fcSubnormal);
5196
5197	// sitofp and uitofp turn into +0.0 for zero.
5198	Known.knownNot(RuleOut: fcNegZero);
5199	if (Op->getOpcode() == Instruction::UIToFP)
5200	Known.signBitMustBeZero();
5201
5202	if (InterestedClasses & fcInf) {
5203	// Get width of largest magnitude integer (remove a bit if signed).
5204	// This still works for a signed minimum value because the largest FP
5205	// value is scaled by some fraction close to 2.0 (1.0 + 0.xxxx).
5206	int IntSize = Op->getOperand(i: 0)->getType()->getScalarSizeInBits();
5207	if (Op->getOpcode() == Instruction::SIToFP)
5208	--IntSize;
5209
5210	// If the exponent of the largest finite FP value can hold the largest
5211	// integer, the result of the cast must be finite.
5212	Type *FPTy = Op->getType()->getScalarType();
5213	if (ilogb(Arg: APFloat::getLargest(Sem: FPTy->getFltSemantics())) >= IntSize)
5214	Known.knownNot(RuleOut: fcInf);
5215	}
5216
5217	break;
5218	}
5219	case Instruction::ExtractElement: {
5220	// Look through extract element. If the index is non-constant or
5221	// out-of-range demand all elements, otherwise just the extracted element.
5222	const Value *Vec = Op->getOperand(i: 0);
5223	const Value *Idx = Op->getOperand(i: 1);
5224	auto *CIdx = dyn_cast<ConstantInt>(Val: Idx);
5225
5226	if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType())) {
5227	unsigned NumElts = VecTy->getNumElements();
5228	APInt DemandedVecElts = APInt::getAllOnes(numBits: NumElts);
5229	if (CIdx && CIdx->getValue().ult(RHS: NumElts))
5230	DemandedVecElts = APInt::getOneBitSet(numBits: NumElts, BitNo: CIdx->getZExtValue());
5231	return computeKnownFPClass(V: Vec, DemandedElts: DemandedVecElts, InterestedClasses, Known,
5232	Depth: Depth + 1, Q);
5233	}
5234
5235	break;
5236	}
5237	case Instruction::InsertElement: {
5238	if (isa<ScalableVectorType>(Val: Op->getType()))
5239	return;
5240
5241	const Value *Vec = Op->getOperand(i: 0);
5242	const Value *Elt = Op->getOperand(i: 1);
5243	auto *CIdx = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 2));
5244	// Early out if the index is non-constant or out-of-range.
5245	unsigned NumElts = DemandedElts.getBitWidth();
5246	if (!CIdx || CIdx->getValue().uge(RHS: NumElts))
5247	return;
5248
5249	unsigned EltIdx = CIdx->getZExtValue();
5250	// Do we demand the inserted element?
5251	if (DemandedElts[EltIdx]) {
5252	computeKnownFPClass(V: Elt, Known, InterestedClasses, Depth: Depth + 1, Q);
5253	// If we don't know any bits, early out.
5254	if (Known.isUnknown())
5255	break;
5256	} else {
5257	Known.KnownFPClasses = fcNone;
5258	}
5259
5260	// We don't need the base vector element that has been inserted.
5261	APInt DemandedVecElts = DemandedElts;
5262	DemandedVecElts.clearBit(BitPosition: EltIdx);
5263	if (!!DemandedVecElts) {
5264	KnownFPClass Known2;
5265	computeKnownFPClass(V: Vec, DemandedElts: DemandedVecElts, InterestedClasses, Known&: Known2,
5266	Depth: Depth + 1, Q);
5267	Known |= Known2;
5268	}
5269
5270	break;
5271	}
5272	case Instruction::ShuffleVector: {
5273	// For undef elements, we don't know anything about the common state of
5274	// the shuffle result.
5275	APInt DemandedLHS, DemandedRHS;
5276	auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: Op);
5277	if (!Shuf || !getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS))
5278	return;
5279
5280	if (!!DemandedLHS) {
5281	const Value *LHS = Shuf->getOperand(i_nocapture: 0);
5282	computeKnownFPClass(V: LHS, DemandedElts: DemandedLHS, InterestedClasses, Known,
5283	Depth: Depth + 1, Q);
5284
5285	// If we don't know any bits, early out.
5286	if (Known.isUnknown())
5287	break;
5288	} else {
5289	Known.KnownFPClasses = fcNone;
5290	}
5291
5292	if (!!DemandedRHS) {
5293	KnownFPClass Known2;
5294	const Value *RHS = Shuf->getOperand(i_nocapture: 1);
5295	computeKnownFPClass(V: RHS, DemandedElts: DemandedRHS, InterestedClasses, Known&: Known2,
5296	Depth: Depth + 1, Q);
5297	Known |= Known2;
5298	}
5299
5300	break;
5301	}
5302	case Instruction::ExtractValue: {
5303	const ExtractValueInst *Extract = cast<ExtractValueInst>(Val: Op);
5304	ArrayRef<unsigned> Indices = Extract->getIndices();
5305	const Value *Src = Extract->getAggregateOperand();
5306	if (isa<StructType>(Val: Src->getType()) && Indices.size() == 1 &&
5307	Indices[0] == 0) {
5308	if (const auto *II = dyn_cast<IntrinsicInst>(Val: Src)) {
5309	switch (II->getIntrinsicID()) {
5310	case Intrinsic::frexp: {
5311	Known.knownNot(RuleOut: fcSubnormal);
5312
5313	KnownFPClass KnownSrc;
5314	computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
5315	InterestedClasses, Known&: KnownSrc, Depth: Depth + 1, Q);
5316
5317	const Function *F = cast<Instruction>(Val: Op)->getFunction();
5318
5319	if (KnownSrc.isKnownNever(Mask: fcNegative))
5320	Known.knownNot(RuleOut: fcNegative);
5321	else {
5322	if (F && KnownSrc.isKnownNeverLogicalNegZero(F: *F, Ty: Op->getType()))
5323	Known.knownNot(RuleOut: fcNegZero);
5324	if (KnownSrc.isKnownNever(Mask: fcNegInf))
5325	Known.knownNot(RuleOut: fcNegInf);
5326	}
5327
5328	if (KnownSrc.isKnownNever(Mask: fcPositive))
5329	Known.knownNot(RuleOut: fcPositive);
5330	else {
5331	if (F && KnownSrc.isKnownNeverLogicalPosZero(F: *F, Ty: Op->getType()))
5332	Known.knownNot(RuleOut: fcPosZero);
5333	if (KnownSrc.isKnownNever(Mask: fcPosInf))
5334	Known.knownNot(RuleOut: fcPosInf);
5335	}
5336
5337	Known.propagateNaN(Src: KnownSrc);
5338	return;
5339	}
5340	default:
5341	break;
5342	}
5343	}
5344	}
5345
5346	computeKnownFPClass(V: Src, DemandedElts, InterestedClasses, Known, Depth: Depth + 1,
5347	Q);
5348	break;
5349	}
5350	case Instruction::PHI: {
5351	const PHINode *P = cast<PHINode>(Val: Op);
5352	// Unreachable blocks may have zero-operand PHI nodes.
5353	if (P->getNumIncomingValues() == 0)
5354	break;
5355
5356	// Otherwise take the unions of the known bit sets of the operands,
5357	// taking conservative care to avoid excessive recursion.
5358	const unsigned PhiRecursionLimit = MaxAnalysisRecursionDepth - 2;
5359
5360	if (Depth < PhiRecursionLimit) {
5361	// Skip if every incoming value references to ourself.
5362	if (isa_and_nonnull<UndefValue>(Val: P->hasConstantValue()))
5363	break;
5364
5365	bool First = true;
5366
5367	for (const Use &U : P->operands()) {
5368	Value *IncValue = U.get();
5369	// Skip direct self references.
5370	if (IncValue == P)
5371	continue;
5372
5373	KnownFPClass KnownSrc;
5374	// Recurse, but cap the recursion to two levels, because we don't want
5375	// to waste time spinning around in loops. We need at least depth 2 to
5376	// detect known sign bits.
5377	computeKnownFPClass(
5378	V: IncValue, DemandedElts, InterestedClasses, Known&: KnownSrc,
5379	Depth: PhiRecursionLimit,
5380	Q: Q.getWithInstruction(I: P->getIncomingBlock(U)->getTerminator()));
5381
5382	if (First) {
5383	Known = KnownSrc;
5384	First = false;
5385	} else {
5386	Known |= KnownSrc;
5387	}
5388
5389	if (Known.KnownFPClasses == fcAllFlags)
5390	break;
5391	}
5392	}
5393
5394	break;
5395	}
5396	default:
5397	break;
5398	}
5399	}
5400
5401	KnownFPClass llvm::computeKnownFPClass(const Value *V,
5402	const APInt &DemandedElts,
5403	FPClassTest InterestedClasses,
5404	unsigned Depth,
5405	const SimplifyQuery &SQ) {
5406	KnownFPClass KnownClasses;
5407	::computeKnownFPClass(V, DemandedElts, InterestedClasses, Known&: KnownClasses, Depth,
5408	Q: SQ);
5409	return KnownClasses;
5410	}
5411
5412	KnownFPClass llvm::computeKnownFPClass(const Value *V,
5413	FPClassTest InterestedClasses,
5414	unsigned Depth,
5415	const SimplifyQuery &SQ) {
5416	KnownFPClass Known;
5417	::computeKnownFPClass(V, Known, InterestedClasses, Depth, Q: SQ);
5418	return Known;
5419	}
5420
5421	Value *llvm::isBytewiseValue(Value *V, const DataLayout &DL) {
5422
5423	// All byte-wide stores are splatable, even of arbitrary variables.
5424	if (V->getType()->isIntegerTy(Bitwidth: 8))
5425	return V;
5426
5427	LLVMContext &Ctx = V->getContext();
5428
5429	// Undef don't care.
5430	auto *UndefInt8 = UndefValue::get(T: Type::getInt8Ty(C&: Ctx));
5431	if (isa<UndefValue>(Val: V))
5432	return UndefInt8;
5433
5434	// Return Undef for zero-sized type.
5435	if (DL.getTypeStoreSize(Ty: V->getType()).isZero())
5436	return UndefInt8;
5437
5438	Constant *C = dyn_cast<Constant>(Val: V);
5439	if (!C) {
5440	// Conceptually, we could handle things like:
5441	// %a = zext i8 %X to i16
5442	// %b = shl i16 %a, 8
5443	// %c = or i16 %a, %b
5444	// but until there is an example that actually needs this, it doesn't seem
5445	// worth worrying about.
5446	return nullptr;
5447	}
5448
5449	// Handle 'null' ConstantArrayZero etc.
5450	if (C->isNullValue())
5451	return Constant::getNullValue(Ty: Type::getInt8Ty(C&: Ctx));
5452
5453	// Constant floating-point values can be handled as integer values if the
5454	// corresponding integer value is "byteable". An important case is 0.0.
5455	if (ConstantFP *CFP = dyn_cast<ConstantFP>(Val: C)) {
5456	Type *Ty = nullptr;
5457	if (CFP->getType()->isHalfTy())
5458	Ty = Type::getInt16Ty(C&: Ctx);
5459	else if (CFP->getType()->isFloatTy())
5460	Ty = Type::getInt32Ty(C&: Ctx);
5461	else if (CFP->getType()->isDoubleTy())
5462	Ty = Type::getInt64Ty(C&: Ctx);
5463	// Don't handle long double formats, which have strange constraints.
5464	return Ty ? isBytewiseValue(V: ConstantExpr::getBitCast(C: CFP, Ty), DL)
5465	: nullptr;
5466	}
5467
5468	// We can handle constant integers that are multiple of 8 bits.
5469	if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: C)) {
5470	if (CI->getBitWidth() % 8 == 0) {
5471	assert(CI->getBitWidth() > 8 && "8 bits should be handled above!");
5472	if (!CI->getValue().isSplat(SplatSizeInBits: 8))
5473	return nullptr;
5474	return ConstantInt::get(Context&: Ctx, V: CI->getValue().trunc(width: 8));
5475	}
5476	}
5477
5478	if (auto *CE = dyn_cast<ConstantExpr>(Val: C)) {
5479	if (CE->getOpcode() == Instruction::IntToPtr) {
5480	if (auto *PtrTy = dyn_cast<PointerType>(Val: CE->getType())) {
5481	unsigned BitWidth = DL.getPointerSizeInBits(AS: PtrTy->getAddressSpace());
5482	if (Constant *Op = ConstantFoldIntegerCast(
5483	C: CE->getOperand(i_nocapture: 0), DestTy: Type::getIntNTy(C&: Ctx, N: BitWidth), IsSigned: false, DL))
5484	return isBytewiseValue(V: Op, DL);
5485	}
5486	}
5487	}
5488
5489	auto Merge = [&](Value *LHS, Value *RHS) -> Value * {
5490	if (LHS == RHS)
5491	return LHS;
5492	if (!LHS || !RHS)
5493	return nullptr;
5494	if (LHS == UndefInt8)
5495	return RHS;
5496	if (RHS == UndefInt8)
5497	return LHS;
5498	return nullptr;
5499	};
5500
5501	if (ConstantDataSequential *CA = dyn_cast<ConstantDataSequential>(Val: C)) {
5502	Value *Val = UndefInt8;
5503	for (unsigned I = 0, E = CA->getNumElements(); I != E; ++I)
5504	if (!(Val = Merge(Val, isBytewiseValue(V: CA->getElementAsConstant(i: I), DL))))
5505	return nullptr;
5506	return Val;
5507	}
5508
5509	if (isa<ConstantAggregate>(Val: C)) {
5510	Value *Val = UndefInt8;
5511	for (unsigned I = 0, E = C->getNumOperands(); I != E; ++I)
5512	if (!(Val = Merge(Val, isBytewiseValue(V: C->getOperand(i: I), DL))))
5513	return nullptr;
5514	return Val;
5515	}
5516
5517	// Don't try to handle the handful of other constants.
5518	return nullptr;
5519	}
5520
5521	// This is the recursive version of BuildSubAggregate. It takes a few different
5522	// arguments. Idxs is the index within the nested struct From that we are
5523	// looking at now (which is of type IndexedType). IdxSkip is the number of
5524	// indices from Idxs that should be left out when inserting into the resulting
5525	// struct. To is the result struct built so far, new insertvalue instructions
5526	// build on that.
5527	static Value *BuildSubAggregate(Value *From, Value* To, Type *IndexedType,
5528	SmallVectorImpl<unsigned> &Idxs,
5529	unsigned IdxSkip,
5530	Instruction *InsertBefore) {
5531	StructType *STy = dyn_cast<StructType>(Val: IndexedType);
5532	if (STy) {
5533	// Save the original To argument so we can modify it
5534	Value *OrigTo = To;
5535	// General case, the type indexed by Idxs is a struct
5536	for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
5537	// Process each struct element recursively
5538	Idxs.push_back(Elt: i);
5539	Value *PrevTo = To;
5540	To = BuildSubAggregate(From, To, IndexedType: STy->getElementType(N: i), Idxs, IdxSkip,
5541	InsertBefore);
5542	Idxs.pop_back();
5543	if (!To) {
5544	// Couldn't find any inserted value for this index? Cleanup
5545	while (PrevTo != OrigTo) {
5546	InsertValueInst* Del = cast<InsertValueInst>(Val: PrevTo);
5547	PrevTo = Del->getAggregateOperand();
5548	Del->eraseFromParent();
5549	}
5550	// Stop processing elements
5551	break;
5552	}
5553	}
5554	// If we successfully found a value for each of our subaggregates
5555	if (To)
5556	return To;
5557	}
5558	// Base case, the type indexed by SourceIdxs is not a struct, or not all of
5559	// the struct's elements had a value that was inserted directly. In the latter
5560	// case, perhaps we can't determine each of the subelements individually, but
5561	// we might be able to find the complete struct somewhere.
5562
5563	// Find the value that is at that particular spot
5564	Value *V = FindInsertedValue(V: From, idx_range: Idxs);
5565
5566	if (!V)
5567	return nullptr;
5568
5569	// Insert the value in the new (sub) aggregate
5570	return InsertValueInst::Create(Agg: To, Val: V, Idxs: ArrayRef(Idxs).slice(N: IdxSkip), NameStr: "tmp",
5571	InsertBefore);
5572	}
5573
5574	// This helper takes a nested struct and extracts a part of it (which is again a
5575	// struct) into a new value. For example, given the struct:
5576	// { a, { b, { c, d }, e } }
5577	// and the indices "1, 1" this returns
5578	// { c, d }.
5579	//
5580	// It does this by inserting an insertvalue for each element in the resulting
5581	// struct, as opposed to just inserting a single struct. This will only work if
5582	// each of the elements of the substruct are known (ie, inserted into From by an
5583	// insertvalue instruction somewhere).
5584	//
5585	// All inserted insertvalue instructions are inserted before InsertBefore
5586	static Value *BuildSubAggregate(Value *From, ArrayRef<unsigned> idx_range,
5587	Instruction *InsertBefore) {
5588	assert(InsertBefore && "Must have someplace to insert!");
5589	Type *IndexedType = ExtractValueInst::getIndexedType(Agg: From->getType(),
5590	Idxs: idx_range);
5591	Value *To = PoisonValue::get(T: IndexedType);
5592	SmallVector<unsigned, 10> Idxs(idx_range.begin(), idx_range.end());
5593	unsigned IdxSkip = Idxs.size();
5594
5595	return BuildSubAggregate(From, To, IndexedType, Idxs, IdxSkip, InsertBefore);
5596	}
5597
5598	/// Given an aggregate and a sequence of indices, see if the scalar value
5599	/// indexed is already around as a register, for example if it was inserted
5600	/// directly into the aggregate.
5601	///
5602	/// If InsertBefore is not null, this function will duplicate (modified)
5603	/// insertvalues when a part of a nested struct is extracted.
5604	Value *llvm::FindInsertedValue(Value *V, ArrayRef<unsigned> idx_range,
5605	Instruction *InsertBefore) {
5606	// Nothing to index? Just return V then (this is useful at the end of our
5607	// recursion).
5608	if (idx_range.empty())
5609	return V;
5610	// We have indices, so V should have an indexable type.
5611	assert((V->getType()->isStructTy() || V->getType()->isArrayTy()) &&
5612	"Not looking at a struct or array?");
5613	assert(ExtractValueInst::getIndexedType(V->getType(), idx_range) &&
5614	"Invalid indices for type?");
5615
5616	if (Constant *C = dyn_cast<Constant>(Val: V)) {
5617	C = C->getAggregateElement(Elt: idx_range[0]);
5618	if (!C) return nullptr;
5619	return FindInsertedValue(V: C, idx_range: idx_range.slice(N: 1), InsertBefore);
5620	}
5621
5622	if (InsertValueInst *I = dyn_cast<InsertValueInst>(Val: V)) {
5623	// Loop the indices for the insertvalue instruction in parallel with the
5624	// requested indices
5625	const unsigned *req_idx = idx_range.begin();
5626	for (const unsigned *i = I->idx_begin(), *e = I->idx_end();
5627	i != e; ++i, ++req_idx) {
5628	if (req_idx == idx_range.end()) {
5629	// We can't handle this without inserting insertvalues
5630	if (!InsertBefore)
5631	return nullptr;
5632
5633	// The requested index identifies a part of a nested aggregate. Handle
5634	// this specially. For example,
5635	// %A = insertvalue { i32, {i32, i32 } } undef, i32 10, 1, 0
5636	// %B = insertvalue { i32, {i32, i32 } } %A, i32 11, 1, 1
5637	// %C = extractvalue {i32, { i32, i32 } } %B, 1
5638	// This can be changed into
5639	// %A = insertvalue {i32, i32 } undef, i32 10, 0
5640	// %C = insertvalue {i32, i32 } %A, i32 11, 1
5641	// which allows the unused 0,0 element from the nested struct to be
5642	// removed.
5643	return BuildSubAggregate(From: V, idx_range: ArrayRef(idx_range.begin(), req_idx),
5644	InsertBefore);
5645	}
5646
5647	// This insert value inserts something else than what we are looking for.
5648	// See if the (aggregate) value inserted into has the value we are
5649	// looking for, then.
5650	if (*req_idx != *i)
5651	return FindInsertedValue(V: I->getAggregateOperand(), idx_range,
5652	InsertBefore);
5653	}
5654	// If we end up here, the indices of the insertvalue match with those
5655	// requested (though possibly only partially). Now we recursively look at
5656	// the inserted value, passing any remaining indices.
5657	return FindInsertedValue(V: I->getInsertedValueOperand(),
5658	idx_range: ArrayRef(req_idx, idx_range.end()), InsertBefore);
5659	}
5660
5661	if (ExtractValueInst *I = dyn_cast<ExtractValueInst>(Val: V)) {
5662	// If we're extracting a value from an aggregate that was extracted from
5663	// something else, we can extract from that something else directly instead.
5664	// However, we will need to chain I's indices with the requested indices.
5665
5666	// Calculate the number of indices required
5667	unsigned size = I->getNumIndices() + idx_range.size();
5668	// Allocate some space to put the new indices in
5669	SmallVector<unsigned, 5> Idxs;
5670	Idxs.reserve(N: size);
5671	// Add indices from the extract value instruction
5672	Idxs.append(in_start: I->idx_begin(), in_end: I->idx_end());
5673
5674	// Add requested indices
5675	Idxs.append(in_start: idx_range.begin(), in_end: idx_range.end());
5676
5677	assert(Idxs.size() == size
5678	&& "Number of indices added not correct?");
5679
5680	return FindInsertedValue(V: I->getAggregateOperand(), idx_range: Idxs, InsertBefore);
5681	}
5682	// Otherwise, we don't know (such as, extracting from a function return value
5683	// or load instruction)
5684	return nullptr;
5685	}
5686
5687	bool llvm::isGEPBasedOnPointerToString(const GEPOperator *GEP,
5688	unsigned CharSize) {
5689	// Make sure the GEP has exactly three arguments.
5690	if (GEP->getNumOperands() != 3)
5691	return false;
5692
5693	// Make sure the index-ee is a pointer to array of \p CharSize integers.
5694	// CharSize.
5695	ArrayType *AT = dyn_cast<ArrayType>(Val: GEP->getSourceElementType());
5696	if (!AT || !AT->getElementType()->isIntegerTy(Bitwidth: CharSize))
5697	return false;
5698
5699	// Check to make sure that the first operand of the GEP is an integer and
5700	// has value 0 so that we are sure we're indexing into the initializer.
5701	const ConstantInt *FirstIdx = dyn_cast<ConstantInt>(Val: GEP->getOperand(i_nocapture: 1));
5702	if (!FirstIdx || !FirstIdx->isZero())
5703	return false;
5704
5705	return true;
5706	}
5707
5708	// If V refers to an initialized global constant, set Slice either to
5709	// its initializer if the size of its elements equals ElementSize, or,
5710	// for ElementSize == 8, to its representation as an array of unsiged
5711	// char. Return true on success.
5712	// Offset is in the unit "nr of ElementSize sized elements".
5713	bool llvm::getConstantDataArrayInfo(const Value *V,
5714	ConstantDataArraySlice &Slice,
5715	unsigned ElementSize, uint64_t Offset) {
5716	assert(V && "V should not be null.");
5717	assert((ElementSize % 8) == 0 &&
5718	"ElementSize expected to be a multiple of the size of a byte.");
5719	unsigned ElementSizeInBytes = ElementSize / 8;
5720
5721	// Drill down into the pointer expression V, ignoring any intervening
5722	// casts, and determine the identity of the object it references along
5723	// with the cumulative byte offset into it.
5724	const GlobalVariable *GV =
5725	dyn_cast<GlobalVariable>(Val: getUnderlyingObject(V));
5726	if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer())
5727	// Fail if V is not based on constant global object.
5728	return false;
5729
5730	const DataLayout &DL = GV->getParent()->getDataLayout();
5731	APInt Off(DL.getIndexTypeSizeInBits(Ty: V->getType()), 0);
5732
5733	if (GV != V->stripAndAccumulateConstantOffsets(DL, Offset&: Off,
5734	/*AllowNonInbounds*/ true))
5735	// Fail if a constant offset could not be determined.
5736	return false;
5737
5738	uint64_t StartIdx = Off.getLimitedValue();
5739	if (StartIdx == UINT64_MAX)
5740	// Fail if the constant offset is excessive.
5741	return false;
5742
5743	// Off/StartIdx is in the unit of bytes. So we need to convert to number of
5744	// elements. Simply bail out if that isn't possible.
5745	if ((StartIdx % ElementSizeInBytes) != 0)
5746	return false;
5747
5748	Offset += StartIdx / ElementSizeInBytes;
5749	ConstantDataArray *Array = nullptr;
5750	ArrayType *ArrayTy = nullptr;
5751
5752	if (GV->getInitializer()->isNullValue()) {
5753	Type *GVTy = GV->getValueType();
5754	uint64_t SizeInBytes = DL.getTypeStoreSize(Ty: GVTy).getFixedValue();
5755	uint64_t Length = SizeInBytes / ElementSizeInBytes;
5756
5757	Slice.Array = nullptr;
5758	Slice.Offset = 0;
5759	// Return an empty Slice for undersized constants to let callers
5760	// transform even undefined library calls into simpler, well-defined
5761	// expressions. This is preferable to making the calls although it
5762	// prevents sanitizers from detecting such calls.
5763	Slice.Length = Length < Offset ? 0 : Length - Offset;
5764	return true;
5765	}
5766
5767	auto *Init = const_cast<Constant *>(GV->getInitializer());
5768	if (auto *ArrayInit = dyn_cast<ConstantDataArray>(Val: Init)) {
5769	Type *InitElTy = ArrayInit->getElementType();
5770	if (InitElTy->isIntegerTy(Bitwidth: ElementSize)) {
5771	// If Init is an initializer for an array of the expected type
5772	// and size, use it as is.
5773	Array = ArrayInit;
5774	ArrayTy = ArrayInit->getType();
5775	}
5776	}
5777
5778	if (!Array) {
5779	if (ElementSize != 8)
5780	// TODO: Handle conversions to larger integral types.
5781	return false;
5782
5783	// Otherwise extract the portion of the initializer starting
5784	// at Offset as an array of bytes, and reset Offset.
5785	Init = ReadByteArrayFromGlobal(GV, Offset);
5786	if (!Init)
5787	return false;
5788
5789	Offset = 0;
5790	Array = dyn_cast<ConstantDataArray>(Val: Init);
5791	ArrayTy = dyn_cast<ArrayType>(Val: Init->getType());
5792	}
5793
5794	uint64_t NumElts = ArrayTy->getArrayNumElements();
5795	if (Offset > NumElts)
5796	return false;
5797
5798	Slice.Array = Array;
5799	Slice.Offset = Offset;
5800	Slice.Length = NumElts - Offset;
5801	return true;
5802	}
5803
5804	/// Extract bytes from the initializer of the constant array V, which need
5805	/// not be a nul-terminated string. On success, store the bytes in Str and
5806	/// return true. When TrimAtNul is set, Str will contain only the bytes up
5807	/// to but not including the first nul. Return false on failure.
5808	bool llvm::getConstantStringInfo(const Value *V, StringRef &Str,
5809	bool TrimAtNul) {
5810	ConstantDataArraySlice Slice;
5811	if (!getConstantDataArrayInfo(V, Slice, ElementSize: 8))
5812	return false;
5813
5814	if (Slice.Array == nullptr) {
5815	if (TrimAtNul) {
5816	// Return a nul-terminated string even for an empty Slice. This is
5817	// safe because all existing SimplifyLibcalls callers require string
5818	// arguments and the behavior of the functions they fold is undefined
5819	// otherwise. Folding the calls this way is preferable to making
5820	// the undefined library calls, even though it prevents sanitizers
5821	// from reporting such calls.
5822	Str = StringRef();
5823	return true;
5824	}
5825	if (Slice.Length == 1) {
5826	Str = StringRef("", 1);
5827	return true;
5828	}
5829	// We cannot instantiate a StringRef as we do not have an appropriate string
5830	// of 0s at hand.
5831	return false;
5832	}
5833
5834	// Start out with the entire array in the StringRef.
5835	Str = Slice.Array->getAsString();
5836	// Skip over 'offset' bytes.
5837	Str = Str.substr(Start: Slice.Offset);
5838
5839	if (TrimAtNul) {
5840	// Trim off the \0 and anything after it. If the array is not nul
5841	// terminated, we just return the whole end of string. The client may know
5842	// some other way that the string is length-bound.
5843	Str = Str.substr(Start: 0, N: Str.find(C: '\0'));
5844	}
5845	return true;
5846	}
5847
5848	// These next two are very similar to the above, but also look through PHI
5849	// nodes.
5850	// TODO: See if we can integrate these two together.
5851
5852	/// If we can compute the length of the string pointed to by
5853	/// the specified pointer, return 'len+1'. If we can't, return 0.
5854	static uint64_t GetStringLengthH(const Value *V,
5855	SmallPtrSetImpl<const PHINode*> &PHIs,
5856	unsigned CharSize) {
5857	// Look through noop bitcast instructions.
5858	V = V->stripPointerCasts();
5859
5860	// If this is a PHI node, there are two cases: either we have already seen it
5861	// or we haven't.
5862	if (const PHINode *PN = dyn_cast<PHINode>(Val: V)) {
5863	if (!PHIs.insert(Ptr: PN).second)
5864	return ~0ULL; // already in the set.
5865
5866	// If it was new, see if all the input strings are the same length.
5867	uint64_t LenSoFar = ~0ULL;
5868	for (Value *IncValue : PN->incoming_values()) {
5869	uint64_t Len = GetStringLengthH(V: IncValue, PHIs, CharSize);
5870	if (Len == 0) return 0; // Unknown length -> unknown.
5871
5872	if (Len == ~0ULL) continue;
5873
5874	if (Len != LenSoFar && LenSoFar != ~0ULL)
5875	return 0; // Disagree -> unknown.
5876	LenSoFar = Len;
5877	}
5878
5879	// Success, all agree.
5880	return LenSoFar;
5881	}
5882
5883	// strlen(select(c,x,y)) -> strlen(x) ^ strlen(y)
5884	if (const SelectInst *SI = dyn_cast<SelectInst>(Val: V)) {
5885	uint64_t Len1 = GetStringLengthH(V: SI->getTrueValue(), PHIs, CharSize);
5886	if (Len1 == 0) return 0;
5887	uint64_t Len2 = GetStringLengthH(V: SI->getFalseValue(), PHIs, CharSize);
5888	if (Len2 == 0) return 0;
5889	if (Len1 == ~0ULL) return Len2;
5890	if (Len2 == ~0ULL) return Len1;
5891	if (Len1 != Len2) return 0;
5892	return Len1;
5893	}
5894
5895	// Otherwise, see if we can read the string.
5896	ConstantDataArraySlice Slice;
5897	if (!getConstantDataArrayInfo(V, Slice, ElementSize: CharSize))
5898	return 0;
5899
5900	if (Slice.Array == nullptr)
5901	// Zeroinitializer (including an empty one).
5902	return 1;
5903
5904	// Search for the first nul character. Return a conservative result even
5905	// when there is no nul. This is safe since otherwise the string function
5906	// being folded such as strlen is undefined, and can be preferable to
5907	// making the undefined library call.
5908	unsigned NullIndex = 0;
5909	for (unsigned E = Slice.Length; NullIndex < E; ++NullIndex) {
5910	if (Slice.Array->getElementAsInteger(i: Slice.Offset + NullIndex) == 0)
5911	break;
5912	}
5913
5914	return NullIndex + 1;
5915	}
5916
5917	/// If we can compute the length of the string pointed to by
5918	/// the specified pointer, return 'len+1'. If we can't, return 0.
5919	uint64_t llvm::GetStringLength(const Value *V, unsigned CharSize) {
5920	if (!V->getType()->isPointerTy())
5921	return 0;
5922
5923	SmallPtrSet<const PHINode*, 32> PHIs;
5924	uint64_t Len = GetStringLengthH(V, PHIs, CharSize);
5925	// If Len is ~0ULL, we had an infinite phi cycle: this is dead code, so return
5926	// an empty string as a length.
5927	return Len == ~0ULL ? 1 : Len;
5928	}
5929
5930	const Value *
5931	llvm::getArgumentAliasingToReturnedPointer(const CallBase *Call,
5932	bool MustPreserveNullness) {
5933	assert(Call &&
5934	"getArgumentAliasingToReturnedPointer only works on nonnull calls");
5935	if (const Value *RV = Call->getReturnedArgOperand())
5936	return RV;
5937	// This can be used only as a aliasing property.
5938	if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
5939	Call, MustPreserveNullness))
5940	return Call->getArgOperand(i: 0);
5941	return nullptr;
5942	}
5943
5944	bool llvm::isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
5945	const CallBase *Call, bool MustPreserveNullness) {
5946	switch (Call->getIntrinsicID()) {
5947	case Intrinsic::launder_invariant_group:
5948	case Intrinsic::strip_invariant_group:
5949	case Intrinsic::aarch64_irg:
5950	case Intrinsic::aarch64_tagp:
5951	// The amdgcn_make_buffer_rsrc function does not alter the address of the
5952	// input pointer (and thus preserve null-ness for the purposes of escape
5953	// analysis, which is where the MustPreserveNullness flag comes in to play).
5954	// However, it will not necessarily map ptr addrspace(N) null to ptr
5955	// addrspace(8) null, aka the "null descriptor", which has "all loads return
5956	// 0, all stores are dropped" semantics. Given the context of this intrinsic
5957	// list, no one should be relying on such a strict interpretation of
5958	// MustPreserveNullness (and, at time of writing, they are not), but we
5959	// document this fact out of an abundance of caution.
5960	case Intrinsic::amdgcn_make_buffer_rsrc:
5961	return true;
5962	case Intrinsic::ptrmask:
5963	return !MustPreserveNullness;
5964	default:
5965	return false;
5966	}
5967	}
5968
5969	/// \p PN defines a loop-variant pointer to an object. Check if the
5970	/// previous iteration of the loop was referring to the same object as \p PN.
5971	static bool isSameUnderlyingObjectInLoop(const PHINode *PN,
5972	const LoopInfo *LI) {
5973	// Find the loop-defined value.
5974	Loop *L = LI->getLoopFor(BB: PN->getParent());
5975	if (PN->getNumIncomingValues() != 2)
5976	return true;
5977
5978	// Find the value from previous iteration.
5979	auto *PrevValue = dyn_cast<Instruction>(Val: PN->getIncomingValue(i: 0));
5980	if (!PrevValue || LI->getLoopFor(BB: PrevValue->getParent()) != L)
5981	PrevValue = dyn_cast<Instruction>(Val: PN->getIncomingValue(i: 1));
5982	if (!PrevValue || LI->getLoopFor(BB: PrevValue->getParent()) != L)
5983	return true;
5984
5985	// If a new pointer is loaded in the loop, the pointer references a different
5986	// object in every iteration. E.g.:
5987	// for (i)
5988	// int *p = a[i];
5989	// ...
5990	if (auto *Load = dyn_cast<LoadInst>(Val: PrevValue))
5991	if (!L->isLoopInvariant(V: Load->getPointerOperand()))
5992	return false;
5993	return true;
5994	}
5995
5996	const Value *llvm::getUnderlyingObject(const Value *V, unsigned MaxLookup) {
5997	if (!V->getType()->isPointerTy())
5998	return V;
5999	for (unsigned Count = 0; MaxLookup == 0 || Count < MaxLookup; ++Count) {
6000	if (auto *GEP = dyn_cast<GEPOperator>(Val: V)) {
6001	V = GEP->getPointerOperand();
6002	} else if (Operator::getOpcode(V) == Instruction::BitCast ||
6003	Operator::getOpcode(V) == Instruction::AddrSpaceCast) {
6004	V = cast<Operator>(Val: V)->getOperand(i: 0);
6005	if (!V->getType()->isPointerTy())
6006	return V;
6007	} else if (auto *GA = dyn_cast<GlobalAlias>(Val: V)) {
6008	if (GA->isInterposable())
6009	return V;
6010	V = GA->getAliasee();
6011	} else {
6012	if (auto *PHI = dyn_cast<PHINode>(Val: V)) {
6013	// Look through single-arg phi nodes created by LCSSA.
6014	if (PHI->getNumIncomingValues() == 1) {
6015	V = PHI->getIncomingValue(i: 0);
6016	continue;
6017	}
6018	} else if (auto *Call = dyn_cast<CallBase>(Val: V)) {
6019	// CaptureTracking can know about special capturing properties of some
6020	// intrinsics like launder.invariant.group, that can't be expressed with
6021	// the attributes, but have properties like returning aliasing pointer.
6022	// Because some analysis may assume that nocaptured pointer is not
6023	// returned from some special intrinsic (because function would have to
6024	// be marked with returns attribute), it is crucial to use this function
6025	// because it should be in sync with CaptureTracking. Not using it may
6026	// cause weird miscompilations where 2 aliasing pointers are assumed to
6027	// noalias.
6028	if (auto *RP = getArgumentAliasingToReturnedPointer(Call, MustPreserveNullness: false)) {
6029	V = RP;
6030	continue;
6031	}
6032	}
6033
6034	return V;
6035	}
6036	assert(V->getType()->isPointerTy() && "Unexpected operand type!");
6037	}
6038	return V;
6039	}
6040
6041	void llvm::getUnderlyingObjects(const Value *V,
6042	SmallVectorImpl<const Value *> &Objects,
6043	LoopInfo *LI, unsigned MaxLookup) {
6044	SmallPtrSet<const Value *, 4> Visited;
6045	SmallVector<const Value *, 4> Worklist;
6046	Worklist.push_back(Elt: V);
6047	do {
6048	const Value *P = Worklist.pop_back_val();
6049	P = getUnderlyingObject(V: P, MaxLookup);
6050
6051	if (!Visited.insert(Ptr: P).second)
6052	continue;
6053
6054	if (auto *SI = dyn_cast<SelectInst>(Val: P)) {
6055	Worklist.push_back(Elt: SI->getTrueValue());
6056	Worklist.push_back(Elt: SI->getFalseValue());
6057	continue;
6058	}
6059
6060	if (auto *PN = dyn_cast<PHINode>(Val: P)) {
6061	// If this PHI changes the underlying object in every iteration of the
6062	// loop, don't look through it. Consider:
6063	// int **A;
6064	// for (i) {
6065	// Prev = Curr; // Prev = PHI (Prev_0, Curr)
6066	// Curr = A[i];
6067	// *Prev, *Curr;
6068	//
6069	// Prev is tracking Curr one iteration behind so they refer to different
6070	// underlying objects.
6071	if (!LI || !LI->isLoopHeader(BB: PN->getParent()) ||
6072	isSameUnderlyingObjectInLoop(PN, LI))
6073	append_range(C&: Worklist, R: PN->incoming_values());
6074	continue;
6075	}
6076
6077	Objects.push_back(Elt: P);
6078	} while (!Worklist.empty());
6079	}
6080
6081	/// This is the function that does the work of looking through basic
6082	/// ptrtoint+arithmetic+inttoptr sequences.
6083	static const Value *getUnderlyingObjectFromInt(const Value *V) {
6084	do {
6085	if (const Operator *U = dyn_cast<Operator>(Val: V)) {
6086	// If we find a ptrtoint, we can transfer control back to the
6087	// regular getUnderlyingObjectFromInt.
6088	if (U->getOpcode() == Instruction::PtrToInt)
6089	return U->getOperand(i: 0);
6090	// If we find an add of a constant, a multiplied value, or a phi, it's
6091	// likely that the other operand will lead us to the base
6092	// object. We don't have to worry about the case where the
6093	// object address is somehow being computed by the multiply,
6094	// because our callers only care when the result is an
6095	// identifiable object.
6096	if (U->getOpcode() != Instruction::Add ||
6097	(!isa<ConstantInt>(Val: U->getOperand(i: 1)) &&
6098	Operator::getOpcode(V: U->getOperand(i: 1)) != Instruction::Mul &&
6099	!isa<PHINode>(Val: U->getOperand(i: 1))))
6100	return V;
6101	V = U->getOperand(i: 0);
6102	} else {
6103	return V;
6104	}
6105	assert(V->getType()->isIntegerTy() && "Unexpected operand type!");
6106	} while (true);
6107	}
6108
6109	/// This is a wrapper around getUnderlyingObjects and adds support for basic
6110	/// ptrtoint+arithmetic+inttoptr sequences.
6111	/// It returns false if unidentified object is found in getUnderlyingObjects.
6112	bool llvm::getUnderlyingObjectsForCodeGen(const Value *V,
6113	SmallVectorImpl<Value *> &Objects) {
6114	SmallPtrSet<const Value *, 16> Visited;
6115	SmallVector<const Value *, 4> Working(1, V);
6116	do {
6117	V = Working.pop_back_val();
6118
6119	SmallVector<const Value *, 4> Objs;
6120	getUnderlyingObjects(V, Objects&: Objs);
6121
6122	for (const Value *V : Objs) {
6123	if (!Visited.insert(Ptr: V).second)
6124	continue;
6125	if (Operator::getOpcode(V) == Instruction::IntToPtr) {
6126	const Value *O =
6127	getUnderlyingObjectFromInt(V: cast<User>(Val: V)->getOperand(i: 0));
6128	if (O->getType()->isPointerTy()) {
6129	Working.push_back(Elt: O);
6130	continue;
6131	}
6132	}
6133	// If getUnderlyingObjects fails to find an identifiable object,
6134	// getUnderlyingObjectsForCodeGen also fails for safety.
6135	if (!isIdentifiedObject(V)) {
6136	Objects.clear();
6137	return false;
6138	}
6139	Objects.push_back(Elt: const_cast<Value *>(V));
6140	}
6141	} while (!Working.empty());
6142	return true;
6143	}
6144
6145	AllocaInst *llvm::findAllocaForValue(Value *V, bool OffsetZero) {
6146	AllocaInst *Result = nullptr;
6147	SmallPtrSet<Value *, 4> Visited;
6148	SmallVector<Value *, 4> Worklist;
6149
6150	auto AddWork = [&](Value *V) {
6151	if (Visited.insert(Ptr: V).second)
6152	Worklist.push_back(Elt: V);
6153	};
6154
6155	AddWork(V);
6156	do {
6157	V = Worklist.pop_back_val();
6158	assert(Visited.count(V));
6159
6160	if (AllocaInst *AI = dyn_cast<AllocaInst>(Val: V)) {
6161	if (Result && Result != AI)
6162	return nullptr;
6163	Result = AI;
6164	} else if (CastInst *CI = dyn_cast<CastInst>(Val: V)) {
6165	AddWork(CI->getOperand(i_nocapture: 0));
6166	} else if (PHINode *PN = dyn_cast<PHINode>(Val: V)) {
6167	for (Value *IncValue : PN->incoming_values())
6168	AddWork(IncValue);
6169	} else if (auto *SI = dyn_cast<SelectInst>(Val: V)) {
6170	AddWork(SI->getTrueValue());
6171	AddWork(SI->getFalseValue());
6172	} else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: V)) {
6173	if (OffsetZero && !GEP->hasAllZeroIndices())
6174	return nullptr;
6175	AddWork(GEP->getPointerOperand());
6176	} else if (CallBase *CB = dyn_cast<CallBase>(Val: V)) {
6177	Value *Returned = CB->getReturnedArgOperand();
6178	if (Returned)
6179	AddWork(Returned);
6180	else
6181	return nullptr;
6182	} else {
6183	return nullptr;
6184	}
6185	} while (!Worklist.empty());
6186
6187	return Result;
6188	}
6189
6190	static bool onlyUsedByLifetimeMarkersOrDroppableInstsHelper(
6191	const Value *V, bool AllowLifetime, bool AllowDroppable) {
6192	for (const User *U : V->users()) {
6193	const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U);
6194	if (!II)
6195	return false;
6196
6197	if (AllowLifetime && II->isLifetimeStartOrEnd())
6198	continue;
6199
6200	if (AllowDroppable && II->isDroppable())
6201	continue;
6202
6203	return false;
6204	}
6205	return true;
6206	}
6207
6208	bool llvm::onlyUsedByLifetimeMarkers(const Value *V) {
6209	return onlyUsedByLifetimeMarkersOrDroppableInstsHelper(
6210	V, /* AllowLifetime */ true, /* AllowDroppable */ false);
6211	}
6212	bool llvm::onlyUsedByLifetimeMarkersOrDroppableInsts(const Value *V) {
6213	return onlyUsedByLifetimeMarkersOrDroppableInstsHelper(
6214	V, /* AllowLifetime */ true, /* AllowDroppable */ true);
6215	}
6216
6217	bool llvm::mustSuppressSpeculation(const LoadInst &LI) {
6218	if (!LI.isUnordered())
6219	return true;
6220	const Function &F = *LI.getFunction();
6221	// Speculative load may create a race that did not exist in the source.
6222	return F.hasFnAttribute(Attribute::SanitizeThread) ||
6223	// Speculative load may load data from dirty regions.
6224	F.hasFnAttribute(Attribute::SanitizeAddress) ||
6225	F.hasFnAttribute(Attribute::SanitizeHWAddress);
6226	}
6227
6228	bool llvm::isSafeToSpeculativelyExecute(const Instruction *Inst,
6229	const Instruction *CtxI,
6230	AssumptionCache *AC,
6231	const DominatorTree *DT,
6232	const TargetLibraryInfo *TLI) {
6233	return isSafeToSpeculativelyExecuteWithOpcode(Opcode: Inst->getOpcode(), Inst, CtxI,
6234	AC, DT, TLI);
6235	}
6236
6237	bool llvm::isSafeToSpeculativelyExecuteWithOpcode(
6238	unsigned Opcode, const Instruction *Inst, const Instruction *CtxI,
6239	AssumptionCache *AC, const DominatorTree *DT,
6240	const TargetLibraryInfo *TLI) {
6241	#ifndef NDEBUG
6242	if (Inst->getOpcode() != Opcode) {
6243	// Check that the operands are actually compatible with the Opcode override.
6244	auto hasEqualReturnAndLeadingOperandTypes =
6245	[](const Instruction *Inst, unsigned NumLeadingOperands) {
6246	if (Inst->getNumOperands() < NumLeadingOperands)
6247	return false;
6248	const Type *ExpectedType = Inst->getType();
6249	for (unsigned ItOp = 0; ItOp < NumLeadingOperands; ++ItOp)
6250	if (Inst->getOperand(i: ItOp)->getType() != ExpectedType)
6251	return false;
6252	return true;
6253	};
6254	assert(!Instruction::isBinaryOp(Opcode) ||
6255	hasEqualReturnAndLeadingOperandTypes(Inst, 2));
6256	assert(!Instruction::isUnaryOp(Opcode) ||
6257	hasEqualReturnAndLeadingOperandTypes(Inst, 1));
6258	}
6259	#endif
6260
6261	switch (Opcode) {
6262	default:
6263	return true;
6264	case Instruction::UDiv:
6265	case Instruction::URem: {
6266	// x / y is undefined if y == 0.
6267	const APInt *V;
6268	if (match(V: Inst->getOperand(i: 1), P: m_APInt(Res&: V)))
6269	return *V != 0;
6270	return false;
6271	}
6272	case Instruction::SDiv:
6273	case Instruction::SRem: {
6274	// x / y is undefined if y == 0 or x == INT_MIN and y == -1
6275	const APInt *Numerator, *Denominator;
6276	if (!match(V: Inst->getOperand(i: 1), P: m_APInt(Res&: Denominator)))
6277	return false;
6278	// We cannot hoist this division if the denominator is 0.
6279	if (*Denominator == 0)
6280	return false;
6281	// It's safe to hoist if the denominator is not 0 or -1.
6282	if (!Denominator->isAllOnes())
6283	return true;
6284	// At this point we know that the denominator is -1. It is safe to hoist as
6285	// long we know that the numerator is not INT_MIN.
6286	if (match(V: Inst->getOperand(i: 0), P: m_APInt(Res&: Numerator)))
6287	return !Numerator->isMinSignedValue();
6288	// The numerator *might* be MinSignedValue.
6289	return false;
6290	}
6291	case Instruction::Load: {
6292	const LoadInst *LI = dyn_cast<LoadInst>(Val: Inst);
6293	if (!LI)
6294	return false;
6295	if (mustSuppressSpeculation(LI: *LI))
6296	return false;
6297	const DataLayout &DL = LI->getModule()->getDataLayout();
6298	return isDereferenceableAndAlignedPointer(V: LI->getPointerOperand(),
6299	Ty: LI->getType(), Alignment: LI->getAlign(), DL,
6300	CtxI, AC, DT, TLI);
6301	}
6302	case Instruction::Call: {
6303	auto *CI = dyn_cast<const CallInst>(Val: Inst);
6304	if (!CI)
6305	return false;
6306	const Function *Callee = CI->getCalledFunction();
6307
6308	// The called function could have undefined behavior or side-effects, even
6309	// if marked readnone nounwind.
6310	return Callee && Callee->isSpeculatable();
6311	}
6312	case Instruction::VAArg:
6313	case Instruction::Alloca:
6314	case Instruction::Invoke:
6315	case Instruction::CallBr:
6316	case Instruction::PHI:
6317	case Instruction::Store:
6318	case Instruction::Ret:
6319	case Instruction::Br:
6320	case Instruction::IndirectBr:
6321	case Instruction::Switch:
6322	case Instruction::Unreachable:
6323	case Instruction::Fence:
6324	case Instruction::AtomicRMW:
6325	case Instruction::AtomicCmpXchg:
6326	case Instruction::LandingPad:
6327	case Instruction::Resume:
6328	case Instruction::CatchSwitch:
6329	case Instruction::CatchPad:
6330	case Instruction::CatchRet:
6331	case Instruction::CleanupPad:
6332	case Instruction::CleanupRet:
6333	return false; // Misc instructions which have effects
6334	}
6335	}
6336
6337	bool llvm::mayHaveNonDefUseDependency(const Instruction &I) {
6338	if (I.mayReadOrWriteMemory())
6339	// Memory dependency possible
6340	return true;
6341	if (!isSafeToSpeculativelyExecute(Inst: &I))
6342	// Can't move above a maythrow call or infinite loop. Or if an
6343	// inalloca alloca, above a stacksave call.
6344	return true;
6345	if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
6346	// 1) Can't reorder two inf-loop calls, even if readonly
6347	// 2) Also can't reorder an inf-loop call below a instruction which isn't
6348	// safe to speculative execute. (Inverse of above)
6349	return true;
6350	return false;
6351	}
6352
6353	/// Convert ConstantRange OverflowResult into ValueTracking OverflowResult.
6354	static OverflowResult mapOverflowResult(ConstantRange::OverflowResult OR) {
6355	switch (OR) {
6356	case ConstantRange::OverflowResult::MayOverflow:
6357	return OverflowResult::MayOverflow;
6358	case ConstantRange::OverflowResult::AlwaysOverflowsLow:
6359	return OverflowResult::AlwaysOverflowsLow;
6360	case ConstantRange::OverflowResult::AlwaysOverflowsHigh:
6361	return OverflowResult::AlwaysOverflowsHigh;
6362	case ConstantRange::OverflowResult::NeverOverflows:
6363	return OverflowResult::NeverOverflows;
6364	}
6365	llvm_unreachable("Unknown OverflowResult");
6366	}
6367
6368	/// Combine constant ranges from computeConstantRange() and computeKnownBits().
6369	ConstantRange
6370	llvm::computeConstantRangeIncludingKnownBits(const WithCache<const Value *> &V,
6371	bool ForSigned,
6372	const SimplifyQuery &SQ) {
6373	ConstantRange CR1 =
6374	ConstantRange::fromKnownBits(Known: V.getKnownBits(Q: SQ), IsSigned: ForSigned);
6375	ConstantRange CR2 = computeConstantRange(V, ForSigned, UseInstrInfo: SQ.IIQ.UseInstrInfo);
6376	ConstantRange::PreferredRangeType RangeType =
6377	ForSigned ? ConstantRange::Signed : ConstantRange::Unsigned;
6378	return CR1.intersectWith(CR: CR2, Type: RangeType);
6379	}
6380
6381	OverflowResult llvm::computeOverflowForUnsignedMul(const Value *LHS,
6382	const Value *RHS,
6383	const SimplifyQuery &SQ) {
6384	KnownBits LHSKnown = computeKnownBits(V: LHS, /*Depth=*/0, Q: SQ);
6385	KnownBits RHSKnown = computeKnownBits(V: RHS, /*Depth=*/0, Q: SQ);
6386	ConstantRange LHSRange = ConstantRange::fromKnownBits(Known: LHSKnown, IsSigned: false);
6387	ConstantRange RHSRange = ConstantRange::fromKnownBits(Known: RHSKnown, IsSigned: false);
6388	return mapOverflowResult(OR: LHSRange.unsignedMulMayOverflow(Other: RHSRange));
6389	}
6390
6391	OverflowResult llvm::computeOverflowForSignedMul(const Value *LHS,
6392	const Value *RHS,
6393	const SimplifyQuery &SQ) {
6394	// Multiplying n * m significant bits yields a result of n + m significant
6395	// bits. If the total number of significant bits does not exceed the
6396	// result bit width (minus 1), there is no overflow.
6397	// This means if we have enough leading sign bits in the operands
6398	// we can guarantee that the result does not overflow.
6399	// Ref: "Hacker's Delight" by Henry Warren
6400	unsigned BitWidth = LHS->getType()->getScalarSizeInBits();
6401
6402	// Note that underestimating the number of sign bits gives a more
6403	// conservative answer.
6404	unsigned SignBits =
6405	::ComputeNumSignBits(V: LHS, Depth: 0, Q: SQ) + ::ComputeNumSignBits(V: RHS, Depth: 0, Q: SQ);
6406
6407	// First handle the easy case: if we have enough sign bits there's
6408	// definitely no overflow.
6409	if (SignBits > BitWidth + 1)
6410	return OverflowResult::NeverOverflows;
6411
6412	// There are two ambiguous cases where there can be no overflow:
6413	// SignBits == BitWidth + 1 and
6414	// SignBits == BitWidth
6415	// The second case is difficult to check, therefore we only handle the
6416	// first case.
6417	if (SignBits == BitWidth + 1) {
6418	// It overflows only when both arguments are negative and the true
6419	// product is exactly the minimum negative number.
6420	// E.g. mul i16 with 17 sign bits: 0xff00 * 0xff80 = 0x8000
6421	// For simplicity we just check if at least one side is not negative.
6422	KnownBits LHSKnown = computeKnownBits(V: LHS, /*Depth=*/0, Q: SQ);
6423	KnownBits RHSKnown = computeKnownBits(V: RHS, /*Depth=*/0, Q: SQ);
6424	if (LHSKnown.isNonNegative() || RHSKnown.isNonNegative())
6425	return OverflowResult::NeverOverflows;
6426	}
6427	return OverflowResult::MayOverflow;
6428	}
6429
6430	OverflowResult
6431	llvm::computeOverflowForUnsignedAdd(const WithCache<const Value *> &LHS,
6432	const WithCache<const Value *> &RHS,
6433	const SimplifyQuery &SQ) {
6434	ConstantRange LHSRange =
6435	computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/false, SQ);
6436	ConstantRange RHSRange =
6437	computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/false, SQ);
6438	return mapOverflowResult(OR: LHSRange.unsignedAddMayOverflow(Other: RHSRange));
6439	}
6440
6441	static OverflowResult
6442	computeOverflowForSignedAdd(const WithCache<const Value *> &LHS,
6443	const WithCache<const Value *> &RHS,
6444	const AddOperator *Add, const SimplifyQuery &SQ) {
6445	if (Add && Add->hasNoSignedWrap()) {
6446	return OverflowResult::NeverOverflows;
6447	}
6448
6449	// If LHS and RHS each have at least two sign bits, the addition will look
6450	// like
6451	//
6452	// XX..... +
6453	// YY.....
6454	//
6455	// If the carry into the most significant position is 0, X and Y can't both
6456	// be 1 and therefore the carry out of the addition is also 0.
6457	//
6458	// If the carry into the most significant position is 1, X and Y can't both
6459	// be 0 and therefore the carry out of the addition is also 1.
6460	//
6461	// Since the carry into the most significant position is always equal to
6462	// the carry out of the addition, there is no signed overflow.
6463	if (::ComputeNumSignBits(V: LHS, Depth: 0, Q: SQ) > 1 &&
6464	::ComputeNumSignBits(V: RHS, Depth: 0, Q: SQ) > 1)
6465	return OverflowResult::NeverOverflows;
6466
6467	ConstantRange LHSRange =
6468	computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/true, SQ);
6469	ConstantRange RHSRange =
6470	computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/true, SQ);
6471	OverflowResult OR =
6472	mapOverflowResult(OR: LHSRange.signedAddMayOverflow(Other: RHSRange));
6473	if (OR != OverflowResult::MayOverflow)
6474	return OR;
6475
6476	// The remaining code needs Add to be available. Early returns if not so.
6477	if (!Add)
6478	return OverflowResult::MayOverflow;
6479
6480	// If the sign of Add is the same as at least one of the operands, this add
6481	// CANNOT overflow. If this can be determined from the known bits of the
6482	// operands the above signedAddMayOverflow() check will have already done so.
6483	// The only other way to improve on the known bits is from an assumption, so
6484	// call computeKnownBitsFromContext() directly.
6485	bool LHSOrRHSKnownNonNegative =
6486	(LHSRange.isAllNonNegative() || RHSRange.isAllNonNegative());
6487	bool LHSOrRHSKnownNegative =
6488	(LHSRange.isAllNegative() || RHSRange.isAllNegative());
6489	if (LHSOrRHSKnownNonNegative || LHSOrRHSKnownNegative) {
6490	KnownBits AddKnown(LHSRange.getBitWidth());
6491	computeKnownBitsFromContext(V: Add, Known&: AddKnown, /*Depth=*/0, Q: SQ);
6492	if ((AddKnown.isNonNegative() && LHSOrRHSKnownNonNegative) ||
6493	(AddKnown.isNegative() && LHSOrRHSKnownNegative))
6494	return OverflowResult::NeverOverflows;
6495	}
6496
6497	return OverflowResult::MayOverflow;
6498	}
6499
6500	OverflowResult llvm::computeOverflowForUnsignedSub(const Value *LHS,
6501	const Value *RHS,
6502	const SimplifyQuery &SQ) {
6503	// X - (X % ?)
6504	// The remainder of a value can't have greater magnitude than itself,
6505	// so the subtraction can't overflow.
6506
6507	// X - (X -nuw ?)
6508	// In the minimal case, this would simplify to "?", so there's no subtract
6509	// at all. But if this analysis is used to peek through casts, for example,
6510	// then determining no-overflow may allow other transforms.
6511
6512	// TODO: There are other patterns like this.
6513	// See simplifyICmpWithBinOpOnLHS() for candidates.
6514	if (match(V: RHS, P: m_URem(L: m_Specific(V: LHS), R: m_Value())) ||
6515	match(V: RHS, P: m_NUWSub(L: m_Specific(V: LHS), R: m_Value())))
6516	if (isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT))
6517	return OverflowResult::NeverOverflows;
6518
6519	// Checking for conditions implied by dominating conditions may be expensive.
6520	// Limit it to usub_with_overflow calls for now.
6521	if (match(SQ.CxtI,
6522	m_Intrinsic<Intrinsic::usub_with_overflow>(m_Value(), m_Value())))
6523	if (auto C = isImpliedByDomCondition(Pred: CmpInst::ICMP_UGE, LHS, RHS, ContextI: SQ.CxtI,
6524	DL: SQ.DL)) {
6525	if (*C)
6526	return OverflowResult::NeverOverflows;
6527	return OverflowResult::AlwaysOverflowsLow;
6528	}
6529	ConstantRange LHSRange =
6530	computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/false, SQ);
6531	ConstantRange RHSRange =
6532	computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/false, SQ);
6533	return mapOverflowResult(OR: LHSRange.unsignedSubMayOverflow(Other: RHSRange));
6534	}
6535
6536	OverflowResult llvm::computeOverflowForSignedSub(const Value *LHS,
6537	const Value *RHS,
6538	const SimplifyQuery &SQ) {
6539	// X - (X % ?)
6540	// The remainder of a value can't have greater magnitude than itself,
6541	// so the subtraction can't overflow.
6542
6543	// X - (X -nsw ?)
6544	// In the minimal case, this would simplify to "?", so there's no subtract
6545	// at all. But if this analysis is used to peek through casts, for example,
6546	// then determining no-overflow may allow other transforms.
6547	if (match(V: RHS, P: m_SRem(L: m_Specific(V: LHS), R: m_Value())) ||
6548	match(V: RHS, P: m_NSWSub(L: m_Specific(V: LHS), R: m_Value())))
6549	if (isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT))
6550	return OverflowResult::NeverOverflows;
6551
6552	// If LHS and RHS each have at least two sign bits, the subtraction
6553	// cannot overflow.
6554	if (::ComputeNumSignBits(V: LHS, Depth: 0, Q: SQ) > 1 &&
6555	::ComputeNumSignBits(V: RHS, Depth: 0, Q: SQ) > 1)
6556	return OverflowResult::NeverOverflows;
6557
6558	ConstantRange LHSRange =
6559	computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/true, SQ);
6560	ConstantRange RHSRange =
6561	computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/true, SQ);
6562	return mapOverflowResult(OR: LHSRange.signedSubMayOverflow(Other: RHSRange));
6563	}
6564
6565	bool llvm::isOverflowIntrinsicNoWrap(const WithOverflowInst *WO,
6566	const DominatorTree &DT) {
6567	SmallVector<const BranchInst *, 2> GuardingBranches;
6568	SmallVector<const ExtractValueInst *, 2> Results;
6569
6570	for (const User *U : WO->users()) {
6571	if (const auto *EVI = dyn_cast<ExtractValueInst>(Val: U)) {
6572	assert(EVI->getNumIndices() == 1 && "Obvious from CI's type");
6573
6574	if (EVI->getIndices()[0] == 0)
6575	Results.push_back(Elt: EVI);
6576	else {
6577	assert(EVI->getIndices()[0] == 1 && "Obvious from CI's type");
6578
6579	for (const auto *U : EVI->users())
6580	if (const auto *B = dyn_cast<BranchInst>(Val: U)) {
6581	assert(B->isConditional() && "How else is it using an i1?");
6582	GuardingBranches.push_back(Elt: B);
6583	}
6584	}
6585	} else {
6586	// We are using the aggregate directly in a way we don't want to analyze
6587	// here (storing it to a global, say).
6588	return false;
6589	}
6590	}
6591
6592	auto AllUsesGuardedByBranch = [&](const BranchInst *BI) {
6593	BasicBlockEdge NoWrapEdge(BI->getParent(), BI->getSuccessor(i: 1));
6594	if (!NoWrapEdge.isSingleEdge())
6595	return false;
6596
6597	// Check if all users of the add are provably no-wrap.
6598	for (const auto *Result : Results) {
6599	// If the extractvalue itself is not executed on overflow, the we don't
6600	// need to check each use separately, since domination is transitive.
6601	if (DT.dominates(BBE: NoWrapEdge, BB: Result->getParent()))
6602	continue;
6603
6604	for (const auto &RU : Result->uses())
6605	if (!DT.dominates(BBE: NoWrapEdge, U: RU))
6606	return false;
6607	}
6608
6609	return true;
6610	};
6611
6612	return llvm::any_of(Range&: GuardingBranches, P: AllUsesGuardedByBranch);
6613	}
6614
6615	/// Shifts return poison if shiftwidth is larger than the bitwidth.
6616	static bool shiftAmountKnownInRange(const Value *ShiftAmount) {
6617	auto *C = dyn_cast<Constant>(Val: ShiftAmount);
6618	if (!C)
6619	return false;
6620
6621	// Shifts return poison if shiftwidth is larger than the bitwidth.
6622	SmallVector<const Constant *, 4> ShiftAmounts;
6623	if (auto *FVTy = dyn_cast<FixedVectorType>(Val: C->getType())) {
6624	unsigned NumElts = FVTy->getNumElements();
6625	for (unsigned i = 0; i < NumElts; ++i)
6626	ShiftAmounts.push_back(Elt: C->getAggregateElement(Elt: i));
6627	} else if (isa<ScalableVectorType>(Val: C->getType()))
6628	return false; // Can't tell, just return false to be safe
6629	else
6630	ShiftAmounts.push_back(Elt: C);
6631
6632	bool Safe = llvm::all_of(Range&: ShiftAmounts, P: [](const Constant *C) {
6633	auto *CI = dyn_cast_or_null<ConstantInt>(Val: C);
6634	return CI && CI->getValue().ult(RHS: C->getType()->getIntegerBitWidth());
6635	});
6636
6637	return Safe;
6638	}
6639
6640	enum class UndefPoisonKind {
6641	PoisonOnly = (1 << 0),
6642	UndefOnly = (1 << 1),
6643	UndefOrPoison = PoisonOnly | UndefOnly,
6644	};
6645
6646	static bool includesPoison(UndefPoisonKind Kind) {
6647	return (unsigned(Kind) & unsigned(UndefPoisonKind::PoisonOnly)) != 0;
6648	}
6649
6650	static bool includesUndef(UndefPoisonKind Kind) {
6651	return (unsigned(Kind) & unsigned(UndefPoisonKind::UndefOnly)) != 0;
6652	}
6653
6654	static bool canCreateUndefOrPoison(const Operator *Op, UndefPoisonKind Kind,
6655	bool ConsiderFlagsAndMetadata) {
6656
6657	if (ConsiderFlagsAndMetadata && includesPoison(Kind) &&
6658	Op->hasPoisonGeneratingFlagsOrMetadata())
6659	return true;
6660
6661	unsigned Opcode = Op->getOpcode();
6662
6663	// Check whether opcode is a poison/undef-generating operation
6664	switch (Opcode) {
6665	case Instruction::Shl:
6666	case Instruction::AShr:
6667	case Instruction::LShr:
6668	return includesPoison(Kind) && !shiftAmountKnownInRange(ShiftAmount: Op->getOperand(i: 1));
6669	case Instruction::FPToSI:
6670	case Instruction::FPToUI:
6671	// fptosi/ui yields poison if the resulting value does not fit in the
6672	// destination type.
6673	return true;
6674	case Instruction::Call:
6675	if (auto *II = dyn_cast<IntrinsicInst>(Val: Op)) {
6676	switch (II->getIntrinsicID()) {
6677	// TODO: Add more intrinsics.
6678	case Intrinsic::ctlz:
6679	case Intrinsic::cttz:
6680	case Intrinsic::abs:
6681	if (cast<ConstantInt>(Val: II->getArgOperand(i: 1))->isNullValue())
6682	return false;
6683	break;
6684	case Intrinsic::ctpop:
6685	case Intrinsic::bswap:
6686	case Intrinsic::bitreverse:
6687	case Intrinsic::fshl:
6688	case Intrinsic::fshr:
6689	case Intrinsic::smax:
6690	case Intrinsic::smin:
6691	case Intrinsic::umax:
6692	case Intrinsic::umin:
6693	case Intrinsic::ptrmask:
6694	case Intrinsic::fptoui_sat:
6695	case Intrinsic::fptosi_sat:
6696	case Intrinsic::sadd_with_overflow:
6697	case Intrinsic::ssub_with_overflow:
6698	case Intrinsic::smul_with_overflow:
6699	case Intrinsic::uadd_with_overflow:
6700	case Intrinsic::usub_with_overflow:
6701	case Intrinsic::umul_with_overflow:
6702	case Intrinsic::sadd_sat:
6703	case Intrinsic::uadd_sat:
6704	case Intrinsic::ssub_sat:
6705	case Intrinsic::usub_sat:
6706	return false;
6707	case Intrinsic::sshl_sat:
6708	case Intrinsic::ushl_sat:
6709	return includesPoison(Kind) &&
6710	!shiftAmountKnownInRange(ShiftAmount: II->getArgOperand(i: 1));
6711	case Intrinsic::fma:
6712	case Intrinsic::fmuladd:
6713	case Intrinsic::sqrt:
6714	case Intrinsic::powi:
6715	case Intrinsic::sin:
6716	case Intrinsic::cos:
6717	case Intrinsic::pow:
6718	case Intrinsic::log:
6719	case Intrinsic::log10:
6720	case Intrinsic::log2:
6721	case Intrinsic::exp:
6722	case Intrinsic::exp2:
6723	case Intrinsic::exp10:
6724	case Intrinsic::fabs:
6725	case Intrinsic::copysign:
6726	case Intrinsic::floor:
6727	case Intrinsic::ceil:
6728	case Intrinsic::trunc:
6729	case Intrinsic::rint:
6730	case Intrinsic::nearbyint:
6731	case Intrinsic::round:
6732	case Intrinsic::roundeven:
6733	case Intrinsic::fptrunc_round:
6734	case Intrinsic::canonicalize:
6735	case Intrinsic::arithmetic_fence:
6736	case Intrinsic::minnum:
6737	case Intrinsic::maxnum:
6738	case Intrinsic::minimum:
6739	case Intrinsic::maximum:
6740	case Intrinsic::is_fpclass:
6741	case Intrinsic::ldexp:
6742	case Intrinsic::frexp:
6743	return false;
6744	case Intrinsic::lround:
6745	case Intrinsic::llround:
6746	case Intrinsic::lrint:
6747	case Intrinsic::llrint:
6748	// If the value doesn't fit an unspecified value is returned (but this
6749	// is not poison).
6750	return false;
6751	}
6752	}
6753	[[fallthrough]];
6754	case Instruction::CallBr:
6755	case Instruction::Invoke: {
6756	const auto *CB = cast<CallBase>(Val: Op);
6757	return !CB->hasRetAttr(Attribute::NoUndef);
6758	}
6759	case Instruction::InsertElement:
6760	case Instruction::ExtractElement: {
6761	// If index exceeds the length of the vector, it returns poison
6762	auto *VTy = cast<VectorType>(Val: Op->getOperand(i: 0)->getType());
6763	unsigned IdxOp = Op->getOpcode() == Instruction::InsertElement ? 2 : 1;
6764	auto *Idx = dyn_cast<ConstantInt>(Val: Op->getOperand(i: IdxOp));
6765	if (includesPoison(Kind))
6766	return !Idx ||
6767	Idx->getValue().uge(RHS: VTy->getElementCount().getKnownMinValue());
6768	return false;
6769	}
6770	case Instruction::ShuffleVector: {
6771	ArrayRef<int> Mask = isa<ConstantExpr>(Val: Op)
6772	? cast<ConstantExpr>(Val: Op)->getShuffleMask()
6773	: cast<ShuffleVectorInst>(Val: Op)->getShuffleMask();
6774	return includesPoison(Kind) && is_contained(Range&: Mask, Element: PoisonMaskElem);
6775	}
6776	case Instruction::FNeg:
6777	case Instruction::PHI:
6778	case Instruction::Select:
6779	case Instruction::URem:
6780	case Instruction::SRem:
6781	case Instruction::ExtractValue:
6782	case Instruction::InsertValue:
6783	case Instruction::Freeze:
6784	case Instruction::ICmp:
6785	case Instruction::FCmp:
6786	case Instruction::FAdd:
6787	case Instruction::FSub:
6788	case Instruction::FMul:
6789	case Instruction::FDiv:
6790	case Instruction::FRem:
6791	return false;
6792	case Instruction::GetElementPtr:
6793	// inbounds is handled above
6794	// TODO: what about inrange on constexpr?
6795	return false;
6796	default: {
6797	const auto *CE = dyn_cast<ConstantExpr>(Val: Op);
6798	if (isa<CastInst>(Val: Op) || (CE && CE->isCast()))
6799	return false;
6800	else if (Instruction::isBinaryOp(Opcode))
6801	return false;
6802	// Be conservative and return true.
6803	return true;
6804	}
6805	}
6806	}
6807
6808	bool llvm::canCreateUndefOrPoison(const Operator *Op,
6809	bool ConsiderFlagsAndMetadata) {
6810	return ::canCreateUndefOrPoison(Op, Kind: UndefPoisonKind::UndefOrPoison,
6811	ConsiderFlagsAndMetadata);
6812	}
6813
6814	bool llvm::canCreatePoison(const Operator *Op, bool ConsiderFlagsAndMetadata) {
6815	return ::canCreateUndefOrPoison(Op, Kind: UndefPoisonKind::PoisonOnly,
6816	ConsiderFlagsAndMetadata);
6817	}
6818
6819	static bool directlyImpliesPoison(const Value *ValAssumedPoison, const Value *V,
6820	unsigned Depth) {
6821	if (ValAssumedPoison == V)
6822	return true;
6823
6824	const unsigned MaxDepth = 2;
6825	if (Depth >= MaxDepth)
6826	return false;
6827
6828	if (const auto *I = dyn_cast<Instruction>(Val: V)) {
6829	if (any_of(Range: I->operands(), P: [=](const Use &Op) {
6830	return propagatesPoison(PoisonOp: Op) &&
6831	directlyImpliesPoison(ValAssumedPoison, V: Op, Depth: Depth + 1);
6832	}))
6833	return true;
6834
6835	// V = extractvalue V0, idx
6836	// V2 = extractvalue V0, idx2
6837	// V0's elements are all poison or not. (e.g., add_with_overflow)
6838	const WithOverflowInst *II;
6839	if (match(V: I, P: m_ExtractValue(V: m_WithOverflowInst(I&: II))) &&
6840	(match(V: ValAssumedPoison, P: m_ExtractValue(V: m_Specific(V: II))) ||
6841	llvm::is_contained(Range: II->args(), Element: ValAssumedPoison)))
6842	return true;
6843	}
6844	return false;
6845	}
6846
6847	static bool impliesPoison(const Value *ValAssumedPoison, const Value *V,
6848	unsigned Depth) {
6849	if (isGuaranteedNotToBePoison(V: ValAssumedPoison))
6850	return true;
6851
6852	if (directlyImpliesPoison(ValAssumedPoison, V, /* Depth */ 0))
6853	return true;
6854
6855	const unsigned MaxDepth = 2;
6856	if (Depth >= MaxDepth)
6857	return false;
6858
6859	const auto *I = dyn_cast<Instruction>(Val: ValAssumedPoison);
6860	if (I && !canCreatePoison(Op: cast<Operator>(Val: I))) {
6861	return all_of(Range: I->operands(), P: [=](const Value *Op) {
6862	return impliesPoison(ValAssumedPoison: Op, V, Depth: Depth + 1);
6863	});
6864	}
6865	return false;
6866	}
6867
6868	bool llvm::impliesPoison(const Value *ValAssumedPoison, const Value *V) {
6869	return ::impliesPoison(ValAssumedPoison, V, /* Depth */ 0);
6870	}
6871
6872	static bool programUndefinedIfUndefOrPoison(const Value *V, bool PoisonOnly);
6873
6874	static bool isGuaranteedNotToBeUndefOrPoison(
6875	const Value *V, AssumptionCache *AC, const Instruction *CtxI,
6876	const DominatorTree *DT, unsigned Depth, UndefPoisonKind Kind) {
6877	if (Depth >= MaxAnalysisRecursionDepth)
6878	return false;
6879
6880	if (isa<MetadataAsValue>(Val: V))
6881	return false;
6882
6883	if (const auto *A = dyn_cast<Argument>(Val: V)) {
6884	if (A->hasAttribute(Attribute::NoUndef) ||
6885	A->hasAttribute(Attribute::Dereferenceable) ||
6886	A->hasAttribute(Attribute::DereferenceableOrNull))
6887	return true;
6888	}
6889
6890	if (auto *C = dyn_cast<Constant>(Val: V)) {
6891	if (isa<PoisonValue>(Val: C))
6892	return !includesPoison(Kind);
6893
6894	if (isa<UndefValue>(Val: C))
6895	return !includesUndef(Kind);
6896
6897	if (isa<ConstantInt>(Val: C) || isa<GlobalVariable>(Val: C) || isa<ConstantFP>(Val: V) ||
6898	isa<ConstantPointerNull>(Val: C) || isa<Function>(Val: C))
6899	return true;
6900
6901	if (C->getType()->isVectorTy() && !isa<ConstantExpr>(Val: C))
6902	return (!includesUndef(Kind) ? !C->containsPoisonElement()
6903	: !C->containsUndefOrPoisonElement()) &&
6904	!C->containsConstantExpression();
6905	}
6906
6907	// Strip cast operations from a pointer value.
6908	// Note that stripPointerCastsSameRepresentation can strip off getelementptr
6909	// inbounds with zero offset. To guarantee that the result isn't poison, the
6910	// stripped pointer is checked as it has to be pointing into an allocated
6911	// object or be null `null` to ensure `inbounds` getelement pointers with a
6912	// zero offset could not produce poison.
6913	// It can strip off addrspacecast that do not change bit representation as
6914	// well. We believe that such addrspacecast is equivalent to no-op.
6915	auto *StrippedV = V->stripPointerCastsSameRepresentation();
6916	if (isa<AllocaInst>(Val: StrippedV) || isa<GlobalVariable>(Val: StrippedV) ||
6917	isa<Function>(Val: StrippedV) || isa<ConstantPointerNull>(Val: StrippedV))
6918	return true;
6919
6920	auto OpCheck = [&](const Value *V) {
6921	return isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth: Depth + 1, Kind);
6922	};
6923
6924	if (auto *Opr = dyn_cast<Operator>(Val: V)) {
6925	// If the value is a freeze instruction, then it can never
6926	// be undef or poison.
6927	if (isa<FreezeInst>(Val: V))
6928	return true;
6929
6930	if (const auto *CB = dyn_cast<CallBase>(Val: V)) {
6931	if (CB->hasRetAttr(Attribute::NoUndef) ||
6932	CB->hasRetAttr(Attribute::Dereferenceable) ||
6933	CB->hasRetAttr(Attribute::DereferenceableOrNull))
6934	return true;
6935	}
6936
6937	if (const auto *PN = dyn_cast<PHINode>(Val: V)) {
6938	unsigned Num = PN->getNumIncomingValues();
6939	bool IsWellDefined = true;
6940	for (unsigned i = 0; i < Num; ++i) {
6941	auto *TI = PN->getIncomingBlock(i)->getTerminator();
6942	if (!isGuaranteedNotToBeUndefOrPoison(V: PN->getIncomingValue(i), AC, CtxI: TI,
6943	DT, Depth: Depth + 1, Kind)) {
6944	IsWellDefined = false;
6945	break;
6946	}
6947	}
6948	if (IsWellDefined)
6949	return true;
6950	} else if (!::canCreateUndefOrPoison(Op: Opr, Kind,
6951	/*ConsiderFlagsAndMetadata*/ true) &&
6952	all_of(Range: Opr->operands(), P: OpCheck))
6953	return true;
6954	}
6955
6956	if (auto *I = dyn_cast<LoadInst>(Val: V))
6957	if (I->hasMetadata(KindID: LLVMContext::MD_noundef) ||
6958	I->hasMetadata(KindID: LLVMContext::MD_dereferenceable) ||
6959	I->hasMetadata(KindID: LLVMContext::MD_dereferenceable_or_null))
6960	return true;
6961
6962	if (programUndefinedIfUndefOrPoison(V, PoisonOnly: !includesUndef(Kind)))
6963	return true;
6964
6965	// CxtI may be null or a cloned instruction.
6966	if (!CtxI || !CtxI->getParent() || !DT)
6967	return false;
6968
6969	auto *DNode = DT->getNode(BB: CtxI->getParent());
6970	if (!DNode)
6971	// Unreachable block
6972	return false;
6973
6974	// If V is used as a branch condition before reaching CtxI, V cannot be
6975	// undef or poison.
6976	// br V, BB1, BB2
6977	// BB1:
6978	// CtxI ; V cannot be undef or poison here
6979	auto *Dominator = DNode->getIDom();
6980	while (Dominator) {
6981	auto *TI = Dominator->getBlock()->getTerminator();
6982
6983	Value *Cond = nullptr;
6984	if (auto BI = dyn_cast_or_null<BranchInst>(Val: TI)) {
6985	if (BI->isConditional())
6986	Cond = BI->getCondition();
6987	} else if (auto SI = dyn_cast_or_null<SwitchInst>(Val: TI)) {
6988	Cond = SI->getCondition();
6989	}
6990
6991	if (Cond) {
6992	if (Cond == V)
6993	return true;
6994	else if (!includesUndef(Kind) && isa<Operator>(Val: Cond)) {
6995	// For poison, we can analyze further
6996	auto *Opr = cast<Operator>(Val: Cond);
6997	if (any_of(Range: Opr->operands(),
6998	P: [V](const Use &U) { return V == U && propagatesPoison(PoisonOp: U); }))
6999	return true;
7000	}
7001	}
7002
7003	Dominator = Dominator->getIDom();
7004	}
7005
7006	if (getKnowledgeValidInContext(V, {Attribute::NoUndef}, CtxI, DT, AC))
7007	return true;
7008
7009	return false;
7010	}
7011
7012	bool llvm::isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC,
7013	const Instruction *CtxI,
7014	const DominatorTree *DT,
7015	unsigned Depth) {
7016	return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth,
7017	Kind: UndefPoisonKind::UndefOrPoison);
7018	}
7019
7020	bool llvm::isGuaranteedNotToBePoison(const Value *V, AssumptionCache *AC,
7021	const Instruction *CtxI,
7022	const DominatorTree *DT, unsigned Depth) {
7023	return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth,
7024	Kind: UndefPoisonKind::PoisonOnly);
7025	}
7026
7027	bool llvm::isGuaranteedNotToBeUndef(const Value *V, AssumptionCache *AC,
7028	const Instruction *CtxI,
7029	const DominatorTree *DT, unsigned Depth) {
7030	return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth,
7031	Kind: UndefPoisonKind::UndefOnly);
7032	}
7033
7034	/// Return true if undefined behavior would provably be executed on the path to
7035	/// OnPathTo if Root produced a posion result. Note that this doesn't say
7036	/// anything about whether OnPathTo is actually executed or whether Root is
7037	/// actually poison. This can be used to assess whether a new use of Root can
7038	/// be added at a location which is control equivalent with OnPathTo (such as
7039	/// immediately before it) without introducing UB which didn't previously
7040	/// exist. Note that a false result conveys no information.
7041	bool llvm::mustExecuteUBIfPoisonOnPathTo(Instruction *Root,
7042	Instruction *OnPathTo,
7043	DominatorTree *DT) {
7044	// Basic approach is to assume Root is poison, propagate poison forward
7045	// through all users we can easily track, and then check whether any of those
7046	// users are provable UB and must execute before out exiting block might
7047	// exit.
7048
7049	// The set of all recursive users we've visited (which are assumed to all be
7050	// poison because of said visit)
7051	SmallSet<const Value *, 16> KnownPoison;
7052	SmallVector<const Instruction*, 16> Worklist;
7053	Worklist.push_back(Elt: Root);
7054	while (!Worklist.empty()) {
7055	const Instruction *I = Worklist.pop_back_val();
7056
7057	// If we know this must trigger UB on a path leading our target.
7058	if (mustTriggerUB(I, KnownPoison) && DT->dominates(Def: I, User: OnPathTo))
7059	return true;
7060
7061	// If we can't analyze propagation through this instruction, just skip it
7062	// and transitive users. Safe as false is a conservative result.
7063	if (I != Root && !any_of(Range: I->operands(), P: [&KnownPoison](const Use &U) {
7064	return KnownPoison.contains(Ptr: U) && propagatesPoison(PoisonOp: U);
7065	}))
7066	continue;
7067
7068	if (KnownPoison.insert(Ptr: I).second)
7069	for (const User *User : I->users())
7070	Worklist.push_back(Elt: cast<Instruction>(Val: User));
7071	}
7072
7073	// Might be non-UB, or might have a path we couldn't prove must execute on
7074	// way to exiting bb.
7075	return false;
7076	}
7077
7078	OverflowResult llvm::computeOverflowForSignedAdd(const AddOperator *Add,
7079	const SimplifyQuery &SQ) {
7080	return ::computeOverflowForSignedAdd(LHS: Add->getOperand(i_nocapture: 0), RHS: Add->getOperand(i_nocapture: 1),
7081	Add, SQ);
7082	}
7083
7084	OverflowResult
7085	llvm::computeOverflowForSignedAdd(const WithCache<const Value *> &LHS,
7086	const WithCache<const Value *> &RHS,
7087	const SimplifyQuery &SQ) {
7088	return ::computeOverflowForSignedAdd(LHS, RHS, Add: nullptr, SQ);
7089	}
7090
7091	bool llvm::isGuaranteedToTransferExecutionToSuccessor(const Instruction *I) {
7092	// Note: An atomic operation isn't guaranteed to return in a reasonable amount
7093	// of time because it's possible for another thread to interfere with it for an
7094	// arbitrary length of time, but programs aren't allowed to rely on that.
7095
7096	// If there is no successor, then execution can't transfer to it.
7097	if (isa<ReturnInst>(Val: I))
7098	return false;
7099	if (isa<UnreachableInst>(Val: I))
7100	return false;
7101
7102	// Note: Do not add new checks here; instead, change Instruction::mayThrow or
7103	// Instruction::willReturn.
7104	//
7105	// FIXME: Move this check into Instruction::willReturn.
7106	if (isa<CatchPadInst>(Val: I)) {
7107	switch (classifyEHPersonality(Pers: I->getFunction()->getPersonalityFn())) {
7108	default:
7109	// A catchpad may invoke exception object constructors and such, which
7110	// in some languages can be arbitrary code, so be conservative by default.
7111	return false;
7112	case EHPersonality::CoreCLR:
7113	// For CoreCLR, it just involves a type test.
7114	return true;
7115	}
7116	}
7117
7118	// An instruction that returns without throwing must transfer control flow
7119	// to a successor.
7120	return !I->mayThrow() && I->willReturn();
7121	}
7122
7123	bool llvm::isGuaranteedToTransferExecutionToSuccessor(const BasicBlock *BB) {
7124	// TODO: This is slightly conservative for invoke instruction since exiting
7125	// via an exception *is* normal control for them.
7126	for (const Instruction &I : *BB)
7127	if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
7128	return false;
7129	return true;
7130	}
7131
7132	bool llvm::isGuaranteedToTransferExecutionToSuccessor(
7133	BasicBlock::const_iterator Begin, BasicBlock::const_iterator End,
7134	unsigned ScanLimit) {
7135	return isGuaranteedToTransferExecutionToSuccessor(Range: make_range(x: Begin, y: End),
7136	ScanLimit);
7137	}
7138
7139	bool llvm::isGuaranteedToTransferExecutionToSuccessor(
7140	iterator_range<BasicBlock::const_iterator> Range, unsigned ScanLimit) {
7141	assert(ScanLimit && "scan limit must be non-zero");
7142	for (const Instruction &I : Range) {
7143	if (isa<DbgInfoIntrinsic>(Val: I))
7144	continue;
7145	if (--ScanLimit == 0)
7146	return false;
7147	if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
7148	return false;
7149	}
7150	return true;
7151	}
7152
7153	bool llvm::isGuaranteedToExecuteForEveryIteration(const Instruction *I,
7154	const Loop *L) {
7155	// The loop header is guaranteed to be executed for every iteration.
7156	//
7157	// FIXME: Relax this constraint to cover all basic blocks that are
7158	// guaranteed to be executed at every iteration.
7159	if (I->getParent() != L->getHeader()) return false;
7160
7161	for (const Instruction &LI : *L->getHeader()) {
7162	if (&LI == I) return true;
7163	if (!isGuaranteedToTransferExecutionToSuccessor(I: &LI)) return false;
7164	}
7165	llvm_unreachable("Instruction not contained in its own parent basic block.");
7166	}
7167
7168	bool llvm::propagatesPoison(const Use &PoisonOp) {
7169	const Operator *I = cast<Operator>(Val: PoisonOp.getUser());
7170	switch (I->getOpcode()) {
7171	case Instruction::Freeze:
7172	case Instruction::PHI:
7173	case Instruction::Invoke:
7174	return false;
7175	case Instruction::Select:
7176	return PoisonOp.getOperandNo() == 0;
7177	case Instruction::Call:
7178	if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
7179	switch (II->getIntrinsicID()) {
7180	// TODO: Add more intrinsics.
7181	case Intrinsic::sadd_with_overflow:
7182	case Intrinsic::ssub_with_overflow:
7183	case Intrinsic::smul_with_overflow:
7184	case Intrinsic::uadd_with_overflow:
7185	case Intrinsic::usub_with_overflow:
7186	case Intrinsic::umul_with_overflow:
7187	// If an input is a vector containing a poison element, the
7188	// two output vectors (calculated results, overflow bits)'
7189	// corresponding lanes are poison.
7190	return true;
7191	case Intrinsic::ctpop:
7192	return true;
7193	}
7194	}
7195	return false;
7196	case Instruction::ICmp:
7197	case Instruction::FCmp:
7198	case Instruction::GetElementPtr:
7199	return true;
7200	default:
7201	if (isa<BinaryOperator>(Val: I) || isa<UnaryOperator>(Val: I) || isa<CastInst>(Val: I))
7202	return true;
7203
7204	// Be conservative and return false.
7205	return false;
7206	}
7207	}
7208
7209	void llvm::getGuaranteedWellDefinedOps(
7210	const Instruction *I, SmallVectorImpl<const Value *> &Operands) {
7211	switch (I->getOpcode()) {
7212	case Instruction::Store:
7213	Operands.push_back(Elt: cast<StoreInst>(Val: I)->getPointerOperand());
7214	break;
7215
7216	case Instruction::Load:
7217	Operands.push_back(Elt: cast<LoadInst>(Val: I)->getPointerOperand());
7218	break;
7219
7220	// Since dereferenceable attribute imply noundef, atomic operations
7221	// also implicitly have noundef pointers too
7222	case Instruction::AtomicCmpXchg:
7223	Operands.push_back(Elt: cast<AtomicCmpXchgInst>(Val: I)->getPointerOperand());
7224	break;
7225
7226	case Instruction::AtomicRMW:
7227	Operands.push_back(Elt: cast<AtomicRMWInst>(Val: I)->getPointerOperand());
7228	break;
7229
7230	case Instruction::Call:
7231	case Instruction::Invoke: {
7232	const CallBase *CB = cast<CallBase>(Val: I);
7233	if (CB->isIndirectCall())
7234	Operands.push_back(Elt: CB->getCalledOperand());
7235	for (unsigned i = 0; i < CB->arg_size(); ++i) {
7236	if (CB->paramHasAttr(i, Attribute::NoUndef) ||
7237	CB->paramHasAttr(i, Attribute::Dereferenceable) ||
7238	CB->paramHasAttr(i, Attribute::DereferenceableOrNull))
7239	Operands.push_back(Elt: CB->getArgOperand(i));
7240	}
7241	break;
7242	}
7243	case Instruction::Ret:
7244	if (I->getFunction()->hasRetAttribute(Attribute::NoUndef))
7245	Operands.push_back(Elt: I->getOperand(i: 0));
7246	break;
7247	case Instruction::Switch:
7248	Operands.push_back(Elt: cast<SwitchInst>(Val: I)->getCondition());
7249	break;
7250	case Instruction::Br: {
7251	auto *BR = cast<BranchInst>(Val: I);
7252	if (BR->isConditional())
7253	Operands.push_back(Elt: BR->getCondition());
7254	break;
7255	}
7256	default:
7257	break;
7258	}
7259	}
7260
7261	void llvm::getGuaranteedNonPoisonOps(const Instruction *I,
7262	SmallVectorImpl<const Value *> &Operands) {
7263	getGuaranteedWellDefinedOps(I, Operands);
7264	switch (I->getOpcode()) {
7265	// Divisors of these operations are allowed to be partially undef.
7266	case Instruction::UDiv:
7267	case Instruction::SDiv:
7268	case Instruction::URem:
7269	case Instruction::SRem:
7270	Operands.push_back(Elt: I->getOperand(i: 1));
7271	break;
7272	default:
7273	break;
7274	}
7275	}
7276
7277	bool llvm::mustTriggerUB(const Instruction *I,
7278	const SmallPtrSetImpl<const Value *> &KnownPoison) {
7279	SmallVector<const Value *, 4> NonPoisonOps;
7280	getGuaranteedNonPoisonOps(I, Operands&: NonPoisonOps);
7281
7282	for (const auto *V : NonPoisonOps)
7283	if (KnownPoison.count(Ptr: V))
7284	return true;
7285
7286	return false;
7287	}
7288
7289	static bool programUndefinedIfUndefOrPoison(const Value *V,
7290	bool PoisonOnly) {
7291	// We currently only look for uses of values within the same basic
7292	// block, as that makes it easier to guarantee that the uses will be
7293	// executed given that Inst is executed.
7294	//
7295	// FIXME: Expand this to consider uses beyond the same basic block. To do
7296	// this, look out for the distinction between post-dominance and strong
7297	// post-dominance.
7298	const BasicBlock *BB = nullptr;
7299	BasicBlock::const_iterator Begin;
7300	if (const auto *Inst = dyn_cast<Instruction>(Val: V)) {
7301	BB = Inst->getParent();
7302	Begin = Inst->getIterator();
7303	Begin++;
7304	} else if (const auto *Arg = dyn_cast<Argument>(Val: V)) {
7305	if (Arg->getParent()->isDeclaration())
7306	return false;
7307	BB = &Arg->getParent()->getEntryBlock();
7308	Begin = BB->begin();
7309	} else {
7310	return false;
7311	}
7312
7313	// Limit number of instructions we look at, to avoid scanning through large
7314	// blocks. The current limit is chosen arbitrarily.
7315	unsigned ScanLimit = 32;
7316	BasicBlock::const_iterator End = BB->end();
7317
7318	if (!PoisonOnly) {
7319	// Since undef does not propagate eagerly, be conservative & just check
7320	// whether a value is directly passed to an instruction that must take
7321	// well-defined operands.
7322
7323	for (const auto &I : make_range(x: Begin, y: End)) {
7324	if (isa<DbgInfoIntrinsic>(Val: I))
7325	continue;
7326	if (--ScanLimit == 0)
7327	break;
7328
7329	SmallVector<const Value *, 4> WellDefinedOps;
7330	getGuaranteedWellDefinedOps(I: &I, Operands&: WellDefinedOps);
7331	if (is_contained(Range&: WellDefinedOps, Element: V))
7332	return true;
7333
7334	if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
7335	break;
7336	}
7337	return false;
7338	}
7339
7340	// Set of instructions that we have proved will yield poison if Inst
7341	// does.
7342	SmallSet<const Value *, 16> YieldsPoison;
7343	SmallSet<const BasicBlock *, 4> Visited;
7344
7345	YieldsPoison.insert(Ptr: V);
7346	Visited.insert(Ptr: BB);
7347
7348	while (true) {
7349	for (const auto &I : make_range(x: Begin, y: End)) {
7350	if (isa<DbgInfoIntrinsic>(Val: I))
7351	continue;
7352	if (--ScanLimit == 0)
7353	return false;
7354	if (mustTriggerUB(I: &I, KnownPoison: YieldsPoison))
7355	return true;
7356	if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
7357	return false;
7358
7359	// If an operand is poison and propagates it, mark I as yielding poison.
7360	for (const Use &Op : I.operands()) {
7361	if (YieldsPoison.count(Ptr: Op) && propagatesPoison(PoisonOp: Op)) {
7362	YieldsPoison.insert(Ptr: &I);
7363	break;
7364	}
7365	}
7366
7367	// Special handling for select, which returns poison if its operand 0 is
7368	// poison (handled in the loop above) *or* if both its true/false operands
7369	// are poison (handled here).
7370	if (I.getOpcode() == Instruction::Select &&
7371	YieldsPoison.count(Ptr: I.getOperand(i: 1)) &&
7372	YieldsPoison.count(Ptr: I.getOperand(i: 2))) {
7373	YieldsPoison.insert(Ptr: &I);
7374	}
7375	}
7376
7377	BB = BB->getSingleSuccessor();
7378	if (!BB || !Visited.insert(Ptr: BB).second)
7379	break;
7380
7381	Begin = BB->getFirstNonPHI()->getIterator();
7382	End = BB->end();
7383	}
7384	return false;
7385	}
7386
7387	bool llvm::programUndefinedIfUndefOrPoison(const Instruction *Inst) {
7388	return ::programUndefinedIfUndefOrPoison(V: Inst, PoisonOnly: false);
7389	}
7390
7391	bool llvm::programUndefinedIfPoison(const Instruction *Inst) {
7392	return ::programUndefinedIfUndefOrPoison(V: Inst, PoisonOnly: true);
7393	}
7394
7395	static bool isKnownNonNaN(const Value *V, FastMathFlags FMF) {
7396	if (FMF.noNaNs())
7397	return true;
7398
7399	if (auto *C = dyn_cast<ConstantFP>(Val: V))
7400	return !C->isNaN();
7401
7402	if (auto *C = dyn_cast<ConstantDataVector>(Val: V)) {
7403	if (!C->getElementType()->isFloatingPointTy())
7404	return false;
7405	for (unsigned I = 0, E = C->getNumElements(); I < E; ++I) {
7406	if (C->getElementAsAPFloat(i: I).isNaN())
7407	return false;
7408	}
7409	return true;
7410	}
7411
7412	if (isa<ConstantAggregateZero>(Val: V))
7413	return true;
7414
7415	return false;
7416	}
7417
7418	static bool isKnownNonZero(const Value *V) {
7419	if (auto *C = dyn_cast<ConstantFP>(Val: V))
7420	return !C->isZero();
7421
7422	if (auto *C = dyn_cast<ConstantDataVector>(Val: V)) {
7423	if (!C->getElementType()->isFloatingPointTy())
7424	return false;
7425	for (unsigned I = 0, E = C->getNumElements(); I < E; ++I) {
7426	if (C->getElementAsAPFloat(i: I).isZero())
7427	return false;
7428	}
7429	return true;
7430	}
7431
7432	return false;
7433	}
7434
7435	/// Match clamp pattern for float types without care about NaNs or signed zeros.
7436	/// Given non-min/max outer cmp/select from the clamp pattern this
7437	/// function recognizes if it can be substitued by a "canonical" min/max
7438	/// pattern.
7439	static SelectPatternResult matchFastFloatClamp(CmpInst::Predicate Pred,
7440	Value *CmpLHS, Value *CmpRHS,
7441	Value *TrueVal, Value *FalseVal,
7442	Value *&LHS, Value *&RHS) {
7443	// Try to match
7444	// X < C1 ? C1 : Min(X, C2) --> Max(C1, Min(X, C2))
7445	// X > C1 ? C1 : Max(X, C2) --> Min(C1, Max(X, C2))
7446	// and return description of the outer Max/Min.
7447
7448	// First, check if select has inverse order:
7449	if (CmpRHS == FalseVal) {
7450	std::swap(a&: TrueVal, b&: FalseVal);
7451	Pred = CmpInst::getInversePredicate(pred: Pred);
7452	}
7453
7454	// Assume success now. If there's no match, callers should not use these anyway.
7455	LHS = TrueVal;
7456	RHS = FalseVal;
7457
7458	const APFloat *FC1;
7459	if (CmpRHS != TrueVal || !match(V: CmpRHS, P: m_APFloat(Res&: FC1)) || !FC1->isFinite())
7460	return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
7461
7462	const APFloat *FC2;
7463	switch (Pred) {
7464	case CmpInst::FCMP_OLT:
7465	case CmpInst::FCMP_OLE:
7466	case CmpInst::FCMP_ULT:
7467	case CmpInst::FCMP_ULE:
7468	if (match(V: FalseVal,
7469	P: m_CombineOr(L: m_OrdFMin(L: m_Specific(V: CmpLHS), R: m_APFloat(Res&: FC2)),
7470	R: m_UnordFMin(L: m_Specific(V: CmpLHS), R: m_APFloat(Res&: FC2)))) &&
7471	*FC1 < *FC2)
7472	return {.Flavor: SPF_FMAXNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
7473	break;
7474	case CmpInst::FCMP_OGT:
7475	case CmpInst::FCMP_OGE:
7476	case CmpInst::FCMP_UGT:
7477	case CmpInst::FCMP_UGE:
7478	if (match(V: FalseVal,
7479	P: m_CombineOr(L: m_OrdFMax(L: m_Specific(V: CmpLHS), R: m_APFloat(Res&: FC2)),
7480	R: m_UnordFMax(L: m_Specific(V: CmpLHS), R: m_APFloat(Res&: FC2)))) &&
7481	*FC1 > *FC2)
7482	return {.Flavor: SPF_FMINNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
7483	break;
7484	default:
7485	break;
7486	}
7487
7488	return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
7489	}
7490
7491	/// Recognize variations of:
7492	/// CLAMP(v,l,h) ==> ((v) < (l) ? (l) : ((v) > (h) ? (h) : (v)))
7493	static SelectPatternResult matchClamp(CmpInst::Predicate Pred,
7494	Value *CmpLHS, Value *CmpRHS,
7495	Value *TrueVal, Value *FalseVal) {
7496	// Swap the select operands and predicate to match the patterns below.
7497	if (CmpRHS != TrueVal) {
7498	Pred = ICmpInst::getSwappedPredicate(pred: Pred);
7499	std::swap(a&: TrueVal, b&: FalseVal);
7500	}
7501	const APInt *C1;
7502	if (CmpRHS == TrueVal && match(V: CmpRHS, P: m_APInt(Res&: C1))) {
7503	const APInt *C2;
7504	// (X <s C1) ? C1 : SMIN(X, C2) ==> SMAX(SMIN(X, C2), C1)
7505	if (match(V: FalseVal, P: m_SMin(L: m_Specific(V: CmpLHS), R: m_APInt(Res&: C2))) &&
7506	C1->slt(RHS: *C2) && Pred == CmpInst::ICMP_SLT)
7507	return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
7508
7509	// (X >s C1) ? C1 : SMAX(X, C2) ==> SMIN(SMAX(X, C2), C1)
7510	if (match(V: FalseVal, P: m_SMax(L: m_Specific(V: CmpLHS), R: m_APInt(Res&: C2))) &&
7511	C1->sgt(RHS: *C2) && Pred == CmpInst::ICMP_SGT)
7512	return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
7513
7514	// (X <u C1) ? C1 : UMIN(X, C2) ==> UMAX(UMIN(X, C2), C1)
7515	if (match(V: FalseVal, P: m_UMin(L: m_Specific(V: CmpLHS), R: m_APInt(Res&: C2))) &&
7516	C1->ult(RHS: *C2) && Pred == CmpInst::ICMP_ULT)
7517	return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
7518
7519	// (X >u C1) ? C1 : UMAX(X, C2) ==> UMIN(UMAX(X, C2), C1)
7520	if (match(V: FalseVal, P: m_UMax(L: m_Specific(V: CmpLHS), R: m_APInt(Res&: C2))) &&
7521	C1->ugt(RHS: *C2) && Pred == CmpInst::ICMP_UGT)
7522	return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
7523	}
7524	return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
7525	}
7526
7527	/// Recognize variations of:
7528	/// a < c ? min(a,b) : min(b,c) ==> min(min(a,b),min(b,c))
7529	static SelectPatternResult matchMinMaxOfMinMax(CmpInst::Predicate Pred,
7530	Value *CmpLHS, Value *CmpRHS,
7531	Value *TVal, Value *FVal,
7532	unsigned Depth) {
7533	// TODO: Allow FP min/max with nnan/nsz.
7534	assert(CmpInst::isIntPredicate(Pred) && "Expected integer comparison");
7535
7536	Value *A = nullptr, *B = nullptr;
7537	SelectPatternResult L = matchSelectPattern(V: TVal, LHS&: A, RHS&: B, CastOp: nullptr, Depth: Depth + 1);
7538	if (!SelectPatternResult::isMinOrMax(SPF: L.Flavor))
7539	return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
7540
7541	Value *C = nullptr, *D = nullptr;
7542	SelectPatternResult R = matchSelectPattern(V: FVal, LHS&: C, RHS&: D, CastOp: nullptr, Depth: Depth + 1);
7543	if (L.Flavor != R.Flavor)
7544	return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
7545
7546	// We have something like: x Pred y ? min(a, b) : min(c, d).
7547	// Try to match the compare to the min/max operations of the select operands.
7548	// First, make sure we have the right compare predicate.
7549	switch (L.Flavor) {
7550	case SPF_SMIN:
7551	if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE) {
7552	Pred = ICmpInst::getSwappedPredicate(pred: Pred);
7553	std::swap(a&: CmpLHS, b&: CmpRHS);
7554	}
7555	if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE)
7556	break;
7557	return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
7558	case SPF_SMAX:
7559	if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE) {
7560	Pred = ICmpInst::getSwappedPredicate(pred: Pred);
7561	std::swap(a&: CmpLHS, b&: CmpRHS);
7562	}
7563	if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE)
7564	break;
7565	return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
7566	case SPF_UMIN:
7567	if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) {
7568	Pred = ICmpInst::getSwappedPredicate(pred: Pred);
7569	std::swap(a&: CmpLHS, b&: CmpRHS);
7570	}
7571	if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE)
7572	break;
7573	return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
7574	case SPF_UMAX:
7575	if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) {
7576	Pred = ICmpInst::getSwappedPredicate(pred: Pred);
7577	std::swap(a&: CmpLHS, b&: CmpRHS);
7578	}
7579	if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE)
7580	break;
7581	return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
7582	default:
7583	return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
7584	}
7585
7586	// If there is a common operand in the already matched min/max and the other
7587	// min/max operands match the compare operands (either directly or inverted),
7588	// then this is min/max of the same flavor.
7589
7590	// a pred c ? m(a, b) : m(c, b) --> m(m(a, b), m(c, b))
7591	// ~c pred ~a ? m(a, b) : m(c, b) --> m(m(a, b), m(c, b))
7592	if (D == B) {
7593	if ((CmpLHS == A && CmpRHS == C) || (match(V: C, P: m_Not(V: m_Specific(V: CmpLHS))) &&
7594	match(V: A, P: m_Not(V: m_Specific(V: CmpRHS)))))
7595	return {.Flavor: L.Flavor, .NaNBehavior: SPNB_NA, .Ordered: false};
7596	}
7597	// a pred d ? m(a, b) : m(b, d) --> m(m(a, b), m(b, d))
7598	// ~d pred ~a ? m(a, b) : m(b, d) --> m(m(a, b), m(b, d))
7599	if (C == B) {
7600	if ((CmpLHS == A && CmpRHS == D) || (match(V: D, P: m_Not(V: m_Specific(V: CmpLHS))) &&
7601	match(V: A, P: m_Not(V: m_Specific(V: CmpRHS)))))
7602	return {.Flavor: L.Flavor, .NaNBehavior: SPNB_NA, .Ordered: false};
7603	}
7604	// b pred c ? m(a, b) : m(c, a) --> m(m(a, b), m(c, a))
7605	// ~c pred ~b ? m(a, b) : m(c, a) --> m(m(a, b), m(c, a))
7606	if (D == A) {
7607	if ((CmpLHS == B && CmpRHS == C) || (match(V: C, P: m_Not(V: m_Specific(V: CmpLHS))) &&
7608	match(V: B, P: m_Not(V: m_Specific(V: CmpRHS)))))
7609	return {.Flavor: L.Flavor, .NaNBehavior: SPNB_NA, .Ordered: false};
7610	}
7611	// b pred d ? m(a, b) : m(a, d) --> m(m(a, b), m(a, d))
7612	// ~d pred ~b ? m(a, b) : m(a, d) --> m(m(a, b), m(a, d))
7613	if (C == A) {
7614	if ((CmpLHS == B && CmpRHS == D) || (match(V: D, P: m_Not(V: m_Specific(V: CmpLHS))) &&
7615	match(V: B, P: m_Not(V: m_Specific(V: CmpRHS)))))
7616	return {.Flavor: L.Flavor, .NaNBehavior: SPNB_NA, .Ordered: false};
7617	}
7618
7619	return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
7620	}
7621
7622	/// If the input value is the result of a 'not' op, constant integer, or vector
7623	/// splat of a constant integer, return the bitwise-not source value.
7624	/// TODO: This could be extended to handle non-splat vector integer constants.
7625	static Value *getNotValue(Value *V) {
7626	Value *NotV;
7627	if (match(V, P: m_Not(V: m_Value(V&: NotV))))
7628	return NotV;
7629
7630	const APInt *C;
7631	if (match(V, P: m_APInt(Res&: C)))
7632	return ConstantInt::get(Ty: V->getType(), V: ~(*C));
7633
7634	return nullptr;
7635	}
7636
7637	/// Match non-obvious integer minimum and maximum sequences.
7638	static SelectPatternResult matchMinMax(CmpInst::Predicate Pred,
7639	Value *CmpLHS, Value *CmpRHS,
7640	Value *TrueVal, Value *FalseVal,
7641	Value *&LHS, Value *&RHS,
7642	unsigned Depth) {
7643	// Assume success. If there's no match, callers should not use these anyway.
7644	LHS = TrueVal;
7645	RHS = FalseVal;
7646
7647	SelectPatternResult SPR = matchClamp(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal);
7648	if (SPR.Flavor != SelectPatternFlavor::SPF_UNKNOWN)
7649	return SPR;
7650
7651	SPR = matchMinMaxOfMinMax(Pred, CmpLHS, CmpRHS, TVal: TrueVal, FVal: FalseVal, Depth);
7652	if (SPR.Flavor != SelectPatternFlavor::SPF_UNKNOWN)
7653	return SPR;
7654
7655	// Look through 'not' ops to find disguised min/max.
7656	// (X > Y) ? ~X : ~Y ==> (~X < ~Y) ? ~X : ~Y ==> MIN(~X, ~Y)
7657	// (X < Y) ? ~X : ~Y ==> (~X > ~Y) ? ~X : ~Y ==> MAX(~X, ~Y)
7658	if (CmpLHS == getNotValue(V: TrueVal) && CmpRHS == getNotValue(V: FalseVal)) {
7659	switch (Pred) {
7660	case CmpInst::ICMP_SGT: return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
7661	case CmpInst::ICMP_SLT: return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
7662	case CmpInst::ICMP_UGT: return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
7663	case CmpInst::ICMP_ULT: return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
7664	default: break;
7665	}
7666	}
7667
7668	// (X > Y) ? ~Y : ~X ==> (~X < ~Y) ? ~Y : ~X ==> MAX(~Y, ~X)
7669	// (X < Y) ? ~Y : ~X ==> (~X > ~Y) ? ~Y : ~X ==> MIN(~Y, ~X)
7670	if (CmpLHS == getNotValue(V: FalseVal) && CmpRHS == getNotValue(V: TrueVal)) {
7671	switch (Pred) {
7672	case CmpInst::ICMP_SGT: return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
7673	case CmpInst::ICMP_SLT: return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
7674	case CmpInst::ICMP_UGT: return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
7675	case CmpInst::ICMP_ULT: return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
7676	default: break;
7677	}
7678	}
7679
7680	if (Pred != CmpInst::ICMP_SGT && Pred != CmpInst::ICMP_SLT)
7681	return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
7682
7683	const APInt *C1;
7684	if (!match(V: CmpRHS, P: m_APInt(Res&: C1)))
7685	return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
7686
7687	// An unsigned min/max can be written with a signed compare.
7688	const APInt *C2;
7689	if ((CmpLHS == TrueVal && match(V: FalseVal, P: m_APInt(Res&: C2))) ||
7690	(CmpLHS == FalseVal && match(V: TrueVal, P: m_APInt(Res&: C2)))) {
7691	// Is the sign bit set?
7692	// (X <s 0) ? X : MAXVAL ==> (X >u MAXVAL) ? X : MAXVAL ==> UMAX
7693	// (X <s 0) ? MAXVAL : X ==> (X >u MAXVAL) ? MAXVAL : X ==> UMIN
7694	if (Pred == CmpInst::ICMP_SLT && C1->isZero() && C2->isMaxSignedValue())
7695	return {.Flavor: CmpLHS == TrueVal ? SPF_UMAX : SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
7696
7697	// Is the sign bit clear?
7698	// (X >s -1) ? MINVAL : X ==> (X <u MINVAL) ? MINVAL : X ==> UMAX
7699	// (X >s -1) ? X : MINVAL ==> (X <u MINVAL) ? X : MINVAL ==> UMIN
7700	if (Pred == CmpInst::ICMP_SGT && C1->isAllOnes() && C2->isMinSignedValue())
7701	return {.Flavor: CmpLHS == FalseVal ? SPF_UMAX : SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
7702	}
7703
7704	return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
7705	}
7706
7707	bool llvm::isKnownNegation(const Value *X, const Value *Y, bool NeedNSW) {
7708	assert(X && Y && "Invalid operand");
7709
7710	// X = sub (0, Y) || X = sub nsw (0, Y)
7711	if ((!NeedNSW && match(V: X, P: m_Sub(L: m_ZeroInt(), R: m_Specific(V: Y)))) ||
7712	(NeedNSW && match(V: X, P: m_NSWSub(L: m_ZeroInt(), R: m_Specific(V: Y)))))
7713	return true;
7714
7715	// Y = sub (0, X) || Y = sub nsw (0, X)
7716	if ((!NeedNSW && match(V: Y, P: m_Sub(L: m_ZeroInt(), R: m_Specific(V: X)))) ||
7717	(NeedNSW && match(V: Y, P: m_NSWSub(L: m_ZeroInt(), R: m_Specific(V: X)))))
7718	return true;
7719
7720	// X = sub (A, B), Y = sub (B, A) || X = sub nsw (A, B), Y = sub nsw (B, A)
7721	Value *A, *B;
7722	return (!NeedNSW && (match(V: X, P: m_Sub(L: m_Value(V&: A), R: m_Value(V&: B))) &&
7723	match(V: Y, P: m_Sub(L: m_Specific(V: B), R: m_Specific(V: A))))) ||
7724	(NeedNSW && (match(V: X, P: m_NSWSub(L: m_Value(V&: A), R: m_Value(V&: B))) &&
7725	match(V: Y, P: m_NSWSub(L: m_Specific(V: B), R: m_Specific(V: A)))));
7726	}
7727
7728	static SelectPatternResult matchSelectPattern(CmpInst::Predicate Pred,
7729	FastMathFlags FMF,
7730	Value *CmpLHS, Value *CmpRHS,
7731	Value *TrueVal, Value *FalseVal,
7732	Value *&LHS, Value *&RHS,
7733	unsigned Depth) {
7734	bool HasMismatchedZeros = false;
7735	if (CmpInst::isFPPredicate(P: Pred)) {
7736	// IEEE-754 ignores the sign of 0.0 in comparisons. So if the select has one
7737	// 0.0 operand, set the compare's 0.0 operands to that same value for the
7738	// purpose of identifying min/max. Disregard vector constants with undefined
7739	// elements because those can not be back-propagated for analysis.
7740	Value *OutputZeroVal = nullptr;
7741	if (match(V: TrueVal, P: m_AnyZeroFP()) && !match(V: FalseVal, P: m_AnyZeroFP()) &&
7742	!cast<Constant>(Val: TrueVal)->containsUndefOrPoisonElement())
7743	OutputZeroVal = TrueVal;
7744	else if (match(V: FalseVal, P: m_AnyZeroFP()) && !match(V: TrueVal, P: m_AnyZeroFP()) &&
7745	!cast<Constant>(Val: FalseVal)->containsUndefOrPoisonElement())
7746	OutputZeroVal = FalseVal;
7747
7748	if (OutputZeroVal) {
7749	if (match(V: CmpLHS, P: m_AnyZeroFP()) && CmpLHS != OutputZeroVal) {
7750	HasMismatchedZeros = true;
7751	CmpLHS = OutputZeroVal;
7752	}
7753	if (match(V: CmpRHS, P: m_AnyZeroFP()) && CmpRHS != OutputZeroVal) {
7754	HasMismatchedZeros = true;
7755	CmpRHS = OutputZeroVal;
7756	}
7757	}
7758	}
7759
7760	LHS = CmpLHS;
7761	RHS = CmpRHS;
7762
7763	// Signed zero may return inconsistent results between implementations.
7764	// (0.0 <= -0.0) ? 0.0 : -0.0 // Returns 0.0
7765	// minNum(0.0, -0.0) // May return -0.0 or 0.0 (IEEE 754-2008 5.3.1)
7766	// Therefore, we behave conservatively and only proceed if at least one of the
7767	// operands is known to not be zero or if we don't care about signed zero.
7768	switch (Pred) {
7769	default: break;
7770	case CmpInst::FCMP_OGT: case CmpInst::FCMP_OLT:
7771	case CmpInst::FCMP_UGT: case CmpInst::FCMP_ULT:
7772	if (!HasMismatchedZeros)
7773	break;
7774	[[fallthrough]];
7775	case CmpInst::FCMP_OGE: case CmpInst::FCMP_OLE:
7776	case CmpInst::FCMP_UGE: case CmpInst::FCMP_ULE:
7777	if (!FMF.noSignedZeros() && !isKnownNonZero(V: CmpLHS) &&
7778	!isKnownNonZero(V: CmpRHS))
7779	return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
7780	}
7781
7782	SelectPatternNaNBehavior NaNBehavior = SPNB_NA;
7783	bool Ordered = false;
7784
7785	// When given one NaN and one non-NaN input:
7786	// - maxnum/minnum (C99 fmaxf()/fminf()) return the non-NaN input.
7787	// - A simple C99 (a < b ? a : b) construction will return 'b' (as the
7788	// ordered comparison fails), which could be NaN or non-NaN.
7789	// so here we discover exactly what NaN behavior is required/accepted.
7790	if (CmpInst::isFPPredicate(P: Pred)) {
7791	bool LHSSafe = isKnownNonNaN(V: CmpLHS, FMF);
7792	bool RHSSafe = isKnownNonNaN(V: CmpRHS, FMF);
7793
7794	if (LHSSafe && RHSSafe) {
7795	// Both operands are known non-NaN.
7796	NaNBehavior = SPNB_RETURNS_ANY;
7797	} else if (CmpInst::isOrdered(predicate: Pred)) {
7798	// An ordered comparison will return false when given a NaN, so it
7799	// returns the RHS.
7800	Ordered = true;
7801	if (LHSSafe)
7802	// LHS is non-NaN, so if RHS is NaN then NaN will be returned.
7803	NaNBehavior = SPNB_RETURNS_NAN;
7804	else if (RHSSafe)
7805	NaNBehavior = SPNB_RETURNS_OTHER;
7806	else
7807	// Completely unsafe.
7808	return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
7809	} else {
7810	Ordered = false;
7811	// An unordered comparison will return true when given a NaN, so it
7812	// returns the LHS.
7813	if (LHSSafe)
7814	// LHS is non-NaN, so if RHS is NaN then non-NaN will be returned.
7815	NaNBehavior = SPNB_RETURNS_OTHER;
7816	else if (RHSSafe)
7817	NaNBehavior = SPNB_RETURNS_NAN;
7818	else
7819	// Completely unsafe.
7820	return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
7821	}
7822	}
7823
7824	if (TrueVal == CmpRHS && FalseVal == CmpLHS) {
7825	std::swap(a&: CmpLHS, b&: CmpRHS);
7826	Pred = CmpInst::getSwappedPredicate(pred: Pred);
7827	if (NaNBehavior == SPNB_RETURNS_NAN)
7828	NaNBehavior = SPNB_RETURNS_OTHER;
7829	else if (NaNBehavior == SPNB_RETURNS_OTHER)
7830	NaNBehavior = SPNB_RETURNS_NAN;
7831	Ordered = !Ordered;
7832	}
7833
7834	// ([if]cmp X, Y) ? X : Y
7835	if (TrueVal == CmpLHS && FalseVal == CmpRHS) {
7836	switch (Pred) {
7837	default: return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false}; // Equality.
7838	case ICmpInst::ICMP_UGT:
7839	case ICmpInst::ICMP_UGE: return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
7840	case ICmpInst::ICMP_SGT:
7841	case ICmpInst::ICMP_SGE: return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
7842	case ICmpInst::ICMP_ULT:
7843	case ICmpInst::ICMP_ULE: return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
7844	case ICmpInst::ICMP_SLT:
7845	case ICmpInst::ICMP_SLE: return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
7846	case FCmpInst::FCMP_UGT:
7847	case FCmpInst::FCMP_UGE:
7848	case FCmpInst::FCMP_OGT:
7849	case FCmpInst::FCMP_OGE: return {.Flavor: SPF_FMAXNUM, .NaNBehavior: NaNBehavior, .Ordered: Ordered};
7850	case FCmpInst::FCMP_ULT:
7851	case FCmpInst::FCMP_ULE:
7852	case FCmpInst::FCMP_OLT:
7853	case FCmpInst::FCMP_OLE: return {.Flavor: SPF_FMINNUM, .NaNBehavior: NaNBehavior, .Ordered: Ordered};
7854	}
7855	}
7856
7857	if (isKnownNegation(X: TrueVal, Y: FalseVal)) {
7858	// Sign-extending LHS does not change its sign, so TrueVal/FalseVal can
7859	// match against either LHS or sext(LHS).
7860	auto MaybeSExtCmpLHS =
7861	m_CombineOr(L: m_Specific(V: CmpLHS), R: m_SExt(Op: m_Specific(V: CmpLHS)));
7862	auto ZeroOrAllOnes = m_CombineOr(L: m_ZeroInt(), R: m_AllOnes());
7863	auto ZeroOrOne = m_CombineOr(L: m_ZeroInt(), R: m_One());
7864	if (match(V: TrueVal, P: MaybeSExtCmpLHS)) {
7865	// Set the return values. If the compare uses the negated value (-X >s 0),
7866	// swap the return values because the negated value is always 'RHS'.
7867	LHS = TrueVal;
7868	RHS = FalseVal;
7869	if (match(V: CmpLHS, P: m_Neg(V: m_Specific(V: FalseVal))))
7870	std::swap(a&: LHS, b&: RHS);
7871
7872	// (X >s 0) ? X : -X or (X >s -1) ? X : -X --> ABS(X)
7873	// (-X >s 0) ? -X : X or (-X >s -1) ? -X : X --> ABS(X)
7874	if (Pred == ICmpInst::ICMP_SGT && match(V: CmpRHS, P: ZeroOrAllOnes))
7875	return {.Flavor: SPF_ABS, .NaNBehavior: SPNB_NA, .Ordered: false};
7876
7877	// (X >=s 0) ? X : -X or (X >=s 1) ? X : -X --> ABS(X)
7878	if (Pred == ICmpInst::ICMP_SGE && match(V: CmpRHS, P: ZeroOrOne))
7879	return {.Flavor: SPF_ABS, .NaNBehavior: SPNB_NA, .Ordered: false};
7880
7881	// (X <s 0) ? X : -X or (X <s 1) ? X : -X --> NABS(X)
7882	// (-X <s 0) ? -X : X or (-X <s 1) ? -X : X --> NABS(X)
7883	if (Pred == ICmpInst::ICMP_SLT && match(V: CmpRHS, P: ZeroOrOne))
7884	return {.Flavor: SPF_NABS, .NaNBehavior: SPNB_NA, .Ordered: false};
7885	}
7886	else if (match(V: FalseVal, P: MaybeSExtCmpLHS)) {
7887	// Set the return values. If the compare uses the negated value (-X >s 0),
7888	// swap the return values because the negated value is always 'RHS'.
7889	LHS = FalseVal;
7890	RHS = TrueVal;
7891	if (match(V: CmpLHS, P: m_Neg(V: m_Specific(V: TrueVal))))
7892	std::swap(a&: LHS, b&: RHS);
7893
7894	// (X >s 0) ? -X : X or (X >s -1) ? -X : X --> NABS(X)
7895	// (-X >s 0) ? X : -X or (-X >s -1) ? X : -X --> NABS(X)
7896	if (Pred == ICmpInst::ICMP_SGT && match(V: CmpRHS, P: ZeroOrAllOnes))
7897	return {.Flavor: SPF_NABS, .NaNBehavior: SPNB_NA, .Ordered: false};
7898
7899	// (X <s 0) ? -X : X or (X <s 1) ? -X : X --> ABS(X)
7900	// (-X <s 0) ? X : -X or (-X <s 1) ? X : -X --> ABS(X)
7901	if (Pred == ICmpInst::ICMP_SLT && match(V: CmpRHS, P: ZeroOrOne))
7902	return {.Flavor: SPF_ABS, .NaNBehavior: SPNB_NA, .Ordered: false};
7903	}
7904	}
7905
7906	if (CmpInst::isIntPredicate(P: Pred))
7907	return matchMinMax(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal, LHS, RHS, Depth);
7908
7909	// According to (IEEE 754-2008 5.3.1), minNum(0.0, -0.0) and similar
7910	// may return either -0.0 or 0.0, so fcmp/select pair has stricter
7911	// semantics than minNum. Be conservative in such case.
7912	if (NaNBehavior != SPNB_RETURNS_ANY ||
7913	(!FMF.noSignedZeros() && !isKnownNonZero(V: CmpLHS) &&
7914	!isKnownNonZero(V: CmpRHS)))
7915	return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
7916
7917	return matchFastFloatClamp(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal, LHS, RHS);
7918	}
7919
7920	/// Helps to match a select pattern in case of a type mismatch.
7921	///
7922	/// The function processes the case when type of true and false values of a
7923	/// select instruction differs from type of the cmp instruction operands because
7924	/// of a cast instruction. The function checks if it is legal to move the cast
7925	/// operation after "select". If yes, it returns the new second value of
7926	/// "select" (with the assumption that cast is moved):
7927	/// 1. As operand of cast instruction when both values of "select" are same cast
7928	/// instructions.
7929	/// 2. As restored constant (by applying reverse cast operation) when the first
7930	/// value of the "select" is a cast operation and the second value is a
7931	/// constant.
7932	/// NOTE: We return only the new second value because the first value could be
7933	/// accessed as operand of cast instruction.
7934	static Value *lookThroughCast(CmpInst *CmpI, Value *V1, Value *V2,
7935	Instruction::CastOps *CastOp) {
7936	auto *Cast1 = dyn_cast<CastInst>(Val: V1);
7937	if (!Cast1)
7938	return nullptr;
7939
7940	*CastOp = Cast1->getOpcode();
7941	Type *SrcTy = Cast1->getSrcTy();
7942	if (auto *Cast2 = dyn_cast<CastInst>(Val: V2)) {
7943	// If V1 and V2 are both the same cast from the same type, look through V1.
7944	if (*CastOp == Cast2->getOpcode() && SrcTy == Cast2->getSrcTy())
7945	return Cast2->getOperand(i_nocapture: 0);
7946	return nullptr;
7947	}
7948
7949	auto *C = dyn_cast<Constant>(Val: V2);
7950	if (!C)
7951	return nullptr;
7952
7953	const DataLayout &DL = CmpI->getModule()->getDataLayout();
7954	Constant *CastedTo = nullptr;
7955	switch (*CastOp) {
7956	case Instruction::ZExt:
7957	if (CmpI->isUnsigned())
7958	CastedTo = ConstantExpr::getTrunc(C, Ty: SrcTy);
7959	break;
7960	case Instruction::SExt:
7961	if (CmpI->isSigned())
7962	CastedTo = ConstantExpr::getTrunc(C, Ty: SrcTy, OnlyIfReduced: true);
7963	break;
7964	case Instruction::Trunc:
7965	Constant *CmpConst;
7966	if (match(V: CmpI->getOperand(i_nocapture: 1), P: m_Constant(C&: CmpConst)) &&
7967	CmpConst->getType() == SrcTy) {
7968	// Here we have the following case:
7969	//
7970	// %cond = cmp iN %x, CmpConst
7971	// %tr = trunc iN %x to iK
7972	// %narrowsel = select i1 %cond, iK %t, iK C
7973	//
7974	// We can always move trunc after select operation:
7975	//
7976	// %cond = cmp iN %x, CmpConst
7977	// %widesel = select i1 %cond, iN %x, iN CmpConst
7978	// %tr = trunc iN %widesel to iK
7979	//
7980	// Note that C could be extended in any way because we don't care about
7981	// upper bits after truncation. It can't be abs pattern, because it would
7982	// look like:
7983	//
7984	// select i1 %cond, x, -x.
7985	//
7986	// So only min/max pattern could be matched. Such match requires widened C
7987	// == CmpConst. That is why set widened C = CmpConst, condition trunc
7988	// CmpConst == C is checked below.
7989	CastedTo = CmpConst;
7990	} else {
7991	unsigned ExtOp = CmpI->isSigned() ? Instruction::SExt : Instruction::ZExt;
7992	CastedTo = ConstantFoldCastOperand(Opcode: ExtOp, C, DestTy: SrcTy, DL);
7993	}
7994	break;
7995	case Instruction::FPTrunc:
7996	CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPExt, C, DestTy: SrcTy, DL);
7997	break;
7998	case Instruction::FPExt:
7999	CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPTrunc, C, DestTy: SrcTy, DL);
8000	break;
8001	case Instruction::FPToUI:
8002	CastedTo = ConstantFoldCastOperand(Opcode: Instruction::UIToFP, C, DestTy: SrcTy, DL);
8003	break;
8004	case Instruction::FPToSI:
8005	CastedTo = ConstantFoldCastOperand(Opcode: Instruction::SIToFP, C, DestTy: SrcTy, DL);
8006	break;
8007	case Instruction::UIToFP:
8008	CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPToUI, C, DestTy: SrcTy, DL);
8009	break;
8010	case Instruction::SIToFP:
8011	CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPToSI, C, DestTy: SrcTy, DL);
8012	break;
8013	default:
8014	break;
8015	}
8016
8017	if (!CastedTo)
8018	return nullptr;
8019
8020	// Make sure the cast doesn't lose any information.
8021	Constant *CastedBack =
8022	ConstantFoldCastOperand(Opcode: *CastOp, C: CastedTo, DestTy: C->getType(), DL);
8023	if (CastedBack && CastedBack != C)
8024	return nullptr;
8025
8026	return CastedTo;
8027	}
8028
8029	SelectPatternResult llvm::matchSelectPattern(Value *V, Value *&LHS, Value *&RHS,
8030	Instruction::CastOps *CastOp,
8031	unsigned Depth) {
8032	if (Depth >= MaxAnalysisRecursionDepth)
8033	return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8034
8035	SelectInst *SI = dyn_cast<SelectInst>(Val: V);
8036	if (!SI) return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8037
8038	CmpInst *CmpI = dyn_cast<CmpInst>(Val: SI->getCondition());
8039	if (!CmpI) return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8040
8041	Value *TrueVal = SI->getTrueValue();
8042	Value *FalseVal = SI->getFalseValue();
8043
8044	return llvm::matchDecomposedSelectPattern(CmpI, TrueVal, FalseVal, LHS, RHS,
8045	CastOp, Depth);
8046	}
8047
8048	SelectPatternResult llvm::matchDecomposedSelectPattern(
8049	CmpInst *CmpI, Value *TrueVal, Value *FalseVal, Value *&LHS, Value *&RHS,
8050	Instruction::CastOps *CastOp, unsigned Depth) {
8051	CmpInst::Predicate Pred = CmpI->getPredicate();
8052	Value *CmpLHS = CmpI->getOperand(i_nocapture: 0);
8053	Value *CmpRHS = CmpI->getOperand(i_nocapture: 1);
8054	FastMathFlags FMF;
8055	if (isa<FPMathOperator>(Val: CmpI))
8056	FMF = CmpI->getFastMathFlags();
8057
8058	// Bail out early.
8059	if (CmpI->isEquality())
8060	return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8061
8062	// Deal with type mismatches.
8063	if (CastOp && CmpLHS->getType() != TrueVal->getType()) {
8064	if (Value *C = lookThroughCast(CmpI, V1: TrueVal, V2: FalseVal, CastOp)) {
8065	// If this is a potential fmin/fmax with a cast to integer, then ignore
8066	// -0.0 because there is no corresponding integer value.
8067	if (*CastOp == Instruction::FPToSI || *CastOp == Instruction::FPToUI)
8068	FMF.setNoSignedZeros();
8069	return ::matchSelectPattern(Pred, FMF, CmpLHS, CmpRHS,
8070	TrueVal: cast<CastInst>(Val: TrueVal)->getOperand(i_nocapture: 0), FalseVal: C,
8071	LHS, RHS, Depth);
8072	}
8073	if (Value *C = lookThroughCast(CmpI, V1: FalseVal, V2: TrueVal, CastOp)) {
8074	// If this is a potential fmin/fmax with a cast to integer, then ignore
8075	// -0.0 because there is no corresponding integer value.
8076	if (*CastOp == Instruction::FPToSI || *CastOp == Instruction::FPToUI)
8077	FMF.setNoSignedZeros();
8078	return ::matchSelectPattern(Pred, FMF, CmpLHS, CmpRHS,
8079	TrueVal: C, FalseVal: cast<CastInst>(Val: FalseVal)->getOperand(i_nocapture: 0),
8080	LHS, RHS, Depth);
8081	}
8082	}
8083	return ::matchSelectPattern(Pred, FMF, CmpLHS, CmpRHS, TrueVal, FalseVal,
8084	LHS, RHS, Depth);
8085	}
8086
8087	CmpInst::Predicate llvm::getMinMaxPred(SelectPatternFlavor SPF, bool Ordered) {
8088	if (SPF == SPF_SMIN) return ICmpInst::ICMP_SLT;
8089	if (SPF == SPF_UMIN) return ICmpInst::ICMP_ULT;
8090	if (SPF == SPF_SMAX) return ICmpInst::ICMP_SGT;
8091	if (SPF == SPF_UMAX) return ICmpInst::ICMP_UGT;
8092	if (SPF == SPF_FMINNUM)
8093	return Ordered ? FCmpInst::FCMP_OLT : FCmpInst::FCMP_ULT;
8094	if (SPF == SPF_FMAXNUM)
8095	return Ordered ? FCmpInst::FCMP_OGT : FCmpInst::FCMP_UGT;
8096	llvm_unreachable("unhandled!");
8097	}
8098
8099	SelectPatternFlavor llvm::getInverseMinMaxFlavor(SelectPatternFlavor SPF) {
8100	if (SPF == SPF_SMIN) return SPF_SMAX;
8101	if (SPF == SPF_UMIN) return SPF_UMAX;
8102	if (SPF == SPF_SMAX) return SPF_SMIN;
8103	if (SPF == SPF_UMAX) return SPF_UMIN;
8104	llvm_unreachable("unhandled!");
8105	}
8106
8107	Intrinsic::ID llvm::getInverseMinMaxIntrinsic(Intrinsic::ID MinMaxID) {
8108	switch (MinMaxID) {
8109	case Intrinsic::smax: return Intrinsic::smin;
8110	case Intrinsic::smin: return Intrinsic::smax;
8111	case Intrinsic::umax: return Intrinsic::umin;
8112	case Intrinsic::umin: return Intrinsic::umax;
8113	// Please note that next four intrinsics may produce the same result for
8114	// original and inverted case even if X != Y due to NaN is handled specially.
8115	case Intrinsic::maximum: return Intrinsic::minimum;
8116	case Intrinsic::minimum: return Intrinsic::maximum;
8117	case Intrinsic::maxnum: return Intrinsic::minnum;
8118	case Intrinsic::minnum: return Intrinsic::maxnum;
8119	default: llvm_unreachable("Unexpected intrinsic");
8120	}
8121	}
8122
8123	APInt llvm::getMinMaxLimit(SelectPatternFlavor SPF, unsigned BitWidth) {
8124	switch (SPF) {
8125	case SPF_SMAX: return APInt::getSignedMaxValue(numBits: BitWidth);
8126	case SPF_SMIN: return APInt::getSignedMinValue(numBits: BitWidth);
8127	case SPF_UMAX: return APInt::getMaxValue(numBits: BitWidth);
8128	case SPF_UMIN: return APInt::getMinValue(numBits: BitWidth);
8129	default: llvm_unreachable("Unexpected flavor");
8130	}
8131	}
8132
8133	std::pair<Intrinsic::ID, bool>
8134	llvm::canConvertToMinOrMaxIntrinsic(ArrayRef<Value *> VL) {
8135	// Check if VL contains select instructions that can be folded into a min/max
8136	// vector intrinsic and return the intrinsic if it is possible.
8137	// TODO: Support floating point min/max.
8138	bool AllCmpSingleUse = true;
8139	SelectPatternResult SelectPattern;
8140	SelectPattern.Flavor = SPF_UNKNOWN;
8141	if (all_of(Range&: VL, P: [&SelectPattern, &AllCmpSingleUse](Value *I) {
8142	Value *LHS, *RHS;
8143	auto CurrentPattern = matchSelectPattern(V: I, LHS, RHS);
8144	if (!SelectPatternResult::isMinOrMax(SPF: CurrentPattern.Flavor) ||
8145	CurrentPattern.Flavor == SPF_FMINNUM ||
8146	CurrentPattern.Flavor == SPF_FMAXNUM ||
8147	!I->getType()->isIntOrIntVectorTy())
8148	return false;
8149	if (SelectPattern.Flavor != SPF_UNKNOWN &&
8150	SelectPattern.Flavor != CurrentPattern.Flavor)
8151	return false;
8152	SelectPattern = CurrentPattern;
8153	AllCmpSingleUse &=
8154	match(V: I, P: m_Select(C: m_OneUse(SubPattern: m_Value()), L: m_Value(), R: m_Value()));
8155	return true;
8156	})) {
8157	switch (SelectPattern.Flavor) {
8158	case SPF_SMIN:
8159	return {Intrinsic::smin, AllCmpSingleUse};
8160	case SPF_UMIN:
8161	return {Intrinsic::umin, AllCmpSingleUse};
8162	case SPF_SMAX:
8163	return {Intrinsic::smax, AllCmpSingleUse};
8164	case SPF_UMAX:
8165	return {Intrinsic::umax, AllCmpSingleUse};
8166	default:
8167	llvm_unreachable("unexpected select pattern flavor");
8168	}
8169	}
8170	return {Intrinsic::not_intrinsic, false};
8171	}
8172
8173	bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO,
8174	Value *&Start, Value *&Step) {
8175	// Handle the case of a simple two-predecessor recurrence PHI.
8176	// There's a lot more that could theoretically be done here, but
8177	// this is sufficient to catch some interesting cases.
8178	if (P->getNumIncomingValues() != 2)
8179	return false;
8180
8181	for (unsigned i = 0; i != 2; ++i) {
8182	Value *L = P->getIncomingValue(i);
8183	Value *R = P->getIncomingValue(i: !i);
8184	auto *LU = dyn_cast<BinaryOperator>(Val: L);
8185	if (!LU)
8186	continue;
8187	unsigned Opcode = LU->getOpcode();
8188
8189	switch (Opcode) {
8190	default:
8191	continue;
8192	// TODO: Expand list -- xor, div, gep, uaddo, etc..
8193	case Instruction::LShr:
8194	case Instruction::AShr:
8195	case Instruction::Shl:
8196	case Instruction::Add:
8197	case Instruction::Sub:
8198	case Instruction::And:
8199	case Instruction::Or:
8200	case Instruction::Mul:
8201	case Instruction::FMul: {
8202	Value *LL = LU->getOperand(i_nocapture: 0);
8203	Value *LR = LU->getOperand(i_nocapture: 1);
8204	// Find a recurrence.
8205	if (LL == P)
8206	L = LR;
8207	else if (LR == P)
8208	L = LL;
8209	else
8210	continue; // Check for recurrence with L and R flipped.
8211
8212	break; // Match!
8213	}
8214	};
8215
8216	// We have matched a recurrence of the form:
8217	// %iv = [R, %entry], [%iv.next, %backedge]
8218	// %iv.next = binop %iv, L
8219	// OR
8220	// %iv = [R, %entry], [%iv.next, %backedge]
8221	// %iv.next = binop L, %iv
8222	BO = LU;
8223	Start = R;
8224	Step = L;
8225	return true;
8226	}
8227	return false;
8228	}
8229
8230	bool llvm::matchSimpleRecurrence(const BinaryOperator *I, PHINode *&P,
8231	Value *&Start, Value *&Step) {
8232	BinaryOperator *BO = nullptr;
8233	P = dyn_cast<PHINode>(Val: I->getOperand(i_nocapture: 0));
8234	if (!P)
8235	P = dyn_cast<PHINode>(Val: I->getOperand(i_nocapture: 1));
8236	return P && matchSimpleRecurrence(P, BO, Start, Step) && BO == I;
8237	}
8238
8239	/// Return true if "icmp Pred LHS RHS" is always true.
8240	static bool isTruePredicate(CmpInst::Predicate Pred, const Value *LHS,
8241	const Value *RHS, const DataLayout &DL,
8242	unsigned Depth) {
8243	if (ICmpInst::isTrueWhenEqual(predicate: Pred) && LHS == RHS)
8244	return true;
8245
8246	switch (Pred) {
8247	default:
8248	return false;
8249
8250	case CmpInst::ICMP_SLE: {
8251	const APInt *C;
8252
8253	// LHS s<= LHS +_{nsw} C if C >= 0
8254	if (match(V: RHS, P: m_NSWAdd(L: m_Specific(V: LHS), R: m_APInt(Res&: C))))
8255	return !C->isNegative();
8256	return false;
8257	}
8258
8259	case CmpInst::ICMP_ULE: {
8260	// LHS u<= LHS +_{nuw} V for any V
8261	if (match(V: RHS, P: m_c_Add(L: m_Specific(V: LHS), R: m_Value())) &&
8262	cast<OverflowingBinaryOperator>(Val: RHS)->hasNoUnsignedWrap())
8263	return true;
8264
8265	// RHS >> V u<= RHS for any V
8266	if (match(V: LHS, P: m_LShr(L: m_Specific(V: RHS), R: m_Value())))
8267	return true;
8268
8269	// Match A to (X +_{nuw} CA) and B to (X +_{nuw} CB)
8270	auto MatchNUWAddsToSameValue = [&](const Value *A, const Value *B,
8271	const Value *&X,
8272	const APInt *&CA, const APInt *&CB) {
8273	if (match(V: A, P: m_NUWAdd(L: m_Value(V&: X), R: m_APInt(Res&: CA))) &&
8274	match(V: B, P: m_NUWAdd(L: m_Specific(V: X), R: m_APInt(Res&: CB))))
8275	return true;
8276
8277	// If X & C == 0 then (X | C) == X +_{nuw} C
8278	if (match(V: A, P: m_Or(L: m_Value(V&: X), R: m_APInt(Res&: CA))) &&
8279	match(V: B, P: m_Or(L: m_Specific(V: X), R: m_APInt(Res&: CB)))) {
8280	KnownBits Known(CA->getBitWidth());
8281	computeKnownBits(V: X, Known, DL, Depth: Depth + 1, /*AC*/ nullptr,
8282	/*CxtI*/ nullptr, /*DT*/ nullptr);
8283	if (CA->isSubsetOf(RHS: Known.Zero) && CB->isSubsetOf(RHS: Known.Zero))
8284	return true;
8285	}
8286
8287	return false;
8288	};
8289
8290	const Value *X;
8291	const APInt *CLHS, *CRHS;
8292	if (MatchNUWAddsToSameValue(LHS, RHS, X, CLHS, CRHS))
8293	return CLHS->ule(RHS: *CRHS);
8294
8295	return false;
8296	}
8297	}
8298	}
8299
8300	/// Return true if "icmp Pred BLHS BRHS" is true whenever "icmp Pred
8301	/// ALHS ARHS" is true. Otherwise, return std::nullopt.
8302	static std::optional<bool>
8303	isImpliedCondOperands(CmpInst::Predicate Pred, const Value *ALHS,
8304	const Value *ARHS, const Value *BLHS, const Value *BRHS,
8305	const DataLayout &DL, unsigned Depth) {
8306	switch (Pred) {
8307	default:
8308	return std::nullopt;
8309
8310	case CmpInst::ICMP_SLT:
8311	case CmpInst::ICMP_SLE:
8312	if (isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: BLHS, RHS: ALHS, DL, Depth) &&
8313	isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: ARHS, RHS: BRHS, DL, Depth))
8314	return true;
8315	return std::nullopt;
8316
8317	case CmpInst::ICMP_SGT:
8318	case CmpInst::ICMP_SGE:
8319	if (isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: ALHS, RHS: BLHS, DL, Depth) &&
8320	isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: BRHS, RHS: ARHS, DL, Depth))
8321	return true;
8322	return std::nullopt;
8323
8324	case CmpInst::ICMP_ULT:
8325	case CmpInst::ICMP_ULE:
8326	if (isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: BLHS, RHS: ALHS, DL, Depth) &&
8327	isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: ARHS, RHS: BRHS, DL, Depth))
8328	return true;
8329	return std::nullopt;
8330
8331	case CmpInst::ICMP_UGT:
8332	case CmpInst::ICMP_UGE:
8333	if (isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: ALHS, RHS: BLHS, DL, Depth) &&
8334	isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: BRHS, RHS: ARHS, DL, Depth))
8335	return true;
8336	return std::nullopt;
8337	}
8338	}
8339
8340	/// Return true if the operands of two compares (expanded as "L0 pred L1" and
8341	/// "R0 pred R1") match. IsSwappedOps is true when the operands match, but are
8342	/// swapped.
8343	static bool areMatchingOperands(const Value *L0, const Value *L1, const Value *R0,
8344	const Value *R1, bool &AreSwappedOps) {
8345	bool AreMatchingOps = (L0 == R0 && L1 == R1);
8346	AreSwappedOps = (L0 == R1 && L1 == R0);
8347	return AreMatchingOps || AreSwappedOps;
8348	}
8349
8350	/// Return true if "icmp1 LPred X, Y" implies "icmp2 RPred X, Y" is true.
8351	/// Return false if "icmp1 LPred X, Y" implies "icmp2 RPred X, Y" is false.
8352	/// Otherwise, return std::nullopt if we can't infer anything.
8353	static std::optional<bool>
8354	isImpliedCondMatchingOperands(CmpInst::Predicate LPred,
8355	CmpInst::Predicate RPred, bool AreSwappedOps) {
8356	// Canonicalize the predicate as if the operands were not commuted.
8357	if (AreSwappedOps)
8358	RPred = ICmpInst::getSwappedPredicate(pred: RPred);
8359
8360	if (CmpInst::isImpliedTrueByMatchingCmp(Pred1: LPred, Pred2: RPred))
8361	return true;
8362	if (CmpInst::isImpliedFalseByMatchingCmp(Pred1: LPred, Pred2: RPred))
8363	return false;
8364
8365	return std::nullopt;
8366	}
8367
8368	/// Return true if "icmp LPred X, LC" implies "icmp RPred X, RC" is true.
8369	/// Return false if "icmp LPred X, LC" implies "icmp RPred X, RC" is false.
8370	/// Otherwise, return std::nullopt if we can't infer anything.
8371	static std::optional<bool> isImpliedCondCommonOperandWithConstants(
8372	CmpInst::Predicate LPred, const APInt &LC, CmpInst::Predicate RPred,
8373	const APInt &RC) {
8374	ConstantRange DomCR = ConstantRange::makeExactICmpRegion(Pred: LPred, Other: LC);
8375	ConstantRange CR = ConstantRange::makeExactICmpRegion(Pred: RPred, Other: RC);
8376	ConstantRange Intersection = DomCR.intersectWith(CR);
8377	ConstantRange Difference = DomCR.difference(CR);
8378	if (Intersection.isEmptySet())
8379	return false;
8380	if (Difference.isEmptySet())
8381	return true;
8382	return std::nullopt;
8383	}
8384
8385	/// Return true if LHS implies RHS (expanded to its components as "R0 RPred R1")
8386	/// is true. Return false if LHS implies RHS is false. Otherwise, return
8387	/// std::nullopt if we can't infer anything.
8388	static std::optional<bool> isImpliedCondICmps(const ICmpInst *LHS,
8389	CmpInst::Predicate RPred,
8390	const Value *R0, const Value *R1,
8391	const DataLayout &DL,
8392	bool LHSIsTrue, unsigned Depth) {
8393	Value *L0 = LHS->getOperand(i_nocapture: 0);
8394	Value *L1 = LHS->getOperand(i_nocapture: 1);
8395
8396	// The rest of the logic assumes the LHS condition is true. If that's not the
8397	// case, invert the predicate to make it so.
8398	CmpInst::Predicate LPred =
8399	LHSIsTrue ? LHS->getPredicate() : LHS->getInversePredicate();
8400
8401	// Can we infer anything when the 0-operands match and the 1-operands are
8402	// constants (not necessarily matching)?
8403	const APInt *LC, *RC;
8404	if (L0 == R0 && match(V: L1, P: m_APInt(Res&: LC)) && match(V: R1, P: m_APInt(Res&: RC)))
8405	return isImpliedCondCommonOperandWithConstants(LPred, LC: *LC, RPred, RC: *RC);
8406
8407	// Can we infer anything when the two compares have matching operands?
8408	bool AreSwappedOps;
8409	if (areMatchingOperands(L0, L1, R0, R1, AreSwappedOps))
8410	return isImpliedCondMatchingOperands(LPred, RPred, AreSwappedOps);
8411
8412	// L0 = R0 = L1 + R1, L0 >=u L1 implies R0 >=u R1, L0 <u L1 implies R0 <u R1
8413	if (ICmpInst::isUnsigned(predicate: LPred) && ICmpInst::isUnsigned(predicate: RPred)) {
8414	if (L0 == R1) {
8415	std::swap(a&: R0, b&: R1);
8416	RPred = ICmpInst::getSwappedPredicate(pred: RPred);
8417	}
8418	if (L1 == R0) {
8419	std::swap(a&: L0, b&: L1);
8420	LPred = ICmpInst::getSwappedPredicate(pred: LPred);
8421	}
8422	if (L1 == R1) {
8423	std::swap(a&: L0, b&: L1);
8424	LPred = ICmpInst::getSwappedPredicate(pred: LPred);
8425	std::swap(a&: R0, b&: R1);
8426	RPred = ICmpInst::getSwappedPredicate(pred: RPred);
8427	}
8428	if (L0 == R0 &&
8429	(LPred == ICmpInst::ICMP_ULT || LPred == ICmpInst::ICMP_UGE) &&
8430	(RPred == ICmpInst::ICMP_ULT || RPred == ICmpInst::ICMP_UGE) &&
8431	match(V: L0, P: m_c_Add(L: m_Specific(V: L1), R: m_Specific(V: R1))))
8432	return LPred == RPred;
8433	}
8434
8435	if (LPred == RPred)
8436	return isImpliedCondOperands(Pred: LPred, ALHS: L0, ARHS: L1, BLHS: R0, BRHS: R1, DL, Depth);
8437
8438	return std::nullopt;
8439	}
8440
8441	/// Return true if LHS implies RHS is true. Return false if LHS implies RHS is
8442	/// false. Otherwise, return std::nullopt if we can't infer anything. We
8443	/// expect the RHS to be an icmp and the LHS to be an 'and', 'or', or a 'select'
8444	/// instruction.
8445	static std::optional<bool>
8446	isImpliedCondAndOr(const Instruction *LHS, CmpInst::Predicate RHSPred,
8447	const Value *RHSOp0, const Value *RHSOp1,
8448	const DataLayout &DL, bool LHSIsTrue, unsigned Depth) {
8449	// The LHS must be an 'or', 'and', or a 'select' instruction.
8450	assert((LHS->getOpcode() == Instruction::And ||
8451	LHS->getOpcode() == Instruction::Or ||
8452	LHS->getOpcode() == Instruction::Select) &&
8453	"Expected LHS to be 'and', 'or', or 'select'.");
8454
8455	assert(Depth <= MaxAnalysisRecursionDepth && "Hit recursion limit");
8456
8457	// If the result of an 'or' is false, then we know both legs of the 'or' are
8458	// false. Similarly, if the result of an 'and' is true, then we know both
8459	// legs of the 'and' are true.
8460	const Value *ALHS, *ARHS;
8461	if ((!LHSIsTrue && match(V: LHS, P: m_LogicalOr(L: m_Value(V&: ALHS), R: m_Value(V&: ARHS)))) ||
8462	(LHSIsTrue && match(V: LHS, P: m_LogicalAnd(L: m_Value(V&: ALHS), R: m_Value(V&: ARHS))))) {
8463	// FIXME: Make this non-recursion.
8464	if (std::optional<bool> Implication = isImpliedCondition(
8465	LHS: ALHS, RHSPred, RHSOp0, RHSOp1, DL, LHSIsTrue, Depth: Depth + 1))
8466	return Implication;
8467	if (std::optional<bool> Implication = isImpliedCondition(
8468	LHS: ARHS, RHSPred, RHSOp0, RHSOp1, DL, LHSIsTrue, Depth: Depth + 1))
8469	return Implication;
8470	return std::nullopt;
8471	}
8472	return std::nullopt;
8473	}
8474
8475	std::optional<bool>
8476	llvm::isImpliedCondition(const Value *LHS, CmpInst::Predicate RHSPred,
8477	const Value *RHSOp0, const Value *RHSOp1,
8478	const DataLayout &DL, bool LHSIsTrue, unsigned Depth) {
8479	// Bail out when we hit the limit.
8480	if (Depth == MaxAnalysisRecursionDepth)
8481	return std::nullopt;
8482
8483	// A mismatch occurs when we compare a scalar cmp to a vector cmp, for
8484	// example.
8485	if (RHSOp0->getType()->isVectorTy() != LHS->getType()->isVectorTy())
8486	return std::nullopt;
8487
8488	assert(LHS->getType()->isIntOrIntVectorTy(1) &&
8489	"Expected integer type only!");
8490
8491	// Both LHS and RHS are icmps.
8492	const ICmpInst *LHSCmp = dyn_cast<ICmpInst>(Val: LHS);
8493	if (LHSCmp)
8494	return isImpliedCondICmps(LHS: LHSCmp, RPred: RHSPred, R0: RHSOp0, R1: RHSOp1, DL, LHSIsTrue,
8495	Depth);
8496
8497	/// The LHS should be an 'or', 'and', or a 'select' instruction. We expect
8498	/// the RHS to be an icmp.
8499	/// FIXME: Add support for and/or/select on the RHS.
8500	if (const Instruction *LHSI = dyn_cast<Instruction>(Val: LHS)) {
8501	if ((LHSI->getOpcode() == Instruction::And ||
8502	LHSI->getOpcode() == Instruction::Or ||
8503	LHSI->getOpcode() == Instruction::Select))
8504	return isImpliedCondAndOr(LHS: LHSI, RHSPred, RHSOp0, RHSOp1, DL, LHSIsTrue,
8505	Depth);
8506	}
8507	return std::nullopt;
8508	}
8509
8510	std::optional<bool> llvm::isImpliedCondition(const Value *LHS, const Value *RHS,
8511	const DataLayout &DL,
8512	bool LHSIsTrue, unsigned Depth) {
8513	// LHS ==> RHS by definition
8514	if (LHS == RHS)
8515	return LHSIsTrue;
8516
8517	if (const ICmpInst *RHSCmp = dyn_cast<ICmpInst>(Val: RHS))
8518	return isImpliedCondition(LHS, RHSPred: RHSCmp->getPredicate(),
8519	RHSOp0: RHSCmp->getOperand(i_nocapture: 0), RHSOp1: RHSCmp->getOperand(i_nocapture: 1), DL,
8520	LHSIsTrue, Depth);
8521
8522	if (Depth == MaxAnalysisRecursionDepth)
8523	return std::nullopt;
8524
8525	// LHS ==> (RHS1 || RHS2) if LHS ==> RHS1 or LHS ==> RHS2
8526	// LHS ==> !(RHS1 && RHS2) if LHS ==> !RHS1 or LHS ==> !RHS2
8527	const Value *RHS1, *RHS2;
8528	if (match(V: RHS, P: m_LogicalOr(L: m_Value(V&: RHS1), R: m_Value(V&: RHS2)))) {
8529	if (std::optional<bool> Imp =
8530	isImpliedCondition(LHS, RHS: RHS1, DL, LHSIsTrue, Depth: Depth + 1))
8531	if (*Imp == true)
8532	return true;
8533	if (std::optional<bool> Imp =
8534	isImpliedCondition(LHS, RHS: RHS2, DL, LHSIsTrue, Depth: Depth + 1))
8535	if (*Imp == true)
8536	return true;
8537	}
8538	if (match(V: RHS, P: m_LogicalAnd(L: m_Value(V&: RHS1), R: m_Value(V&: RHS2)))) {
8539	if (std::optional<bool> Imp =
8540	isImpliedCondition(LHS, RHS: RHS1, DL, LHSIsTrue, Depth: Depth + 1))
8541	if (*Imp == false)
8542	return false;
8543	if (std::optional<bool> Imp =
8544	isImpliedCondition(LHS, RHS: RHS2, DL, LHSIsTrue, Depth: Depth + 1))
8545	if (*Imp == false)
8546	return false;
8547	}
8548
8549	return std::nullopt;
8550	}
8551
8552	// Returns a pair (Condition, ConditionIsTrue), where Condition is a branch
8553	// condition dominating ContextI or nullptr, if no condition is found.
8554	static std::pair<Value *, bool>
8555	getDomPredecessorCondition(const Instruction *ContextI) {
8556	if (!ContextI || !ContextI->getParent())
8557	return {nullptr, false};
8558
8559	// TODO: This is a poor/cheap way to determine dominance. Should we use a
8560	// dominator tree (eg, from a SimplifyQuery) instead?
8561	const BasicBlock *ContextBB = ContextI->getParent();
8562	const BasicBlock *PredBB = ContextBB->getSinglePredecessor();
8563	if (!PredBB)
8564	return {nullptr, false};
8565
8566	// We need a conditional branch in the predecessor.
8567	Value *PredCond;
8568	BasicBlock *TrueBB, *FalseBB;
8569	if (!match(V: PredBB->getTerminator(), P: m_Br(C: m_Value(V&: PredCond), T&: TrueBB, F&: FalseBB)))
8570	return {nullptr, false};
8571
8572	// The branch should get simplified. Don't bother simplifying this condition.
8573	if (TrueBB == FalseBB)
8574	return {nullptr, false};
8575
8576	assert((TrueBB == ContextBB || FalseBB == ContextBB) &&
8577	"Predecessor block does not point to successor?");
8578
8579	// Is this condition implied by the predecessor condition?
8580	return {PredCond, TrueBB == ContextBB};
8581	}
8582
8583	std::optional<bool> llvm::isImpliedByDomCondition(const Value *Cond,
8584	const Instruction *ContextI,
8585	const DataLayout &DL) {
8586	assert(Cond->getType()->isIntOrIntVectorTy(1) && "Condition must be bool");
8587	auto PredCond = getDomPredecessorCondition(ContextI);
8588	if (PredCond.first)
8589	return isImpliedCondition(LHS: PredCond.first, RHS: Cond, DL, LHSIsTrue: PredCond.second);
8590	return std::nullopt;
8591	}
8592
8593	std::optional<bool> llvm::isImpliedByDomCondition(CmpInst::Predicate Pred,
8594	const Value *LHS,
8595	const Value *RHS,
8596	const Instruction *ContextI,
8597	const DataLayout &DL) {
8598	auto PredCond = getDomPredecessorCondition(ContextI);
8599	if (PredCond.first)
8600	return isImpliedCondition(LHS: PredCond.first, RHSPred: Pred, RHSOp0: LHS, RHSOp1: RHS, DL,
8601	LHSIsTrue: PredCond.second);
8602	return std::nullopt;
8603	}
8604
8605	static void setLimitsForBinOp(const BinaryOperator &BO, APInt &Lower,
8606	APInt &Upper, const InstrInfoQuery &IIQ,
8607	bool PreferSignedRange) {
8608	unsigned Width = Lower.getBitWidth();
8609	const APInt *C;
8610	switch (BO.getOpcode()) {
8611	case Instruction::Add:
8612	if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && !C->isZero()) {
8613	bool HasNSW = IIQ.hasNoSignedWrap(Op: &BO);
8614	bool HasNUW = IIQ.hasNoUnsignedWrap(Op: &BO);
8615
8616	// If the caller expects a signed compare, then try to use a signed range.
8617	// Otherwise if both no-wraps are set, use the unsigned range because it
8618	// is never larger than the signed range. Example:
8619	// "add nuw nsw i8 X, -2" is unsigned [254,255] vs. signed [-128, 125].
8620	if (PreferSignedRange && HasNSW && HasNUW)
8621	HasNUW = false;
8622
8623	if (HasNUW) {
8624	// 'add nuw x, C' produces [C, UINT_MAX].
8625	Lower = *C;
8626	} else if (HasNSW) {
8627	if (C->isNegative()) {
8628	// 'add nsw x, -C' produces [SINT_MIN, SINT_MAX - C].
8629	Lower = APInt::getSignedMinValue(numBits: Width);
8630	Upper = APInt::getSignedMaxValue(numBits: Width) + *C + 1;
8631	} else {
8632	// 'add nsw x, +C' produces [SINT_MIN + C, SINT_MAX].
8633	Lower = APInt::getSignedMinValue(numBits: Width) + *C;
8634	Upper = APInt::getSignedMaxValue(numBits: Width) + 1;
8635	}
8636	}
8637	}
8638	break;
8639
8640	case Instruction::And:
8641	if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
8642	// 'and x, C' produces [0, C].
8643	Upper = *C + 1;
8644	// X & -X is a power of two or zero. So we can cap the value at max power of
8645	// two.
8646	if (match(V: BO.getOperand(i_nocapture: 0), P: m_Neg(V: m_Specific(V: BO.getOperand(i_nocapture: 1)))) ||
8647	match(V: BO.getOperand(i_nocapture: 1), P: m_Neg(V: m_Specific(V: BO.getOperand(i_nocapture: 0)))))
8648	Upper = APInt::getSignedMinValue(numBits: Width) + 1;
8649	break;
8650
8651	case Instruction::Or:
8652	if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
8653	// 'or x, C' produces [C, UINT_MAX].
8654	Lower = *C;
8655	break;
8656
8657	case Instruction::AShr:
8658	if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && C->ult(RHS: Width)) {
8659	// 'ashr x, C' produces [INT_MIN >> C, INT_MAX >> C].
8660	Lower = APInt::getSignedMinValue(numBits: Width).ashr(ShiftAmt: *C);
8661	Upper = APInt::getSignedMaxValue(numBits: Width).ashr(ShiftAmt: *C) + 1;
8662	} else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
8663	unsigned ShiftAmount = Width - 1;
8664	if (!C->isZero() && IIQ.isExact(Op: &BO))
8665	ShiftAmount = C->countr_zero();
8666	if (C->isNegative()) {
8667	// 'ashr C, x' produces [C, C >> (Width-1)]
8668	Lower = *C;
8669	Upper = C->ashr(ShiftAmt: ShiftAmount) + 1;
8670	} else {
8671	// 'ashr C, x' produces [C >> (Width-1), C]
8672	Lower = C->ashr(ShiftAmt: ShiftAmount);
8673	Upper = *C + 1;
8674	}
8675	}
8676	break;
8677
8678	case Instruction::LShr:
8679	if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && C->ult(RHS: Width)) {
8680	// 'lshr x, C' produces [0, UINT_MAX >> C].
8681	Upper = APInt::getAllOnes(numBits: Width).lshr(ShiftAmt: *C) + 1;
8682	} else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
8683	// 'lshr C, x' produces [C >> (Width-1), C].
8684	unsigned ShiftAmount = Width - 1;
8685	if (!C->isZero() && IIQ.isExact(Op: &BO))
8686	ShiftAmount = C->countr_zero();
8687	Lower = C->lshr(shiftAmt: ShiftAmount);
8688	Upper = *C + 1;
8689	}
8690	break;
8691
8692	case Instruction::Shl:
8693	if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
8694	if (IIQ.hasNoUnsignedWrap(Op: &BO)) {
8695	// 'shl nuw C, x' produces [C, C << CLZ(C)]
8696	Lower = *C;
8697	Upper = Lower.shl(shiftAmt: Lower.countl_zero()) + 1;
8698	} else if (BO.hasNoSignedWrap()) { // TODO: What if both nuw+nsw?
8699	if (C->isNegative()) {
8700	// 'shl nsw C, x' produces [C << CLO(C)-1, C]
8701	unsigned ShiftAmount = C->countl_one() - 1;
8702	Lower = C->shl(shiftAmt: ShiftAmount);
8703	Upper = *C + 1;
8704	} else {
8705	// 'shl nsw C, x' produces [C, C << CLZ(C)-1]
8706	unsigned ShiftAmount = C->countl_zero() - 1;
8707	Lower = *C;
8708	Upper = C->shl(shiftAmt: ShiftAmount) + 1;
8709	}
8710	} else {
8711	// If lowbit is set, value can never be zero.
8712	if ((*C)[0])
8713	Lower = APInt::getOneBitSet(numBits: Width, BitNo: 0);
8714	// If we are shifting a constant the largest it can be is if the longest
8715	// sequence of consecutive ones is shifted to the highbits (breaking
8716	// ties for which sequence is higher). At the moment we take a liberal
8717	// upper bound on this by just popcounting the constant.
8718	// TODO: There may be a bitwise trick for it longest/highest
8719	// consecutative sequence of ones (naive method is O(Width) loop).
8720	Upper = APInt::getHighBitsSet(numBits: Width, hiBitsSet: C->popcount()) + 1;
8721	}
8722	} else if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && C->ult(RHS: Width)) {
8723	Upper = APInt::getBitsSetFrom(numBits: Width, loBit: C->getZExtValue()) + 1;
8724	}
8725	break;
8726
8727	case Instruction::SDiv:
8728	if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
8729	APInt IntMin = APInt::getSignedMinValue(numBits: Width);
8730	APInt IntMax = APInt::getSignedMaxValue(numBits: Width);
8731	if (C->isAllOnes()) {
8732	// 'sdiv x, -1' produces [INT_MIN + 1, INT_MAX]
8733	// where C != -1 and C != 0 and C != 1
8734	Lower = IntMin + 1;
8735	Upper = IntMax + 1;
8736	} else if (C->countl_zero() < Width - 1) {
8737	// 'sdiv x, C' produces [INT_MIN / C, INT_MAX / C]
8738	// where C != -1 and C != 0 and C != 1
8739	Lower = IntMin.sdiv(RHS: *C);
8740	Upper = IntMax.sdiv(RHS: *C);
8741	if (Lower.sgt(RHS: Upper))
8742	std::swap(a&: Lower, b&: Upper);
8743	Upper = Upper + 1;
8744	assert(Upper != Lower && "Upper part of range has wrapped!");
8745	}
8746	} else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
8747	if (C->isMinSignedValue()) {
8748	// 'sdiv INT_MIN, x' produces [INT_MIN, INT_MIN / -2].
8749	Lower = *C;
8750	Upper = Lower.lshr(shiftAmt: 1) + 1;
8751	} else {
8752	// 'sdiv C, x' produces [-|C|, |C|].
8753	Upper = C->abs() + 1;
8754	Lower = (-Upper) + 1;
8755	}
8756	}
8757	break;
8758
8759	case Instruction::UDiv:
8760	if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && !C->isZero()) {
8761	// 'udiv x, C' produces [0, UINT_MAX / C].
8762	Upper = APInt::getMaxValue(numBits: Width).udiv(RHS: *C) + 1;
8763	} else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
8764	// 'udiv C, x' produces [0, C].
8765	Upper = *C + 1;
8766	}
8767	break;
8768
8769	case Instruction::SRem:
8770	if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
8771	// 'srem x, C' produces (-|C|, |C|).
8772	Upper = C->abs();
8773	Lower = (-Upper) + 1;
8774	}
8775	break;
8776
8777	case Instruction::URem:
8778	if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
8779	// 'urem x, C' produces [0, C).
8780	Upper = *C;
8781	break;
8782
8783	default:
8784	break;
8785	}
8786	}
8787
8788	static ConstantRange getRangeForIntrinsic(const IntrinsicInst &II) {
8789	unsigned Width = II.getType()->getScalarSizeInBits();
8790	const APInt *C;
8791	switch (II.getIntrinsicID()) {
8792	case Intrinsic::ctpop:
8793	case Intrinsic::ctlz:
8794	case Intrinsic::cttz:
8795	// Maximum of set/clear bits is the bit width.
8796	return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
8797	Upper: APInt(Width, Width + 1));
8798	case Intrinsic::uadd_sat:
8799	// uadd.sat(x, C) produces [C, UINT_MAX].
8800	if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)) ||
8801	match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
8802	return ConstantRange::getNonEmpty(Lower: *C, Upper: APInt::getZero(numBits: Width));
8803	break;
8804	case Intrinsic::sadd_sat:
8805	if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)) ||
8806	match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
8807	if (C->isNegative())
8808	// sadd.sat(x, -C) produces [SINT_MIN, SINT_MAX + (-C)].
8809	return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
8810	Upper: APInt::getSignedMaxValue(numBits: Width) + *C +
8811	1);
8812
8813	// sadd.sat(x, +C) produces [SINT_MIN + C, SINT_MAX].
8814	return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width) + *C,
8815	Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
8816	}
8817	break;
8818	case Intrinsic::usub_sat:
8819	// usub.sat(C, x) produces [0, C].
8820	if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)))
8821	return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width), Upper: *C + 1);
8822
8823	// usub.sat(x, C) produces [0, UINT_MAX - C].
8824	if (match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
8825	return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
8826	Upper: APInt::getMaxValue(numBits: Width) - *C + 1);
8827	break;
8828	case Intrinsic::ssub_sat:
8829	if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
8830	if (C->isNegative())
8831	// ssub.sat(-C, x) produces [SINT_MIN, -SINT_MIN + (-C)].
8832	return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
8833	Upper: *C - APInt::getSignedMinValue(numBits: Width) +
8834	1);
8835
8836	// ssub.sat(+C, x) produces [-SINT_MAX + C, SINT_MAX].
8837	return ConstantRange::getNonEmpty(Lower: *C - APInt::getSignedMaxValue(numBits: Width),
8838	Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
8839	} else if (match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
8840	if (C->isNegative())
8841	// ssub.sat(x, -C) produces [SINT_MIN - (-C), SINT_MAX]:
8842	return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width) - *C,
8843	Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
8844
8845	// ssub.sat(x, +C) produces [SINT_MIN, SINT_MAX - C].
8846	return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
8847	Upper: APInt::getSignedMaxValue(numBits: Width) - *C +
8848	1);
8849	}
8850	break;
8851	case Intrinsic::umin:
8852	case Intrinsic::umax:
8853	case Intrinsic::smin:
8854	case Intrinsic::smax:
8855	if (!match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)) &&
8856	!match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
8857	break;
8858
8859	switch (II.getIntrinsicID()) {
8860	case Intrinsic::umin:
8861	return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width), Upper: *C + 1);
8862	case Intrinsic::umax:
8863	return ConstantRange::getNonEmpty(Lower: *C, Upper: APInt::getZero(numBits: Width));
8864	case Intrinsic::smin:
8865	return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
8866	Upper: *C + 1);
8867	case Intrinsic::smax:
8868	return ConstantRange::getNonEmpty(Lower: *C,
8869	Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
8870	default:
8871	llvm_unreachable("Must be min/max intrinsic");
8872	}
8873	break;
8874	case Intrinsic::abs:
8875	// If abs of SIGNED_MIN is poison, then the result is [0..SIGNED_MAX],
8876	// otherwise it is [0..SIGNED_MIN], as -SIGNED_MIN == SIGNED_MIN.
8877	if (match(V: II.getOperand(i_nocapture: 1), P: m_One()))
8878	return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
8879	Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
8880
8881	return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
8882	Upper: APInt::getSignedMinValue(numBits: Width) + 1);
8883	case Intrinsic::vscale:
8884	if (!II.getParent() || !II.getFunction())
8885	break;
8886	return getVScaleRange(F: II.getFunction(), BitWidth: Width);
8887	default:
8888	break;
8889	}
8890
8891	return ConstantRange::getFull(BitWidth: Width);
8892	}
8893
8894	static ConstantRange getRangeForSelectPattern(const SelectInst &SI,
8895	const InstrInfoQuery &IIQ) {
8896	unsigned BitWidth = SI.getType()->getScalarSizeInBits();
8897	const Value *LHS = nullptr, *RHS = nullptr;
8898	SelectPatternResult R = matchSelectPattern(V: &SI, LHS, RHS);
8899	if (R.Flavor == SPF_UNKNOWN)
8900	return ConstantRange::getFull(BitWidth);
8901
8902	if (R.Flavor == SelectPatternFlavor::SPF_ABS) {
8903	// If the negation part of the abs (in RHS) has the NSW flag,
8904	// then the result of abs(X) is [0..SIGNED_MAX],
8905	// otherwise it is [0..SIGNED_MIN], as -SIGNED_MIN == SIGNED_MIN.
8906	if (match(V: RHS, P: m_Neg(V: m_Specific(V: LHS))) &&
8907	IIQ.hasNoSignedWrap(Op: cast<Instruction>(Val: RHS)))
8908	return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: BitWidth),
8909	Upper: APInt::getSignedMaxValue(numBits: BitWidth) + 1);
8910
8911	return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: BitWidth),
8912	Upper: APInt::getSignedMinValue(numBits: BitWidth) + 1);
8913	}
8914
8915	if (R.Flavor == SelectPatternFlavor::SPF_NABS) {
8916	// The result of -abs(X) is <= 0.
8917	return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: BitWidth),
8918	Upper: APInt(BitWidth, 1));
8919	}
8920
8921	const APInt *C;
8922	if (!match(V: LHS, P: m_APInt(Res&: C)) && !match(V: RHS, P: m_APInt(Res&: C)))
8923	return ConstantRange::getFull(BitWidth);
8924
8925	switch (R.Flavor) {
8926	case SPF_UMIN:
8927	return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: BitWidth), Upper: *C + 1);
8928	case SPF_UMAX:
8929	return ConstantRange::getNonEmpty(Lower: *C, Upper: APInt::getZero(numBits: BitWidth));
8930	case SPF_SMIN:
8931	return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: BitWidth),
8932	Upper: *C + 1);
8933	case SPF_SMAX:
8934	return ConstantRange::getNonEmpty(Lower: *C,
8935	Upper: APInt::getSignedMaxValue(numBits: BitWidth) + 1);
8936	default:
8937	return ConstantRange::getFull(BitWidth);
8938	}
8939	}
8940
8941	static void setLimitForFPToI(const Instruction *I, APInt &Lower, APInt &Upper) {
8942	// The maximum representable value of a half is 65504. For floats the maximum
8943	// value is 3.4e38 which requires roughly 129 bits.
8944	unsigned BitWidth = I->getType()->getScalarSizeInBits();
8945	if (!I->getOperand(i: 0)->getType()->getScalarType()->isHalfTy())
8946	return;
8947	if (isa<FPToSIInst>(Val: I) && BitWidth >= 17) {
8948	Lower = APInt(BitWidth, -65504);
8949	Upper = APInt(BitWidth, 65505);
8950	}
8951
8952	if (isa<FPToUIInst>(Val: I) && BitWidth >= 16) {
8953	// For a fptoui the lower limit is left as 0.
8954	Upper = APInt(BitWidth, 65505);
8955	}
8956	}
8957
8958	ConstantRange llvm::computeConstantRange(const Value *V, bool ForSigned,
8959	bool UseInstrInfo, AssumptionCache *AC,
8960	const Instruction *CtxI,
8961	const DominatorTree *DT,
8962	unsigned Depth) {
8963	assert(V->getType()->isIntOrIntVectorTy() && "Expected integer instruction");
8964
8965	if (Depth == MaxAnalysisRecursionDepth)
8966	return ConstantRange::getFull(BitWidth: V->getType()->getScalarSizeInBits());
8967
8968	const APInt *C;
8969	if (match(V, P: m_APInt(Res&: C)))
8970	return ConstantRange(*C);
8971	unsigned BitWidth = V->getType()->getScalarSizeInBits();
8972
8973	if (auto *VC = dyn_cast<ConstantDataVector>(Val: V)) {
8974	ConstantRange CR = ConstantRange::getEmpty(BitWidth);
8975	for (unsigned ElemIdx = 0, NElem = VC->getNumElements(); ElemIdx < NElem;
8976	++ElemIdx)
8977	CR = CR.unionWith(CR: VC->getElementAsAPInt(i: ElemIdx));
8978	return CR;
8979	}
8980
8981	InstrInfoQuery IIQ(UseInstrInfo);
8982	ConstantRange CR = ConstantRange::getFull(BitWidth);
8983	if (auto *BO = dyn_cast<BinaryOperator>(Val: V)) {
8984	APInt Lower = APInt(BitWidth, 0);
8985	APInt Upper = APInt(BitWidth, 0);
8986	// TODO: Return ConstantRange.
8987	setLimitsForBinOp(BO: *BO, Lower, Upper, IIQ, PreferSignedRange: ForSigned);
8988	CR = ConstantRange::getNonEmpty(Lower, Upper);
8989	} else if (auto *II = dyn_cast<IntrinsicInst>(Val: V))
8990	CR = getRangeForIntrinsic(II: *II);
8991	else if (auto *SI = dyn_cast<SelectInst>(Val: V)) {
8992	ConstantRange CRTrue = computeConstantRange(
8993	V: SI->getTrueValue(), ForSigned, UseInstrInfo, AC, CtxI, DT, Depth: Depth + 1);
8994	ConstantRange CRFalse = computeConstantRange(
8995	V: SI->getFalseValue(), ForSigned, UseInstrInfo, AC, CtxI, DT, Depth: Depth + 1);
8996	CR = CRTrue.unionWith(CR: CRFalse);
8997	CR = CR.intersectWith(CR: getRangeForSelectPattern(SI: *SI, IIQ));
8998	} else if (isa<FPToUIInst>(Val: V) || isa<FPToSIInst>(Val: V)) {
8999	APInt Lower = APInt(BitWidth, 0);
9000	APInt Upper = APInt(BitWidth, 0);
9001	// TODO: Return ConstantRange.
9002	setLimitForFPToI(I: cast<Instruction>(Val: V), Lower, Upper);
9003	CR = ConstantRange::getNonEmpty(Lower, Upper);
9004	}
9005
9006	if (auto *I = dyn_cast<Instruction>(Val: V))
9007	if (auto *Range = IIQ.getMetadata(I, KindID: LLVMContext::MD_range))
9008	CR = CR.intersectWith(CR: getConstantRangeFromMetadata(RangeMD: *Range));
9009
9010	if (CtxI && AC) {
9011	// Try to restrict the range based on information from assumptions.
9012	for (auto &AssumeVH : AC->assumptionsFor(V)) {
9013	if (!AssumeVH)
9014	continue;
9015	CallInst *I = cast<CallInst>(Val&: AssumeVH);
9016	assert(I->getParent()->getParent() == CtxI->getParent()->getParent() &&
9017	"Got assumption for the wrong function!");
9018	assert(I->getCalledFunction()->getIntrinsicID() == Intrinsic::assume &&
9019	"must be an assume intrinsic");
9020
9021	if (!isValidAssumeForContext(Inv: I, CxtI: CtxI, DT))
9022	continue;
9023	Value *Arg = I->getArgOperand(i: 0);
9024	ICmpInst *Cmp = dyn_cast<ICmpInst>(Val: Arg);
9025	// Currently we just use information from comparisons.
9026	if (!Cmp || Cmp->getOperand(i_nocapture: 0) != V)
9027	continue;
9028	// TODO: Set "ForSigned" parameter via Cmp->isSigned()?
9029	ConstantRange RHS =
9030	computeConstantRange(V: Cmp->getOperand(i_nocapture: 1), /* ForSigned */ false,
9031	UseInstrInfo, AC, CtxI: I, DT, Depth: Depth + 1);
9032	CR = CR.intersectWith(
9033	CR: ConstantRange::makeAllowedICmpRegion(Pred: Cmp->getPredicate(), Other: RHS));
9034	}
9035	}
9036
9037	return CR;
9038	}
9039