1	//===- AttributorAttributes.cpp - Attributes for Attributor deduction -----===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// See the Attributor.h file comment and the class descriptions in that file for
10	// more information.
11	//
12	//===----------------------------------------------------------------------===//
13
14	#include "llvm/Transforms/IPO/Attributor.h"
15
16	#include "llvm/ADT/APInt.h"
17	#include "llvm/ADT/ArrayRef.h"
18	#include "llvm/ADT/DenseMapInfo.h"
19	#include "llvm/ADT/MapVector.h"
20	#include "llvm/ADT/SCCIterator.h"
21	#include "llvm/ADT/STLExtras.h"
22	#include "llvm/ADT/SetOperations.h"
23	#include "llvm/ADT/SetVector.h"
24	#include "llvm/ADT/SmallPtrSet.h"
25	#include "llvm/ADT/SmallVector.h"
26	#include "llvm/ADT/Statistic.h"
27	#include "llvm/ADT/StringExtras.h"
28	#include "llvm/Analysis/AliasAnalysis.h"
29	#include "llvm/Analysis/AssumeBundleQueries.h"
30	#include "llvm/Analysis/AssumptionCache.h"
31	#include "llvm/Analysis/CaptureTracking.h"
32	#include "llvm/Analysis/CycleAnalysis.h"
33	#include "llvm/Analysis/InstructionSimplify.h"
34	#include "llvm/Analysis/LazyValueInfo.h"
35	#include "llvm/Analysis/MemoryBuiltins.h"
36	#include "llvm/Analysis/OptimizationRemarkEmitter.h"
37	#include "llvm/Analysis/ScalarEvolution.h"
38	#include "llvm/Analysis/TargetTransformInfo.h"
39	#include "llvm/Analysis/ValueTracking.h"
40	#include "llvm/IR/Argument.h"
41	#include "llvm/IR/Assumptions.h"
42	#include "llvm/IR/Attributes.h"
43	#include "llvm/IR/BasicBlock.h"
44	#include "llvm/IR/Constant.h"
45	#include "llvm/IR/Constants.h"
46	#include "llvm/IR/DataLayout.h"
47	#include "llvm/IR/DerivedTypes.h"
48	#include "llvm/IR/GlobalValue.h"
49	#include "llvm/IR/IRBuilder.h"
50	#include "llvm/IR/InlineAsm.h"
51	#include "llvm/IR/InstrTypes.h"
52	#include "llvm/IR/Instruction.h"
53	#include "llvm/IR/Instructions.h"
54	#include "llvm/IR/IntrinsicInst.h"
55	#include "llvm/IR/IntrinsicsAMDGPU.h"
56	#include "llvm/IR/IntrinsicsNVPTX.h"
57	#include "llvm/IR/LLVMContext.h"
58	#include "llvm/IR/MDBuilder.h"
59	#include "llvm/IR/NoFolder.h"
60	#include "llvm/IR/Value.h"
61	#include "llvm/IR/ValueHandle.h"
62	#include "llvm/Support/Alignment.h"
63	#include "llvm/Support/Casting.h"
64	#include "llvm/Support/CommandLine.h"
65	#include "llvm/Support/ErrorHandling.h"
66	#include "llvm/Support/GraphWriter.h"
67	#include "llvm/Support/MathExtras.h"
68	#include "llvm/Support/TypeSize.h"
69	#include "llvm/Support/raw_ostream.h"
70	#include "llvm/Transforms/Utils/BasicBlockUtils.h"
71	#include "llvm/Transforms/Utils/CallPromotionUtils.h"
72	#include "llvm/Transforms/Utils/Local.h"
73	#include "llvm/Transforms/Utils/ValueMapper.h"
74	#include <cassert>
75	#include <numeric>
76	#include <optional>
77	#include <string>
78
79	using namespace llvm;
80
81	#define DEBUG_TYPE "attributor"
82
83	static cl::opt<bool> ManifestInternal(
84	"attributor-manifest-internal", cl::Hidden,
85	cl::desc("Manifest Attributor internal string attributes."),
86	cl::init(Val: false));
87
88	static cl::opt<int> MaxHeapToStackSize("max-heap-to-stack-size", cl::init(Val: 128),
89	cl::Hidden);
90
91	template <>
92	unsigned llvm::PotentialConstantIntValuesState::MaxPotentialValues = 0;
93
94	template <> unsigned llvm::PotentialLLVMValuesState::MaxPotentialValues = -1;
95
96	static cl::opt<unsigned, true> MaxPotentialValues(
97	"attributor-max-potential-values", cl::Hidden,
98	cl::desc("Maximum number of potential values to be "
99	"tracked for each position."),
100	cl::location(L&: llvm::PotentialConstantIntValuesState::MaxPotentialValues),
101	cl::init(Val: 7));
102
103	static cl::opt<int> MaxPotentialValuesIterations(
104	"attributor-max-potential-values-iterations", cl::Hidden,
105	cl::desc(
106	"Maximum number of iterations we keep dismantling potential values."),
107	cl::init(Val: 64));
108
109	STATISTIC(NumAAs, "Number of abstract attributes created");
110
111	// Some helper macros to deal with statistics tracking.
112	//
113	// Usage:
114	// For simple IR attribute tracking overload trackStatistics in the abstract
115	// attribute and choose the right STATS_DECLTRACK_********* macro,
116	// e.g.,:
117	// void trackStatistics() const override {
118	// STATS_DECLTRACK_ARG_ATTR(returned)
119	// }
120	// If there is a single "increment" side one can use the macro
121	// STATS_DECLTRACK with a custom message. If there are multiple increment
122	// sides, STATS_DECL and STATS_TRACK can also be used separately.
123	//
124	#define BUILD_STAT_MSG_IR_ATTR(TYPE, NAME) \
125	("Number of " #TYPE " marked '" #NAME "'")
126	#define BUILD_STAT_NAME(NAME, TYPE) NumIR##TYPE##_##NAME
127	#define STATS_DECL_(NAME, MSG) STATISTIC(NAME, MSG);
128	#define STATS_DECL(NAME, TYPE, MSG) \
129	STATS_DECL_(BUILD_STAT_NAME(NAME, TYPE), MSG);
130	#define STATS_TRACK(NAME, TYPE) ++(BUILD_STAT_NAME(NAME, TYPE));
131	#define STATS_DECLTRACK(NAME, TYPE, MSG) \
132	{ \
133	STATS_DECL(NAME, TYPE, MSG) \
134	STATS_TRACK(NAME, TYPE) \
135	}
136	#define STATS_DECLTRACK_ARG_ATTR(NAME) \
137	STATS_DECLTRACK(NAME, Arguments, BUILD_STAT_MSG_IR_ATTR(arguments, NAME))
138	#define STATS_DECLTRACK_CSARG_ATTR(NAME) \
139	STATS_DECLTRACK(NAME, CSArguments, \
140	BUILD_STAT_MSG_IR_ATTR(call site arguments, NAME))
141	#define STATS_DECLTRACK_FN_ATTR(NAME) \
142	STATS_DECLTRACK(NAME, Function, BUILD_STAT_MSG_IR_ATTR(functions, NAME))
143	#define STATS_DECLTRACK_CS_ATTR(NAME) \
144	STATS_DECLTRACK(NAME, CS, BUILD_STAT_MSG_IR_ATTR(call site, NAME))
145	#define STATS_DECLTRACK_FNRET_ATTR(NAME) \
146	STATS_DECLTRACK(NAME, FunctionReturn, \
147	BUILD_STAT_MSG_IR_ATTR(function returns, NAME))
148	#define STATS_DECLTRACK_CSRET_ATTR(NAME) \
149	STATS_DECLTRACK(NAME, CSReturn, \
150	BUILD_STAT_MSG_IR_ATTR(call site returns, NAME))
151	#define STATS_DECLTRACK_FLOATING_ATTR(NAME) \
152	STATS_DECLTRACK(NAME, Floating, \
153	("Number of floating values known to be '" #NAME "'"))
154
155	// Specialization of the operator<< for abstract attributes subclasses. This
156	// disambiguates situations where multiple operators are applicable.
157	namespace llvm {
158	#define PIPE_OPERATOR(CLASS) \
159	raw_ostream &operator<<(raw_ostream &OS, const CLASS &AA) { \
160	return OS << static_cast<const AbstractAttribute &>(AA); \
161	}
162
163	PIPE_OPERATOR(AAIsDead)
164	PIPE_OPERATOR(AANoUnwind)
165	PIPE_OPERATOR(AANoSync)
166	PIPE_OPERATOR(AANoRecurse)
167	PIPE_OPERATOR(AANonConvergent)
168	PIPE_OPERATOR(AAWillReturn)
169	PIPE_OPERATOR(AANoReturn)
170	PIPE_OPERATOR(AANonNull)
171	PIPE_OPERATOR(AAMustProgress)
172	PIPE_OPERATOR(AANoAlias)
173	PIPE_OPERATOR(AADereferenceable)
174	PIPE_OPERATOR(AAAlign)
175	PIPE_OPERATOR(AAInstanceInfo)
176	PIPE_OPERATOR(AANoCapture)
177	PIPE_OPERATOR(AAValueSimplify)
178	PIPE_OPERATOR(AANoFree)
179	PIPE_OPERATOR(AAHeapToStack)
180	PIPE_OPERATOR(AAIntraFnReachability)
181	PIPE_OPERATOR(AAMemoryBehavior)
182	PIPE_OPERATOR(AAMemoryLocation)
183	PIPE_OPERATOR(AAValueConstantRange)
184	PIPE_OPERATOR(AAPrivatizablePtr)
185	PIPE_OPERATOR(AAUndefinedBehavior)
186	PIPE_OPERATOR(AAPotentialConstantValues)
187	PIPE_OPERATOR(AAPotentialValues)
188	PIPE_OPERATOR(AANoUndef)
189	PIPE_OPERATOR(AANoFPClass)
190	PIPE_OPERATOR(AACallEdges)
191	PIPE_OPERATOR(AAInterFnReachability)
192	PIPE_OPERATOR(AAPointerInfo)
193	PIPE_OPERATOR(AAAssumptionInfo)
194	PIPE_OPERATOR(AAUnderlyingObjects)
195	PIPE_OPERATOR(AAAddressSpace)
196	PIPE_OPERATOR(AAAllocationInfo)
197	PIPE_OPERATOR(AAIndirectCallInfo)
198	PIPE_OPERATOR(AAGlobalValueInfo)
199	PIPE_OPERATOR(AADenormalFPMath)
200
201	#undef PIPE_OPERATOR
202
203	template <>
204	ChangeStatus clampStateAndIndicateChange<DerefState>(DerefState &S,
205	const DerefState &R) {
206	ChangeStatus CS0 =
207	clampStateAndIndicateChange(S&: S.DerefBytesState, R: R.DerefBytesState);
208	ChangeStatus CS1 = clampStateAndIndicateChange(S&: S.GlobalState, R: R.GlobalState);
209	return CS0 | CS1;
210	}
211
212	} // namespace llvm
213
214	static bool mayBeInCycle(const CycleInfo *CI, const Instruction *I,
215	bool HeaderOnly, Cycle **CPtr = nullptr) {
216	if (!CI)
217	return true;
218	auto *BB = I->getParent();
219	auto *C = CI->getCycle(Block: BB);
220	if (!C)
221	return false;
222	if (CPtr)
223	*CPtr = C;
224	return !HeaderOnly || BB == C->getHeader();
225	}
226
227	/// Checks if a type could have padding bytes.
228	static bool isDenselyPacked(Type *Ty, const DataLayout &DL) {
229	// There is no size information, so be conservative.
230	if (!Ty->isSized())
231	return false;
232
233	// If the alloc size is not equal to the storage size, then there are padding
234	// bytes. For x86_fp80 on x86-64, size: 80 alloc size: 128.
235	if (DL.getTypeSizeInBits(Ty) != DL.getTypeAllocSizeInBits(Ty))
236	return false;
237
238	// FIXME: This isn't the right way to check for padding in vectors with
239	// non-byte-size elements.
240	if (VectorType *SeqTy = dyn_cast<VectorType>(Val: Ty))
241	return isDenselyPacked(Ty: SeqTy->getElementType(), DL);
242
243	// For array types, check for padding within members.
244	if (ArrayType *SeqTy = dyn_cast<ArrayType>(Val: Ty))
245	return isDenselyPacked(Ty: SeqTy->getElementType(), DL);
246
247	if (!isa<StructType>(Val: Ty))
248	return true;
249
250	// Check for padding within and between elements of a struct.
251	StructType *StructTy = cast<StructType>(Val: Ty);
252	const StructLayout *Layout = DL.getStructLayout(Ty: StructTy);
253	uint64_t StartPos = 0;
254	for (unsigned I = 0, E = StructTy->getNumElements(); I < E; ++I) {
255	Type *ElTy = StructTy->getElementType(N: I);
256	if (!isDenselyPacked(Ty: ElTy, DL))
257	return false;
258	if (StartPos != Layout->getElementOffsetInBits(Idx: I))
259	return false;
260	StartPos += DL.getTypeAllocSizeInBits(Ty: ElTy);
261	}
262
263	return true;
264	}
265
266	/// Get pointer operand of memory accessing instruction. If \p I is
267	/// not a memory accessing instruction, return nullptr. If \p AllowVolatile,
268	/// is set to false and the instruction is volatile, return nullptr.
269	static const Value *getPointerOperand(const Instruction *I,
270	bool AllowVolatile) {
271	if (!AllowVolatile && I->isVolatile())
272	return nullptr;
273
274	if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
275	return LI->getPointerOperand();
276	}
277
278	if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
279	return SI->getPointerOperand();
280	}
281
282	if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
283	return CXI->getPointerOperand();
284	}
285
286	if (auto *RMWI = dyn_cast<AtomicRMWInst>(Val: I)) {
287	return RMWI->getPointerOperand();
288	}
289
290	return nullptr;
291	}
292
293	/// Helper function to create a pointer based on \p Ptr, and advanced by \p
294	/// Offset bytes.
295	static Value *constructPointer(Value *Ptr, int64_t Offset,
296	IRBuilder<NoFolder> &IRB) {
297	LLVM_DEBUG(dbgs() << "Construct pointer: " << *Ptr << " + " << Offset
298	<< "-bytes\n");
299
300	if (Offset)
301	Ptr = IRB.CreatePtrAdd(Ptr, Offset: IRB.getInt64(C: Offset),
302	Name: Ptr->getName() + ".b" + Twine(Offset));
303	return Ptr;
304	}
305
306	static const Value *
307	stripAndAccumulateOffsets(Attributor &A, const AbstractAttribute &QueryingAA,
308	const Value *Val, const DataLayout &DL, APInt &Offset,
309	bool GetMinOffset, bool AllowNonInbounds,
310	bool UseAssumed = false) {
311
312	auto AttributorAnalysis = [&](Value &V, APInt &ROffset) -> bool {
313	const IRPosition &Pos = IRPosition::value(V);
314	// Only track dependence if we are going to use the assumed info.
315	const AAValueConstantRange *ValueConstantRangeAA =
316	A.getAAFor<AAValueConstantRange>(QueryingAA, IRP: Pos,
317	DepClass: UseAssumed ? DepClassTy::OPTIONAL
318	: DepClassTy::NONE);
319	if (!ValueConstantRangeAA)
320	return false;
321	ConstantRange Range = UseAssumed ? ValueConstantRangeAA->getAssumed()
322	: ValueConstantRangeAA->getKnown();
323	if (Range.isFullSet())
324	return false;
325
326	// We can only use the lower part of the range because the upper part can
327	// be higher than what the value can really be.
328	if (GetMinOffset)
329	ROffset = Range.getSignedMin();
330	else
331	ROffset = Range.getSignedMax();
332	return true;
333	};
334
335	return Val->stripAndAccumulateConstantOffsets(DL, Offset, AllowNonInbounds,
336	/* AllowInvariant */ AllowInvariantGroup: true,
337	ExternalAnalysis: AttributorAnalysis);
338	}
339
340	static const Value *
341	getMinimalBaseOfPointer(Attributor &A, const AbstractAttribute &QueryingAA,
342	const Value *Ptr, int64_t &BytesOffset,
343	const DataLayout &DL, bool AllowNonInbounds = false) {
344	APInt OffsetAPInt(DL.getIndexTypeSizeInBits(Ty: Ptr->getType()), 0);
345	const Value *Base =
346	stripAndAccumulateOffsets(A, QueryingAA, Val: Ptr, DL, Offset&: OffsetAPInt,
347	/* GetMinOffset */ true, AllowNonInbounds);
348
349	BytesOffset = OffsetAPInt.getSExtValue();
350	return Base;
351	}
352
353	/// Clamp the information known for all returned values of a function
354	/// (identified by \p QueryingAA) into \p S.
355	template <typename AAType, typename StateType = typename AAType::StateType,
356	Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind,
357	bool RecurseForSelectAndPHI = true>
358	static void clampReturnedValueStates(
359	Attributor &A, const AAType &QueryingAA, StateType &S,
360	const IRPosition::CallBaseContext *CBContext = nullptr) {
361	LLVM_DEBUG(dbgs() << "[Attributor] Clamp return value states for "
362	<< QueryingAA << " into " << S << "\n");
363
364	assert((QueryingAA.getIRPosition().getPositionKind() ==
365	IRPosition::IRP_RETURNED ||
366	QueryingAA.getIRPosition().getPositionKind() ==
367	IRPosition::IRP_CALL_SITE_RETURNED) &&
368	"Can only clamp returned value states for a function returned or call "
369	"site returned position!");
370
371	// Use an optional state as there might not be any return values and we want
372	// to join (IntegerState::operator&) the state of all there are.
373	std::optional<StateType> T;
374
375	// Callback for each possibly returned value.
376	auto CheckReturnValue = [&](Value &RV) -> bool {
377	const IRPosition &RVPos = IRPosition::value(V: RV, CBContext);
378	// If possible, use the hasAssumedIRAttr interface.
379	if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
380	bool IsKnown;
381	return AA::hasAssumedIRAttr<IRAttributeKind>(
382	A, &QueryingAA, RVPos, DepClassTy::REQUIRED, IsKnown);
383	}
384
385	const AAType *AA =
386	A.getAAFor<AAType>(QueryingAA, RVPos, DepClassTy::REQUIRED);
387	if (!AA)
388	return false;
389	LLVM_DEBUG(dbgs() << "[Attributor] RV: " << RV
390	<< " AA: " << AA->getAsStr(&A) << " @ " << RVPos << "\n");
391	const StateType &AAS = AA->getState();
392	if (!T)
393	T = StateType::getBestState(AAS);
394	*T &= AAS;
395	LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " RV State: " << T
396	<< "\n");
397	return T->isValidState();
398	};
399
400	if (!A.checkForAllReturnedValues(Pred: CheckReturnValue, QueryingAA,
401	S: AA::ValueScope::Intraprocedural,
402	RecurseForSelectAndPHI))
403	S.indicatePessimisticFixpoint();
404	else if (T)
405	S ^= *T;
406	}
407
408	namespace {
409	/// Helper class for generic deduction: return value -> returned position.
410	template <typename AAType, typename BaseType,
411	typename StateType = typename BaseType::StateType,
412	bool PropagateCallBaseContext = false,
413	Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind,
414	bool RecurseForSelectAndPHI = true>
415	struct AAReturnedFromReturnedValues : public BaseType {
416	AAReturnedFromReturnedValues(const IRPosition &IRP, Attributor &A)
417	: BaseType(IRP, A) {}
418
419	/// See AbstractAttribute::updateImpl(...).
420	ChangeStatus updateImpl(Attributor &A) override {
421	StateType S(StateType::getBestState(this->getState()));
422	clampReturnedValueStates<AAType, StateType, IRAttributeKind, RecurseForSelectAndPHI>(
423	A, *this, S,
424	PropagateCallBaseContext ? this->getCallBaseContext() : nullptr);
425	// TODO: If we know we visited all returned values, thus no are assumed
426	// dead, we can take the known information from the state T.
427	return clampStateAndIndicateChange<StateType>(this->getState(), S);
428	}
429	};
430
431	/// Clamp the information known at all call sites for a given argument
432	/// (identified by \p QueryingAA) into \p S.
433	template <typename AAType, typename StateType = typename AAType::StateType,
434	Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
435	static void clampCallSiteArgumentStates(Attributor &A, const AAType &QueryingAA,
436	StateType &S) {
437	LLVM_DEBUG(dbgs() << "[Attributor] Clamp call site argument states for "
438	<< QueryingAA << " into " << S << "\n");
439
440	assert(QueryingAA.getIRPosition().getPositionKind() ==
441	IRPosition::IRP_ARGUMENT &&
442	"Can only clamp call site argument states for an argument position!");
443
444	// Use an optional state as there might not be any return values and we want
445	// to join (IntegerState::operator&) the state of all there are.
446	std::optional<StateType> T;
447
448	// The argument number which is also the call site argument number.
449	unsigned ArgNo = QueryingAA.getIRPosition().getCallSiteArgNo();
450
451	auto CallSiteCheck = [&](AbstractCallSite ACS) {
452	const IRPosition &ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo);
453	// Check if a coresponding argument was found or if it is on not associated
454	// (which can happen for callback calls).
455	if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
456	return false;
457
458	// If possible, use the hasAssumedIRAttr interface.
459	if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
460	bool IsKnown;
461	return AA::hasAssumedIRAttr<IRAttributeKind>(
462	A, &QueryingAA, ACSArgPos, DepClassTy::REQUIRED, IsKnown);
463	}
464
465	const AAType *AA =
466	A.getAAFor<AAType>(QueryingAA, ACSArgPos, DepClassTy::REQUIRED);
467	if (!AA)
468	return false;
469	LLVM_DEBUG(dbgs() << "[Attributor] ACS: " << *ACS.getInstruction()
470	<< " AA: " << AA->getAsStr(&A) << " @" << ACSArgPos
471	<< "\n");
472	const StateType &AAS = AA->getState();
473	if (!T)
474	T = StateType::getBestState(AAS);
475	*T &= AAS;
476	LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " CSA State: " << T
477	<< "\n");
478	return T->isValidState();
479	};
480
481	bool UsedAssumedInformation = false;
482	if (!A.checkForAllCallSites(CallSiteCheck, QueryingAA, true,
483	UsedAssumedInformation))
484	S.indicatePessimisticFixpoint();
485	else if (T)
486	S ^= *T;
487	}
488
489	/// This function is the bridge between argument position and the call base
490	/// context.
491	template <typename AAType, typename BaseType,
492	typename StateType = typename AAType::StateType,
493	Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
494	bool getArgumentStateFromCallBaseContext(Attributor &A,
495	BaseType &QueryingAttribute,
496	IRPosition &Pos, StateType &State) {
497	assert((Pos.getPositionKind() == IRPosition::IRP_ARGUMENT) &&
498	"Expected an 'argument' position !");
499	const CallBase *CBContext = Pos.getCallBaseContext();
500	if (!CBContext)
501	return false;
502
503	int ArgNo = Pos.getCallSiteArgNo();
504	assert(ArgNo >= 0 && "Invalid Arg No!");
505	const IRPosition CBArgPos = IRPosition::callsite_argument(CB: *CBContext, ArgNo);
506
507	// If possible, use the hasAssumedIRAttr interface.
508	if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
509	bool IsKnown;
510	return AA::hasAssumedIRAttr<IRAttributeKind>(
511	A, &QueryingAttribute, CBArgPos, DepClassTy::REQUIRED, IsKnown);
512	}
513
514	const auto *AA =
515	A.getAAFor<AAType>(QueryingAttribute, CBArgPos, DepClassTy::REQUIRED);
516	if (!AA)
517	return false;
518	const StateType &CBArgumentState =
519	static_cast<const StateType &>(AA->getState());
520
521	LLVM_DEBUG(dbgs() << "[Attributor] Briding Call site context to argument"
522	<< "Position:" << Pos << "CB Arg state:" << CBArgumentState
523	<< "\n");
524
525	// NOTE: If we want to do call site grouping it should happen here.
526	State ^= CBArgumentState;
527	return true;
528	}
529
530	/// Helper class for generic deduction: call site argument -> argument position.
531	template <typename AAType, typename BaseType,
532	typename StateType = typename AAType::StateType,
533	bool BridgeCallBaseContext = false,
534	Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
535	struct AAArgumentFromCallSiteArguments : public BaseType {
536	AAArgumentFromCallSiteArguments(const IRPosition &IRP, Attributor &A)
537	: BaseType(IRP, A) {}
538
539	/// See AbstractAttribute::updateImpl(...).
540	ChangeStatus updateImpl(Attributor &A) override {
541	StateType S = StateType::getBestState(this->getState());
542
543	if (BridgeCallBaseContext) {
544	bool Success =
545	getArgumentStateFromCallBaseContext<AAType, BaseType, StateType,
546	IRAttributeKind>(
547	A, *this, this->getIRPosition(), S);
548	if (Success)
549	return clampStateAndIndicateChange<StateType>(this->getState(), S);
550	}
551	clampCallSiteArgumentStates<AAType, StateType, IRAttributeKind>(A, *this,
552	S);
553
554	// TODO: If we know we visited all incoming values, thus no are assumed
555	// dead, we can take the known information from the state T.
556	return clampStateAndIndicateChange<StateType>(this->getState(), S);
557	}
558	};
559
560	/// Helper class for generic replication: function returned -> cs returned.
561	template <typename AAType, typename BaseType,
562	typename StateType = typename BaseType::StateType,
563	bool IntroduceCallBaseContext = false,
564	Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
565	struct AACalleeToCallSite : public BaseType {
566	AACalleeToCallSite(const IRPosition &IRP, Attributor &A) : BaseType(IRP, A) {}
567
568	/// See AbstractAttribute::updateImpl(...).
569	ChangeStatus updateImpl(Attributor &A) override {
570	auto IRPKind = this->getIRPosition().getPositionKind();
571	assert((IRPKind == IRPosition::IRP_CALL_SITE_RETURNED ||
572	IRPKind == IRPosition::IRP_CALL_SITE) &&
573	"Can only wrap function returned positions for call site "
574	"returned positions!");
575	auto &S = this->getState();
576
577	CallBase &CB = cast<CallBase>(this->getAnchorValue());
578	if (IntroduceCallBaseContext)
579	LLVM_DEBUG(dbgs() << "[Attributor] Introducing call base context:" << CB
580	<< "\n");
581
582	ChangeStatus Changed = ChangeStatus::UNCHANGED;
583	auto CalleePred = [&](ArrayRef<const Function *> Callees) {
584	for (const Function *Callee : Callees) {
585	IRPosition FnPos =
586	IRPKind == llvm::IRPosition::IRP_CALL_SITE_RETURNED
587	? IRPosition::returned(F: *Callee,
588	CBContext: IntroduceCallBaseContext ? &CB : nullptr)
589	: IRPosition::function(
590	F: *Callee, CBContext: IntroduceCallBaseContext ? &CB : nullptr);
591	// If possible, use the hasAssumedIRAttr interface.
592	if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
593	bool IsKnown;
594	if (!AA::hasAssumedIRAttr<IRAttributeKind>(
595	A, this, FnPos, DepClassTy::REQUIRED, IsKnown))
596	return false;
597	continue;
598	}
599
600	const AAType *AA =
601	A.getAAFor<AAType>(*this, FnPos, DepClassTy::REQUIRED);
602	if (!AA)
603	return false;
604	Changed |= clampStateAndIndicateChange(S, AA->getState());
605	if (S.isAtFixpoint())
606	return S.isValidState();
607	}
608	return true;
609	};
610	if (!A.checkForAllCallees(Pred: CalleePred, QueryingAA: *this, CB))
611	return S.indicatePessimisticFixpoint();
612	return Changed;
613	}
614	};
615
616	/// Helper function to accumulate uses.
617	template <class AAType, typename StateType = typename AAType::StateType>
618	static void followUsesInContext(AAType &AA, Attributor &A,
619	MustBeExecutedContextExplorer &Explorer,
620	const Instruction *CtxI,
621	SetVector<const Use *> &Uses,
622	StateType &State) {
623	auto EIt = Explorer.begin(PP: CtxI), EEnd = Explorer.end(CtxI);
624	for (unsigned u = 0; u < Uses.size(); ++u) {
625	const Use *U = Uses[u];
626	if (const Instruction *UserI = dyn_cast<Instruction>(Val: U->getUser())) {
627	bool Found = Explorer.findInContextOf(I: UserI, EIt, EEnd);
628	if (Found && AA.followUseInMBEC(A, U, UserI, State))
629	for (const Use &Us : UserI->uses())
630	Uses.insert(X: &Us);
631	}
632	}
633	}
634
635	/// Use the must-be-executed-context around \p I to add information into \p S.
636	/// The AAType class is required to have `followUseInMBEC` method with the
637	/// following signature and behaviour:
638	///
639	/// bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I)
640	/// U - Underlying use.
641	/// I - The user of the \p U.
642	/// Returns true if the value should be tracked transitively.
643	///
644	template <class AAType, typename StateType = typename AAType::StateType>
645	static void followUsesInMBEC(AAType &AA, Attributor &A, StateType &S,
646	Instruction &CtxI) {
647	MustBeExecutedContextExplorer *Explorer =
648	A.getInfoCache().getMustBeExecutedContextExplorer();
649	if (!Explorer)
650	return;
651
652	// Container for (transitive) uses of the associated value.
653	SetVector<const Use *> Uses;
654	for (const Use &U : AA.getIRPosition().getAssociatedValue().uses())
655	Uses.insert(X: &U);
656
657	followUsesInContext<AAType>(AA, A, *Explorer, &CtxI, Uses, S);
658
659	if (S.isAtFixpoint())
660	return;
661
662	SmallVector<const BranchInst *, 4> BrInsts;
663	auto Pred = [&](const Instruction *I) {
664	if (const BranchInst *Br = dyn_cast<BranchInst>(Val: I))
665	if (Br->isConditional())
666	BrInsts.push_back(Elt: Br);
667	return true;
668	};
669
670	// Here, accumulate conditional branch instructions in the context. We
671	// explore the child paths and collect the known states. The disjunction of
672	// those states can be merged to its own state. Let ParentState_i be a state
673	// to indicate the known information for an i-th branch instruction in the
674	// context. ChildStates are created for its successors respectively.
675	//
676	// ParentS_1 = ChildS_{1, 1} /\ ChildS_{1, 2} /\ ... /\ ChildS_{1, n_1}
677	// ParentS_2 = ChildS_{2, 1} /\ ChildS_{2, 2} /\ ... /\ ChildS_{2, n_2}
678	// ...
679	// ParentS_m = ChildS_{m, 1} /\ ChildS_{m, 2} /\ ... /\ ChildS_{m, n_m}
680	//
681	// Known State |= ParentS_1 \/ ParentS_2 \/... \/ ParentS_m
682	//
683	// FIXME: Currently, recursive branches are not handled. For example, we
684	// can't deduce that ptr must be dereferenced in below function.
685	//
686	// void f(int a, int c, int *ptr) {
687	// if(a)
688	// if (b) {
689	// *ptr = 0;
690	// } else {
691	// *ptr = 1;
692	// }
693	// else {
694	// if (b) {
695	// *ptr = 0;
696	// } else {
697	// *ptr = 1;
698	// }
699	// }
700	// }
701
702	Explorer->checkForAllContext(PP: &CtxI, Pred);
703	for (const BranchInst *Br : BrInsts) {
704	StateType ParentState;
705
706	// The known state of the parent state is a conjunction of children's
707	// known states so it is initialized with a best state.
708	ParentState.indicateOptimisticFixpoint();
709
710	for (const BasicBlock *BB : Br->successors()) {
711	StateType ChildState;
712
713	size_t BeforeSize = Uses.size();
714	followUsesInContext(AA, A, *Explorer, &BB->front(), Uses, ChildState);
715
716	// Erase uses which only appear in the child.
717	for (auto It = Uses.begin() + BeforeSize; It != Uses.end();)
718	It = Uses.erase(I: It);
719
720	ParentState &= ChildState;
721	}
722
723	// Use only known state.
724	S += ParentState;
725	}
726	}
727	} // namespace
728
729	/// ------------------------ PointerInfo ---------------------------------------
730
731	namespace llvm {
732	namespace AA {
733	namespace PointerInfo {
734
735	struct State;
736
737	} // namespace PointerInfo
738	} // namespace AA
739
740	/// Helper for AA::PointerInfo::Access DenseMap/Set usage.
741	template <>
742	struct DenseMapInfo<AAPointerInfo::Access> : DenseMapInfo<Instruction *> {
743	using Access = AAPointerInfo::Access;
744	static inline Access getEmptyKey();
745	static inline Access getTombstoneKey();
746	static unsigned getHashValue(const Access &A);
747	static bool isEqual(const Access &LHS, const Access &RHS);
748	};
749
750	/// Helper that allows RangeTy as a key in a DenseMap.
751	template <> struct DenseMapInfo<AA::RangeTy> {
752	static inline AA::RangeTy getEmptyKey() {
753	auto EmptyKey = DenseMapInfo<int64_t>::getEmptyKey();
754	return AA::RangeTy{EmptyKey, EmptyKey};
755	}
756
757	static inline AA::RangeTy getTombstoneKey() {
758	auto TombstoneKey = DenseMapInfo<int64_t>::getTombstoneKey();
759	return AA::RangeTy{TombstoneKey, TombstoneKey};
760	}
761
762	static unsigned getHashValue(const AA::RangeTy &Range) {
763	return detail::combineHashValue(
764	a: DenseMapInfo<int64_t>::getHashValue(Val: Range.Offset),
765	b: DenseMapInfo<int64_t>::getHashValue(Val: Range.Size));
766	}
767
768	static bool isEqual(const AA::RangeTy &A, const AA::RangeTy B) {
769	return A == B;
770	}
771	};
772
773	/// Helper for AA::PointerInfo::Access DenseMap/Set usage ignoring everythign
774	/// but the instruction
775	struct AccessAsInstructionInfo : DenseMapInfo<Instruction *> {
776	using Base = DenseMapInfo<Instruction *>;
777	using Access = AAPointerInfo::Access;
778	static inline Access getEmptyKey();
779	static inline Access getTombstoneKey();
780	static unsigned getHashValue(const Access &A);
781	static bool isEqual(const Access &LHS, const Access &RHS);
782	};
783
784	} // namespace llvm
785
786	/// A type to track pointer/struct usage and accesses for AAPointerInfo.
787	struct AA::PointerInfo::State : public AbstractState {
788	/// Return the best possible representable state.
789	static State getBestState(const State &SIS) { return State(); }
790
791	/// Return the worst possible representable state.
792	static State getWorstState(const State &SIS) {
793	State R;
794	R.indicatePessimisticFixpoint();
795	return R;
796	}
797
798	State() = default;
799	State(State &&SIS) = default;
800
801	const State &getAssumed() const { return *this; }
802
803	/// See AbstractState::isValidState().
804	bool isValidState() const override { return BS.isValidState(); }
805
806	/// See AbstractState::isAtFixpoint().
807	bool isAtFixpoint() const override { return BS.isAtFixpoint(); }
808
809	/// See AbstractState::indicateOptimisticFixpoint().
810	ChangeStatus indicateOptimisticFixpoint() override {
811	BS.indicateOptimisticFixpoint();
812	return ChangeStatus::UNCHANGED;
813	}
814
815	/// See AbstractState::indicatePessimisticFixpoint().
816	ChangeStatus indicatePessimisticFixpoint() override {
817	BS.indicatePessimisticFixpoint();
818	return ChangeStatus::CHANGED;
819	}
820
821	State &operator=(const State &R) {
822	if (this == &R)
823	return *this;
824	BS = R.BS;
825	AccessList = R.AccessList;
826	OffsetBins = R.OffsetBins;
827	RemoteIMap = R.RemoteIMap;
828	return *this;
829	}
830
831	State &operator=(State &&R) {
832	if (this == &R)
833	return *this;
834	std::swap(a&: BS, b&: R.BS);
835	std::swap(LHS&: AccessList, RHS&: R.AccessList);
836	std::swap(a&: OffsetBins, b&: R.OffsetBins);
837	std::swap(a&: RemoteIMap, b&: R.RemoteIMap);
838	return *this;
839	}
840
841	/// Add a new Access to the state at offset \p Offset and with size \p Size.
842	/// The access is associated with \p I, writes \p Content (if anything), and
843	/// is of kind \p Kind. If an Access already exists for the same \p I and same
844	/// \p RemoteI, the two are combined, potentially losing information about
845	/// offset and size. The resulting access must now be moved from its original
846	/// OffsetBin to the bin for its new offset.
847	///
848	/// \Returns CHANGED, if the state changed, UNCHANGED otherwise.
849	ChangeStatus addAccess(Attributor &A, const AAPointerInfo::RangeList &Ranges,
850	Instruction &I, std::optional<Value *> Content,
851	AAPointerInfo::AccessKind Kind, Type *Ty,
852	Instruction *RemoteI = nullptr);
853
854	AAPointerInfo::const_bin_iterator begin() const { return OffsetBins.begin(); }
855	AAPointerInfo::const_bin_iterator end() const { return OffsetBins.end(); }
856	int64_t numOffsetBins() const { return OffsetBins.size(); }
857
858	const AAPointerInfo::Access &getAccess(unsigned Index) const {
859	return AccessList[Index];
860	}
861
862	protected:
863	// Every memory instruction results in an Access object. We maintain a list of
864	// all Access objects that we own, along with the following maps:
865	//
866	// - OffsetBins: RangeTy -> { Access }
867	// - RemoteIMap: RemoteI x LocalI -> Access
868	//
869	// A RemoteI is any instruction that accesses memory. RemoteI is different
870	// from LocalI if and only if LocalI is a call; then RemoteI is some
871	// instruction in the callgraph starting from LocalI. Multiple paths in the
872	// callgraph from LocalI to RemoteI may produce multiple accesses, but these
873	// are all combined into a single Access object. This may result in loss of
874	// information in RangeTy in the Access object.
875	SmallVector<AAPointerInfo::Access> AccessList;
876	AAPointerInfo::OffsetBinsTy OffsetBins;
877	DenseMap<const Instruction *, SmallVector<unsigned>> RemoteIMap;
878
879	/// See AAPointerInfo::forallInterferingAccesses.
880	bool forallInterferingAccesses(
881	AA::RangeTy Range,
882	function_ref<bool(const AAPointerInfo::Access &, bool)> CB) const {
883	if (!isValidState())
884	return false;
885
886	for (const auto &It : OffsetBins) {
887	AA::RangeTy ItRange = It.getFirst();
888	if (!Range.mayOverlap(Range: ItRange))
889	continue;
890	bool IsExact = Range == ItRange && !Range.offsetOrSizeAreUnknown();
891	for (auto Index : It.getSecond()) {
892	auto &Access = AccessList[Index];
893	if (!CB(Access, IsExact))
894	return false;
895	}
896	}
897	return true;
898	}
899
900	/// See AAPointerInfo::forallInterferingAccesses.
901	bool forallInterferingAccesses(
902	Instruction &I,
903	function_ref<bool(const AAPointerInfo::Access &, bool)> CB,
904	AA::RangeTy &Range) const {
905	if (!isValidState())
906	return false;
907
908	auto LocalList = RemoteIMap.find(Val: &I);
909	if (LocalList == RemoteIMap.end()) {
910	return true;
911	}
912
913	for (unsigned Index : LocalList->getSecond()) {
914	for (auto &R : AccessList[Index]) {
915	Range &= R;
916	if (Range.offsetAndSizeAreUnknown())
917	break;
918	}
919	}
920	return forallInterferingAccesses(Range, CB);
921	}
922
923	private:
924	/// State to track fixpoint and validity.
925	BooleanState BS;
926	};
927
928	ChangeStatus AA::PointerInfo::State::addAccess(
929	Attributor &A, const AAPointerInfo::RangeList &Ranges, Instruction &I,
930	std::optional<Value *> Content, AAPointerInfo::AccessKind Kind, Type *Ty,
931	Instruction *RemoteI) {
932	RemoteI = RemoteI ? RemoteI : &I;
933
934	// Check if we have an access for this instruction, if not, simply add it.
935	auto &LocalList = RemoteIMap[RemoteI];
936	bool AccExists = false;
937	unsigned AccIndex = AccessList.size();
938	for (auto Index : LocalList) {
939	auto &A = AccessList[Index];
940	if (A.getLocalInst() == &I) {
941	AccExists = true;
942	AccIndex = Index;
943	break;
944	}
945	}
946
947	auto AddToBins = [&](const AAPointerInfo::RangeList &ToAdd) {
948	LLVM_DEBUG(if (ToAdd.size()) dbgs()
949	<< "[AAPointerInfo] Inserting access in new offset bins\n";);
950
951	for (auto Key : ToAdd) {
952	LLVM_DEBUG(dbgs() << " key " << Key << "\n");
953	OffsetBins[Key].insert(V: AccIndex);
954	}
955	};
956
957	if (!AccExists) {
958	AccessList.emplace_back(Args: &I, Args&: RemoteI, Args: Ranges, Args&: Content, Args&: Kind, Args&: Ty);
959	assert((AccessList.size() == AccIndex + 1) &&
960	"New Access should have been at AccIndex");
961	LocalList.push_back(Elt: AccIndex);
962	AddToBins(AccessList[AccIndex].getRanges());
963	return ChangeStatus::CHANGED;
964	}
965
966	// Combine the new Access with the existing Access, and then update the
967	// mapping in the offset bins.
968	AAPointerInfo::Access Acc(&I, RemoteI, Ranges, Content, Kind, Ty);
969	auto &Current = AccessList[AccIndex];
970	auto Before = Current;
971	Current &= Acc;
972	if (Current == Before)
973	return ChangeStatus::UNCHANGED;
974
975	auto &ExistingRanges = Before.getRanges();
976	auto &NewRanges = Current.getRanges();
977
978	// Ranges that are in the old access but not the new access need to be removed
979	// from the offset bins.
980	AAPointerInfo::RangeList ToRemove;
981	AAPointerInfo::RangeList::set_difference(L: ExistingRanges, R: NewRanges, D&: ToRemove);
982	LLVM_DEBUG(if (ToRemove.size()) dbgs()
983	<< "[AAPointerInfo] Removing access from old offset bins\n";);
984
985	for (auto Key : ToRemove) {
986	LLVM_DEBUG(dbgs() << " key " << Key << "\n");
987	assert(OffsetBins.count(Key) && "Existing Access must be in some bin.");
988	auto &Bin = OffsetBins[Key];
989	assert(Bin.count(AccIndex) &&
990	"Expected bin to actually contain the Access.");
991	Bin.erase(V: AccIndex);
992	}
993
994	// Ranges that are in the new access but not the old access need to be added
995	// to the offset bins.
996	AAPointerInfo::RangeList ToAdd;
997	AAPointerInfo::RangeList::set_difference(L: NewRanges, R: ExistingRanges, D&: ToAdd);
998	AddToBins(ToAdd);
999	return ChangeStatus::CHANGED;
1000	}
1001
1002	namespace {
1003
1004	/// A helper containing a list of offsets computed for a Use. Ideally this
1005	/// list should be strictly ascending, but we ensure that only when we
1006	/// actually translate the list of offsets to a RangeList.
1007	struct OffsetInfo {
1008	using VecTy = SmallVector<int64_t>;
1009	using const_iterator = VecTy::const_iterator;
1010	VecTy Offsets;
1011
1012	const_iterator begin() const { return Offsets.begin(); }
1013	const_iterator end() const { return Offsets.end(); }
1014
1015	bool operator==(const OffsetInfo &RHS) const {
1016	return Offsets == RHS.Offsets;
1017	}
1018
1019	bool operator!=(const OffsetInfo &RHS) const { return !(*this == RHS); }
1020
1021	void insert(int64_t Offset) { Offsets.push_back(Elt: Offset); }
1022	bool isUnassigned() const { return Offsets.size() == 0; }
1023
1024	bool isUnknown() const {
1025	if (isUnassigned())
1026	return false;
1027	if (Offsets.size() == 1)
1028	return Offsets.front() == AA::RangeTy::Unknown;
1029	return false;
1030	}
1031
1032	void setUnknown() {
1033	Offsets.clear();
1034	Offsets.push_back(Elt: AA::RangeTy::Unknown);
1035	}
1036
1037	void addToAll(int64_t Inc) {
1038	for (auto &Offset : Offsets) {
1039	Offset += Inc;
1040	}
1041	}
1042
1043	/// Copy offsets from \p R into the current list.
1044	///
1045	/// Ideally all lists should be strictly ascending, but we defer that to the
1046	/// actual use of the list. So we just blindly append here.
1047	void merge(const OffsetInfo &R) { Offsets.append(RHS: R.Offsets); }
1048	};
1049
1050	#ifndef NDEBUG
1051	static raw_ostream &operator<<(raw_ostream &OS, const OffsetInfo &OI) {
1052	ListSeparator LS;
1053	OS << "[";
1054	for (auto Offset : OI) {
1055	OS << LS << Offset;
1056	}
1057	OS << "]";
1058	return OS;
1059	}
1060	#endif // NDEBUG
1061
1062	struct AAPointerInfoImpl
1063	: public StateWrapper<AA::PointerInfo::State, AAPointerInfo> {
1064	using BaseTy = StateWrapper<AA::PointerInfo::State, AAPointerInfo>;
1065	AAPointerInfoImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {}
1066
1067	/// See AbstractAttribute::getAsStr().
1068	const std::string getAsStr(Attributor *A) const override {
1069	return std::string("PointerInfo ") +
1070	(isValidState() ? (std::string("#") +
1071	std::to_string(val: OffsetBins.size()) + " bins")
1072	: "<invalid>");
1073	}
1074
1075	/// See AbstractAttribute::manifest(...).
1076	ChangeStatus manifest(Attributor &A) override {
1077	return AAPointerInfo::manifest(A);
1078	}
1079
1080	virtual const_bin_iterator begin() const override { return State::begin(); }
1081	virtual const_bin_iterator end() const override { return State::end(); }
1082	virtual int64_t numOffsetBins() const override {
1083	return State::numOffsetBins();
1084	}
1085
1086	bool forallInterferingAccesses(
1087	AA::RangeTy Range,
1088	function_ref<bool(const AAPointerInfo::Access &, bool)> CB)
1089	const override {
1090	return State::forallInterferingAccesses(Range, CB);
1091	}
1092
1093	bool forallInterferingAccesses(
1094	Attributor &A, const AbstractAttribute &QueryingAA, Instruction &I,
1095	bool FindInterferingWrites, bool FindInterferingReads,
1096	function_ref<bool(const Access &, bool)> UserCB, bool &HasBeenWrittenTo,
1097	AA::RangeTy &Range,
1098	function_ref<bool(const Access &)> SkipCB) const override {
1099	HasBeenWrittenTo = false;
1100
1101	SmallPtrSet<const Access *, 8> DominatingWrites;
1102	SmallVector<std::pair<const Access *, bool>, 8> InterferingAccesses;
1103
1104	Function &Scope = *I.getFunction();
1105	bool IsKnownNoSync;
1106	bool IsAssumedNoSync = AA::hasAssumedIRAttr<Attribute::NoSync>(
1107	A, &QueryingAA, IRPosition::function(Scope), DepClassTy::OPTIONAL,
1108	IsKnownNoSync);
1109	const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>(
1110	IRP: IRPosition::function(F: Scope), QueryingAA: &QueryingAA, DepClass: DepClassTy::NONE);
1111	bool AllInSameNoSyncFn = IsAssumedNoSync;
1112	bool InstIsExecutedByInitialThreadOnly =
1113	ExecDomainAA && ExecDomainAA->isExecutedByInitialThreadOnly(I);
1114
1115	// If the function is not ending in aligned barriers, we need the stores to
1116	// be in aligned barriers. The load being in one is not sufficient since the
1117	// store might be executed by a thread that disappears after, causing the
1118	// aligned barrier guarding the load to unblock and the load to read a value
1119	// that has no CFG path to the load.
1120	bool InstIsExecutedInAlignedRegion =
1121	FindInterferingReads && ExecDomainAA &&
1122	ExecDomainAA->isExecutedInAlignedRegion(A, I);
1123
1124	if (InstIsExecutedInAlignedRegion || InstIsExecutedByInitialThreadOnly)
1125	A.recordDependence(FromAA: *ExecDomainAA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
1126
1127	InformationCache &InfoCache = A.getInfoCache();
1128	bool IsThreadLocalObj =
1129	AA::isAssumedThreadLocalObject(A, Obj&: getAssociatedValue(), QueryingAA: *this);
1130
1131	// Helper to determine if we need to consider threading, which we cannot
1132	// right now. However, if the function is (assumed) nosync or the thread
1133	// executing all instructions is the main thread only we can ignore
1134	// threading. Also, thread-local objects do not require threading reasoning.
1135	// Finally, we can ignore threading if either access is executed in an
1136	// aligned region.
1137	auto CanIgnoreThreadingForInst = [&](const Instruction &I) -> bool {
1138	if (IsThreadLocalObj || AllInSameNoSyncFn)
1139	return true;
1140	const auto *FnExecDomainAA =
1141	I.getFunction() == &Scope
1142	? ExecDomainAA
1143	: A.lookupAAFor<AAExecutionDomain>(
1144	IRP: IRPosition::function(F: *I.getFunction()), QueryingAA: &QueryingAA,
1145	DepClass: DepClassTy::NONE);
1146	if (!FnExecDomainAA)
1147	return false;
1148	if (InstIsExecutedInAlignedRegion ||
1149	(FindInterferingWrites &&
1150	FnExecDomainAA->isExecutedInAlignedRegion(A, I))) {
1151	A.recordDependence(FromAA: *FnExecDomainAA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
1152	return true;
1153	}
1154	if (InstIsExecutedByInitialThreadOnly &&
1155	FnExecDomainAA->isExecutedByInitialThreadOnly(I)) {
1156	A.recordDependence(FromAA: *FnExecDomainAA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
1157	return true;
1158	}
1159	return false;
1160	};
1161
1162	// Helper to determine if the access is executed by the same thread as the
1163	// given instruction, for now it is sufficient to avoid any potential
1164	// threading effects as we cannot deal with them anyway.
1165	auto CanIgnoreThreading = [&](const Access &Acc) -> bool {
1166	return CanIgnoreThreadingForInst(*Acc.getRemoteInst()) ||
1167	(Acc.getRemoteInst() != Acc.getLocalInst() &&
1168	CanIgnoreThreadingForInst(*Acc.getLocalInst()));
1169	};
1170
1171	// TODO: Use inter-procedural reachability and dominance.
1172	bool IsKnownNoRecurse;
1173	AA::hasAssumedIRAttr<Attribute::NoRecurse>(
1174	A, this, IRPosition::function(Scope), DepClassTy::OPTIONAL,
1175	IsKnownNoRecurse);
1176
1177	// TODO: Use reaching kernels from AAKernelInfo (or move it to
1178	// AAExecutionDomain) such that we allow scopes other than kernels as long
1179	// as the reaching kernels are disjoint.
1180	bool InstInKernel = Scope.hasFnAttribute(Kind: "kernel");
1181	bool ObjHasKernelLifetime = false;
1182	const bool UseDominanceReasoning =
1183	FindInterferingWrites && IsKnownNoRecurse;
1184	const DominatorTree *DT =
1185	InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: Scope);
1186
1187	// Helper to check if a value has "kernel lifetime", that is it will not
1188	// outlive a GPU kernel. This is true for shared, constant, and local
1189	// globals on AMD and NVIDIA GPUs.
1190	auto HasKernelLifetime = [&](Value *V, Module &M) {
1191	if (!AA::isGPU(M))
1192	return false;
1193	switch (AA::GPUAddressSpace(V->getType()->getPointerAddressSpace())) {
1194	case AA::GPUAddressSpace::Shared:
1195	case AA::GPUAddressSpace::Constant:
1196	case AA::GPUAddressSpace::Local:
1197	return true;
1198	default:
1199	return false;
1200	};
1201	};
1202
1203	// The IsLiveInCalleeCB will be used by the AA::isPotentiallyReachable query
1204	// to determine if we should look at reachability from the callee. For
1205	// certain pointers we know the lifetime and we do not have to step into the
1206	// callee to determine reachability as the pointer would be dead in the
1207	// callee. See the conditional initialization below.
1208	std::function<bool(const Function &)> IsLiveInCalleeCB;
1209
1210	if (auto *AI = dyn_cast<AllocaInst>(Val: &getAssociatedValue())) {
1211	// If the alloca containing function is not recursive the alloca
1212	// must be dead in the callee.
1213	const Function *AIFn = AI->getFunction();
1214	ObjHasKernelLifetime = AIFn->hasFnAttribute(Kind: "kernel");
1215	bool IsKnownNoRecurse;
1216	if (AA::hasAssumedIRAttr<Attribute::NoRecurse>(
1217	A, this, IRPosition::function(*AIFn), DepClassTy::OPTIONAL,
1218	IsKnownNoRecurse)) {
1219	IsLiveInCalleeCB = [AIFn](const Function &Fn) { return AIFn != &Fn; };
1220	}
1221	} else if (auto *GV = dyn_cast<GlobalValue>(Val: &getAssociatedValue())) {
1222	// If the global has kernel lifetime we can stop if we reach a kernel
1223	// as it is "dead" in the (unknown) callees.
1224	ObjHasKernelLifetime = HasKernelLifetime(GV, *GV->getParent());
1225	if (ObjHasKernelLifetime)
1226	IsLiveInCalleeCB = [](const Function &Fn) {
1227	return !Fn.hasFnAttribute(Kind: "kernel");
1228	};
1229	}
1230
1231	// Set of accesses/instructions that will overwrite the result and are
1232	// therefore blockers in the reachability traversal.
1233	AA::InstExclusionSetTy ExclusionSet;
1234
1235	auto AccessCB = [&](const Access &Acc, bool Exact) {
1236	Function *AccScope = Acc.getRemoteInst()->getFunction();
1237	bool AccInSameScope = AccScope == &Scope;
1238
1239	// If the object has kernel lifetime we can ignore accesses only reachable
1240	// by other kernels. For now we only skip accesses *in* other kernels.
1241	if (InstInKernel && ObjHasKernelLifetime && !AccInSameScope &&
1242	AccScope->hasFnAttribute(Kind: "kernel"))
1243	return true;
1244
1245	if (Exact && Acc.isMustAccess() && Acc.getRemoteInst() != &I) {
1246	if (Acc.isWrite() || (isa<LoadInst>(Val: I) && Acc.isWriteOrAssumption()))
1247	ExclusionSet.insert(Ptr: Acc.getRemoteInst());
1248	}
1249
1250	if ((!FindInterferingWrites || !Acc.isWriteOrAssumption()) &&
1251	(!FindInterferingReads || !Acc.isRead()))
1252	return true;
1253
1254	bool Dominates = FindInterferingWrites && DT && Exact &&
1255	Acc.isMustAccess() && AccInSameScope &&
1256	DT->dominates(Def: Acc.getRemoteInst(), User: &I);
1257	if (Dominates)
1258	DominatingWrites.insert(Ptr: &Acc);
1259
1260	// Track if all interesting accesses are in the same `nosync` function as
1261	// the given instruction.
1262	AllInSameNoSyncFn &= Acc.getRemoteInst()->getFunction() == &Scope;
1263
1264	InterferingAccesses.push_back(Elt: {&Acc, Exact});
1265	return true;
1266	};
1267	if (!State::forallInterferingAccesses(I, CB: AccessCB, Range))
1268	return false;
1269
1270	HasBeenWrittenTo = !DominatingWrites.empty();
1271
1272	// Dominating writes form a chain, find the least/lowest member.
1273	Instruction *LeastDominatingWriteInst = nullptr;
1274	for (const Access *Acc : DominatingWrites) {
1275	if (!LeastDominatingWriteInst) {
1276	LeastDominatingWriteInst = Acc->getRemoteInst();
1277	} else if (DT->dominates(Def: LeastDominatingWriteInst,
1278	User: Acc->getRemoteInst())) {
1279	LeastDominatingWriteInst = Acc->getRemoteInst();
1280	}
1281	}
1282
1283	// Helper to determine if we can skip a specific write access.
1284	auto CanSkipAccess = [&](const Access &Acc, bool Exact) {
1285	if (SkipCB && SkipCB(Acc))
1286	return true;
1287	if (!CanIgnoreThreading(Acc))
1288	return false;
1289
1290	// Check read (RAW) dependences and write (WAR) dependences as necessary.
1291	// If we successfully excluded all effects we are interested in, the
1292	// access can be skipped.
1293	bool ReadChecked = !FindInterferingReads;
1294	bool WriteChecked = !FindInterferingWrites;
1295
1296	// If the instruction cannot reach the access, the former does not
1297	// interfere with what the access reads.
1298	if (!ReadChecked) {
1299	if (!AA::isPotentiallyReachable(A, FromI: I, ToI: *Acc.getRemoteInst(), QueryingAA,
1300	ExclusionSet: &ExclusionSet, GoBackwardsCB: IsLiveInCalleeCB))
1301	ReadChecked = true;
1302	}
1303	// If the instruction cannot be reach from the access, the latter does not
1304	// interfere with what the instruction reads.
1305	if (!WriteChecked) {
1306	if (!AA::isPotentiallyReachable(A, FromI: *Acc.getRemoteInst(), ToI: I, QueryingAA,
1307	ExclusionSet: &ExclusionSet, GoBackwardsCB: IsLiveInCalleeCB))
1308	WriteChecked = true;
1309	}
1310
1311	// If we still might be affected by the write of the access but there are
1312	// dominating writes in the function of the instruction
1313	// (HasBeenWrittenTo), we can try to reason that the access is overwritten
1314	// by them. This would have happend above if they are all in the same
1315	// function, so we only check the inter-procedural case. Effectively, we
1316	// want to show that there is no call after the dominting write that might
1317	// reach the access, and when it returns reach the instruction with the
1318	// updated value. To this end, we iterate all call sites, check if they
1319	// might reach the instruction without going through another access
1320	// (ExclusionSet) and at the same time might reach the access. However,
1321	// that is all part of AAInterFnReachability.
1322	if (!WriteChecked && HasBeenWrittenTo &&
1323	Acc.getRemoteInst()->getFunction() != &Scope) {
1324
1325	const auto *FnReachabilityAA = A.getAAFor<AAInterFnReachability>(
1326	QueryingAA, IRP: IRPosition::function(F: Scope), DepClass: DepClassTy::OPTIONAL);
1327
1328	// Without going backwards in the call tree, can we reach the access
1329	// from the least dominating write. Do not allow to pass the instruction
1330	// itself either.
1331	bool Inserted = ExclusionSet.insert(Ptr: &I).second;
1332
1333	if (!FnReachabilityAA ||
1334	!FnReachabilityAA->instructionCanReach(
1335	A, Inst: *LeastDominatingWriteInst,
1336	Fn: *Acc.getRemoteInst()->getFunction(), ExclusionSet: &ExclusionSet))
1337	WriteChecked = true;
1338
1339	if (Inserted)
1340	ExclusionSet.erase(Ptr: &I);
1341	}
1342
1343	if (ReadChecked && WriteChecked)
1344	return true;
1345
1346	if (!DT || !UseDominanceReasoning)
1347	return false;
1348	if (!DominatingWrites.count(Ptr: &Acc))
1349	return false;
1350	return LeastDominatingWriteInst != Acc.getRemoteInst();
1351	};
1352
1353	// Run the user callback on all accesses we cannot skip and return if
1354	// that succeeded for all or not.
1355	for (auto &It : InterferingAccesses) {
1356	if ((!AllInSameNoSyncFn && !IsThreadLocalObj && !ExecDomainAA) ||
1357	!CanSkipAccess(*It.first, It.second)) {
1358	if (!UserCB(*It.first, It.second))
1359	return false;
1360	}
1361	}
1362	return true;
1363	}
1364
1365	ChangeStatus translateAndAddStateFromCallee(Attributor &A,
1366	const AAPointerInfo &OtherAA,
1367	CallBase &CB) {
1368	using namespace AA::PointerInfo;
1369	if (!OtherAA.getState().isValidState() || !isValidState())
1370	return indicatePessimisticFixpoint();
1371
1372	const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA);
1373	bool IsByval = OtherAAImpl.getAssociatedArgument()->hasByValAttr();
1374
1375	// Combine the accesses bin by bin.
1376	ChangeStatus Changed = ChangeStatus::UNCHANGED;
1377	const auto &State = OtherAAImpl.getState();
1378	for (const auto &It : State) {
1379	for (auto Index : It.getSecond()) {
1380	const auto &RAcc = State.getAccess(Index);
1381	if (IsByval && !RAcc.isRead())
1382	continue;
1383	bool UsedAssumedInformation = false;
1384	AccessKind AK = RAcc.getKind();
1385	auto Content = A.translateArgumentToCallSiteContent(
1386	V: RAcc.getContent(), CB, AA: *this, UsedAssumedInformation);
1387	AK = AccessKind(AK & (IsByval ? AccessKind::AK_R : AccessKind::AK_RW));
1388	AK = AccessKind(AK | (RAcc.isMayAccess() ? AK_MAY : AK_MUST));
1389
1390	Changed |= addAccess(A, Ranges: RAcc.getRanges(), I&: CB, Content, Kind: AK,
1391	Ty: RAcc.getType(), RemoteI: RAcc.getRemoteInst());
1392	}
1393	}
1394	return Changed;
1395	}
1396
1397	ChangeStatus translateAndAddState(Attributor &A, const AAPointerInfo &OtherAA,
1398	const OffsetInfo &Offsets, CallBase &CB) {
1399	using namespace AA::PointerInfo;
1400	if (!OtherAA.getState().isValidState() || !isValidState())
1401	return indicatePessimisticFixpoint();
1402
1403	const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA);
1404
1405	// Combine the accesses bin by bin.
1406	ChangeStatus Changed = ChangeStatus::UNCHANGED;
1407	const auto &State = OtherAAImpl.getState();
1408	for (const auto &It : State) {
1409	for (auto Index : It.getSecond()) {
1410	const auto &RAcc = State.getAccess(Index);
1411	for (auto Offset : Offsets) {
1412	auto NewRanges = Offset == AA::RangeTy::Unknown
1413	? AA::RangeTy::getUnknown()
1414	: RAcc.getRanges();
1415	if (!NewRanges.isUnknown()) {
1416	NewRanges.addToAllOffsets(Inc: Offset);
1417	}
1418	Changed |=
1419	addAccess(A, Ranges: NewRanges, I&: CB, Content: RAcc.getContent(), Kind: RAcc.getKind(),
1420	Ty: RAcc.getType(), RemoteI: RAcc.getRemoteInst());
1421	}
1422	}
1423	}
1424	return Changed;
1425	}
1426
1427	/// Statistic tracking for all AAPointerInfo implementations.
1428	/// See AbstractAttribute::trackStatistics().
1429	void trackPointerInfoStatistics(const IRPosition &IRP) const {}
1430
1431	/// Dump the state into \p O.
1432	void dumpState(raw_ostream &O) {
1433	for (auto &It : OffsetBins) {
1434	O << "[" << It.first.Offset << "-" << It.first.Offset + It.first.Size
1435	<< "] : " << It.getSecond().size() << "\n";
1436	for (auto AccIndex : It.getSecond()) {
1437	auto &Acc = AccessList[AccIndex];
1438	O << " - " << Acc.getKind() << " - " << *Acc.getLocalInst() << "\n";
1439	if (Acc.getLocalInst() != Acc.getRemoteInst())
1440	O << " --> " << *Acc.getRemoteInst()
1441	<< "\n";
1442	if (!Acc.isWrittenValueYetUndetermined()) {
1443	if (isa_and_nonnull<Function>(Val: Acc.getWrittenValue()))
1444	O << " - c: func " << Acc.getWrittenValue()->getName()
1445	<< "\n";
1446	else if (Acc.getWrittenValue())
1447	O << " - c: " << *Acc.getWrittenValue() << "\n";
1448	else
1449	O << " - c: <unknown>\n";
1450	}
1451	}
1452	}
1453	}
1454	};
1455
1456	struct AAPointerInfoFloating : public AAPointerInfoImpl {
1457	using AccessKind = AAPointerInfo::AccessKind;
1458	AAPointerInfoFloating(const IRPosition &IRP, Attributor &A)
1459	: AAPointerInfoImpl(IRP, A) {}
1460
1461	/// Deal with an access and signal if it was handled successfully.
1462	bool handleAccess(Attributor &A, Instruction &I,
1463	std::optional<Value *> Content, AccessKind Kind,
1464	SmallVectorImpl<int64_t> &Offsets, ChangeStatus &Changed,
1465	Type &Ty) {
1466	using namespace AA::PointerInfo;
1467	auto Size = AA::RangeTy::Unknown;
1468	const DataLayout &DL = A.getDataLayout();
1469	TypeSize AccessSize = DL.getTypeStoreSize(Ty: &Ty);
1470	if (!AccessSize.isScalable())
1471	Size = AccessSize.getFixedValue();
1472
1473	// Make a strictly ascending list of offsets as required by addAccess()
1474	llvm::sort(C&: Offsets);
1475	auto *Last = std::unique(first: Offsets.begin(), last: Offsets.end());
1476	Offsets.erase(CS: Last, CE: Offsets.end());
1477
1478	VectorType *VT = dyn_cast<VectorType>(Val: &Ty);
1479	if (!VT || VT->getElementCount().isScalable() ||
1480	!Content.value_or(u: nullptr) || !isa<Constant>(Val: *Content) ||
1481	(*Content)->getType() != VT ||
1482	DL.getTypeStoreSize(Ty: VT->getElementType()).isScalable()) {
1483	Changed = Changed | addAccess(A, Ranges: {Offsets, Size}, I, Content, Kind, Ty: &Ty);
1484	} else {
1485	// Handle vector stores with constant content element-wise.
1486	// TODO: We could look for the elements or create instructions
1487	// representing them.
1488	// TODO: We need to push the Content into the range abstraction
1489	// (AA::RangeTy) to allow different content values for different
1490	// ranges. ranges. Hence, support vectors storing different values.
1491	Type *ElementType = VT->getElementType();
1492	int64_t ElementSize = DL.getTypeStoreSize(Ty: ElementType).getFixedValue();
1493	auto *ConstContent = cast<Constant>(Val: *Content);
1494	Type *Int32Ty = Type::getInt32Ty(C&: ElementType->getContext());
1495	SmallVector<int64_t> ElementOffsets(Offsets.begin(), Offsets.end());
1496
1497	for (int i = 0, e = VT->getElementCount().getFixedValue(); i != e; ++i) {
1498	Value *ElementContent = ConstantExpr::getExtractElement(
1499	Vec: ConstContent, Idx: ConstantInt::get(Ty: Int32Ty, V: i));
1500
1501	// Add the element access.
1502	Changed = Changed | addAccess(A, Ranges: {ElementOffsets, ElementSize}, I,
1503	Content: ElementContent, Kind, Ty: ElementType);
1504
1505	// Advance the offsets for the next element.
1506	for (auto &ElementOffset : ElementOffsets)
1507	ElementOffset += ElementSize;
1508	}
1509	}
1510	return true;
1511	};
1512
1513	/// See AbstractAttribute::updateImpl(...).
1514	ChangeStatus updateImpl(Attributor &A) override;
1515
1516	/// If the indices to \p GEP can be traced to constants, incorporate all
1517	/// of these into \p UsrOI.
1518	///
1519	/// \return true iff \p UsrOI is updated.
1520	bool collectConstantsForGEP(Attributor &A, const DataLayout &DL,
1521	OffsetInfo &UsrOI, const OffsetInfo &PtrOI,
1522	const GEPOperator *GEP);
1523
1524	/// See AbstractAttribute::trackStatistics()
1525	void trackStatistics() const override {
1526	AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
1527	}
1528	};
1529
1530	bool AAPointerInfoFloating::collectConstantsForGEP(Attributor &A,
1531	const DataLayout &DL,
1532	OffsetInfo &UsrOI,
1533	const OffsetInfo &PtrOI,
1534	const GEPOperator *GEP) {
1535	unsigned BitWidth = DL.getIndexTypeSizeInBits(Ty: GEP->getType());
1536	MapVector<Value *, APInt> VariableOffsets;
1537	APInt ConstantOffset(BitWidth, 0);
1538
1539	assert(!UsrOI.isUnknown() && !PtrOI.isUnknown() &&
1540	"Don't look for constant values if the offset has already been "
1541	"determined to be unknown.");
1542
1543	if (!GEP->collectOffset(DL, BitWidth, VariableOffsets, ConstantOffset)) {
1544	UsrOI.setUnknown();
1545	return true;
1546	}
1547
1548	LLVM_DEBUG(dbgs() << "[AAPointerInfo] GEP offset is "
1549	<< (VariableOffsets.empty() ? "" : "not") << " constant "
1550	<< *GEP << "\n");
1551
1552	auto Union = PtrOI;
1553	Union.addToAll(Inc: ConstantOffset.getSExtValue());
1554
1555	// Each VI in VariableOffsets has a set of potential constant values. Every
1556	// combination of elements, picked one each from these sets, is separately
1557	// added to the original set of offsets, thus resulting in more offsets.
1558	for (const auto &VI : VariableOffsets) {
1559	auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>(
1560	QueryingAA: *this, IRP: IRPosition::value(V: *VI.first), DepClass: DepClassTy::OPTIONAL);
1561	if (!PotentialConstantsAA || !PotentialConstantsAA->isValidState()) {
1562	UsrOI.setUnknown();
1563	return true;
1564	}
1565
1566	// UndefValue is treated as a zero, which leaves Union as is.
1567	if (PotentialConstantsAA->undefIsContained())
1568	continue;
1569
1570	// We need at least one constant in every set to compute an actual offset.
1571	// Otherwise, we end up pessimizing AAPointerInfo by respecting offsets that
1572	// don't actually exist. In other words, the absence of constant values
1573	// implies that the operation can be assumed dead for now.
1574	auto &AssumedSet = PotentialConstantsAA->getAssumedSet();
1575	if (AssumedSet.empty())
1576	return false;
1577
1578	OffsetInfo Product;
1579	for (const auto &ConstOffset : AssumedSet) {
1580	auto CopyPerOffset = Union;
1581	CopyPerOffset.addToAll(Inc: ConstOffset.getSExtValue() *
1582	VI.second.getZExtValue());
1583	Product.merge(R: CopyPerOffset);
1584	}
1585	Union = Product;
1586	}
1587
1588	UsrOI = std::move(Union);
1589	return true;
1590	}
1591
1592	ChangeStatus AAPointerInfoFloating::updateImpl(Attributor &A) {
1593	using namespace AA::PointerInfo;
1594	ChangeStatus Changed = ChangeStatus::UNCHANGED;
1595	const DataLayout &DL = A.getDataLayout();
1596	Value &AssociatedValue = getAssociatedValue();
1597
1598	DenseMap<Value *, OffsetInfo> OffsetInfoMap;
1599	OffsetInfoMap[&AssociatedValue].insert(Offset: 0);
1600
1601	auto HandlePassthroughUser = [&](Value *Usr, Value *CurPtr, bool &Follow) {
1602	// One does not simply walk into a map and assign a reference to a possibly
1603	// new location. That can cause an invalidation before the assignment
1604	// happens, like so:
1605	//
1606	// OffsetInfoMap[Usr] = OffsetInfoMap[CurPtr]; /* bad idea! */
1607	//
1608	// The RHS is a reference that may be invalidated by an insertion caused by
1609	// the LHS. So we ensure that the side-effect of the LHS happens first.
1610	auto &UsrOI = OffsetInfoMap[Usr];
1611	auto &PtrOI = OffsetInfoMap[CurPtr];
1612	assert(!PtrOI.isUnassigned() &&
1613	"Cannot pass through if the input Ptr was not visited!");
1614	UsrOI = PtrOI;
1615	Follow = true;
1616	return true;
1617	};
1618
1619	const auto *F = getAnchorScope();
1620	const auto *CI =
1621	F ? A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(F: *F)
1622	: nullptr;
1623	const auto *TLI =
1624	F ? A.getInfoCache().getTargetLibraryInfoForFunction(F: *F) : nullptr;
1625
1626	auto UsePred = [&](const Use &U, bool &Follow) -> bool {
1627	Value *CurPtr = U.get();
1628	User *Usr = U.getUser();
1629	LLVM_DEBUG(dbgs() << "[AAPointerInfo] Analyze " << *CurPtr << " in " << *Usr
1630	<< "\n");
1631	assert(OffsetInfoMap.count(CurPtr) &&
1632	"The current pointer offset should have been seeded!");
1633
1634	if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Val: Usr)) {
1635	if (CE->isCast())
1636	return HandlePassthroughUser(Usr, CurPtr, Follow);
1637	if (CE->isCompare())
1638	return true;
1639	if (!isa<GEPOperator>(Val: CE)) {
1640	LLVM_DEBUG(dbgs() << "[AAPointerInfo] Unhandled constant user " << *CE
1641	<< "\n");
1642	return false;
1643	}
1644	}
1645	if (auto *GEP = dyn_cast<GEPOperator>(Val: Usr)) {
1646	// Note the order here, the Usr access might change the map, CurPtr is
1647	// already in it though.
1648	auto &UsrOI = OffsetInfoMap[Usr];
1649	auto &PtrOI = OffsetInfoMap[CurPtr];
1650
1651	if (UsrOI.isUnknown())
1652	return true;
1653
1654	if (PtrOI.isUnknown()) {
1655	Follow = true;
1656	UsrOI.setUnknown();
1657	return true;
1658	}
1659
1660	Follow = collectConstantsForGEP(A, DL, UsrOI, PtrOI, GEP);
1661	return true;
1662	}
1663	if (isa<PtrToIntInst>(Val: Usr))
1664	return false;
1665	if (isa<CastInst>(Val: Usr) || isa<SelectInst>(Val: Usr) || isa<ReturnInst>(Val: Usr))
1666	return HandlePassthroughUser(Usr, CurPtr, Follow);
1667
1668	// For PHIs we need to take care of the recurrence explicitly as the value
1669	// might change while we iterate through a loop. For now, we give up if
1670	// the PHI is not invariant.
1671	if (isa<PHINode>(Val: Usr)) {
1672	// Note the order here, the Usr access might change the map, CurPtr is
1673	// already in it though.
1674	bool IsFirstPHIUser = !OffsetInfoMap.count(Val: Usr);
1675	auto &UsrOI = OffsetInfoMap[Usr];
1676	auto &PtrOI = OffsetInfoMap[CurPtr];
1677
1678	// Check if the PHI operand has already an unknown offset as we can't
1679	// improve on that anymore.
1680	if (PtrOI.isUnknown()) {
1681	LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand offset unknown "
1682	<< *CurPtr << " in " << *Usr << "\n");
1683	Follow = !UsrOI.isUnknown();
1684	UsrOI.setUnknown();
1685	return true;
1686	}
1687
1688	// Check if the PHI is invariant (so far).
1689	if (UsrOI == PtrOI) {
1690	assert(!PtrOI.isUnassigned() &&
1691	"Cannot assign if the current Ptr was not visited!");
1692	LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant (so far)");
1693	return true;
1694	}
1695
1696	// Check if the PHI operand can be traced back to AssociatedValue.
1697	APInt Offset(
1698	DL.getIndexSizeInBits(AS: CurPtr->getType()->getPointerAddressSpace()),
1699	0);
1700	Value *CurPtrBase = CurPtr->stripAndAccumulateConstantOffsets(
1701	DL, Offset, /* AllowNonInbounds */ true);
1702	auto It = OffsetInfoMap.find(Val: CurPtrBase);
1703	if (It == OffsetInfoMap.end()) {
1704	LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand is too complex "
1705	<< *CurPtr << " in " << *Usr << "\n");
1706	UsrOI.setUnknown();
1707	Follow = true;
1708	return true;
1709	}
1710
1711	// Check if the PHI operand is not dependent on the PHI itself. Every
1712	// recurrence is a cyclic net of PHIs in the data flow, and has an
1713	// equivalent Cycle in the control flow. One of those PHIs must be in the
1714	// header of that control flow Cycle. This is independent of the choice of
1715	// Cycles reported by CycleInfo. It is sufficient to check the PHIs in
1716	// every Cycle header; if such a node is marked unknown, this will
1717	// eventually propagate through the whole net of PHIs in the recurrence.
1718	if (mayBeInCycle(CI, I: cast<Instruction>(Val: Usr), /* HeaderOnly */ true)) {
1719	auto BaseOI = It->getSecond();
1720	BaseOI.addToAll(Inc: Offset.getZExtValue());
1721	if (IsFirstPHIUser || BaseOI == UsrOI) {
1722	LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant " << *CurPtr
1723	<< " in " << *Usr << "\n");
1724	return HandlePassthroughUser(Usr, CurPtr, Follow);
1725	}
1726
1727	LLVM_DEBUG(
1728	dbgs() << "[AAPointerInfo] PHI operand pointer offset mismatch "
1729	<< *CurPtr << " in " << *Usr << "\n");
1730	UsrOI.setUnknown();
1731	Follow = true;
1732	return true;
1733	}
1734
1735	UsrOI.merge(R: PtrOI);
1736	Follow = true;
1737	return true;
1738	}
1739
1740	if (auto *LoadI = dyn_cast<LoadInst>(Val: Usr)) {
1741	// If the access is to a pointer that may or may not be the associated
1742	// value, e.g. due to a PHI, we cannot assume it will be read.
1743	AccessKind AK = AccessKind::AK_R;
1744	if (getUnderlyingObject(V: CurPtr) == &AssociatedValue)
1745	AK = AccessKind(AK | AccessKind::AK_MUST);
1746	else
1747	AK = AccessKind(AK | AccessKind::AK_MAY);
1748	if (!handleAccess(A, I&: *LoadI, /* Content */ nullptr, Kind: AK,
1749	Offsets&: OffsetInfoMap[CurPtr].Offsets, Changed,
1750	Ty&: *LoadI->getType()))
1751	return false;
1752
1753	auto IsAssumption = [](Instruction &I) {
1754	if (auto *II = dyn_cast<IntrinsicInst>(Val: &I))
1755	return II->isAssumeLikeIntrinsic();
1756	return false;
1757	};
1758
1759	auto IsImpactedInRange = [&](Instruction *FromI, Instruction *ToI) {
1760	// Check if the assumption and the load are executed together without
1761	// memory modification.
1762	do {
1763	if (FromI->mayWriteToMemory() && !IsAssumption(*FromI))
1764	return true;
1765	FromI = FromI->getNextNonDebugInstruction();
1766	} while (FromI && FromI != ToI);
1767	return false;
1768	};
1769
1770	BasicBlock *BB = LoadI->getParent();
1771	auto IsValidAssume = [&](IntrinsicInst &IntrI) {
1772	if (IntrI.getIntrinsicID() != Intrinsic::assume)
1773	return false;
1774	BasicBlock *IntrBB = IntrI.getParent();
1775	if (IntrI.getParent() == BB) {
1776	if (IsImpactedInRange(LoadI->getNextNonDebugInstruction(), &IntrI))
1777	return false;
1778	} else {
1779	auto PredIt = pred_begin(BB: IntrBB);
1780	if (PredIt == pred_end(BB: IntrBB))
1781	return false;
1782	if ((*PredIt) != BB)
1783	return false;
1784	if (++PredIt != pred_end(BB: IntrBB))
1785	return false;
1786	for (auto *SuccBB : successors(BB)) {
1787	if (SuccBB == IntrBB)
1788	continue;
1789	if (isa<UnreachableInst>(Val: SuccBB->getTerminator()))
1790	continue;
1791	return false;
1792	}
1793	if (IsImpactedInRange(LoadI->getNextNonDebugInstruction(),
1794	BB->getTerminator()))
1795	return false;
1796	if (IsImpactedInRange(&IntrBB->front(), &IntrI))
1797	return false;
1798	}
1799	return true;
1800	};
1801
1802	std::pair<Value *, IntrinsicInst *> Assumption;
1803	for (const Use &LoadU : LoadI->uses()) {
1804	if (auto *CmpI = dyn_cast<CmpInst>(Val: LoadU.getUser())) {
1805	if (!CmpI->isEquality() || !CmpI->isTrueWhenEqual())
1806	continue;
1807	for (const Use &CmpU : CmpI->uses()) {
1808	if (auto *IntrI = dyn_cast<IntrinsicInst>(Val: CmpU.getUser())) {
1809	if (!IsValidAssume(*IntrI))
1810	continue;
1811	int Idx = CmpI->getOperandUse(i: 0) == LoadU;
1812	Assumption = {CmpI->getOperand(i_nocapture: Idx), IntrI};
1813	break;
1814	}
1815	}
1816	}
1817	if (Assumption.first)
1818	break;
1819	}
1820
1821	// Check if we found an assumption associated with this load.
1822	if (!Assumption.first || !Assumption.second)
1823	return true;
1824
1825	LLVM_DEBUG(dbgs() << "[AAPointerInfo] Assumption found "
1826	<< *Assumption.second << ": " << *LoadI
1827	<< " == " << *Assumption.first << "\n");
1828	bool UsedAssumedInformation = false;
1829	std::optional<Value *> Content = nullptr;
1830	if (Assumption.first)
1831	Content =
1832	A.getAssumedSimplified(V: *Assumption.first, AA: *this,
1833	UsedAssumedInformation, S: AA::Interprocedural);
1834	return handleAccess(
1835	A, I&: *Assumption.second, Content, Kind: AccessKind::AK_ASSUMPTION,
1836	Offsets&: OffsetInfoMap[CurPtr].Offsets, Changed, Ty&: *LoadI->getType());
1837	}
1838
1839	auto HandleStoreLike = [&](Instruction &I, Value *ValueOp, Type &ValueTy,
1840	ArrayRef<Value *> OtherOps, AccessKind AK) {
1841	for (auto *OtherOp : OtherOps) {
1842	if (OtherOp == CurPtr) {
1843	LLVM_DEBUG(
1844	dbgs()
1845	<< "[AAPointerInfo] Escaping use in store like instruction " << I
1846	<< "\n");
1847	return false;
1848	}
1849	}
1850
1851	// If the access is to a pointer that may or may not be the associated
1852	// value, e.g. due to a PHI, we cannot assume it will be written.
1853	if (getUnderlyingObject(V: CurPtr) == &AssociatedValue)
1854	AK = AccessKind(AK | AccessKind::AK_MUST);
1855	else
1856	AK = AccessKind(AK | AccessKind::AK_MAY);
1857	bool UsedAssumedInformation = false;
1858	std::optional<Value *> Content = nullptr;
1859	if (ValueOp)
1860	Content = A.getAssumedSimplified(
1861	V: *ValueOp, AA: *this, UsedAssumedInformation, S: AA::Interprocedural);
1862	return handleAccess(A, I, Content, Kind: AK, Offsets&: OffsetInfoMap[CurPtr].Offsets,
1863	Changed, Ty&: ValueTy);
1864	};
1865
1866	if (auto *StoreI = dyn_cast<StoreInst>(Val: Usr))
1867	return HandleStoreLike(*StoreI, StoreI->getValueOperand(),
1868	*StoreI->getValueOperand()->getType(),
1869	{StoreI->getValueOperand()}, AccessKind::AK_W);
1870	if (auto *RMWI = dyn_cast<AtomicRMWInst>(Val: Usr))
1871	return HandleStoreLike(*RMWI, nullptr, *RMWI->getValOperand()->getType(),
1872	{RMWI->getValOperand()}, AccessKind::AK_RW);
1873	if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Val: Usr))
1874	return HandleStoreLike(
1875	*CXI, nullptr, *CXI->getNewValOperand()->getType(),
1876	{CXI->getCompareOperand(), CXI->getNewValOperand()},
1877	AccessKind::AK_RW);
1878
1879	if (auto *CB = dyn_cast<CallBase>(Val: Usr)) {
1880	if (CB->isLifetimeStartOrEnd())
1881	return true;
1882	if (getFreedOperand(CB, TLI) == U)
1883	return true;
1884	if (CB->isArgOperand(U: &U)) {
1885	unsigned ArgNo = CB->getArgOperandNo(U: &U);
1886	const auto *CSArgPI = A.getAAFor<AAPointerInfo>(
1887	QueryingAA: *this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
1888	DepClass: DepClassTy::REQUIRED);
1889	if (!CSArgPI)
1890	return false;
1891	Changed =
1892	translateAndAddState(A, OtherAA: *CSArgPI, Offsets: OffsetInfoMap[CurPtr], CB&: *CB) |
1893	Changed;
1894	return isValidState();
1895	}
1896	LLVM_DEBUG(dbgs() << "[AAPointerInfo] Call user not handled " << *CB
1897	<< "\n");
1898	// TODO: Allow some call uses
1899	return false;
1900	}
1901
1902	LLVM_DEBUG(dbgs() << "[AAPointerInfo] User not handled " << *Usr << "\n");
1903	return false;
1904	};
1905	auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
1906	assert(OffsetInfoMap.count(OldU) && "Old use should be known already!");
1907	if (OffsetInfoMap.count(Val: NewU)) {
1908	LLVM_DEBUG({
1909	if (!(OffsetInfoMap[NewU] == OffsetInfoMap[OldU])) {
1910	dbgs() << "[AAPointerInfo] Equivalent use callback failed: "
1911	<< OffsetInfoMap[NewU] << " vs " << OffsetInfoMap[OldU]
1912	<< "\n";
1913	}
1914	});
1915	return OffsetInfoMap[NewU] == OffsetInfoMap[OldU];
1916	}
1917	OffsetInfoMap[NewU] = OffsetInfoMap[OldU];
1918	return true;
1919	};
1920	if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: AssociatedValue,
1921	/* CheckBBLivenessOnly */ true, LivenessDepClass: DepClassTy::OPTIONAL,
1922	/* IgnoreDroppableUses */ true, EquivalentUseCB)) {
1923	LLVM_DEBUG(dbgs() << "[AAPointerInfo] Check for all uses failed, abort!\n");
1924	return indicatePessimisticFixpoint();
1925	}
1926
1927	LLVM_DEBUG({
1928	dbgs() << "Accesses by bin after update:\n";
1929	dumpState(dbgs());
1930	});
1931
1932	return Changed;
1933	}
1934
1935	struct AAPointerInfoReturned final : AAPointerInfoImpl {
1936	AAPointerInfoReturned(const IRPosition &IRP, Attributor &A)
1937	: AAPointerInfoImpl(IRP, A) {}
1938
1939	/// See AbstractAttribute::updateImpl(...).
1940	ChangeStatus updateImpl(Attributor &A) override {
1941	return indicatePessimisticFixpoint();
1942	}
1943
1944	/// See AbstractAttribute::trackStatistics()
1945	void trackStatistics() const override {
1946	AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
1947	}
1948	};
1949
1950	struct AAPointerInfoArgument final : AAPointerInfoFloating {
1951	AAPointerInfoArgument(const IRPosition &IRP, Attributor &A)
1952	: AAPointerInfoFloating(IRP, A) {}
1953
1954	/// See AbstractAttribute::trackStatistics()
1955	void trackStatistics() const override {
1956	AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
1957	}
1958	};
1959
1960	struct AAPointerInfoCallSiteArgument final : AAPointerInfoFloating {
1961	AAPointerInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
1962	: AAPointerInfoFloating(IRP, A) {}
1963
1964	/// See AbstractAttribute::updateImpl(...).
1965	ChangeStatus updateImpl(Attributor &A) override {
1966	using namespace AA::PointerInfo;
1967	// We handle memory intrinsics explicitly, at least the first (=
1968	// destination) and second (=source) arguments as we know how they are
1969	// accessed.
1970	if (auto *MI = dyn_cast_or_null<MemIntrinsic>(Val: getCtxI())) {
1971	ConstantInt *Length = dyn_cast<ConstantInt>(Val: MI->getLength());
1972	int64_t LengthVal = AA::RangeTy::Unknown;
1973	if (Length)
1974	LengthVal = Length->getSExtValue();
1975	unsigned ArgNo = getIRPosition().getCallSiteArgNo();
1976	ChangeStatus Changed = ChangeStatus::UNCHANGED;
1977	if (ArgNo > 1) {
1978	LLVM_DEBUG(dbgs() << "[AAPointerInfo] Unhandled memory intrinsic "
1979	<< *MI << "\n");
1980	return indicatePessimisticFixpoint();
1981	} else {
1982	auto Kind =
1983	ArgNo == 0 ? AccessKind::AK_MUST_WRITE : AccessKind::AK_MUST_READ;
1984	Changed =
1985	Changed | addAccess(A, Ranges: {0, LengthVal}, I&: *MI, Content: nullptr, Kind, Ty: nullptr);
1986	}
1987	LLVM_DEBUG({
1988	dbgs() << "Accesses by bin after update:\n";
1989	dumpState(dbgs());
1990	});
1991
1992	return Changed;
1993	}
1994
1995	// TODO: Once we have call site specific value information we can provide
1996	// call site specific liveness information and then it makes
1997	// sense to specialize attributes for call sites arguments instead of
1998	// redirecting requests to the callee argument.
1999	Argument *Arg = getAssociatedArgument();
2000	if (Arg) {
2001	const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
2002	auto *ArgAA =
2003	A.getAAFor<AAPointerInfo>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
2004	if (ArgAA && ArgAA->getState().isValidState())
2005	return translateAndAddStateFromCallee(A, OtherAA: *ArgAA,
2006	CB&: *cast<CallBase>(Val: getCtxI()));
2007	if (!Arg->getParent()->isDeclaration())
2008	return indicatePessimisticFixpoint();
2009	}
2010
2011	bool IsKnownNoCapture;
2012	if (!AA::hasAssumedIRAttr<Attribute::NoCapture>(
2013	A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnownNoCapture))
2014	return indicatePessimisticFixpoint();
2015
2016	bool IsKnown = false;
2017	if (AA::isAssumedReadNone(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown))
2018	return ChangeStatus::UNCHANGED;
2019	bool ReadOnly = AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown);
2020	auto Kind =
2021	ReadOnly ? AccessKind::AK_MAY_READ : AccessKind::AK_MAY_READ_WRITE;
2022	return addAccess(A, Ranges: AA::RangeTy::getUnknown(), I&: *getCtxI(), Content: nullptr, Kind,
2023	Ty: nullptr);
2024	}
2025
2026	/// See AbstractAttribute::trackStatistics()
2027	void trackStatistics() const override {
2028	AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
2029	}
2030	};
2031
2032	struct AAPointerInfoCallSiteReturned final : AAPointerInfoFloating {
2033	AAPointerInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
2034	: AAPointerInfoFloating(IRP, A) {}
2035
2036	/// See AbstractAttribute::trackStatistics()
2037	void trackStatistics() const override {
2038	AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
2039	}
2040	};
2041	} // namespace
2042
2043	/// -----------------------NoUnwind Function Attribute--------------------------
2044
2045	namespace {
2046	struct AANoUnwindImpl : AANoUnwind {
2047	AANoUnwindImpl(const IRPosition &IRP, Attributor &A) : AANoUnwind(IRP, A) {}
2048
2049	/// See AbstractAttribute::initialize(...).
2050	void initialize(Attributor &A) override {
2051	bool IsKnown;
2052	assert(!AA::hasAssumedIRAttr<Attribute::NoUnwind>(
2053	A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2054	(void)IsKnown;
2055	}
2056
2057	const std::string getAsStr(Attributor *A) const override {
2058	return getAssumed() ? "nounwind" : "may-unwind";
2059	}
2060
2061	/// See AbstractAttribute::updateImpl(...).
2062	ChangeStatus updateImpl(Attributor &A) override {
2063	auto Opcodes = {
2064	(unsigned)Instruction::Invoke, (unsigned)Instruction::CallBr,
2065	(unsigned)Instruction::Call, (unsigned)Instruction::CleanupRet,
2066	(unsigned)Instruction::CatchSwitch, (unsigned)Instruction::Resume};
2067
2068	auto CheckForNoUnwind = [&](Instruction &I) {
2069	if (!I.mayThrow(/* IncludePhaseOneUnwind */ true))
2070	return true;
2071
2072	if (const auto *CB = dyn_cast<CallBase>(Val: &I)) {
2073	bool IsKnownNoUnwind;
2074	return AA::hasAssumedIRAttr<Attribute::NoUnwind>(
2075	A, this, IRPosition::callsite_function(*CB), DepClassTy::REQUIRED,
2076	IsKnownNoUnwind);
2077	}
2078	return false;
2079	};
2080
2081	bool UsedAssumedInformation = false;
2082	if (!A.checkForAllInstructions(CheckForNoUnwind, *this, Opcodes,
2083	UsedAssumedInformation))
2084	return indicatePessimisticFixpoint();
2085
2086	return ChangeStatus::UNCHANGED;
2087	}
2088	};
2089
2090	struct AANoUnwindFunction final : public AANoUnwindImpl {
2091	AANoUnwindFunction(const IRPosition &IRP, Attributor &A)
2092	: AANoUnwindImpl(IRP, A) {}
2093
2094	/// See AbstractAttribute::trackStatistics()
2095	void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nounwind) }
2096	};
2097
2098	/// NoUnwind attribute deduction for a call sites.
2099	struct AANoUnwindCallSite final
2100	: AACalleeToCallSite<AANoUnwind, AANoUnwindImpl> {
2101	AANoUnwindCallSite(const IRPosition &IRP, Attributor &A)
2102	: AACalleeToCallSite<AANoUnwind, AANoUnwindImpl>(IRP, A) {}
2103
2104	/// See AbstractAttribute::trackStatistics()
2105	void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nounwind); }
2106	};
2107	} // namespace
2108
2109	/// ------------------------ NoSync Function Attribute -------------------------
2110
2111	bool AANoSync::isAlignedBarrier(const CallBase &CB, bool ExecutedAligned) {
2112	switch (CB.getIntrinsicID()) {
2113	case Intrinsic::nvvm_barrier0:
2114	case Intrinsic::nvvm_barrier0_and:
2115	case Intrinsic::nvvm_barrier0_or:
2116	case Intrinsic::nvvm_barrier0_popc:
2117	return true;
2118	case Intrinsic::amdgcn_s_barrier:
2119	if (ExecutedAligned)
2120	return true;
2121	break;
2122	default:
2123	break;
2124	}
2125	return hasAssumption(CB, AssumptionStr: KnownAssumptionString("ompx_aligned_barrier"));
2126	}
2127
2128	bool AANoSync::isNonRelaxedAtomic(const Instruction *I) {
2129	if (!I->isAtomic())
2130	return false;
2131
2132	if (auto *FI = dyn_cast<FenceInst>(Val: I))
2133	// All legal orderings for fence are stronger than monotonic.
2134	return FI->getSyncScopeID() != SyncScope::SingleThread;
2135	if (auto *AI = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
2136	// Unordered is not a legal ordering for cmpxchg.
2137	return (AI->getSuccessOrdering() != AtomicOrdering::Monotonic ||
2138	AI->getFailureOrdering() != AtomicOrdering::Monotonic);
2139	}
2140
2141	AtomicOrdering Ordering;
2142	switch (I->getOpcode()) {
2143	case Instruction::AtomicRMW:
2144	Ordering = cast<AtomicRMWInst>(Val: I)->getOrdering();
2145	break;
2146	case Instruction::Store:
2147	Ordering = cast<StoreInst>(Val: I)->getOrdering();
2148	break;
2149	case Instruction::Load:
2150	Ordering = cast<LoadInst>(Val: I)->getOrdering();
2151	break;
2152	default:
2153	llvm_unreachable(
2154	"New atomic operations need to be known in the attributor.");
2155	}
2156
2157	return (Ordering != AtomicOrdering::Unordered &&
2158	Ordering != AtomicOrdering::Monotonic);
2159	}
2160
2161	/// Return true if this intrinsic is nosync. This is only used for intrinsics
2162	/// which would be nosync except that they have a volatile flag. All other
2163	/// intrinsics are simply annotated with the nosync attribute in Intrinsics.td.
2164	bool AANoSync::isNoSyncIntrinsic(const Instruction *I) {
2165	if (auto *MI = dyn_cast<MemIntrinsic>(Val: I))
2166	return !MI->isVolatile();
2167	return false;
2168	}
2169
2170	namespace {
2171	struct AANoSyncImpl : AANoSync {
2172	AANoSyncImpl(const IRPosition &IRP, Attributor &A) : AANoSync(IRP, A) {}
2173
2174	/// See AbstractAttribute::initialize(...).
2175	void initialize(Attributor &A) override {
2176	bool IsKnown;
2177	assert(!AA::hasAssumedIRAttr<Attribute::NoSync>(A, nullptr, getIRPosition(),
2178	DepClassTy::NONE, IsKnown));
2179	(void)IsKnown;
2180	}
2181
2182	const std::string getAsStr(Attributor *A) const override {
2183	return getAssumed() ? "nosync" : "may-sync";
2184	}
2185
2186	/// See AbstractAttribute::updateImpl(...).
2187	ChangeStatus updateImpl(Attributor &A) override;
2188	};
2189
2190	ChangeStatus AANoSyncImpl::updateImpl(Attributor &A) {
2191
2192	auto CheckRWInstForNoSync = [&](Instruction &I) {
2193	return AA::isNoSyncInst(A, I, *this);
2194	};
2195
2196	auto CheckForNoSync = [&](Instruction &I) {
2197	// At this point we handled all read/write effects and they are all
2198	// nosync, so they can be skipped.
2199	if (I.mayReadOrWriteMemory())
2200	return true;
2201
2202	bool IsKnown;
2203	CallBase &CB = cast<CallBase>(Val&: I);
2204	if (AA::hasAssumedIRAttr<Attribute::NoSync>(
2205	A, this, IRPosition::callsite_function(CB), DepClassTy::OPTIONAL,
2206	IsKnown))
2207	return true;
2208
2209	// non-convergent and readnone imply nosync.
2210	return !CB.isConvergent();
2211	};
2212
2213	bool UsedAssumedInformation = false;
2214	if (!A.checkForAllReadWriteInstructions(CheckRWInstForNoSync, *this,
2215	UsedAssumedInformation) ||
2216	!A.checkForAllCallLikeInstructions(CheckForNoSync, *this,
2217	UsedAssumedInformation))
2218	return indicatePessimisticFixpoint();
2219
2220	return ChangeStatus::UNCHANGED;
2221	}
2222
2223	struct AANoSyncFunction final : public AANoSyncImpl {
2224	AANoSyncFunction(const IRPosition &IRP, Attributor &A)
2225	: AANoSyncImpl(IRP, A) {}
2226
2227	/// See AbstractAttribute::trackStatistics()
2228	void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nosync) }
2229	};
2230
2231	/// NoSync attribute deduction for a call sites.
2232	struct AANoSyncCallSite final : AACalleeToCallSite<AANoSync, AANoSyncImpl> {
2233	AANoSyncCallSite(const IRPosition &IRP, Attributor &A)
2234	: AACalleeToCallSite<AANoSync, AANoSyncImpl>(IRP, A) {}
2235
2236	/// See AbstractAttribute::trackStatistics()
2237	void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nosync); }
2238	};
2239	} // namespace
2240
2241	/// ------------------------ No-Free Attributes ----------------------------
2242
2243	namespace {
2244	struct AANoFreeImpl : public AANoFree {
2245	AANoFreeImpl(const IRPosition &IRP, Attributor &A) : AANoFree(IRP, A) {}
2246
2247	/// See AbstractAttribute::initialize(...).
2248	void initialize(Attributor &A) override {
2249	bool IsKnown;
2250	assert(!AA::hasAssumedIRAttr<Attribute::NoFree>(A, nullptr, getIRPosition(),
2251	DepClassTy::NONE, IsKnown));
2252	(void)IsKnown;
2253	}
2254
2255	/// See AbstractAttribute::updateImpl(...).
2256	ChangeStatus updateImpl(Attributor &A) override {
2257	auto CheckForNoFree = [&](Instruction &I) {
2258	bool IsKnown;
2259	return AA::hasAssumedIRAttr<Attribute::NoFree>(
2260	A, this, IRPosition::callsite_function(cast<CallBase>(I)),
2261	DepClassTy::REQUIRED, IsKnown);
2262	};
2263
2264	bool UsedAssumedInformation = false;
2265	if (!A.checkForAllCallLikeInstructions(CheckForNoFree, *this,
2266	UsedAssumedInformation))
2267	return indicatePessimisticFixpoint();
2268	return ChangeStatus::UNCHANGED;
2269	}
2270
2271	/// See AbstractAttribute::getAsStr().
2272	const std::string getAsStr(Attributor *A) const override {
2273	return getAssumed() ? "nofree" : "may-free";
2274	}
2275	};
2276
2277	struct AANoFreeFunction final : public AANoFreeImpl {
2278	AANoFreeFunction(const IRPosition &IRP, Attributor &A)
2279	: AANoFreeImpl(IRP, A) {}
2280
2281	/// See AbstractAttribute::trackStatistics()
2282	void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nofree) }
2283	};
2284
2285	/// NoFree attribute deduction for a call sites.
2286	struct AANoFreeCallSite final : AACalleeToCallSite<AANoFree, AANoFreeImpl> {
2287	AANoFreeCallSite(const IRPosition &IRP, Attributor &A)
2288	: AACalleeToCallSite<AANoFree, AANoFreeImpl>(IRP, A) {}
2289
2290	/// See AbstractAttribute::trackStatistics()
2291	void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nofree); }
2292	};
2293
2294	/// NoFree attribute for floating values.
2295	struct AANoFreeFloating : AANoFreeImpl {
2296	AANoFreeFloating(const IRPosition &IRP, Attributor &A)
2297	: AANoFreeImpl(IRP, A) {}
2298
2299	/// See AbstractAttribute::trackStatistics()
2300	void trackStatistics() const override{STATS_DECLTRACK_FLOATING_ATTR(nofree)}
2301
2302	/// See Abstract Attribute::updateImpl(...).
2303	ChangeStatus updateImpl(Attributor &A) override {
2304	const IRPosition &IRP = getIRPosition();
2305
2306	bool IsKnown;
2307	if (AA::hasAssumedIRAttr<Attribute::NoFree>(A, this,
2308	IRPosition::function_scope(IRP),
2309	DepClassTy::OPTIONAL, IsKnown))
2310	return ChangeStatus::UNCHANGED;
2311
2312	Value &AssociatedValue = getIRPosition().getAssociatedValue();
2313	auto Pred = [&](const Use &U, bool &Follow) -> bool {
2314	Instruction *UserI = cast<Instruction>(Val: U.getUser());
2315	if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
2316	if (CB->isBundleOperand(U: &U))
2317	return false;
2318	if (!CB->isArgOperand(U: &U))
2319	return true;
2320	unsigned ArgNo = CB->getArgOperandNo(U: &U);
2321
2322	bool IsKnown;
2323	return AA::hasAssumedIRAttr<Attribute::NoFree>(
2324	A, this, IRPosition::callsite_argument(*CB, ArgNo),
2325	DepClassTy::REQUIRED, IsKnown);
2326	}
2327
2328	if (isa<GetElementPtrInst>(Val: UserI) || isa<BitCastInst>(Val: UserI) ||
2329	isa<PHINode>(Val: UserI) || isa<SelectInst>(Val: UserI)) {
2330	Follow = true;
2331	return true;
2332	}
2333	if (isa<StoreInst>(Val: UserI) || isa<LoadInst>(Val: UserI) ||
2334	isa<ReturnInst>(Val: UserI))
2335	return true;
2336
2337	// Unknown user.
2338	return false;
2339	};
2340	if (!A.checkForAllUses(Pred, *this, AssociatedValue))
2341	return indicatePessimisticFixpoint();
2342
2343	return ChangeStatus::UNCHANGED;
2344	}
2345	};
2346
2347	/// NoFree attribute for a call site argument.
2348	struct AANoFreeArgument final : AANoFreeFloating {
2349	AANoFreeArgument(const IRPosition &IRP, Attributor &A)
2350	: AANoFreeFloating(IRP, A) {}
2351
2352	/// See AbstractAttribute::trackStatistics()
2353	void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofree) }
2354	};
2355
2356	/// NoFree attribute for call site arguments.
2357	struct AANoFreeCallSiteArgument final : AANoFreeFloating {
2358	AANoFreeCallSiteArgument(const IRPosition &IRP, Attributor &A)
2359	: AANoFreeFloating(IRP, A) {}
2360
2361	/// See AbstractAttribute::updateImpl(...).
2362	ChangeStatus updateImpl(Attributor &A) override {
2363	// TODO: Once we have call site specific value information we can provide
2364	// call site specific liveness information and then it makes
2365	// sense to specialize attributes for call sites arguments instead of
2366	// redirecting requests to the callee argument.
2367	Argument *Arg = getAssociatedArgument();
2368	if (!Arg)
2369	return indicatePessimisticFixpoint();
2370	const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
2371	bool IsKnown;
2372	if (AA::hasAssumedIRAttr<Attribute::NoFree>(A, this, ArgPos,
2373	DepClassTy::REQUIRED, IsKnown))
2374	return ChangeStatus::UNCHANGED;
2375	return indicatePessimisticFixpoint();
2376	}
2377
2378	/// See AbstractAttribute::trackStatistics()
2379	void trackStatistics() const override{STATS_DECLTRACK_CSARG_ATTR(nofree)};
2380	};
2381
2382	/// NoFree attribute for function return value.
2383	struct AANoFreeReturned final : AANoFreeFloating {
2384	AANoFreeReturned(const IRPosition &IRP, Attributor &A)
2385	: AANoFreeFloating(IRP, A) {
2386	llvm_unreachable("NoFree is not applicable to function returns!");
2387	}
2388
2389	/// See AbstractAttribute::initialize(...).
2390	void initialize(Attributor &A) override {
2391	llvm_unreachable("NoFree is not applicable to function returns!");
2392	}
2393
2394	/// See AbstractAttribute::updateImpl(...).
2395	ChangeStatus updateImpl(Attributor &A) override {
2396	llvm_unreachable("NoFree is not applicable to function returns!");
2397	}
2398
2399	/// See AbstractAttribute::trackStatistics()
2400	void trackStatistics() const override {}
2401	};
2402
2403	/// NoFree attribute deduction for a call site return value.
2404	struct AANoFreeCallSiteReturned final : AANoFreeFloating {
2405	AANoFreeCallSiteReturned(const IRPosition &IRP, Attributor &A)
2406	: AANoFreeFloating(IRP, A) {}
2407
2408	ChangeStatus manifest(Attributor &A) override {
2409	return ChangeStatus::UNCHANGED;
2410	}
2411	/// See AbstractAttribute::trackStatistics()
2412	void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nofree) }
2413	};
2414	} // namespace
2415
2416	/// ------------------------ NonNull Argument Attribute ------------------------
2417
2418	bool AANonNull::isImpliedByIR(Attributor &A, const IRPosition &IRP,
2419	Attribute::AttrKind ImpliedAttributeKind,
2420	bool IgnoreSubsumingPositions) {
2421	SmallVector<Attribute::AttrKind, 2> AttrKinds;
2422	AttrKinds.push_back(Attribute::NonNull);
2423	if (!NullPointerIsDefined(IRP.getAnchorScope(),
2424	IRP.getAssociatedType()->getPointerAddressSpace()))
2425	AttrKinds.push_back(Attribute::Dereferenceable);
2426	if (A.hasAttr(IRP, AttrKinds, IgnoreSubsumingPositions, Attribute::NonNull))
2427	return true;
2428
2429	DominatorTree *DT = nullptr;
2430	AssumptionCache *AC = nullptr;
2431	InformationCache &InfoCache = A.getInfoCache();
2432	if (const Function *Fn = IRP.getAnchorScope()) {
2433	if (!Fn->isDeclaration()) {
2434	DT = InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *Fn);
2435	AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *Fn);
2436	}
2437	}
2438
2439	SmallVector<AA::ValueAndContext> Worklist;
2440	if (IRP.getPositionKind() != IRP_RETURNED) {
2441	Worklist.push_back(Elt: {IRP.getAssociatedValue(), IRP.getCtxI()});
2442	} else {
2443	bool UsedAssumedInformation = false;
2444	if (!A.checkForAllInstructions(
2445	Pred: [&](Instruction &I) {
2446	Worklist.push_back(Elt: {*cast<ReturnInst>(Val&: I).getReturnValue(), &I});
2447	return true;
2448	},
2449	Fn: IRP.getAssociatedFunction(), QueryingAA: nullptr, Opcodes: {Instruction::Ret},
2450	UsedAssumedInformation))
2451	return false;
2452	}
2453
2454	if (llvm::any_of(Range&: Worklist, P: [&](AA::ValueAndContext VAC) {
2455	return !isKnownNonZero(
2456	V: VAC.getValue(),
2457	Q: SimplifyQuery(A.getDataLayout(), DT, AC, VAC.getCtxI()));
2458	}))
2459	return false;
2460
2461	A.manifestAttrs(IRP, {Attribute::get(IRP.getAnchorValue().getContext(),
2462	Attribute::NonNull)});
2463	return true;
2464	}
2465
2466	namespace {
2467	static int64_t getKnownNonNullAndDerefBytesForUse(
2468	Attributor &A, const AbstractAttribute &QueryingAA, Value &AssociatedValue,
2469	const Use *U, const Instruction *I, bool &IsNonNull, bool &TrackUse) {
2470	TrackUse = false;
2471
2472	const Value *UseV = U->get();
2473	if (!UseV->getType()->isPointerTy())
2474	return 0;
2475
2476	// We need to follow common pointer manipulation uses to the accesses they
2477	// feed into. We can try to be smart to avoid looking through things we do not
2478	// like for now, e.g., non-inbounds GEPs.
2479	if (isa<CastInst>(Val: I)) {
2480	TrackUse = true;
2481	return 0;
2482	}
2483
2484	if (isa<GetElementPtrInst>(Val: I)) {
2485	TrackUse = true;
2486	return 0;
2487	}
2488
2489	Type *PtrTy = UseV->getType();
2490	const Function *F = I->getFunction();
2491	bool NullPointerIsDefined =
2492	F ? llvm::NullPointerIsDefined(F, AS: PtrTy->getPointerAddressSpace()) : true;
2493	const DataLayout &DL = A.getInfoCache().getDL();
2494	if (const auto *CB = dyn_cast<CallBase>(Val: I)) {
2495	if (CB->isBundleOperand(U)) {
2496	if (RetainedKnowledge RK = getKnowledgeFromUse(
2497	U, {Attribute::NonNull, Attribute::Dereferenceable})) {
2498	IsNonNull |=
2499	(RK.AttrKind == Attribute::NonNull || !NullPointerIsDefined);
2500	return RK.ArgValue;
2501	}
2502	return 0;
2503	}
2504
2505	if (CB->isCallee(U)) {
2506	IsNonNull |= !NullPointerIsDefined;
2507	return 0;
2508	}
2509
2510	unsigned ArgNo = CB->getArgOperandNo(U);
2511	IRPosition IRP = IRPosition::callsite_argument(CB: *CB, ArgNo);
2512	// As long as we only use known information there is no need to track
2513	// dependences here.
2514	bool IsKnownNonNull;
2515	AA::hasAssumedIRAttr<Attribute::NonNull>(A, &QueryingAA, IRP,
2516	DepClassTy::NONE, IsKnownNonNull);
2517	IsNonNull |= IsKnownNonNull;
2518	auto *DerefAA =
2519	A.getAAFor<AADereferenceable>(QueryingAA, IRP, DepClass: DepClassTy::NONE);
2520	return DerefAA ? DerefAA->getKnownDereferenceableBytes() : 0;
2521	}
2522
2523	std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(Inst: I);
2524	if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() ||
2525	Loc->Size.isScalable() || I->isVolatile())
2526	return 0;
2527
2528	int64_t Offset;
2529	const Value *Base =
2530	getMinimalBaseOfPointer(A, QueryingAA, Ptr: Loc->Ptr, BytesOffset&: Offset, DL);
2531	if (Base && Base == &AssociatedValue) {
2532	int64_t DerefBytes = Loc->Size.getValue() + Offset;
2533	IsNonNull |= !NullPointerIsDefined;
2534	return std::max(a: int64_t(0), b: DerefBytes);
2535	}
2536
2537	/// Corner case when an offset is 0.
2538	Base = GetPointerBaseWithConstantOffset(Ptr: Loc->Ptr, Offset, DL,
2539	/*AllowNonInbounds*/ true);
2540	if (Base && Base == &AssociatedValue && Offset == 0) {
2541	int64_t DerefBytes = Loc->Size.getValue();
2542	IsNonNull |= !NullPointerIsDefined;
2543	return std::max(a: int64_t(0), b: DerefBytes);
2544	}
2545
2546	return 0;
2547	}
2548
2549	struct AANonNullImpl : AANonNull {
2550	AANonNullImpl(const IRPosition &IRP, Attributor &A) : AANonNull(IRP, A) {}
2551
2552	/// See AbstractAttribute::initialize(...).
2553	void initialize(Attributor &A) override {
2554	Value &V = *getAssociatedValue().stripPointerCasts();
2555	if (isa<ConstantPointerNull>(Val: V)) {
2556	indicatePessimisticFixpoint();
2557	return;
2558	}
2559
2560	if (Instruction *CtxI = getCtxI())
2561	followUsesInMBEC(*this, A, getState(), *CtxI);
2562	}
2563
2564	/// See followUsesInMBEC
2565	bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
2566	AANonNull::StateType &State) {
2567	bool IsNonNull = false;
2568	bool TrackUse = false;
2569	getKnownNonNullAndDerefBytesForUse(A, *this, getAssociatedValue(), U, I,
2570	IsNonNull, TrackUse);
2571	State.setKnown(IsNonNull);
2572	return TrackUse;
2573	}
2574
2575	/// See AbstractAttribute::getAsStr().
2576	const std::string getAsStr(Attributor *A) const override {
2577	return getAssumed() ? "nonnull" : "may-null";
2578	}
2579	};
2580
2581	/// NonNull attribute for a floating value.
2582	struct AANonNullFloating : public AANonNullImpl {
2583	AANonNullFloating(const IRPosition &IRP, Attributor &A)
2584	: AANonNullImpl(IRP, A) {}
2585
2586	/// See AbstractAttribute::updateImpl(...).
2587	ChangeStatus updateImpl(Attributor &A) override {
2588	auto CheckIRP = [&](const IRPosition &IRP) {
2589	bool IsKnownNonNull;
2590	return AA::hasAssumedIRAttr<Attribute::NonNull>(
2591	A, *this, IRP, DepClassTy::OPTIONAL, IsKnownNonNull);
2592	};
2593
2594	bool Stripped;
2595	bool UsedAssumedInformation = false;
2596	Value *AssociatedValue = &getAssociatedValue();
2597	SmallVector<AA::ValueAndContext> Values;
2598	if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
2599	S: AA::AnyScope, UsedAssumedInformation))
2600	Stripped = false;
2601	else
2602	Stripped =
2603	Values.size() != 1 || Values.front().getValue() != AssociatedValue;
2604
2605	if (!Stripped) {
2606	bool IsKnown;
2607	if (auto *PHI = dyn_cast<PHINode>(AssociatedValue))
2608	if (llvm::all_of(PHI->incoming_values(), [&](Value *Op) {
2609	return AA::hasAssumedIRAttr<Attribute::NonNull>(
2610	A, this, IRPosition::value(*Op), DepClassTy::OPTIONAL,
2611	IsKnown);
2612	}))
2613	return ChangeStatus::UNCHANGED;
2614	if (auto *Select = dyn_cast<SelectInst>(AssociatedValue))
2615	if (AA::hasAssumedIRAttr<Attribute::NonNull>(
2616	A, this, IRPosition::value(*Select->getFalseValue()),
2617	DepClassTy::OPTIONAL, IsKnown) &&
2618	AA::hasAssumedIRAttr<Attribute::NonNull>(
2619	A, this, IRPosition::value(*Select->getTrueValue()),
2620	DepClassTy::OPTIONAL, IsKnown))
2621	return ChangeStatus::UNCHANGED;
2622
2623	// If we haven't stripped anything we might still be able to use a
2624	// different AA, but only if the IRP changes. Effectively when we
2625	// interpret this not as a call site value but as a floating/argument
2626	// value.
2627	const IRPosition AVIRP = IRPosition::value(V: *AssociatedValue);
2628	if (AVIRP == getIRPosition() || !CheckIRP(AVIRP))
2629	return indicatePessimisticFixpoint();
2630	return ChangeStatus::UNCHANGED;
2631	}
2632
2633	for (const auto &VAC : Values)
2634	if (!CheckIRP(IRPosition::value(V: *VAC.getValue())))
2635	return indicatePessimisticFixpoint();
2636
2637	return ChangeStatus::UNCHANGED;
2638	}
2639
2640	/// See AbstractAttribute::trackStatistics()
2641	void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) }
2642	};
2643
2644	/// NonNull attribute for function return value.
2645	struct AANonNullReturned final
2646	: AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,
2647	false, AANonNull::IRAttributeKind, false> {
2648	AANonNullReturned(const IRPosition &IRP, Attributor &A)
2649	: AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,
2650	false, Attribute::NonNull, false>(IRP, A) {
2651	}
2652
2653	/// See AbstractAttribute::getAsStr().
2654	const std::string getAsStr(Attributor *A) const override {
2655	return getAssumed() ? "nonnull" : "may-null";
2656	}
2657
2658	/// See AbstractAttribute::trackStatistics()
2659	void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) }
2660	};
2661
2662	/// NonNull attribute for function argument.
2663	struct AANonNullArgument final
2664	: AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl> {
2665	AANonNullArgument(const IRPosition &IRP, Attributor &A)
2666	: AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl>(IRP, A) {}
2667
2668	/// See AbstractAttribute::trackStatistics()
2669	void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nonnull) }
2670	};
2671
2672	struct AANonNullCallSiteArgument final : AANonNullFloating {
2673	AANonNullCallSiteArgument(const IRPosition &IRP, Attributor &A)
2674	: AANonNullFloating(IRP, A) {}
2675
2676	/// See AbstractAttribute::trackStatistics()
2677	void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(nonnull) }
2678	};
2679
2680	/// NonNull attribute for a call site return position.
2681	struct AANonNullCallSiteReturned final
2682	: AACalleeToCallSite<AANonNull, AANonNullImpl> {
2683	AANonNullCallSiteReturned(const IRPosition &IRP, Attributor &A)
2684	: AACalleeToCallSite<AANonNull, AANonNullImpl>(IRP, A) {}
2685
2686	/// See AbstractAttribute::trackStatistics()
2687	void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nonnull) }
2688	};
2689	} // namespace
2690
2691	/// ------------------------ Must-Progress Attributes --------------------------
2692	namespace {
2693	struct AAMustProgressImpl : public AAMustProgress {
2694	AAMustProgressImpl(const IRPosition &IRP, Attributor &A)
2695	: AAMustProgress(IRP, A) {}
2696
2697	/// See AbstractAttribute::initialize(...).
2698	void initialize(Attributor &A) override {
2699	bool IsKnown;
2700	assert(!AA::hasAssumedIRAttr<Attribute::MustProgress>(
2701	A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2702	(void)IsKnown;
2703	}
2704
2705	/// See AbstractAttribute::getAsStr()
2706	const std::string getAsStr(Attributor *A) const override {
2707	return getAssumed() ? "mustprogress" : "may-not-progress";
2708	}
2709	};
2710
2711	struct AAMustProgressFunction final : AAMustProgressImpl {
2712	AAMustProgressFunction(const IRPosition &IRP, Attributor &A)
2713	: AAMustProgressImpl(IRP, A) {}
2714
2715	/// See AbstractAttribute::updateImpl(...).
2716	ChangeStatus updateImpl(Attributor &A) override {
2717	bool IsKnown;
2718	if (AA::hasAssumedIRAttr<Attribute::WillReturn>(
2719	A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnown)) {
2720	if (IsKnown)
2721	return indicateOptimisticFixpoint();
2722	return ChangeStatus::UNCHANGED;
2723	}
2724
2725	auto CheckForMustProgress = [&](AbstractCallSite ACS) {
2726	IRPosition IPos = IRPosition::callsite_function(CB: *ACS.getInstruction());
2727	bool IsKnownMustProgress;
2728	return AA::hasAssumedIRAttr<Attribute::MustProgress>(
2729	A, this, IPos, DepClassTy::REQUIRED, IsKnownMustProgress,
2730	/* IgnoreSubsumingPositions */ true);
2731	};
2732
2733	bool AllCallSitesKnown = true;
2734	if (!A.checkForAllCallSites(CheckForMustProgress, *this,
2735	/* RequireAllCallSites */ true,
2736	AllCallSitesKnown))
2737	return indicatePessimisticFixpoint();
2738
2739	return ChangeStatus::UNCHANGED;
2740	}
2741
2742	/// See AbstractAttribute::trackStatistics()
2743	void trackStatistics() const override {
2744	STATS_DECLTRACK_FN_ATTR(mustprogress)
2745	}
2746	};
2747
2748	/// MustProgress attribute deduction for a call sites.
2749	struct AAMustProgressCallSite final : AAMustProgressImpl {
2750	AAMustProgressCallSite(const IRPosition &IRP, Attributor &A)
2751	: AAMustProgressImpl(IRP, A) {}
2752
2753	/// See AbstractAttribute::updateImpl(...).
2754	ChangeStatus updateImpl(Attributor &A) override {
2755	// TODO: Once we have call site specific value information we can provide
2756	// call site specific liveness information and then it makes
2757	// sense to specialize attributes for call sites arguments instead of
2758	// redirecting requests to the callee argument.
2759	const IRPosition &FnPos = IRPosition::function(F: *getAnchorScope());
2760	bool IsKnownMustProgress;
2761	if (!AA::hasAssumedIRAttr<Attribute::MustProgress>(
2762	A, this, FnPos, DepClassTy::REQUIRED, IsKnownMustProgress))
2763	return indicatePessimisticFixpoint();
2764	return ChangeStatus::UNCHANGED;
2765	}
2766
2767	/// See AbstractAttribute::trackStatistics()
2768	void trackStatistics() const override {
2769	STATS_DECLTRACK_CS_ATTR(mustprogress);
2770	}
2771	};
2772	} // namespace
2773
2774	/// ------------------------ No-Recurse Attributes ----------------------------
2775
2776	namespace {
2777	struct AANoRecurseImpl : public AANoRecurse {
2778	AANoRecurseImpl(const IRPosition &IRP, Attributor &A) : AANoRecurse(IRP, A) {}
2779
2780	/// See AbstractAttribute::initialize(...).
2781	void initialize(Attributor &A) override {
2782	bool IsKnown;
2783	assert(!AA::hasAssumedIRAttr<Attribute::NoRecurse>(
2784	A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2785	(void)IsKnown;
2786	}
2787
2788	/// See AbstractAttribute::getAsStr()
2789	const std::string getAsStr(Attributor *A) const override {
2790	return getAssumed() ? "norecurse" : "may-recurse";
2791	}
2792	};
2793
2794	struct AANoRecurseFunction final : AANoRecurseImpl {
2795	AANoRecurseFunction(const IRPosition &IRP, Attributor &A)
2796	: AANoRecurseImpl(IRP, A) {}
2797
2798	/// See AbstractAttribute::updateImpl(...).
2799	ChangeStatus updateImpl(Attributor &A) override {
2800
2801	// If all live call sites are known to be no-recurse, we are as well.
2802	auto CallSitePred = [&](AbstractCallSite ACS) {
2803	bool IsKnownNoRecurse;
2804	if (!AA::hasAssumedIRAttr<Attribute::NoRecurse>(
2805	A, this,
2806	IRPosition::function(*ACS.getInstruction()->getFunction()),
2807	DepClassTy::NONE, IsKnownNoRecurse))
2808	return false;
2809	return IsKnownNoRecurse;
2810	};
2811	bool UsedAssumedInformation = false;
2812	if (A.checkForAllCallSites(CallSitePred, *this, true,
2813	UsedAssumedInformation)) {
2814	// If we know all call sites and all are known no-recurse, we are done.
2815	// If all known call sites, which might not be all that exist, are known
2816	// to be no-recurse, we are not done but we can continue to assume
2817	// no-recurse. If one of the call sites we have not visited will become
2818	// live, another update is triggered.
2819	if (!UsedAssumedInformation)
2820	indicateOptimisticFixpoint();
2821	return ChangeStatus::UNCHANGED;
2822	}
2823
2824	const AAInterFnReachability *EdgeReachability =
2825	A.getAAFor<AAInterFnReachability>(*this, getIRPosition(),
2826	DepClassTy::REQUIRED);
2827	if (EdgeReachability && EdgeReachability->canReach(A, Fn: *getAnchorScope()))
2828	return indicatePessimisticFixpoint();
2829	return ChangeStatus::UNCHANGED;
2830	}
2831
2832	void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(norecurse) }
2833	};
2834
2835	/// NoRecurse attribute deduction for a call sites.
2836	struct AANoRecurseCallSite final
2837	: AACalleeToCallSite<AANoRecurse, AANoRecurseImpl> {
2838	AANoRecurseCallSite(const IRPosition &IRP, Attributor &A)
2839	: AACalleeToCallSite<AANoRecurse, AANoRecurseImpl>(IRP, A) {}
2840
2841	/// See AbstractAttribute::trackStatistics()
2842	void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(norecurse); }
2843	};
2844	} // namespace
2845
2846	/// ------------------------ No-Convergent Attribute --------------------------
2847
2848	namespace {
2849	struct AANonConvergentImpl : public AANonConvergent {
2850	AANonConvergentImpl(const IRPosition &IRP, Attributor &A)
2851	: AANonConvergent(IRP, A) {}
2852
2853	/// See AbstractAttribute::getAsStr()
2854	const std::string getAsStr(Attributor *A) const override {
2855	return getAssumed() ? "non-convergent" : "may-be-convergent";
2856	}
2857	};
2858
2859	struct AANonConvergentFunction final : AANonConvergentImpl {
2860	AANonConvergentFunction(const IRPosition &IRP, Attributor &A)
2861	: AANonConvergentImpl(IRP, A) {}
2862
2863	/// See AbstractAttribute::updateImpl(...).
2864	ChangeStatus updateImpl(Attributor &A) override {
2865	// If all function calls are known to not be convergent, we are not
2866	// convergent.
2867	auto CalleeIsNotConvergent = [&](Instruction &Inst) {
2868	CallBase &CB = cast<CallBase>(Val&: Inst);
2869	auto *Callee = dyn_cast_if_present<Function>(Val: CB.getCalledOperand());
2870	if (!Callee || Callee->isIntrinsic()) {
2871	return false;
2872	}
2873	if (Callee->isDeclaration()) {
2874	return !Callee->hasFnAttribute(Attribute::Convergent);
2875	}
2876	const auto *ConvergentAA = A.getAAFor<AANonConvergent>(
2877	QueryingAA: *this, IRP: IRPosition::function(F: *Callee), DepClass: DepClassTy::REQUIRED);
2878	return ConvergentAA && ConvergentAA->isAssumedNotConvergent();
2879	};
2880
2881	bool UsedAssumedInformation = false;
2882	if (!A.checkForAllCallLikeInstructions(Pred: CalleeIsNotConvergent, QueryingAA: *this,
2883	UsedAssumedInformation)) {
2884	return indicatePessimisticFixpoint();
2885	}
2886	return ChangeStatus::UNCHANGED;
2887	}
2888
2889	ChangeStatus manifest(Attributor &A) override {
2890	if (isKnownNotConvergent() &&
2891	A.hasAttr(getIRPosition(), Attribute::Convergent)) {
2892	A.removeAttrs(getIRPosition(), {Attribute::Convergent});
2893	return ChangeStatus::CHANGED;
2894	}
2895	return ChangeStatus::UNCHANGED;
2896	}
2897
2898	void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(convergent) }
2899	};
2900	} // namespace
2901
2902	/// -------------------- Undefined-Behavior Attributes ------------------------
2903
2904	namespace {
2905	struct AAUndefinedBehaviorImpl : public AAUndefinedBehavior {
2906	AAUndefinedBehaviorImpl(const IRPosition &IRP, Attributor &A)
2907	: AAUndefinedBehavior(IRP, A) {}
2908
2909	/// See AbstractAttribute::updateImpl(...).
2910	// through a pointer (i.e. also branches etc.)
2911	ChangeStatus updateImpl(Attributor &A) override {
2912	const size_t UBPrevSize = KnownUBInsts.size();
2913	const size_t NoUBPrevSize = AssumedNoUBInsts.size();
2914
2915	auto InspectMemAccessInstForUB = [&](Instruction &I) {
2916	// Lang ref now states volatile store is not UB, let's skip them.
2917	if (I.isVolatile() && I.mayWriteToMemory())
2918	return true;
2919
2920	// Skip instructions that are already saved.
2921	if (AssumedNoUBInsts.count(Ptr: &I) || KnownUBInsts.count(Ptr: &I))
2922	return true;
2923
2924	// If we reach here, we know we have an instruction
2925	// that accesses memory through a pointer operand,
2926	// for which getPointerOperand() should give it to us.
2927	Value *PtrOp =
2928	const_cast<Value *>(getPointerOperand(I: &I, /* AllowVolatile */ true));
2929	assert(PtrOp &&
2930	"Expected pointer operand of memory accessing instruction");
2931
2932	// Either we stopped and the appropriate action was taken,
2933	// or we got back a simplified value to continue.
2934	std::optional<Value *> SimplifiedPtrOp =
2935	stopOnUndefOrAssumed(A, V: PtrOp, I: &I);
2936	if (!SimplifiedPtrOp || !*SimplifiedPtrOp)
2937	return true;
2938	const Value *PtrOpVal = *SimplifiedPtrOp;
2939
2940	// A memory access through a pointer is considered UB
2941	// only if the pointer has constant null value.
2942	// TODO: Expand it to not only check constant values.
2943	if (!isa<ConstantPointerNull>(Val: PtrOpVal)) {
2944	AssumedNoUBInsts.insert(Ptr: &I);
2945	return true;
2946	}
2947	const Type *PtrTy = PtrOpVal->getType();
2948
2949	// Because we only consider instructions inside functions,
2950	// assume that a parent function exists.
2951	const Function *F = I.getFunction();
2952
2953	// A memory access using constant null pointer is only considered UB
2954	// if null pointer is _not_ defined for the target platform.
2955	if (llvm::NullPointerIsDefined(F, AS: PtrTy->getPointerAddressSpace()))
2956	AssumedNoUBInsts.insert(Ptr: &I);
2957	else
2958	KnownUBInsts.insert(Ptr: &I);
2959	return true;
2960	};
2961
2962	auto InspectBrInstForUB = [&](Instruction &I) {
2963	// A conditional branch instruction is considered UB if it has `undef`
2964	// condition.
2965
2966	// Skip instructions that are already saved.
2967	if (AssumedNoUBInsts.count(Ptr: &I) || KnownUBInsts.count(Ptr: &I))
2968	return true;
2969
2970	// We know we have a branch instruction.
2971	auto *BrInst = cast<BranchInst>(Val: &I);
2972
2973	// Unconditional branches are never considered UB.
2974	if (BrInst->isUnconditional())
2975	return true;
2976
2977	// Either we stopped and the appropriate action was taken,
2978	// or we got back a simplified value to continue.
2979	std::optional<Value *> SimplifiedCond =
2980	stopOnUndefOrAssumed(A, V: BrInst->getCondition(), I: BrInst);
2981	if (!SimplifiedCond || !*SimplifiedCond)
2982	return true;
2983	AssumedNoUBInsts.insert(Ptr: &I);
2984	return true;
2985	};
2986
2987	auto InspectCallSiteForUB = [&](Instruction &I) {
2988	// Check whether a callsite always cause UB or not
2989
2990	// Skip instructions that are already saved.
2991	if (AssumedNoUBInsts.count(Ptr: &I) || KnownUBInsts.count(Ptr: &I))
2992	return true;
2993
2994	// Check nonnull and noundef argument attribute violation for each
2995	// callsite.
2996	CallBase &CB = cast<CallBase>(Val&: I);
2997	auto *Callee = dyn_cast_if_present<Function>(Val: CB.getCalledOperand());
2998	if (!Callee)
2999	return true;
3000	for (unsigned idx = 0; idx < CB.arg_size(); idx++) {
3001	// If current argument is known to be simplified to null pointer and the
3002	// corresponding argument position is known to have nonnull attribute,
3003	// the argument is poison. Furthermore, if the argument is poison and
3004	// the position is known to have noundef attriubte, this callsite is
3005	// considered UB.
3006	if (idx >= Callee->arg_size())
3007	break;
3008	Value *ArgVal = CB.getArgOperand(i: idx);
3009	if (!ArgVal)
3010	continue;
3011	// Here, we handle three cases.
3012	// (1) Not having a value means it is dead. (we can replace the value
3013	// with undef)
3014	// (2) Simplified to undef. The argument violate noundef attriubte.
3015	// (3) Simplified to null pointer where known to be nonnull.
3016	// The argument is a poison value and violate noundef attribute.
3017	IRPosition CalleeArgumentIRP = IRPosition::callsite_argument(CB, ArgNo: idx);
3018	bool IsKnownNoUndef;
3019	AA::hasAssumedIRAttr<Attribute::NoUndef>(
3020	A, this, CalleeArgumentIRP, DepClassTy::NONE, IsKnownNoUndef);
3021	if (!IsKnownNoUndef)
3022	continue;
3023	bool UsedAssumedInformation = false;
3024	std::optional<Value *> SimplifiedVal =
3025	A.getAssumedSimplified(IRP: IRPosition::value(V: *ArgVal), AA: *this,
3026	UsedAssumedInformation, S: AA::Interprocedural);
3027	if (UsedAssumedInformation)
3028	continue;
3029	if (SimplifiedVal && !*SimplifiedVal)
3030	return true;
3031	if (!SimplifiedVal || isa<UndefValue>(Val: **SimplifiedVal)) {
3032	KnownUBInsts.insert(Ptr: &I);
3033	continue;
3034	}
3035	if (!ArgVal->getType()->isPointerTy() ||
3036	!isa<ConstantPointerNull>(Val: **SimplifiedVal))
3037	continue;
3038	bool IsKnownNonNull;
3039	AA::hasAssumedIRAttr<Attribute::NonNull>(
3040	A, this, CalleeArgumentIRP, DepClassTy::NONE, IsKnownNonNull);
3041	if (IsKnownNonNull)
3042	KnownUBInsts.insert(Ptr: &I);
3043	}
3044	return true;
3045	};
3046
3047	auto InspectReturnInstForUB = [&](Instruction &I) {
3048	auto &RI = cast<ReturnInst>(Val&: I);
3049	// Either we stopped and the appropriate action was taken,
3050	// or we got back a simplified return value to continue.
3051	std::optional<Value *> SimplifiedRetValue =
3052	stopOnUndefOrAssumed(A, V: RI.getReturnValue(), I: &I);
3053	if (!SimplifiedRetValue || !*SimplifiedRetValue)
3054	return true;
3055
3056	// Check if a return instruction always cause UB or not
3057	// Note: It is guaranteed that the returned position of the anchor
3058	// scope has noundef attribute when this is called.
3059	// We also ensure the return position is not "assumed dead"
3060	// because the returned value was then potentially simplified to
3061	// `undef` in AAReturnedValues without removing the `noundef`
3062	// attribute yet.
3063
3064	// When the returned position has noundef attriubte, UB occurs in the
3065	// following cases.
3066	// (1) Returned value is known to be undef.
3067	// (2) The value is known to be a null pointer and the returned
3068	// position has nonnull attribute (because the returned value is
3069	// poison).
3070	if (isa<ConstantPointerNull>(Val: *SimplifiedRetValue)) {
3071	bool IsKnownNonNull;
3072	AA::hasAssumedIRAttr<Attribute::NonNull>(
3073	A, this, IRPosition::returned(*getAnchorScope()), DepClassTy::NONE,
3074	IsKnownNonNull);
3075	if (IsKnownNonNull)
3076	KnownUBInsts.insert(Ptr: &I);
3077	}
3078
3079	return true;
3080	};
3081
3082	bool UsedAssumedInformation = false;
3083	A.checkForAllInstructions(Pred: InspectMemAccessInstForUB, QueryingAA: *this,
3084	Opcodes: {Instruction::Load, Instruction::Store,
3085	Instruction::AtomicCmpXchg,
3086	Instruction::AtomicRMW},
3087	UsedAssumedInformation,
3088	/* CheckBBLivenessOnly */ true);
3089	A.checkForAllInstructions(Pred: InspectBrInstForUB, QueryingAA: *this, Opcodes: {Instruction::Br},
3090	UsedAssumedInformation,
3091	/* CheckBBLivenessOnly */ true);
3092	A.checkForAllCallLikeInstructions(Pred: InspectCallSiteForUB, QueryingAA: *this,
3093	UsedAssumedInformation);
3094
3095	// If the returned position of the anchor scope has noundef attriubte, check
3096	// all returned instructions.
3097	if (!getAnchorScope()->getReturnType()->isVoidTy()) {
3098	const IRPosition &ReturnIRP = IRPosition::returned(F: *getAnchorScope());
3099	if (!A.isAssumedDead(IRP: ReturnIRP, QueryingAA: this, FnLivenessAA: nullptr, UsedAssumedInformation)) {
3100	bool IsKnownNoUndef;
3101	AA::hasAssumedIRAttr<Attribute::NoUndef>(
3102	A, this, ReturnIRP, DepClassTy::NONE, IsKnownNoUndef);
3103	if (IsKnownNoUndef)
3104	A.checkForAllInstructions(Pred: InspectReturnInstForUB, QueryingAA: *this,
3105	Opcodes: {Instruction::Ret}, UsedAssumedInformation,
3106	/* CheckBBLivenessOnly */ true);
3107	}
3108	}
3109
3110	if (NoUBPrevSize != AssumedNoUBInsts.size() ||
3111	UBPrevSize != KnownUBInsts.size())
3112	return ChangeStatus::CHANGED;
3113	return ChangeStatus::UNCHANGED;
3114	}
3115
3116	bool isKnownToCauseUB(Instruction *I) const override {
3117	return KnownUBInsts.count(Ptr: I);
3118	}
3119
3120	bool isAssumedToCauseUB(Instruction *I) const override {
3121	// In simple words, if an instruction is not in the assumed to _not_
3122	// cause UB, then it is assumed UB (that includes those
3123	// in the KnownUBInsts set). The rest is boilerplate
3124	// is to ensure that it is one of the instructions we test
3125	// for UB.
3126
3127	switch (I->getOpcode()) {
3128	case Instruction::Load:
3129	case Instruction::Store:
3130	case Instruction::AtomicCmpXchg:
3131	case Instruction::AtomicRMW:
3132	return !AssumedNoUBInsts.count(Ptr: I);
3133	case Instruction::Br: {
3134	auto *BrInst = cast<BranchInst>(Val: I);
3135	if (BrInst->isUnconditional())
3136	return false;
3137	return !AssumedNoUBInsts.count(Ptr: I);
3138	} break;
3139	default:
3140	return false;
3141	}
3142	return false;
3143	}
3144
3145	ChangeStatus manifest(Attributor &A) override {
3146	if (KnownUBInsts.empty())
3147	return ChangeStatus::UNCHANGED;
3148	for (Instruction *I : KnownUBInsts)
3149	A.changeToUnreachableAfterManifest(I);
3150	return ChangeStatus::CHANGED;
3151	}
3152
3153	/// See AbstractAttribute::getAsStr()
3154	const std::string getAsStr(Attributor *A) const override {
3155	return getAssumed() ? "undefined-behavior" : "no-ub";
3156	}
3157
3158	/// Note: The correctness of this analysis depends on the fact that the
3159	/// following 2 sets will stop changing after some point.
3160	/// "Change" here means that their size changes.
3161	/// The size of each set is monotonically increasing
3162	/// (we only add items to them) and it is upper bounded by the number of
3163	/// instructions in the processed function (we can never save more
3164	/// elements in either set than this number). Hence, at some point,
3165	/// they will stop increasing.
3166	/// Consequently, at some point, both sets will have stopped
3167	/// changing, effectively making the analysis reach a fixpoint.
3168
3169	/// Note: These 2 sets are disjoint and an instruction can be considered
3170	/// one of 3 things:
3171	/// 1) Known to cause UB (AAUndefinedBehavior could prove it) and put it in
3172	/// the KnownUBInsts set.
3173	/// 2) Assumed to cause UB (in every updateImpl, AAUndefinedBehavior
3174	/// has a reason to assume it).
3175	/// 3) Assumed to not cause UB. very other instruction - AAUndefinedBehavior
3176	/// could not find a reason to assume or prove that it can cause UB,
3177	/// hence it assumes it doesn't. We have a set for these instructions
3178	/// so that we don't reprocess them in every update.
3179	/// Note however that instructions in this set may cause UB.
3180
3181	protected:
3182	/// A set of all live instructions _known_ to cause UB.
3183	SmallPtrSet<Instruction *, 8> KnownUBInsts;
3184
3185	private:
3186	/// A set of all the (live) instructions that are assumed to _not_ cause UB.
3187	SmallPtrSet<Instruction *, 8> AssumedNoUBInsts;
3188
3189	// Should be called on updates in which if we're processing an instruction
3190	// \p I that depends on a value \p V, one of the following has to happen:
3191	// - If the value is assumed, then stop.
3192	// - If the value is known but undef, then consider it UB.
3193	// - Otherwise, do specific processing with the simplified value.
3194	// We return std::nullopt in the first 2 cases to signify that an appropriate
3195	// action was taken and the caller should stop.
3196	// Otherwise, we return the simplified value that the caller should
3197	// use for specific processing.
3198	std::optional<Value *> stopOnUndefOrAssumed(Attributor &A, Value *V,
3199	Instruction *I) {
3200	bool UsedAssumedInformation = false;
3201	std::optional<Value *> SimplifiedV =
3202	A.getAssumedSimplified(IRP: IRPosition::value(V: *V), AA: *this,
3203	UsedAssumedInformation, S: AA::Interprocedural);
3204	if (!UsedAssumedInformation) {
3205	// Don't depend on assumed values.
3206	if (!SimplifiedV) {
3207	// If it is known (which we tested above) but it doesn't have a value,
3208	// then we can assume `undef` and hence the instruction is UB.
3209	KnownUBInsts.insert(Ptr: I);
3210	return std::nullopt;
3211	}
3212	if (!*SimplifiedV)
3213	return nullptr;
3214	V = *SimplifiedV;
3215	}
3216	if (isa<UndefValue>(Val: V)) {
3217	KnownUBInsts.insert(Ptr: I);
3218	return std::nullopt;
3219	}
3220	return V;
3221	}
3222	};
3223
3224	struct AAUndefinedBehaviorFunction final : AAUndefinedBehaviorImpl {
3225	AAUndefinedBehaviorFunction(const IRPosition &IRP, Attributor &A)
3226	: AAUndefinedBehaviorImpl(IRP, A) {}
3227
3228	/// See AbstractAttribute::trackStatistics()
3229	void trackStatistics() const override {
3230	STATS_DECL(UndefinedBehaviorInstruction, Instruction,
3231	"Number of instructions known to have UB");
3232	BUILD_STAT_NAME(UndefinedBehaviorInstruction, Instruction) +=
3233	KnownUBInsts.size();
3234	}
3235	};
3236	} // namespace
3237
3238	/// ------------------------ Will-Return Attributes ----------------------------
3239
3240	namespace {
3241	// Helper function that checks whether a function has any cycle which we don't
3242	// know if it is bounded or not.
3243	// Loops with maximum trip count are considered bounded, any other cycle not.
3244	static bool mayContainUnboundedCycle(Function &F, Attributor &A) {
3245	ScalarEvolution *SE =
3246	A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(F);
3247	LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(F);
3248	// If either SCEV or LoopInfo is not available for the function then we assume
3249	// any cycle to be unbounded cycle.
3250	// We use scc_iterator which uses Tarjan algorithm to find all the maximal
3251	// SCCs.To detect if there's a cycle, we only need to find the maximal ones.
3252	if (!SE || !LI) {
3253	for (scc_iterator<Function *> SCCI = scc_begin(G: &F); !SCCI.isAtEnd(); ++SCCI)
3254	if (SCCI.hasCycle())
3255	return true;
3256	return false;
3257	}
3258
3259	// If there's irreducible control, the function may contain non-loop cycles.
3260	if (mayContainIrreducibleControl(F, LI))
3261	return true;
3262
3263	// Any loop that does not have a max trip count is considered unbounded cycle.
3264	for (auto *L : LI->getLoopsInPreorder()) {
3265	if (!SE->getSmallConstantMaxTripCount(L))
3266	return true;
3267	}
3268	return false;
3269	}
3270
3271	struct AAWillReturnImpl : public AAWillReturn {
3272	AAWillReturnImpl(const IRPosition &IRP, Attributor &A)
3273	: AAWillReturn(IRP, A) {}
3274
3275	/// See AbstractAttribute::initialize(...).
3276	void initialize(Attributor &A) override {
3277	bool IsKnown;
3278	assert(!AA::hasAssumedIRAttr<Attribute::WillReturn>(
3279	A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
3280	(void)IsKnown;
3281	}
3282
3283	/// Check for `mustprogress` and `readonly` as they imply `willreturn`.
3284	bool isImpliedByMustprogressAndReadonly(Attributor &A, bool KnownOnly) {
3285	if (!A.hasAttr(getIRPosition(), {Attribute::MustProgress}))
3286	return false;
3287
3288	bool IsKnown;
3289	if (AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown))
3290	return IsKnown || !KnownOnly;
3291	return false;
3292	}
3293
3294	/// See AbstractAttribute::updateImpl(...).
3295	ChangeStatus updateImpl(Attributor &A) override {
3296	if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false))
3297	return ChangeStatus::UNCHANGED;
3298
3299	auto CheckForWillReturn = [&](Instruction &I) {
3300	IRPosition IPos = IRPosition::callsite_function(CB: cast<CallBase>(Val&: I));
3301	bool IsKnown;
3302	if (AA::hasAssumedIRAttr<Attribute::WillReturn>(
3303	A, this, IPos, DepClassTy::REQUIRED, IsKnown)) {
3304	if (IsKnown)
3305	return true;
3306	} else {
3307	return false;
3308	}
3309	bool IsKnownNoRecurse;
3310	return AA::hasAssumedIRAttr<Attribute::NoRecurse>(
3311	A, this, IPos, DepClassTy::REQUIRED, IsKnownNoRecurse);
3312	};
3313
3314	bool UsedAssumedInformation = false;
3315	if (!A.checkForAllCallLikeInstructions(CheckForWillReturn, *this,
3316	UsedAssumedInformation))
3317	return indicatePessimisticFixpoint();
3318
3319	return ChangeStatus::UNCHANGED;
3320	}
3321
3322	/// See AbstractAttribute::getAsStr()
3323	const std::string getAsStr(Attributor *A) const override {
3324	return getAssumed() ? "willreturn" : "may-noreturn";
3325	}
3326	};
3327
3328	struct AAWillReturnFunction final : AAWillReturnImpl {
3329	AAWillReturnFunction(const IRPosition &IRP, Attributor &A)
3330	: AAWillReturnImpl(IRP, A) {}
3331
3332	/// See AbstractAttribute::initialize(...).
3333	void initialize(Attributor &A) override {
3334	AAWillReturnImpl::initialize(A);
3335
3336	Function *F = getAnchorScope();
3337	assert(F && "Did expect an anchor function");
3338	if (F->isDeclaration() || mayContainUnboundedCycle(F&: *F, A))
3339	indicatePessimisticFixpoint();
3340	}
3341
3342	/// See AbstractAttribute::trackStatistics()
3343	void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(willreturn) }
3344	};
3345
3346	/// WillReturn attribute deduction for a call sites.
3347	struct AAWillReturnCallSite final
3348	: AACalleeToCallSite<AAWillReturn, AAWillReturnImpl> {
3349	AAWillReturnCallSite(const IRPosition &IRP, Attributor &A)
3350	: AACalleeToCallSite<AAWillReturn, AAWillReturnImpl>(IRP, A) {}
3351
3352	/// See AbstractAttribute::updateImpl(...).
3353	ChangeStatus updateImpl(Attributor &A) override {
3354	if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false))
3355	return ChangeStatus::UNCHANGED;
3356
3357	return AACalleeToCallSite::updateImpl(A);
3358	}
3359
3360	/// See AbstractAttribute::trackStatistics()
3361	void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(willreturn); }
3362	};
3363	} // namespace
3364
3365	/// -------------------AAIntraFnReachability Attribute--------------------------
3366
3367	/// All information associated with a reachability query. This boilerplate code
3368	/// is used by both AAIntraFnReachability and AAInterFnReachability, with
3369	/// different \p ToTy values.
3370	template <typename ToTy> struct ReachabilityQueryInfo {
3371	enum class Reachable {
3372	No,
3373	Yes,
3374	};
3375
3376	/// Start here,
3377	const Instruction *From = nullptr;
3378	/// reach this place,
3379	const ToTy *To = nullptr;
3380	/// without going through any of these instructions,
3381	const AA::InstExclusionSetTy *ExclusionSet = nullptr;
3382	/// and remember if it worked:
3383	Reachable Result = Reachable::No;
3384
3385	/// Precomputed hash for this RQI.
3386	unsigned Hash = 0;
3387
3388	unsigned computeHashValue() const {
3389	assert(Hash == 0 && "Computed hash twice!");
3390	using InstSetDMI = DenseMapInfo<const AA::InstExclusionSetTy *>;
3391	using PairDMI = DenseMapInfo<std::pair<const Instruction *, const ToTy *>>;
3392	return const_cast<ReachabilityQueryInfo<ToTy> *>(this)->Hash =
3393	detail::combineHashValue(a: PairDMI ::getHashValue({From, To}),
3394	b: InstSetDMI::getHashValue(BES: ExclusionSet));
3395	}
3396
3397	ReachabilityQueryInfo(const Instruction *From, const ToTy *To)
3398	: From(From), To(To) {}
3399
3400	/// Constructor replacement to ensure unique and stable sets are used for the
3401	/// cache.
3402	ReachabilityQueryInfo(Attributor &A, const Instruction &From, const ToTy &To,
3403	const AA::InstExclusionSetTy *ES, bool MakeUnique)
3404	: From(&From), To(&To), ExclusionSet(ES) {
3405
3406	if (!ES || ES->empty()) {
3407	ExclusionSet = nullptr;
3408	} else if (MakeUnique) {
3409	ExclusionSet = A.getInfoCache().getOrCreateUniqueBlockExecutionSet(BES: ES);
3410	}
3411	}
3412
3413	ReachabilityQueryInfo(const ReachabilityQueryInfo &RQI)
3414	: From(RQI.From), To(RQI.To), ExclusionSet(RQI.ExclusionSet) {}
3415	};
3416
3417	namespace llvm {
3418	template <typename ToTy> struct DenseMapInfo<ReachabilityQueryInfo<ToTy> *> {
3419	using InstSetDMI = DenseMapInfo<const AA::InstExclusionSetTy *>;
3420	using PairDMI = DenseMapInfo<std::pair<const Instruction *, const ToTy *>>;
3421
3422	static ReachabilityQueryInfo<ToTy> EmptyKey;
3423	static ReachabilityQueryInfo<ToTy> TombstoneKey;
3424
3425	static inline ReachabilityQueryInfo<ToTy> *getEmptyKey() { return &EmptyKey; }
3426	static inline ReachabilityQueryInfo<ToTy> *getTombstoneKey() {
3427	return &TombstoneKey;
3428	}
3429	static unsigned getHashValue(const ReachabilityQueryInfo<ToTy> *RQI) {
3430	return RQI->Hash ? RQI->Hash : RQI->computeHashValue();
3431	}
3432	static bool isEqual(const ReachabilityQueryInfo<ToTy> *LHS,
3433	const ReachabilityQueryInfo<ToTy> *RHS) {
3434	if (!PairDMI::isEqual({LHS->From, LHS->To}, {RHS->From, RHS->To}))
3435	return false;
3436	return InstSetDMI::isEqual(LHS: LHS->ExclusionSet, RHS: RHS->ExclusionSet);
3437	}
3438	};
3439
3440	#define DefineKeys(ToTy) \
3441	template <> \
3442	ReachabilityQueryInfo<ToTy> \
3443	DenseMapInfo<ReachabilityQueryInfo<ToTy> *>::EmptyKey = \
3444	ReachabilityQueryInfo<ToTy>( \
3445	DenseMapInfo<const Instruction *>::getEmptyKey(), \
3446	DenseMapInfo<const ToTy *>::getEmptyKey()); \
3447	template <> \
3448	ReachabilityQueryInfo<ToTy> \
3449	DenseMapInfo<ReachabilityQueryInfo<ToTy> *>::TombstoneKey = \
3450	ReachabilityQueryInfo<ToTy>( \
3451	DenseMapInfo<const Instruction *>::getTombstoneKey(), \
3452	DenseMapInfo<const ToTy *>::getTombstoneKey());
3453
3454	DefineKeys(Instruction) DefineKeys(Function)
3455	#undef DefineKeys
3456
3457	} // namespace llvm
3458
3459	namespace {
3460
3461	template <typename BaseTy, typename ToTy>
3462	struct CachedReachabilityAA : public BaseTy {
3463	using RQITy = ReachabilityQueryInfo<ToTy>;
3464
3465	CachedReachabilityAA(const IRPosition &IRP, Attributor &A) : BaseTy(IRP, A) {}
3466
3467	/// See AbstractAttribute::isQueryAA.
3468	bool isQueryAA() const override { return true; }
3469
3470	/// See AbstractAttribute::updateImpl(...).
3471	ChangeStatus updateImpl(Attributor &A) override {
3472	ChangeStatus Changed = ChangeStatus::UNCHANGED;
3473	for (unsigned u = 0, e = QueryVector.size(); u < e; ++u) {
3474	RQITy *RQI = QueryVector[u];
3475	if (RQI->Result == RQITy::Reachable::No &&
3476	isReachableImpl(A, RQI&: *RQI, /*IsTemporaryRQI=*/false))
3477	Changed = ChangeStatus::CHANGED;
3478	}
3479	return Changed;
3480	}
3481
3482	virtual bool isReachableImpl(Attributor &A, RQITy &RQI,
3483	bool IsTemporaryRQI) = 0;
3484
3485	bool rememberResult(Attributor &A, typename RQITy::Reachable Result,
3486	RQITy &RQI, bool UsedExclusionSet, bool IsTemporaryRQI) {
3487	RQI.Result = Result;
3488
3489	// Remove the temporary RQI from the cache.
3490	if (IsTemporaryRQI)
3491	QueryCache.erase(&RQI);
3492
3493	// Insert a plain RQI (w/o exclusion set) if that makes sense. Two options:
3494	// 1) If it is reachable, it doesn't matter if we have an exclusion set for
3495	// this query. 2) We did not use the exclusion set, potentially because
3496	// there is none.
3497	if (Result == RQITy::Reachable::Yes || !UsedExclusionSet) {
3498	RQITy PlainRQI(RQI.From, RQI.To);
3499	if (!QueryCache.count(&PlainRQI)) {
3500	RQITy *RQIPtr = new (A.Allocator) RQITy(RQI.From, RQI.To);
3501	RQIPtr->Result = Result;
3502	QueryVector.push_back(RQIPtr);
3503	QueryCache.insert(RQIPtr);
3504	}
3505	}
3506
3507	// Check if we need to insert a new permanent RQI with the exclusion set.
3508	if (IsTemporaryRQI && Result != RQITy::Reachable::Yes && UsedExclusionSet) {
3509	assert((!RQI.ExclusionSet || !RQI.ExclusionSet->empty()) &&
3510	"Did not expect empty set!");
3511	RQITy *RQIPtr = new (A.Allocator)
3512	RQITy(A, *RQI.From, *RQI.To, RQI.ExclusionSet, true);
3513	assert(RQIPtr->Result == RQITy::Reachable::No && "Already reachable?");
3514	RQIPtr->Result = Result;
3515	assert(!QueryCache.count(RQIPtr));
3516	QueryVector.push_back(RQIPtr);
3517	QueryCache.insert(RQIPtr);
3518	}
3519
3520	if (Result == RQITy::Reachable::No && IsTemporaryRQI)
3521	A.registerForUpdate(AA&: *this);
3522	return Result == RQITy::Reachable::Yes;
3523	}
3524
3525	const std::string getAsStr(Attributor *A) const override {
3526	// TODO: Return the number of reachable queries.
3527	return "#queries(" + std::to_string(QueryVector.size()) + ")";
3528	}
3529
3530	bool checkQueryCache(Attributor &A, RQITy &StackRQI,
3531	typename RQITy::Reachable &Result) {
3532	if (!this->getState().isValidState()) {
3533	Result = RQITy::Reachable::Yes;
3534	return true;
3535	}
3536
3537	// If we have an exclusion set we might be able to find our answer by
3538	// ignoring it first.
3539	if (StackRQI.ExclusionSet) {
3540	RQITy PlainRQI(StackRQI.From, StackRQI.To);
3541	auto It = QueryCache.find(&PlainRQI);
3542	if (It != QueryCache.end() && (*It)->Result == RQITy::Reachable::No) {
3543	Result = RQITy::Reachable::No;
3544	return true;
3545	}
3546	}
3547
3548	auto It = QueryCache.find(&StackRQI);
3549	if (It != QueryCache.end()) {
3550	Result = (*It)->Result;
3551	return true;
3552	}
3553
3554	// Insert a temporary for recursive queries. We will replace it with a
3555	// permanent entry later.
3556	QueryCache.insert(&StackRQI);
3557	return false;
3558	}
3559
3560	private:
3561	SmallVector<RQITy *> QueryVector;
3562	DenseSet<RQITy *> QueryCache;
3563	};
3564
3565	struct AAIntraFnReachabilityFunction final
3566	: public CachedReachabilityAA<AAIntraFnReachability, Instruction> {
3567	using Base = CachedReachabilityAA<AAIntraFnReachability, Instruction>;
3568	AAIntraFnReachabilityFunction(const IRPosition &IRP, Attributor &A)
3569	: Base(IRP, A) {
3570	DT = A.getInfoCache().getAnalysisResultForFunction<DominatorTreeAnalysis>(
3571	F: *IRP.getAssociatedFunction());
3572	}
3573
3574	bool isAssumedReachable(
3575	Attributor &A, const Instruction &From, const Instruction &To,
3576	const AA::InstExclusionSetTy *ExclusionSet) const override {
3577	auto *NonConstThis = const_cast<AAIntraFnReachabilityFunction *>(this);
3578	if (&From == &To)
3579	return true;
3580
3581	RQITy StackRQI(A, From, To, ExclusionSet, false);
3582	typename RQITy::Reachable Result;
3583	if (!NonConstThis->checkQueryCache(A, StackRQI, Result))
3584	return NonConstThis->isReachableImpl(A, RQI&: StackRQI,
3585	/*IsTemporaryRQI=*/true);
3586	return Result == RQITy::Reachable::Yes;
3587	}
3588
3589	ChangeStatus updateImpl(Attributor &A) override {
3590	// We only depend on liveness. DeadEdges is all we care about, check if any
3591	// of them changed.
3592	auto *LivenessAA =
3593	A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL);
3594	if (LivenessAA &&
3595	llvm::all_of(Range&: DeadEdges,
3596	P: [&](const auto &DeadEdge) {
3597	return LivenessAA->isEdgeDead(From: DeadEdge.first,
3598	To: DeadEdge.second);
3599	}) &&
3600	llvm::all_of(Range&: DeadBlocks, P: [&](const BasicBlock *BB) {
3601	return LivenessAA->isAssumedDead(BB);
3602	})) {
3603	return ChangeStatus::UNCHANGED;
3604	}
3605	DeadEdges.clear();
3606	DeadBlocks.clear();
3607	return Base::updateImpl(A);
3608	}
3609
3610	bool isReachableImpl(Attributor &A, RQITy &RQI,
3611	bool IsTemporaryRQI) override {
3612	const Instruction *Origin = RQI.From;
3613	bool UsedExclusionSet = false;
3614
3615	auto WillReachInBlock = [&](const Instruction &From, const Instruction &To,
3616	const AA::InstExclusionSetTy *ExclusionSet) {
3617	const Instruction *IP = &From;
3618	while (IP && IP != &To) {
3619	if (ExclusionSet && IP != Origin && ExclusionSet->count(Ptr: IP)) {
3620	UsedExclusionSet = true;
3621	break;
3622	}
3623	IP = IP->getNextNode();
3624	}
3625	return IP == &To;
3626	};
3627
3628	const BasicBlock *FromBB = RQI.From->getParent();
3629	const BasicBlock *ToBB = RQI.To->getParent();
3630	assert(FromBB->getParent() == ToBB->getParent() &&
3631	"Not an intra-procedural query!");
3632
3633	// Check intra-block reachability, however, other reaching paths are still
3634	// possible.
3635	if (FromBB == ToBB &&
3636	WillReachInBlock(*RQI.From, *RQI.To, RQI.ExclusionSet))
3637	return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3638	IsTemporaryRQI);
3639
3640	// Check if reaching the ToBB block is sufficient or if even that would not
3641	// ensure reaching the target. In the latter case we are done.
3642	if (!WillReachInBlock(ToBB->front(), *RQI.To, RQI.ExclusionSet))
3643	return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
3644	IsTemporaryRQI);
3645
3646	const Function *Fn = FromBB->getParent();
3647	SmallPtrSet<const BasicBlock *, 16> ExclusionBlocks;
3648	if (RQI.ExclusionSet)
3649	for (auto *I : *RQI.ExclusionSet)
3650	if (I->getFunction() == Fn)
3651	ExclusionBlocks.insert(Ptr: I->getParent());
3652
3653	// Check if we make it out of the FromBB block at all.
3654	if (ExclusionBlocks.count(Ptr: FromBB) &&
3655	!WillReachInBlock(*RQI.From, *FromBB->getTerminator(),
3656	RQI.ExclusionSet))
3657	return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet: true, IsTemporaryRQI);
3658
3659	auto *LivenessAA =
3660	A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL);
3661	if (LivenessAA && LivenessAA->isAssumedDead(BB: ToBB)) {
3662	DeadBlocks.insert(V: ToBB);
3663	return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
3664	IsTemporaryRQI);
3665	}
3666
3667	SmallPtrSet<const BasicBlock *, 16> Visited;
3668	SmallVector<const BasicBlock *, 16> Worklist;
3669	Worklist.push_back(Elt: FromBB);
3670
3671	DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> LocalDeadEdges;
3672	while (!Worklist.empty()) {
3673	const BasicBlock *BB = Worklist.pop_back_val();
3674	if (!Visited.insert(Ptr: BB).second)
3675	continue;
3676	for (const BasicBlock *SuccBB : successors(BB)) {
3677	if (LivenessAA && LivenessAA->isEdgeDead(From: BB, To: SuccBB)) {
3678	LocalDeadEdges.insert(V: {BB, SuccBB});
3679	continue;
3680	}
3681	// We checked before if we just need to reach the ToBB block.
3682	if (SuccBB == ToBB)
3683	return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3684	IsTemporaryRQI);
3685	if (DT && ExclusionBlocks.empty() && DT->dominates(A: BB, B: ToBB))
3686	return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3687	IsTemporaryRQI);
3688
3689	if (ExclusionBlocks.count(Ptr: SuccBB)) {
3690	UsedExclusionSet = true;
3691	continue;
3692	}
3693	Worklist.push_back(Elt: SuccBB);
3694	}
3695	}
3696
3697	DeadEdges.insert(I: LocalDeadEdges.begin(), E: LocalDeadEdges.end());
3698	return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
3699	IsTemporaryRQI);
3700	}
3701
3702	/// See AbstractAttribute::trackStatistics()
3703	void trackStatistics() const override {}
3704
3705	private:
3706	// Set of assumed dead blocks we used in the last query. If any changes we
3707	// update the state.
3708	DenseSet<const BasicBlock *> DeadBlocks;
3709
3710	// Set of assumed dead edges we used in the last query. If any changes we
3711	// update the state.
3712	DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> DeadEdges;
3713
3714	/// The dominator tree of the function to short-circuit reasoning.
3715	const DominatorTree *DT = nullptr;
3716	};
3717	} // namespace
3718
3719	/// ------------------------ NoAlias Argument Attribute ------------------------
3720
3721	bool AANoAlias::isImpliedByIR(Attributor &A, const IRPosition &IRP,
3722	Attribute::AttrKind ImpliedAttributeKind,
3723	bool IgnoreSubsumingPositions) {
3724	assert(ImpliedAttributeKind == Attribute::NoAlias &&
3725	"Unexpected attribute kind");
3726	Value *Val = &IRP.getAssociatedValue();
3727	if (IRP.getPositionKind() != IRP_CALL_SITE_ARGUMENT) {
3728	if (isa<AllocaInst>(Val))
3729	return true;
3730	} else {
3731	IgnoreSubsumingPositions = true;
3732	}
3733
3734	if (isa<UndefValue>(Val))
3735	return true;
3736
3737	if (isa<ConstantPointerNull>(Val) &&
3738	!NullPointerIsDefined(F: IRP.getAnchorScope(),
3739	AS: Val->getType()->getPointerAddressSpace()))
3740	return true;
3741
3742	if (A.hasAttr(IRP, {Attribute::ByVal, Attribute::NoAlias},
3743	IgnoreSubsumingPositions, Attribute::NoAlias))
3744	return true;
3745
3746	return false;
3747	}
3748
3749	namespace {
3750	struct AANoAliasImpl : AANoAlias {
3751	AANoAliasImpl(const IRPosition &IRP, Attributor &A) : AANoAlias(IRP, A) {
3752	assert(getAssociatedType()->isPointerTy() &&
3753	"Noalias is a pointer attribute");
3754	}
3755
3756	const std::string getAsStr(Attributor *A) const override {
3757	return getAssumed() ? "noalias" : "may-alias";
3758	}
3759	};
3760
3761	/// NoAlias attribute for a floating value.
3762	struct AANoAliasFloating final : AANoAliasImpl {
3763	AANoAliasFloating(const IRPosition &IRP, Attributor &A)
3764	: AANoAliasImpl(IRP, A) {}
3765
3766	/// See AbstractAttribute::updateImpl(...).
3767	ChangeStatus updateImpl(Attributor &A) override {
3768	// TODO: Implement this.
3769	return indicatePessimisticFixpoint();
3770	}
3771
3772	/// See AbstractAttribute::trackStatistics()
3773	void trackStatistics() const override {
3774	STATS_DECLTRACK_FLOATING_ATTR(noalias)
3775	}
3776	};
3777
3778	/// NoAlias attribute for an argument.
3779	struct AANoAliasArgument final
3780	: AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl> {
3781	using Base = AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl>;
3782	AANoAliasArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
3783
3784	/// See AbstractAttribute::update(...).
3785	ChangeStatus updateImpl(Attributor &A) override {
3786	// We have to make sure no-alias on the argument does not break
3787	// synchronization when this is a callback argument, see also [1] below.
3788	// If synchronization cannot be affected, we delegate to the base updateImpl
3789	// function, otherwise we give up for now.
3790
3791	// If the function is no-sync, no-alias cannot break synchronization.
3792	bool IsKnownNoSycn;
3793	if (AA::hasAssumedIRAttr<Attribute::NoSync>(
3794	A, this, IRPosition::function_scope(getIRPosition()),
3795	DepClassTy::OPTIONAL, IsKnownNoSycn))
3796	return Base::updateImpl(A);
3797
3798	// If the argument is read-only, no-alias cannot break synchronization.
3799	bool IsKnown;
3800	if (AA::isAssumedReadOnly(A, getIRPosition(), *this, IsKnown))
3801	return Base::updateImpl(A);
3802
3803	// If the argument is never passed through callbacks, no-alias cannot break
3804	// synchronization.
3805	bool UsedAssumedInformation = false;
3806	if (A.checkForAllCallSites(
3807	[](AbstractCallSite ACS) { return !ACS.isCallbackCall(); }, *this,
3808	true, UsedAssumedInformation))
3809	return Base::updateImpl(A);
3810
3811	// TODO: add no-alias but make sure it doesn't break synchronization by
3812	// introducing fake uses. See:
3813	// [1] Compiler Optimizations for OpenMP, J. Doerfert and H. Finkel,
3814	// International Workshop on OpenMP 2018,
3815	// http://compilers.cs.uni-saarland.de/people/doerfert/par_opt18.pdf
3816
3817	return indicatePessimisticFixpoint();
3818	}
3819
3820	/// See AbstractAttribute::trackStatistics()
3821	void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noalias) }
3822	};
3823
3824	struct AANoAliasCallSiteArgument final : AANoAliasImpl {
3825	AANoAliasCallSiteArgument(const IRPosition &IRP, Attributor &A)
3826	: AANoAliasImpl(IRP, A) {}
3827
3828	/// Determine if the underlying value may alias with the call site argument
3829	/// \p OtherArgNo of \p ICS (= the underlying call site).
3830	bool mayAliasWithArgument(Attributor &A, AAResults *&AAR,
3831	const AAMemoryBehavior &MemBehaviorAA,
3832	const CallBase &CB, unsigned OtherArgNo) {
3833	// We do not need to worry about aliasing with the underlying IRP.
3834	if (this->getCalleeArgNo() == (int)OtherArgNo)
3835	return false;
3836
3837	// If it is not a pointer or pointer vector we do not alias.
3838	const Value *ArgOp = CB.getArgOperand(i: OtherArgNo);
3839	if (!ArgOp->getType()->isPtrOrPtrVectorTy())
3840	return false;
3841
3842	auto *CBArgMemBehaviorAA = A.getAAFor<AAMemoryBehavior>(
3843	*this, IRPosition::callsite_argument(CB, ArgNo: OtherArgNo), DepClassTy::NONE);
3844
3845	// If the argument is readnone, there is no read-write aliasing.
3846	if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadNone()) {
3847	A.recordDependence(FromAA: *CBArgMemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
3848	return false;
3849	}
3850
3851	// If the argument is readonly and the underlying value is readonly, there
3852	// is no read-write aliasing.
3853	bool IsReadOnly = MemBehaviorAA.isAssumedReadOnly();
3854	if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadOnly() &&
3855	IsReadOnly) {
3856	A.recordDependence(MemBehaviorAA, *this, DepClassTy::OPTIONAL);
3857	A.recordDependence(FromAA: *CBArgMemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
3858	return false;
3859	}
3860
3861	// We have to utilize actual alias analysis queries so we need the object.
3862	if (!AAR)
3863	AAR = A.getInfoCache().getAnalysisResultForFunction<AAManager>(
3864	*getAnchorScope());
3865
3866	// Try to rule it out at the call site.
3867	bool IsAliasing = !AAR || !AAR->isNoAlias(&getAssociatedValue(), ArgOp);
3868	LLVM_DEBUG(dbgs() << "[NoAliasCSArg] Check alias between "
3869	"callsite arguments: "
3870	<< getAssociatedValue() << " " << *ArgOp << " => "
3871	<< (IsAliasing ? "" : "no-") << "alias \n");
3872
3873	return IsAliasing;
3874	}
3875
3876	bool isKnownNoAliasDueToNoAliasPreservation(
3877	Attributor &A, AAResults *&AAR, const AAMemoryBehavior &MemBehaviorAA) {
3878	// We can deduce "noalias" if the following conditions hold.
3879	// (i) Associated value is assumed to be noalias in the definition.
3880	// (ii) Associated value is assumed to be no-capture in all the uses
3881	// possibly executed before this callsite.
3882	// (iii) There is no other pointer argument which could alias with the
3883	// value.
3884
3885	auto IsDereferenceableOrNull = [&](Value *O, const DataLayout &DL) {
3886	const auto *DerefAA = A.getAAFor<AADereferenceable>(
3887	*this, IRPosition::value(V: *O), DepClassTy::OPTIONAL);
3888	return DerefAA ? DerefAA->getAssumedDereferenceableBytes() : 0;
3889	};
3890
3891	const IRPosition &VIRP = IRPosition::value(V: getAssociatedValue());
3892	const Function *ScopeFn = VIRP.getAnchorScope();
3893	// Check whether the value is captured in the scope using AANoCapture.
3894	// Look at CFG and check only uses possibly executed before this
3895	// callsite.
3896	auto UsePred = [&](const Use &U, bool &Follow) -> bool {
3897	Instruction *UserI = cast<Instruction>(Val: U.getUser());
3898
3899	// If UserI is the curr instruction and there is a single potential use of
3900	// the value in UserI we allow the use.
3901	// TODO: We should inspect the operands and allow those that cannot alias
3902	// with the value.
3903	if (UserI == getCtxI() && UserI->getNumOperands() == 1)
3904	return true;
3905
3906	if (ScopeFn) {
3907	if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
3908	if (CB->isArgOperand(U: &U)) {
3909
3910	unsigned ArgNo = CB->getArgOperandNo(U: &U);
3911
3912	bool IsKnownNoCapture;
3913	if (AA::hasAssumedIRAttr<Attribute::NoCapture>(
3914	A, this, IRPosition::callsite_argument(*CB, ArgNo),
3915	DepClassTy::OPTIONAL, IsKnownNoCapture))
3916	return true;
3917	}
3918	}
3919
3920	if (!AA::isPotentiallyReachable(
3921	A, *UserI, *getCtxI(), *this, /* ExclusionSet */ nullptr,
3922	[ScopeFn](const Function &Fn) { return &Fn != ScopeFn; }))
3923	return true;
3924	}
3925
3926	// TODO: We should track the capturing uses in AANoCapture but the problem
3927	// is CGSCC runs. For those we would need to "allow" AANoCapture for
3928	// a value in the module slice.
3929	switch (DetermineUseCaptureKind(U, IsDereferenceableOrNull)) {
3930	case UseCaptureKind::NO_CAPTURE:
3931	return true;
3932	case UseCaptureKind::MAY_CAPTURE:
3933	LLVM_DEBUG(dbgs() << "[AANoAliasCSArg] Unknown user: " << *UserI
3934	<< "\n");
3935	return false;
3936	case UseCaptureKind::PASSTHROUGH:
3937	Follow = true;
3938	return true;
3939	}
3940	llvm_unreachable("unknown UseCaptureKind");
3941	};
3942
3943	bool IsKnownNoCapture;
3944	const AANoCapture *NoCaptureAA = nullptr;
3945	bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
3946	A, this, VIRP, DepClassTy::NONE, IsKnownNoCapture, false, &NoCaptureAA);
3947	if (!IsAssumedNoCapture &&
3948	(!NoCaptureAA || !NoCaptureAA->isAssumedNoCaptureMaybeReturned())) {
3949	if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: getAssociatedValue())) {
3950	LLVM_DEBUG(
3951	dbgs() << "[AANoAliasCSArg] " << getAssociatedValue()
3952	<< " cannot be noalias as it is potentially captured\n");
3953	return false;
3954	}
3955	}
3956	if (NoCaptureAA)
3957	A.recordDependence(*NoCaptureAA, *this, DepClassTy::OPTIONAL);
3958
3959	// Check there is no other pointer argument which could alias with the
3960	// value passed at this call site.
3961	// TODO: AbstractCallSite
3962	const auto &CB = cast<CallBase>(getAnchorValue());
3963	for (unsigned OtherArgNo = 0; OtherArgNo < CB.arg_size(); OtherArgNo++)
3964	if (mayAliasWithArgument(A, AAR, MemBehaviorAA, CB: CB, OtherArgNo))
3965	return false;
3966
3967	return true;
3968	}
3969
3970	/// See AbstractAttribute::updateImpl(...).
3971	ChangeStatus updateImpl(Attributor &A) override {
3972	// If the argument is readnone we are done as there are no accesses via the
3973	// argument.
3974	auto *MemBehaviorAA =
3975	A.getAAFor<AAMemoryBehavior>(*this, getIRPosition(), DepClassTy::NONE);
3976	if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) {
3977	A.recordDependence(FromAA: *MemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
3978	return ChangeStatus::UNCHANGED;
3979	}
3980
3981	bool IsKnownNoAlias;
3982	const IRPosition &VIRP = IRPosition::value(V: getAssociatedValue());
3983	if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
3984	A, this, VIRP, DepClassTy::REQUIRED, IsKnownNoAlias)) {
3985	LLVM_DEBUG(dbgs() << "[AANoAlias] " << getAssociatedValue()
3986	<< " is not no-alias at the definition\n");
3987	return indicatePessimisticFixpoint();
3988	}
3989
3990	AAResults *AAR = nullptr;
3991	if (MemBehaviorAA &&
3992	isKnownNoAliasDueToNoAliasPreservation(A, AAR, MemBehaviorAA: *MemBehaviorAA)) {
3993	LLVM_DEBUG(
3994	dbgs() << "[AANoAlias] No-Alias deduced via no-alias preservation\n");
3995	return ChangeStatus::UNCHANGED;
3996	}
3997
3998	return indicatePessimisticFixpoint();
3999	}
4000
4001	/// See AbstractAttribute::trackStatistics()
4002	void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noalias) }
4003	};
4004
4005	/// NoAlias attribute for function return value.
4006	struct AANoAliasReturned final : AANoAliasImpl {
4007	AANoAliasReturned(const IRPosition &IRP, Attributor &A)
4008	: AANoAliasImpl(IRP, A) {}
4009
4010	/// See AbstractAttribute::updateImpl(...).
4011	ChangeStatus updateImpl(Attributor &A) override {
4012
4013	auto CheckReturnValue = [&](Value &RV) -> bool {
4014	if (Constant *C = dyn_cast<Constant>(Val: &RV))
4015	if (C->isNullValue() || isa<UndefValue>(Val: C))
4016	return true;
4017
4018	/// For now, we can only deduce noalias if we have call sites.
4019	/// FIXME: add more support.
4020	if (!isa<CallBase>(Val: &RV))
4021	return false;
4022
4023	const IRPosition &RVPos = IRPosition::value(V: RV);
4024	bool IsKnownNoAlias;
4025	if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
4026	A, this, RVPos, DepClassTy::REQUIRED, IsKnownNoAlias))
4027	return false;
4028
4029	bool IsKnownNoCapture;
4030	const AANoCapture *NoCaptureAA = nullptr;
4031	bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
4032	A, this, RVPos, DepClassTy::REQUIRED, IsKnownNoCapture, false,
4033	&NoCaptureAA);
4034	return IsAssumedNoCapture ||
4035	(NoCaptureAA && NoCaptureAA->isAssumedNoCaptureMaybeReturned());
4036	};
4037
4038	if (!A.checkForAllReturnedValues(CheckReturnValue, *this))
4039	return indicatePessimisticFixpoint();
4040
4041	return ChangeStatus::UNCHANGED;
4042	}
4043
4044	/// See AbstractAttribute::trackStatistics()
4045	void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noalias) }
4046	};
4047
4048	/// NoAlias attribute deduction for a call site return value.
4049	struct AANoAliasCallSiteReturned final
4050	: AACalleeToCallSite<AANoAlias, AANoAliasImpl> {
4051	AANoAliasCallSiteReturned(const IRPosition &IRP, Attributor &A)
4052	: AACalleeToCallSite<AANoAlias, AANoAliasImpl>(IRP, A) {}
4053
4054	/// See AbstractAttribute::trackStatistics()
4055	void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noalias); }
4056	};
4057	} // namespace
4058
4059	/// -------------------AAIsDead Function Attribute-----------------------
4060
4061	namespace {
4062	struct AAIsDeadValueImpl : public AAIsDead {
4063	AAIsDeadValueImpl(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {}
4064
4065	/// See AAIsDead::isAssumedDead().
4066	bool isAssumedDead() const override { return isAssumed(BitsEncoding: IS_DEAD); }
4067
4068	/// See AAIsDead::isKnownDead().
4069	bool isKnownDead() const override { return isKnown(BitsEncoding: IS_DEAD); }
4070
4071	/// See AAIsDead::isAssumedDead(BasicBlock *).
4072	bool isAssumedDead(const BasicBlock *BB) const override { return false; }
4073
4074	/// See AAIsDead::isKnownDead(BasicBlock *).
4075	bool isKnownDead(const BasicBlock *BB) const override { return false; }
4076
4077	/// See AAIsDead::isAssumedDead(Instruction *I).
4078	bool isAssumedDead(const Instruction *I) const override {
4079	return I == getCtxI() && isAssumedDead();
4080	}
4081
4082	/// See AAIsDead::isKnownDead(Instruction *I).
4083	bool isKnownDead(const Instruction *I) const override {
4084	return isAssumedDead(I) && isKnownDead();
4085	}
4086
4087	/// See AbstractAttribute::getAsStr().
4088	const std::string getAsStr(Attributor *A) const override {
4089	return isAssumedDead() ? "assumed-dead" : "assumed-live";
4090	}
4091
4092	/// Check if all uses are assumed dead.
4093	bool areAllUsesAssumedDead(Attributor &A, Value &V) {
4094	// Callers might not check the type, void has no uses.
4095	if (V.getType()->isVoidTy() || V.use_empty())
4096	return true;
4097
4098	// If we replace a value with a constant there are no uses left afterwards.
4099	if (!isa<Constant>(Val: V)) {
4100	if (auto *I = dyn_cast<Instruction>(Val: &V))
4101	if (!A.isRunOn(Fn&: *I->getFunction()))
4102	return false;
4103	bool UsedAssumedInformation = false;
4104	std::optional<Constant *> C =
4105	A.getAssumedConstant(V, AA: *this, UsedAssumedInformation);
4106	if (!C || *C)
4107	return true;
4108	}
4109
4110	auto UsePred = [&](const Use &U, bool &Follow) { return false; };
4111	// Explicitly set the dependence class to required because we want a long
4112	// chain of N dependent instructions to be considered live as soon as one is
4113	// without going through N update cycles. This is not required for
4114	// correctness.
4115	return A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V, /* CheckBBLivenessOnly */ false,
4116	LivenessDepClass: DepClassTy::REQUIRED,
4117	/* IgnoreDroppableUses */ false);
4118	}
4119
4120	/// Determine if \p I is assumed to be side-effect free.
4121	bool isAssumedSideEffectFree(Attributor &A, Instruction *I) {
4122	if (!I || wouldInstructionBeTriviallyDead(I))
4123	return true;
4124
4125	auto *CB = dyn_cast<CallBase>(Val: I);
4126	if (!CB || isa<IntrinsicInst>(Val: CB))
4127	return false;
4128
4129	const IRPosition &CallIRP = IRPosition::callsite_function(CB: *CB);
4130
4131	bool IsKnownNoUnwind;
4132	if (!AA::hasAssumedIRAttr<Attribute::NoUnwind>(
4133	A, this, CallIRP, DepClassTy::OPTIONAL, IsKnownNoUnwind))
4134	return false;
4135
4136	bool IsKnown;
4137	return AA::isAssumedReadOnly(A, IRP: CallIRP, QueryingAA: *this, IsKnown);
4138	}
4139	};
4140
4141	struct AAIsDeadFloating : public AAIsDeadValueImpl {
4142	AAIsDeadFloating(const IRPosition &IRP, Attributor &A)
4143	: AAIsDeadValueImpl(IRP, A) {}
4144
4145	/// See AbstractAttribute::initialize(...).
4146	void initialize(Attributor &A) override {
4147	AAIsDeadValueImpl::initialize(A);
4148
4149	if (isa<UndefValue>(Val: getAssociatedValue())) {
4150	indicatePessimisticFixpoint();
4151	return;
4152	}
4153
4154	Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
4155	if (!isAssumedSideEffectFree(A, I)) {
4156	if (!isa_and_nonnull<StoreInst>(Val: I) && !isa_and_nonnull<FenceInst>(Val: I))
4157	indicatePessimisticFixpoint();
4158	else
4159	removeAssumedBits(BitsEncoding: HAS_NO_EFFECT);
4160	}
4161	}
4162
4163	bool isDeadFence(Attributor &A, FenceInst &FI) {
4164	const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>(
4165	IRP: IRPosition::function(F: *FI.getFunction()), QueryingAA: *this, DepClass: DepClassTy::NONE);
4166	if (!ExecDomainAA || !ExecDomainAA->isNoOpFence(FI))
4167	return false;
4168	A.recordDependence(FromAA: *ExecDomainAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
4169	return true;
4170	}
4171
4172	bool isDeadStore(Attributor &A, StoreInst &SI,
4173	SmallSetVector<Instruction *, 8> *AssumeOnlyInst = nullptr) {
4174	// Lang ref now states volatile store is not UB/dead, let's skip them.
4175	if (SI.isVolatile())
4176	return false;
4177
4178	// If we are collecting assumes to be deleted we are in the manifest stage.
4179	// It's problematic to collect the potential copies again now so we use the
4180	// cached ones.
4181	bool UsedAssumedInformation = false;
4182	if (!AssumeOnlyInst) {
4183	PotentialCopies.clear();
4184	if (!AA::getPotentialCopiesOfStoredValue(A, SI, PotentialCopies, QueryingAA: *this,
4185	UsedAssumedInformation)) {
4186	LLVM_DEBUG(
4187	dbgs()
4188	<< "[AAIsDead] Could not determine potential copies of store!\n");
4189	return false;
4190	}
4191	}
4192	LLVM_DEBUG(dbgs() << "[AAIsDead] Store has " << PotentialCopies.size()
4193	<< " potential copies.\n");
4194
4195	InformationCache &InfoCache = A.getInfoCache();
4196	return llvm::all_of(Range&: PotentialCopies, P: [&](Value *V) {
4197	if (A.isAssumedDead(IRP: IRPosition::value(V: *V), QueryingAA: this, FnLivenessAA: nullptr,
4198	UsedAssumedInformation))
4199	return true;
4200	if (auto *LI = dyn_cast<LoadInst>(Val: V)) {
4201	if (llvm::all_of(Range: LI->uses(), P: [&](const Use &U) {
4202	auto &UserI = cast<Instruction>(Val&: *U.getUser());
4203	if (InfoCache.isOnlyUsedByAssume(I: UserI)) {
4204	if (AssumeOnlyInst)
4205	AssumeOnlyInst->insert(X: &UserI);
4206	return true;
4207	}
4208	return A.isAssumedDead(U, QueryingAA: this, FnLivenessAA: nullptr, UsedAssumedInformation);
4209	})) {
4210	return true;
4211	}
4212	}
4213	LLVM_DEBUG(dbgs() << "[AAIsDead] Potential copy " << *V
4214	<< " is assumed live!\n");
4215	return false;
4216	});
4217	}
4218
4219	/// See AbstractAttribute::getAsStr().
4220	const std::string getAsStr(Attributor *A) const override {
4221	Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
4222	if (isa_and_nonnull<StoreInst>(Val: I))
4223	if (isValidState())
4224	return "assumed-dead-store";
4225	if (isa_and_nonnull<FenceInst>(Val: I))
4226	if (isValidState())
4227	return "assumed-dead-fence";
4228	return AAIsDeadValueImpl::getAsStr(A);
4229	}
4230
4231	/// See AbstractAttribute::updateImpl(...).
4232	ChangeStatus updateImpl(Attributor &A) override {
4233	Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
4234	if (auto *SI = dyn_cast_or_null<StoreInst>(Val: I)) {
4235	if (!isDeadStore(A, SI&: *SI))
4236	return indicatePessimisticFixpoint();
4237	} else if (auto *FI = dyn_cast_or_null<FenceInst>(Val: I)) {
4238	if (!isDeadFence(A, FI&: *FI))
4239	return indicatePessimisticFixpoint();
4240	} else {
4241	if (!isAssumedSideEffectFree(A, I))
4242	return indicatePessimisticFixpoint();
4243	if (!areAllUsesAssumedDead(A, V&: getAssociatedValue()))
4244	return indicatePessimisticFixpoint();
4245	}
4246	return ChangeStatus::UNCHANGED;
4247	}
4248
4249	bool isRemovableStore() const override {
4250	return isAssumed(BitsEncoding: IS_REMOVABLE) && isa<StoreInst>(Val: &getAssociatedValue());
4251	}
4252
4253	/// See AbstractAttribute::manifest(...).
4254	ChangeStatus manifest(Attributor &A) override {
4255	Value &V = getAssociatedValue();
4256	if (auto *I = dyn_cast<Instruction>(Val: &V)) {
4257	// If we get here we basically know the users are all dead. We check if
4258	// isAssumedSideEffectFree returns true here again because it might not be
4259	// the case and only the users are dead but the instruction (=call) is
4260	// still needed.
4261	if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
4262	SmallSetVector<Instruction *, 8> AssumeOnlyInst;
4263	bool IsDead = isDeadStore(A, SI&: *SI, AssumeOnlyInst: &AssumeOnlyInst);
4264	(void)IsDead;
4265	assert(IsDead && "Store was assumed to be dead!");
4266	A.deleteAfterManifest(I&: *I);
4267	for (size_t i = 0; i < AssumeOnlyInst.size(); ++i) {
4268	Instruction *AOI = AssumeOnlyInst[i];
4269	for (auto *Usr : AOI->users())
4270	AssumeOnlyInst.insert(X: cast<Instruction>(Val: Usr));
4271	A.deleteAfterManifest(I&: *AOI);
4272	}
4273	return ChangeStatus::CHANGED;
4274	}
4275	if (auto *FI = dyn_cast<FenceInst>(Val: I)) {
4276	assert(isDeadFence(A, *FI));
4277	A.deleteAfterManifest(I&: *FI);
4278	return ChangeStatus::CHANGED;
4279	}
4280	if (isAssumedSideEffectFree(A, I) && !isa<InvokeInst>(Val: I)) {
4281	A.deleteAfterManifest(I&: *I);
4282	return ChangeStatus::CHANGED;
4283	}
4284	}
4285	return ChangeStatus::UNCHANGED;
4286	}
4287
4288	/// See AbstractAttribute::trackStatistics()
4289	void trackStatistics() const override {
4290	STATS_DECLTRACK_FLOATING_ATTR(IsDead)
4291	}
4292
4293	private:
4294	// The potential copies of a dead store, used for deletion during manifest.
4295	SmallSetVector<Value *, 4> PotentialCopies;
4296	};
4297
4298	struct AAIsDeadArgument : public AAIsDeadFloating {
4299	AAIsDeadArgument(const IRPosition &IRP, Attributor &A)
4300	: AAIsDeadFloating(IRP, A) {}
4301
4302	/// See AbstractAttribute::manifest(...).
4303	ChangeStatus manifest(Attributor &A) override {
4304	Argument &Arg = *getAssociatedArgument();
4305	if (A.isValidFunctionSignatureRewrite(Arg, /* ReplacementTypes */ {}))
4306	if (A.registerFunctionSignatureRewrite(
4307	Arg, /* ReplacementTypes */ {},
4308	CalleeRepairCB: Attributor::ArgumentReplacementInfo::CalleeRepairCBTy{},
4309	ACSRepairCB: Attributor::ArgumentReplacementInfo::ACSRepairCBTy{})) {
4310	return ChangeStatus::CHANGED;
4311	}
4312	return ChangeStatus::UNCHANGED;
4313	}
4314
4315	/// See AbstractAttribute::trackStatistics()
4316	void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(IsDead) }
4317	};
4318
4319	struct AAIsDeadCallSiteArgument : public AAIsDeadValueImpl {
4320	AAIsDeadCallSiteArgument(const IRPosition &IRP, Attributor &A)
4321	: AAIsDeadValueImpl(IRP, A) {}
4322
4323	/// See AbstractAttribute::initialize(...).
4324	void initialize(Attributor &A) override {
4325	AAIsDeadValueImpl::initialize(A);
4326	if (isa<UndefValue>(Val: getAssociatedValue()))
4327	indicatePessimisticFixpoint();
4328	}
4329
4330	/// See AbstractAttribute::updateImpl(...).
4331	ChangeStatus updateImpl(Attributor &A) override {
4332	// TODO: Once we have call site specific value information we can provide
4333	// call site specific liveness information and then it makes
4334	// sense to specialize attributes for call sites arguments instead of
4335	// redirecting requests to the callee argument.
4336	Argument *Arg = getAssociatedArgument();
4337	if (!Arg)
4338	return indicatePessimisticFixpoint();
4339	const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
4340	auto *ArgAA = A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
4341	if (!ArgAA)
4342	return indicatePessimisticFixpoint();
4343	return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState());
4344	}
4345
4346	/// See AbstractAttribute::manifest(...).
4347	ChangeStatus manifest(Attributor &A) override {
4348	CallBase &CB = cast<CallBase>(Val&: getAnchorValue());
4349	Use &U = CB.getArgOperandUse(i: getCallSiteArgNo());
4350	assert(!isa<UndefValue>(U.get()) &&
4351	"Expected undef values to be filtered out!");
4352	UndefValue &UV = *UndefValue::get(T: U->getType());
4353	if (A.changeUseAfterManifest(U, NV&: UV))
4354	return ChangeStatus::CHANGED;
4355	return ChangeStatus::UNCHANGED;
4356	}
4357
4358	/// See AbstractAttribute::trackStatistics()
4359	void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(IsDead) }
4360	};
4361
4362	struct AAIsDeadCallSiteReturned : public AAIsDeadFloating {
4363	AAIsDeadCallSiteReturned(const IRPosition &IRP, Attributor &A)
4364	: AAIsDeadFloating(IRP, A) {}
4365
4366	/// See AAIsDead::isAssumedDead().
4367	bool isAssumedDead() const override {
4368	return AAIsDeadFloating::isAssumedDead() && IsAssumedSideEffectFree;
4369	}
4370
4371	/// See AbstractAttribute::initialize(...).
4372	void initialize(Attributor &A) override {
4373	AAIsDeadFloating::initialize(A);
4374	if (isa<UndefValue>(Val: getAssociatedValue())) {
4375	indicatePessimisticFixpoint();
4376	return;
4377	}
4378
4379	// We track this separately as a secondary state.
4380	IsAssumedSideEffectFree = isAssumedSideEffectFree(A, I: getCtxI());
4381	}
4382
4383	/// See AbstractAttribute::updateImpl(...).
4384	ChangeStatus updateImpl(Attributor &A) override {
4385	ChangeStatus Changed = ChangeStatus::UNCHANGED;
4386	if (IsAssumedSideEffectFree && !isAssumedSideEffectFree(A, I: getCtxI())) {
4387	IsAssumedSideEffectFree = false;
4388	Changed = ChangeStatus::CHANGED;
4389	}
4390	if (!areAllUsesAssumedDead(A, V&: getAssociatedValue()))
4391	return indicatePessimisticFixpoint();
4392	return Changed;
4393	}
4394
4395	/// See AbstractAttribute::trackStatistics()
4396	void trackStatistics() const override {
4397	if (IsAssumedSideEffectFree)
4398	STATS_DECLTRACK_CSRET_ATTR(IsDead)
4399	else
4400	STATS_DECLTRACK_CSRET_ATTR(UnusedResult)
4401	}
4402
4403	/// See AbstractAttribute::getAsStr().
4404	const std::string getAsStr(Attributor *A) const override {
4405	return isAssumedDead()
4406	? "assumed-dead"
4407	: (getAssumed() ? "assumed-dead-users" : "assumed-live");
4408	}
4409
4410	private:
4411	bool IsAssumedSideEffectFree = true;
4412	};
4413
4414	struct AAIsDeadReturned : public AAIsDeadValueImpl {
4415	AAIsDeadReturned(const IRPosition &IRP, Attributor &A)
4416	: AAIsDeadValueImpl(IRP, A) {}
4417
4418	/// See AbstractAttribute::updateImpl(...).
4419	ChangeStatus updateImpl(Attributor &A) override {
4420
4421	bool UsedAssumedInformation = false;
4422	A.checkForAllInstructions(Pred: [](Instruction &) { return true; }, QueryingAA: *this,
4423	Opcodes: {Instruction::Ret}, UsedAssumedInformation);
4424
4425	auto PredForCallSite = [&](AbstractCallSite ACS) {
4426	if (ACS.isCallbackCall() || !ACS.getInstruction())
4427	return false;
4428	return areAllUsesAssumedDead(A, V&: *ACS.getInstruction());
4429	};
4430
4431	if (!A.checkForAllCallSites(Pred: PredForCallSite, QueryingAA: *this, RequireAllCallSites: true,
4432	UsedAssumedInformation))
4433	return indicatePessimisticFixpoint();
4434
4435	return ChangeStatus::UNCHANGED;
4436	}
4437
4438	/// See AbstractAttribute::manifest(...).
4439	ChangeStatus manifest(Attributor &A) override {
4440	// TODO: Rewrite the signature to return void?
4441	bool AnyChange = false;
4442	UndefValue &UV = *UndefValue::get(T: getAssociatedFunction()->getReturnType());
4443	auto RetInstPred = [&](Instruction &I) {
4444	ReturnInst &RI = cast<ReturnInst>(Val&: I);
4445	if (!isa<UndefValue>(Val: RI.getReturnValue()))
4446	AnyChange |= A.changeUseAfterManifest(U&: RI.getOperandUse(i: 0), NV&: UV);
4447	return true;
4448	};
4449	bool UsedAssumedInformation = false;
4450	A.checkForAllInstructions(Pred: RetInstPred, QueryingAA: *this, Opcodes: {Instruction::Ret},
4451	UsedAssumedInformation);
4452	return AnyChange ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
4453	}
4454
4455	/// See AbstractAttribute::trackStatistics()
4456	void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(IsDead) }
4457	};
4458
4459	struct AAIsDeadFunction : public AAIsDead {
4460	AAIsDeadFunction(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {}
4461
4462	/// See AbstractAttribute::initialize(...).
4463	void initialize(Attributor &A) override {
4464	Function *F = getAnchorScope();
4465	assert(F && "Did expect an anchor function");
4466	if (!isAssumedDeadInternalFunction(A)) {
4467	ToBeExploredFrom.insert(X: &F->getEntryBlock().front());
4468	assumeLive(A, BB: F->getEntryBlock());
4469	}
4470	}
4471
4472	bool isAssumedDeadInternalFunction(Attributor &A) {
4473	if (!getAnchorScope()->hasLocalLinkage())
4474	return false;
4475	bool UsedAssumedInformation = false;
4476	return A.checkForAllCallSites(Pred: [](AbstractCallSite) { return false; }, QueryingAA: *this,
4477	RequireAllCallSites: true, UsedAssumedInformation);
4478	}
4479
4480	/// See AbstractAttribute::getAsStr().
4481	const std::string getAsStr(Attributor *A) const override {
4482	return "Live[#BB " + std::to_string(val: AssumedLiveBlocks.size()) + "/" +
4483	std::to_string(val: getAnchorScope()->size()) + "][#TBEP " +
4484	std::to_string(val: ToBeExploredFrom.size()) + "][#KDE " +
4485	std::to_string(val: KnownDeadEnds.size()) + "]";
4486	}
4487
4488	/// See AbstractAttribute::manifest(...).
4489	ChangeStatus manifest(Attributor &A) override {
4490	assert(getState().isValidState() &&
4491	"Attempted to manifest an invalid state!");
4492
4493	ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
4494	Function &F = *getAnchorScope();
4495
4496	if (AssumedLiveBlocks.empty()) {
4497	A.deleteAfterManifest(F);
4498	return ChangeStatus::CHANGED;
4499	}
4500
4501	// Flag to determine if we can change an invoke to a call assuming the
4502	// callee is nounwind. This is not possible if the personality of the
4503	// function allows to catch asynchronous exceptions.
4504	bool Invoke2CallAllowed = !mayCatchAsynchronousExceptions(F);
4505
4506	KnownDeadEnds.set_union(ToBeExploredFrom);
4507	for (const Instruction *DeadEndI : KnownDeadEnds) {
4508	auto *CB = dyn_cast<CallBase>(Val: DeadEndI);
4509	if (!CB)
4510	continue;
4511	bool IsKnownNoReturn;
4512	bool MayReturn = !AA::hasAssumedIRAttr<Attribute::NoReturn>(
4513	A, this, IRPosition::callsite_function(*CB), DepClassTy::OPTIONAL,
4514	IsKnownNoReturn);
4515	if (MayReturn && (!Invoke2CallAllowed || !isa<InvokeInst>(Val: CB)))
4516	continue;
4517
4518	if (auto *II = dyn_cast<InvokeInst>(Val: DeadEndI))
4519	A.registerInvokeWithDeadSuccessor(II&: const_cast<InvokeInst &>(*II));
4520	else
4521	A.changeToUnreachableAfterManifest(
4522	I: const_cast<Instruction *>(DeadEndI->getNextNode()));
4523	HasChanged = ChangeStatus::CHANGED;
4524	}
4525
4526	STATS_DECL(AAIsDead, BasicBlock, "Number of dead basic blocks deleted.");
4527	for (BasicBlock &BB : F)
4528	if (!AssumedLiveBlocks.count(V: &BB)) {
4529	A.deleteAfterManifest(BB);
4530	++BUILD_STAT_NAME(AAIsDead, BasicBlock);
4531	HasChanged = ChangeStatus::CHANGED;
4532	}
4533
4534	return HasChanged;
4535	}
4536
4537	/// See AbstractAttribute::updateImpl(...).
4538	ChangeStatus updateImpl(Attributor &A) override;
4539
4540	bool isEdgeDead(const BasicBlock *From, const BasicBlock *To) const override {
4541	assert(From->getParent() == getAnchorScope() &&
4542	To->getParent() == getAnchorScope() &&
4543	"Used AAIsDead of the wrong function");
4544	return isValidState() && !AssumedLiveEdges.count(V: std::make_pair(x&: From, y&: To));
4545	}
4546
4547	/// See AbstractAttribute::trackStatistics()
4548	void trackStatistics() const override {}
4549
4550	/// Returns true if the function is assumed dead.
4551	bool isAssumedDead() const override { return false; }
4552
4553	/// See AAIsDead::isKnownDead().
4554	bool isKnownDead() const override { return false; }
4555
4556	/// See AAIsDead::isAssumedDead(BasicBlock *).
4557	bool isAssumedDead(const BasicBlock *BB) const override {
4558	assert(BB->getParent() == getAnchorScope() &&
4559	"BB must be in the same anchor scope function.");
4560
4561	if (!getAssumed())
4562	return false;
4563	return !AssumedLiveBlocks.count(V: BB);
4564	}
4565
4566	/// See AAIsDead::isKnownDead(BasicBlock *).
4567	bool isKnownDead(const BasicBlock *BB) const override {
4568	return getKnown() && isAssumedDead(BB);
4569	}
4570
4571	/// See AAIsDead::isAssumed(Instruction *I).
4572	bool isAssumedDead(const Instruction *I) const override {
4573	assert(I->getParent()->getParent() == getAnchorScope() &&
4574	"Instruction must be in the same anchor scope function.");
4575
4576	if (!getAssumed())
4577	return false;
4578
4579	// If it is not in AssumedLiveBlocks then it for sure dead.
4580	// Otherwise, it can still be after noreturn call in a live block.
4581	if (!AssumedLiveBlocks.count(V: I->getParent()))
4582	return true;
4583
4584	// If it is not after a liveness barrier it is live.
4585	const Instruction *PrevI = I->getPrevNode();
4586	while (PrevI) {
4587	if (KnownDeadEnds.count(key: PrevI) || ToBeExploredFrom.count(key: PrevI))
4588	return true;
4589	PrevI = PrevI->getPrevNode();
4590	}
4591	return false;
4592	}
4593
4594	/// See AAIsDead::isKnownDead(Instruction *I).
4595	bool isKnownDead(const Instruction *I) const override {
4596	return getKnown() && isAssumedDead(I);
4597	}
4598
4599	/// Assume \p BB is (partially) live now and indicate to the Attributor \p A
4600	/// that internal function called from \p BB should now be looked at.
4601	bool assumeLive(Attributor &A, const BasicBlock &BB) {
4602	if (!AssumedLiveBlocks.insert(V: &BB).second)
4603	return false;
4604
4605	// We assume that all of BB is (probably) live now and if there are calls to
4606	// internal functions we will assume that those are now live as well. This
4607	// is a performance optimization for blocks with calls to a lot of internal
4608	// functions. It can however cause dead functions to be treated as live.
4609	for (const Instruction &I : BB)
4610	if (const auto *CB = dyn_cast<CallBase>(Val: &I))
4611	if (auto *F = dyn_cast_if_present<Function>(Val: CB->getCalledOperand()))
4612	if (F->hasLocalLinkage())
4613	A.markLiveInternalFunction(F: *F);
4614	return true;
4615	}
4616
4617	/// Collection of instructions that need to be explored again, e.g., we
4618	/// did assume they do not transfer control to (one of their) successors.
4619	SmallSetVector<const Instruction *, 8> ToBeExploredFrom;
4620
4621	/// Collection of instructions that are known to not transfer control.
4622	SmallSetVector<const Instruction *, 8> KnownDeadEnds;
4623
4624	/// Collection of all assumed live edges
4625	DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> AssumedLiveEdges;
4626
4627	/// Collection of all assumed live BasicBlocks.
4628	DenseSet<const BasicBlock *> AssumedLiveBlocks;
4629	};
4630
4631	static bool
4632	identifyAliveSuccessors(Attributor &A, const CallBase &CB,
4633	AbstractAttribute &AA,
4634	SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4635	const IRPosition &IPos = IRPosition::callsite_function(CB);
4636
4637	bool IsKnownNoReturn;
4638	if (AA::hasAssumedIRAttr<Attribute::NoReturn>(
4639	A, &AA, IPos, DepClassTy::OPTIONAL, IsKnownNoReturn))
4640	return !IsKnownNoReturn;
4641	if (CB.isTerminator())
4642	AliveSuccessors.push_back(Elt: &CB.getSuccessor(Idx: 0)->front());
4643	else
4644	AliveSuccessors.push_back(Elt: CB.getNextNode());
4645	return false;
4646	}
4647
4648	static bool
4649	identifyAliveSuccessors(Attributor &A, const InvokeInst &II,
4650	AbstractAttribute &AA,
4651	SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4652	bool UsedAssumedInformation =
4653	identifyAliveSuccessors(A, CB: cast<CallBase>(Val: II), AA, AliveSuccessors);
4654
4655	// First, determine if we can change an invoke to a call assuming the
4656	// callee is nounwind. This is not possible if the personality of the
4657	// function allows to catch asynchronous exceptions.
4658	if (AAIsDeadFunction::mayCatchAsynchronousExceptions(F: *II.getFunction())) {
4659	AliveSuccessors.push_back(Elt: &II.getUnwindDest()->front());
4660	} else {
4661	const IRPosition &IPos = IRPosition::callsite_function(CB: II);
4662
4663	bool IsKnownNoUnwind;
4664	if (AA::hasAssumedIRAttr<Attribute::NoUnwind>(
4665	A, &AA, IPos, DepClassTy::OPTIONAL, IsKnownNoUnwind)) {
4666	UsedAssumedInformation |= !IsKnownNoUnwind;
4667	} else {
4668	AliveSuccessors.push_back(Elt: &II.getUnwindDest()->front());
4669	}
4670	}
4671	return UsedAssumedInformation;
4672	}
4673
4674	static bool
4675	identifyAliveSuccessors(Attributor &A, const BranchInst &BI,
4676	AbstractAttribute &AA,
4677	SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4678	bool UsedAssumedInformation = false;
4679	if (BI.getNumSuccessors() == 1) {
4680	AliveSuccessors.push_back(Elt: &BI.getSuccessor(i: 0)->front());
4681	} else {
4682	std::optional<Constant *> C =
4683	A.getAssumedConstant(V: *BI.getCondition(), AA, UsedAssumedInformation);
4684	if (!C || isa_and_nonnull<UndefValue>(Val: *C)) {
4685	// No value yet, assume both edges are dead.
4686	} else if (isa_and_nonnull<ConstantInt>(Val: *C)) {
4687	const BasicBlock *SuccBB =
4688	BI.getSuccessor(i: 1 - cast<ConstantInt>(Val: *C)->getValue().getZExtValue());
4689	AliveSuccessors.push_back(Elt: &SuccBB->front());
4690	} else {
4691	AliveSuccessors.push_back(Elt: &BI.getSuccessor(i: 0)->front());
4692	AliveSuccessors.push_back(Elt: &BI.getSuccessor(i: 1)->front());
4693	UsedAssumedInformation = false;
4694	}
4695	}
4696	return UsedAssumedInformation;
4697	}
4698
4699	static bool
4700	identifyAliveSuccessors(Attributor &A, const SwitchInst &SI,
4701	AbstractAttribute &AA,
4702	SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4703	bool UsedAssumedInformation = false;
4704	SmallVector<AA::ValueAndContext> Values;
4705	if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: *SI.getCondition()), AA: &AA,
4706	Values, S: AA::AnyScope,
4707	UsedAssumedInformation)) {
4708	// Something went wrong, assume all successors are live.
4709	for (const BasicBlock *SuccBB : successors(BB: SI.getParent()))
4710	AliveSuccessors.push_back(Elt: &SuccBB->front());
4711	return false;
4712	}
4713
4714	if (Values.empty() ||
4715	(Values.size() == 1 &&
4716	isa_and_nonnull<UndefValue>(Val: Values.front().getValue()))) {
4717	// No valid value yet, assume all edges are dead.
4718	return UsedAssumedInformation;
4719	}
4720
4721	Type &Ty = *SI.getCondition()->getType();
4722	SmallPtrSet<ConstantInt *, 8> Constants;
4723	auto CheckForConstantInt = [&](Value *V) {
4724	if (auto *CI = dyn_cast_if_present<ConstantInt>(Val: AA::getWithType(V&: *V, Ty))) {
4725	Constants.insert(Ptr: CI);
4726	return true;
4727	}
4728	return false;
4729	};
4730
4731	if (!all_of(Range&: Values, P: [&](AA::ValueAndContext &VAC) {
4732	return CheckForConstantInt(VAC.getValue());
4733	})) {
4734	for (const BasicBlock *SuccBB : successors(BB: SI.getParent()))
4735	AliveSuccessors.push_back(Elt: &SuccBB->front());
4736	return UsedAssumedInformation;
4737	}
4738
4739	unsigned MatchedCases = 0;
4740	for (const auto &CaseIt : SI.cases()) {
4741	if (Constants.count(Ptr: CaseIt.getCaseValue())) {
4742	++MatchedCases;
4743	AliveSuccessors.push_back(Elt: &CaseIt.getCaseSuccessor()->front());
4744	}
4745	}
4746
4747	// If all potential values have been matched, we will not visit the default
4748	// case.
4749	if (MatchedCases < Constants.size())
4750	AliveSuccessors.push_back(Elt: &SI.getDefaultDest()->front());
4751	return UsedAssumedInformation;
4752	}
4753
4754	ChangeStatus AAIsDeadFunction::updateImpl(Attributor &A) {
4755	ChangeStatus Change = ChangeStatus::UNCHANGED;
4756
4757	if (AssumedLiveBlocks.empty()) {
4758	if (isAssumedDeadInternalFunction(A))
4759	return ChangeStatus::UNCHANGED;
4760
4761	Function *F = getAnchorScope();
4762	ToBeExploredFrom.insert(X: &F->getEntryBlock().front());
4763	assumeLive(A, BB: F->getEntryBlock());
4764	Change = ChangeStatus::CHANGED;
4765	}
4766
4767	LLVM_DEBUG(dbgs() << "[AAIsDead] Live [" << AssumedLiveBlocks.size() << "/"
4768	<< getAnchorScope()->size() << "] BBs and "
4769	<< ToBeExploredFrom.size() << " exploration points and "
4770	<< KnownDeadEnds.size() << " known dead ends\n");
4771
4772	// Copy and clear the list of instructions we need to explore from. It is
4773	// refilled with instructions the next update has to look at.
4774	SmallVector<const Instruction *, 8> Worklist(ToBeExploredFrom.begin(),
4775	ToBeExploredFrom.end());
4776	decltype(ToBeExploredFrom) NewToBeExploredFrom;
4777
4778	SmallVector<const Instruction *, 8> AliveSuccessors;
4779	while (!Worklist.empty()) {
4780	const Instruction *I = Worklist.pop_back_val();
4781	LLVM_DEBUG(dbgs() << "[AAIsDead] Exploration inst: " << *I << "\n");
4782
4783	// Fast forward for uninteresting instructions. We could look for UB here
4784	// though.
4785	while (!I->isTerminator() && !isa<CallBase>(Val: I))
4786	I = I->getNextNode();
4787
4788	AliveSuccessors.clear();
4789
4790	bool UsedAssumedInformation = false;
4791	switch (I->getOpcode()) {
4792	// TODO: look for (assumed) UB to backwards propagate "deadness".
4793	default:
4794	assert(I->isTerminator() &&
4795	"Expected non-terminators to be handled already!");
4796	for (const BasicBlock *SuccBB : successors(BB: I->getParent()))
4797	AliveSuccessors.push_back(Elt: &SuccBB->front());
4798	break;
4799	case Instruction::Call:
4800	UsedAssumedInformation = identifyAliveSuccessors(A, CB: cast<CallInst>(Val: *I),
4801	AA&: *this, AliveSuccessors);
4802	break;
4803	case Instruction::Invoke:
4804	UsedAssumedInformation = identifyAliveSuccessors(A, II: cast<InvokeInst>(Val: *I),
4805	AA&: *this, AliveSuccessors);
4806	break;
4807	case Instruction::Br:
4808	UsedAssumedInformation = identifyAliveSuccessors(A, BI: cast<BranchInst>(Val: *I),
4809	AA&: *this, AliveSuccessors);
4810	break;
4811	case Instruction::Switch:
4812	UsedAssumedInformation = identifyAliveSuccessors(A, SI: cast<SwitchInst>(Val: *I),
4813	AA&: *this, AliveSuccessors);
4814	break;
4815	}
4816
4817	if (UsedAssumedInformation) {
4818	NewToBeExploredFrom.insert(X: I);
4819	} else if (AliveSuccessors.empty() ||
4820	(I->isTerminator() &&
4821	AliveSuccessors.size() < I->getNumSuccessors())) {
4822	if (KnownDeadEnds.insert(X: I))
4823	Change = ChangeStatus::CHANGED;
4824	}
4825
4826	LLVM_DEBUG(dbgs() << "[AAIsDead] #AliveSuccessors: "
4827	<< AliveSuccessors.size() << " UsedAssumedInformation: "
4828	<< UsedAssumedInformation << "\n");
4829
4830	for (const Instruction *AliveSuccessor : AliveSuccessors) {
4831	if (!I->isTerminator()) {
4832	assert(AliveSuccessors.size() == 1 &&
4833	"Non-terminator expected to have a single successor!");
4834	Worklist.push_back(Elt: AliveSuccessor);
4835	} else {
4836	// record the assumed live edge
4837	auto Edge = std::make_pair(x: I->getParent(), y: AliveSuccessor->getParent());
4838	if (AssumedLiveEdges.insert(V: Edge).second)
4839	Change = ChangeStatus::CHANGED;
4840	if (assumeLive(A, BB: *AliveSuccessor->getParent()))
4841	Worklist.push_back(Elt: AliveSuccessor);
4842	}
4843	}
4844	}
4845
4846	// Check if the content of ToBeExploredFrom changed, ignore the order.
4847	if (NewToBeExploredFrom.size() != ToBeExploredFrom.size() ||
4848	llvm::any_of(Range&: NewToBeExploredFrom, P: [&](const Instruction *I) {
4849	return !ToBeExploredFrom.count(key: I);
4850	})) {
4851	Change = ChangeStatus::CHANGED;
4852	ToBeExploredFrom = std::move(NewToBeExploredFrom);
4853	}
4854
4855	// If we know everything is live there is no need to query for liveness.
4856	// Instead, indicating a pessimistic fixpoint will cause the state to be
4857	// "invalid" and all queries to be answered conservatively without lookups.
4858	// To be in this state we have to (1) finished the exploration and (3) not
4859	// discovered any non-trivial dead end and (2) not ruled unreachable code
4860	// dead.
4861	if (ToBeExploredFrom.empty() &&
4862	getAnchorScope()->size() == AssumedLiveBlocks.size() &&
4863	llvm::all_of(Range&: KnownDeadEnds, P: [](const Instruction *DeadEndI) {
4864	return DeadEndI->isTerminator() && DeadEndI->getNumSuccessors() == 0;
4865	}))
4866	return indicatePessimisticFixpoint();
4867	return Change;
4868	}
4869
4870	/// Liveness information for a call sites.
4871	struct AAIsDeadCallSite final : AAIsDeadFunction {
4872	AAIsDeadCallSite(const IRPosition &IRP, Attributor &A)
4873	: AAIsDeadFunction(IRP, A) {}
4874
4875	/// See AbstractAttribute::initialize(...).
4876	void initialize(Attributor &A) override {
4877	// TODO: Once we have call site specific value information we can provide
4878	// call site specific liveness information and then it makes
4879	// sense to specialize attributes for call sites instead of
4880	// redirecting requests to the callee.
4881	llvm_unreachable("Abstract attributes for liveness are not "
4882	"supported for call sites yet!");
4883	}
4884
4885	/// See AbstractAttribute::updateImpl(...).
4886	ChangeStatus updateImpl(Attributor &A) override {
4887	return indicatePessimisticFixpoint();
4888	}
4889
4890	/// See AbstractAttribute::trackStatistics()
4891	void trackStatistics() const override {}
4892	};
4893	} // namespace
4894
4895	/// -------------------- Dereferenceable Argument Attribute --------------------
4896
4897	namespace {
4898	struct AADereferenceableImpl : AADereferenceable {
4899	AADereferenceableImpl(const IRPosition &IRP, Attributor &A)
4900	: AADereferenceable(IRP, A) {}
4901	using StateType = DerefState;
4902
4903	/// See AbstractAttribute::initialize(...).
4904	void initialize(Attributor &A) override {
4905	Value &V = *getAssociatedValue().stripPointerCasts();
4906	SmallVector<Attribute, 4> Attrs;
4907	A.getAttrs(getIRPosition(),
4908	{Attribute::Dereferenceable, Attribute::DereferenceableOrNull},
4909	Attrs, /* IgnoreSubsumingPositions */ false);
4910	for (const Attribute &Attr : Attrs)
4911	takeKnownDerefBytesMaximum(Attr.getValueAsInt());
4912
4913	// Ensure we initialize the non-null AA (if necessary).
4914	bool IsKnownNonNull;
4915	AA::hasAssumedIRAttr<Attribute::NonNull>(
4916	A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnownNonNull);
4917
4918	bool CanBeNull, CanBeFreed;
4919	takeKnownDerefBytesMaximum(V.getPointerDereferenceableBytes(
4920	DL: A.getDataLayout(), CanBeNull, CanBeFreed));
4921
4922	if (Instruction *CtxI = getCtxI())
4923	followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
4924	}
4925
4926	/// See AbstractAttribute::getState()
4927	/// {
4928	StateType &getState() override { return *this; }
4929	const StateType &getState() const override { return *this; }
4930	/// }
4931
4932	/// Helper function for collecting accessed bytes in must-be-executed-context
4933	void addAccessedBytesForUse(Attributor &A, const Use *U, const Instruction *I,
4934	DerefState &State) {
4935	const Value *UseV = U->get();
4936	if (!UseV->getType()->isPointerTy())
4937	return;
4938
4939	std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(Inst: I);
4940	if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() || I->isVolatile())
4941	return;
4942
4943	int64_t Offset;
4944	const Value *Base = GetPointerBaseWithConstantOffset(
4945	Ptr: Loc->Ptr, Offset, DL: A.getDataLayout(), /*AllowNonInbounds*/ true);
4946	if (Base && Base == &getAssociatedValue())
4947	State.addAccessedBytes(Offset, Size: Loc->Size.getValue());
4948	}
4949
4950	/// See followUsesInMBEC
4951	bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
4952	AADereferenceable::StateType &State) {
4953	bool IsNonNull = false;
4954	bool TrackUse = false;
4955	int64_t DerefBytes = getKnownNonNullAndDerefBytesForUse(
4956	A, *this, getAssociatedValue(), U, I, IsNonNull, TrackUse);
4957	LLVM_DEBUG(dbgs() << "[AADereferenceable] Deref bytes: " << DerefBytes
4958	<< " for instruction " << *I << "\n");
4959
4960	addAccessedBytesForUse(A, U, I, State&: State);
4961	State.takeKnownDerefBytesMaximum(DerefBytes);
4962	return TrackUse;
4963	}
4964
4965	/// See AbstractAttribute::manifest(...).
4966	ChangeStatus manifest(Attributor &A) override {
4967	ChangeStatus Change = AADereferenceable::manifest(A);
4968	bool IsKnownNonNull;
4969	bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
4970	A, this, getIRPosition(), DepClassTy::NONE, IsKnownNonNull);
4971	if (IsAssumedNonNull &&
4972	A.hasAttr(getIRPosition(), Attribute::DereferenceableOrNull)) {
4973	A.removeAttrs(getIRPosition(), {Attribute::DereferenceableOrNull});
4974	return ChangeStatus::CHANGED;
4975	}
4976	return Change;
4977	}
4978
4979	void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
4980	SmallVectorImpl<Attribute> &Attrs) const override {
4981	// TODO: Add *_globally support
4982	bool IsKnownNonNull;
4983	bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
4984	A, this, getIRPosition(), DepClassTy::NONE, IsKnownNonNull);
4985	if (IsAssumedNonNull)
4986	Attrs.emplace_back(Attribute::getWithDereferenceableBytes(
4987	Context&: Ctx, Bytes: getAssumedDereferenceableBytes()));
4988	else
4989	Attrs.emplace_back(Attribute::getWithDereferenceableOrNullBytes(
4990	Context&: Ctx, Bytes: getAssumedDereferenceableBytes()));
4991	}
4992
4993	/// See AbstractAttribute::getAsStr().
4994	const std::string getAsStr(Attributor *A) const override {
4995	if (!getAssumedDereferenceableBytes())
4996	return "unknown-dereferenceable";
4997	bool IsKnownNonNull;
4998	bool IsAssumedNonNull = false;
4999	if (A)
5000	IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
5001	*A, this, getIRPosition(), DepClassTy::NONE, IsKnownNonNull);
5002	return std::string("dereferenceable") +
5003	(IsAssumedNonNull ? "" : "_or_null") +
5004	(isAssumedGlobal() ? "_globally" : "") + "<" +
5005	std::to_string(getKnownDereferenceableBytes()) + "-" +
5006	std::to_string(getAssumedDereferenceableBytes()) + ">" +
5007	(!A ? " [non-null is unknown]" : "");
5008	}
5009	};
5010
5011	/// Dereferenceable attribute for a floating value.
5012	struct AADereferenceableFloating : AADereferenceableImpl {
5013	AADereferenceableFloating(const IRPosition &IRP, Attributor &A)
5014	: AADereferenceableImpl(IRP, A) {}
5015
5016	/// See AbstractAttribute::updateImpl(...).
5017	ChangeStatus updateImpl(Attributor &A) override {
5018	bool Stripped;
5019	bool UsedAssumedInformation = false;
5020	SmallVector<AA::ValueAndContext> Values;
5021	if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
5022	S: AA::AnyScope, UsedAssumedInformation)) {
5023	Values.push_back(Elt: {getAssociatedValue(), getCtxI()});
5024	Stripped = false;
5025	} else {
5026	Stripped = Values.size() != 1 ||
5027	Values.front().getValue() != &getAssociatedValue();
5028	}
5029
5030	const DataLayout &DL = A.getDataLayout();
5031	DerefState T;
5032
5033	auto VisitValueCB = [&](const Value &V) -> bool {
5034	unsigned IdxWidth =
5035	DL.getIndexSizeInBits(AS: V.getType()->getPointerAddressSpace());
5036	APInt Offset(IdxWidth, 0);
5037	const Value *Base = stripAndAccumulateOffsets(
5038	A, *this, &V, DL, Offset, /* GetMinOffset */ false,
5039	/* AllowNonInbounds */ true);
5040
5041	const auto *AA = A.getAAFor<AADereferenceable>(
5042	*this, IRPosition::value(V: *Base), DepClassTy::REQUIRED);
5043	int64_t DerefBytes = 0;
5044	if (!AA || (!Stripped && this == AA)) {
5045	// Use IR information if we did not strip anything.
5046	// TODO: track globally.
5047	bool CanBeNull, CanBeFreed;
5048	DerefBytes =
5049	Base->getPointerDereferenceableBytes(DL, CanBeNull, CanBeFreed);
5050	T.GlobalState.indicatePessimisticFixpoint();
5051	} else {
5052	const DerefState &DS = AA->getState();
5053	DerefBytes = DS.DerefBytesState.getAssumed();
5054	T.GlobalState &= DS.GlobalState;
5055	}
5056
5057	// For now we do not try to "increase" dereferenceability due to negative
5058	// indices as we first have to come up with code to deal with loops and
5059	// for overflows of the dereferenceable bytes.
5060	int64_t OffsetSExt = Offset.getSExtValue();
5061	if (OffsetSExt < 0)
5062	OffsetSExt = 0;
5063
5064	T.takeAssumedDerefBytesMinimum(
5065	Bytes: std::max(a: int64_t(0), b: DerefBytes - OffsetSExt));
5066
5067	if (this == AA) {
5068	if (!Stripped) {
5069	// If nothing was stripped IR information is all we got.
5070	T.takeKnownDerefBytesMaximum(
5071	Bytes: std::max(a: int64_t(0), b: DerefBytes - OffsetSExt));
5072	T.indicatePessimisticFixpoint();
5073	} else if (OffsetSExt > 0) {
5074	// If something was stripped but there is circular reasoning we look
5075	// for the offset. If it is positive we basically decrease the
5076	// dereferenceable bytes in a circular loop now, which will simply
5077	// drive them down to the known value in a very slow way which we
5078	// can accelerate.
5079	T.indicatePessimisticFixpoint();
5080	}
5081	}
5082
5083	return T.isValidState();
5084	};
5085
5086	for (const auto &VAC : Values)
5087	if (!VisitValueCB(*VAC.getValue()))
5088	return indicatePessimisticFixpoint();
5089
5090	return clampStateAndIndicateChange(getState(), T);
5091	}
5092
5093	/// See AbstractAttribute::trackStatistics()
5094	void trackStatistics() const override {
5095	STATS_DECLTRACK_FLOATING_ATTR(dereferenceable)
5096	}
5097	};
5098
5099	/// Dereferenceable attribute for a return value.
5100	struct AADereferenceableReturned final
5101	: AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl> {
5102	using Base =
5103	AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl>;
5104	AADereferenceableReturned(const IRPosition &IRP, Attributor &A)
5105	: Base(IRP, A) {}
5106
5107	/// See AbstractAttribute::trackStatistics()
5108	void trackStatistics() const override {
5109	STATS_DECLTRACK_FNRET_ATTR(dereferenceable)
5110	}
5111	};
5112
5113	/// Dereferenceable attribute for an argument
5114	struct AADereferenceableArgument final
5115	: AAArgumentFromCallSiteArguments<AADereferenceable,
5116	AADereferenceableImpl> {
5117	using Base =
5118	AAArgumentFromCallSiteArguments<AADereferenceable, AADereferenceableImpl>;
5119	AADereferenceableArgument(const IRPosition &IRP, Attributor &A)
5120	: Base(IRP, A) {}
5121
5122	/// See AbstractAttribute::trackStatistics()
5123	void trackStatistics() const override {
5124	STATS_DECLTRACK_ARG_ATTR(dereferenceable)
5125	}
5126	};
5127
5128	/// Dereferenceable attribute for a call site argument.
5129	struct AADereferenceableCallSiteArgument final : AADereferenceableFloating {
5130	AADereferenceableCallSiteArgument(const IRPosition &IRP, Attributor &A)
5131	: AADereferenceableFloating(IRP, A) {}
5132
5133	/// See AbstractAttribute::trackStatistics()
5134	void trackStatistics() const override {
5135	STATS_DECLTRACK_CSARG_ATTR(dereferenceable)
5136	}
5137	};
5138
5139	/// Dereferenceable attribute deduction for a call site return value.
5140	struct AADereferenceableCallSiteReturned final
5141	: AACalleeToCallSite<AADereferenceable, AADereferenceableImpl> {
5142	using Base = AACalleeToCallSite<AADereferenceable, AADereferenceableImpl>;
5143	AADereferenceableCallSiteReturned(const IRPosition &IRP, Attributor &A)
5144	: Base(IRP, A) {}
5145
5146	/// See AbstractAttribute::trackStatistics()
5147	void trackStatistics() const override {
5148	STATS_DECLTRACK_CS_ATTR(dereferenceable);
5149	}
5150	};
5151	} // namespace
5152
5153	// ------------------------ Align Argument Attribute ------------------------
5154
5155	namespace {
5156	static unsigned getKnownAlignForUse(Attributor &A, AAAlign &QueryingAA,
5157	Value &AssociatedValue, const Use *U,
5158	const Instruction *I, bool &TrackUse) {
5159	// We need to follow common pointer manipulation uses to the accesses they
5160	// feed into.
5161	if (isa<CastInst>(Val: I)) {
5162	// Follow all but ptr2int casts.
5163	TrackUse = !isa<PtrToIntInst>(Val: I);
5164	return 0;
5165	}
5166	if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: I)) {
5167	if (GEP->hasAllConstantIndices())
5168	TrackUse = true;
5169	return 0;
5170	}
5171
5172	MaybeAlign MA;
5173	if (const auto *CB = dyn_cast<CallBase>(Val: I)) {
5174	if (CB->isBundleOperand(U) || CB->isCallee(U))
5175	return 0;
5176
5177	unsigned ArgNo = CB->getArgOperandNo(U);
5178	IRPosition IRP = IRPosition::callsite_argument(CB: *CB, ArgNo);
5179	// As long as we only use known information there is no need to track
5180	// dependences here.
5181	auto *AlignAA = A.getAAFor<AAAlign>(QueryingAA, IRP, DepClassTy::NONE);
5182	if (AlignAA)
5183	MA = MaybeAlign(AlignAA->getKnownAlign());
5184	}
5185
5186	const DataLayout &DL = A.getDataLayout();
5187	const Value *UseV = U->get();
5188	if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
5189	if (SI->getPointerOperand() == UseV)
5190	MA = SI->getAlign();
5191	} else if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
5192	if (LI->getPointerOperand() == UseV)
5193	MA = LI->getAlign();
5194	} else if (auto *AI = dyn_cast<AtomicRMWInst>(Val: I)) {
5195	if (AI->getPointerOperand() == UseV)
5196	MA = AI->getAlign();
5197	} else if (auto *AI = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
5198	if (AI->getPointerOperand() == UseV)
5199	MA = AI->getAlign();
5200	}
5201
5202	if (!MA || *MA <= QueryingAA.getKnownAlign())
5203	return 0;
5204
5205	unsigned Alignment = MA->value();
5206	int64_t Offset;
5207
5208	if (const Value *Base = GetPointerBaseWithConstantOffset(Ptr: UseV, Offset, DL)) {
5209	if (Base == &AssociatedValue) {
5210	// BasePointerAddr + Offset = Alignment * Q for some integer Q.
5211	// So we can say that the maximum power of two which is a divisor of
5212	// gcd(Offset, Alignment) is an alignment.
5213
5214	uint32_t gcd = std::gcd(m: uint32_t(abs(x: (int32_t)Offset)), n: Alignment);
5215	Alignment = llvm::bit_floor(Value: gcd);
5216	}
5217	}
5218
5219	return Alignment;
5220	}
5221
5222	struct AAAlignImpl : AAAlign {
5223	AAAlignImpl(const IRPosition &IRP, Attributor &A) : AAAlign(IRP, A) {}
5224
5225	/// See AbstractAttribute::initialize(...).
5226	void initialize(Attributor &A) override {
5227	SmallVector<Attribute, 4> Attrs;
5228	A.getAttrs(getIRPosition(), {Attribute::Alignment}, Attrs);
5229	for (const Attribute &Attr : Attrs)
5230	takeKnownMaximum(Attr.getValueAsInt());
5231
5232	Value &V = *getAssociatedValue().stripPointerCasts();
5233	takeKnownMaximum(V.getPointerAlignment(DL: A.getDataLayout()).value());
5234
5235	if (Instruction *CtxI = getCtxI())
5236	followUsesInMBEC(*this, A, getState(), *CtxI);
5237	}
5238
5239	/// See AbstractAttribute::manifest(...).
5240	ChangeStatus manifest(Attributor &A) override {
5241	ChangeStatus LoadStoreChanged = ChangeStatus::UNCHANGED;
5242
5243	// Check for users that allow alignment annotations.
5244	Value &AssociatedValue = getAssociatedValue();
5245	for (const Use &U : AssociatedValue.uses()) {
5246	if (auto *SI = dyn_cast<StoreInst>(U.getUser())) {
5247	if (SI->getPointerOperand() == &AssociatedValue)
5248	if (SI->getAlign() < getAssumedAlign()) {
5249	STATS_DECLTRACK(AAAlign, Store,
5250	"Number of times alignment added to a store");
5251	SI->setAlignment(getAssumedAlign());
5252	LoadStoreChanged = ChangeStatus::CHANGED;
5253	}
5254	} else if (auto *LI = dyn_cast<LoadInst>(U.getUser())) {
5255	if (LI->getPointerOperand() == &AssociatedValue)
5256	if (LI->getAlign() < getAssumedAlign()) {
5257	LI->setAlignment(getAssumedAlign());
5258	STATS_DECLTRACK(AAAlign, Load,
5259	"Number of times alignment added to a load");
5260	LoadStoreChanged = ChangeStatus::CHANGED;
5261	}
5262	}
5263	}
5264
5265	ChangeStatus Changed = AAAlign::manifest(A);
5266
5267	Align InheritAlign =
5268	getAssociatedValue().getPointerAlignment(A.getDataLayout());
5269	if (InheritAlign >= getAssumedAlign())
5270	return LoadStoreChanged;
5271	return Changed | LoadStoreChanged;
5272	}
5273
5274	// TODO: Provide a helper to determine the implied ABI alignment and check in
5275	// the existing manifest method and a new one for AAAlignImpl that value
5276	// to avoid making the alignment explicit if it did not improve.
5277
5278	/// See AbstractAttribute::getDeducedAttributes
5279	void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
5280	SmallVectorImpl<Attribute> &Attrs) const override {
5281	if (getAssumedAlign() > 1)
5282	Attrs.emplace_back(
5283	Args: Attribute::getWithAlignment(Context&: Ctx, Alignment: Align(getAssumedAlign())));
5284	}
5285
5286	/// See followUsesInMBEC
5287	bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
5288	AAAlign::StateType &State) {
5289	bool TrackUse = false;
5290
5291	unsigned int KnownAlign =
5292	getKnownAlignForUse(A, *this, getAssociatedValue(), U, I, TrackUse);
5293	State.takeKnownMaximum(KnownAlign);
5294
5295	return TrackUse;
5296	}
5297
5298	/// See AbstractAttribute::getAsStr().
5299	const std::string getAsStr(Attributor *A) const override {
5300	return "align<" + std::to_string(getKnownAlign().value()) + "-" +
5301	std::to_string(getAssumedAlign().value()) + ">";
5302	}
5303	};
5304
5305	/// Align attribute for a floating value.
5306	struct AAAlignFloating : AAAlignImpl {
5307	AAAlignFloating(const IRPosition &IRP, Attributor &A) : AAAlignImpl(IRP, A) {}
5308
5309	/// See AbstractAttribute::updateImpl(...).
5310	ChangeStatus updateImpl(Attributor &A) override {
5311	const DataLayout &DL = A.getDataLayout();
5312
5313	bool Stripped;
5314	bool UsedAssumedInformation = false;
5315	SmallVector<AA::ValueAndContext> Values;
5316	if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
5317	S: AA::AnyScope, UsedAssumedInformation)) {
5318	Values.push_back(Elt: {getAssociatedValue(), getCtxI()});
5319	Stripped = false;
5320	} else {
5321	Stripped = Values.size() != 1 ||
5322	Values.front().getValue() != &getAssociatedValue();
5323	}
5324
5325	StateType T;
5326	auto VisitValueCB = [&](Value &V) -> bool {
5327	if (isa<UndefValue>(Val: V) || isa<ConstantPointerNull>(Val: V))
5328	return true;
5329	const auto *AA = A.getAAFor<AAAlign>(*this, IRPosition::value(V),
5330	DepClassTy::REQUIRED);
5331	if (!AA || (!Stripped && this == AA)) {
5332	int64_t Offset;
5333	unsigned Alignment = 1;
5334	if (const Value *Base =
5335	GetPointerBaseWithConstantOffset(Ptr: &V, Offset, DL)) {
5336	// TODO: Use AAAlign for the base too.
5337	Align PA = Base->getPointerAlignment(DL);
5338	// BasePointerAddr + Offset = Alignment * Q for some integer Q.
5339	// So we can say that the maximum power of two which is a divisor of
5340	// gcd(Offset, Alignment) is an alignment.
5341
5342	uint32_t gcd =
5343	std::gcd(m: uint32_t(abs(x: (int32_t)Offset)), n: uint32_t(PA.value()));
5344	Alignment = llvm::bit_floor(Value: gcd);
5345	} else {
5346	Alignment = V.getPointerAlignment(DL).value();
5347	}
5348	// Use only IR information if we did not strip anything.
5349	T.takeKnownMaximum(Alignment);
5350	T.indicatePessimisticFixpoint();
5351	} else {
5352	// Use abstract attribute information.
5353	const AAAlign::StateType &DS = AA->getState();
5354	T ^= DS;
5355	}
5356	return T.isValidState();
5357	};
5358
5359	for (const auto &VAC : Values) {
5360	if (!VisitValueCB(*VAC.getValue()))
5361	return indicatePessimisticFixpoint();
5362	}
5363
5364	// TODO: If we know we visited all incoming values, thus no are assumed
5365	// dead, we can take the known information from the state T.
5366	return clampStateAndIndicateChange(getState(), T);
5367	}
5368
5369	/// See AbstractAttribute::trackStatistics()
5370	void trackStatistics() const override { STATS_DECLTRACK_FLOATING_ATTR(align) }
5371	};
5372
5373	/// Align attribute for function return value.
5374	struct AAAlignReturned final
5375	: AAReturnedFromReturnedValues<AAAlign, AAAlignImpl> {
5376	using Base = AAReturnedFromReturnedValues<AAAlign, AAAlignImpl>;
5377	AAAlignReturned(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
5378
5379	/// See AbstractAttribute::trackStatistics()
5380	void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(aligned) }
5381	};
5382
5383	/// Align attribute for function argument.
5384	struct AAAlignArgument final
5385	: AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl> {
5386	using Base = AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl>;
5387	AAAlignArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
5388
5389	/// See AbstractAttribute::manifest(...).
5390	ChangeStatus manifest(Attributor &A) override {
5391	// If the associated argument is involved in a must-tail call we give up
5392	// because we would need to keep the argument alignments of caller and
5393	// callee in-sync. Just does not seem worth the trouble right now.
5394	if (A.getInfoCache().isInvolvedInMustTailCall(Arg: *getAssociatedArgument()))
5395	return ChangeStatus::UNCHANGED;
5396	return Base::manifest(A);
5397	}
5398
5399	/// See AbstractAttribute::trackStatistics()
5400	void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(aligned) }
5401	};
5402
5403	struct AAAlignCallSiteArgument final : AAAlignFloating {
5404	AAAlignCallSiteArgument(const IRPosition &IRP, Attributor &A)
5405	: AAAlignFloating(IRP, A) {}
5406
5407	/// See AbstractAttribute::manifest(...).
5408	ChangeStatus manifest(Attributor &A) override {
5409	// If the associated argument is involved in a must-tail call we give up
5410	// because we would need to keep the argument alignments of caller and
5411	// callee in-sync. Just does not seem worth the trouble right now.
5412	if (Argument *Arg = getAssociatedArgument())
5413	if (A.getInfoCache().isInvolvedInMustTailCall(Arg: *Arg))
5414	return ChangeStatus::UNCHANGED;
5415	ChangeStatus Changed = AAAlignImpl::manifest(A);
5416	Align InheritAlign =
5417	getAssociatedValue().getPointerAlignment(A.getDataLayout());
5418	if (InheritAlign >= getAssumedAlign())
5419	Changed = ChangeStatus::UNCHANGED;
5420	return Changed;
5421	}
5422
5423	/// See AbstractAttribute::updateImpl(Attributor &A).
5424	ChangeStatus updateImpl(Attributor &A) override {
5425	ChangeStatus Changed = AAAlignFloating::updateImpl(A);
5426	if (Argument *Arg = getAssociatedArgument()) {
5427	// We only take known information from the argument
5428	// so we do not need to track a dependence.
5429	const auto *ArgAlignAA = A.getAAFor<AAAlign>(
5430	*this, IRPosition::argument(Arg: *Arg), DepClassTy::NONE);
5431	if (ArgAlignAA)
5432	takeKnownMaximum(ArgAlignAA->getKnownAlign().value());
5433	}
5434	return Changed;
5435	}
5436
5437	/// See AbstractAttribute::trackStatistics()
5438	void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(aligned) }
5439	};
5440
5441	/// Align attribute deduction for a call site return value.
5442	struct AAAlignCallSiteReturned final
5443	: AACalleeToCallSite<AAAlign, AAAlignImpl> {
5444	using Base = AACalleeToCallSite<AAAlign, AAAlignImpl>;
5445	AAAlignCallSiteReturned(const IRPosition &IRP, Attributor &A)
5446	: Base(IRP, A) {}
5447
5448	/// See AbstractAttribute::trackStatistics()
5449	void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(align); }
5450	};
5451	} // namespace
5452
5453	/// ------------------ Function No-Return Attribute ----------------------------
5454	namespace {
5455	struct AANoReturnImpl : public AANoReturn {
5456	AANoReturnImpl(const IRPosition &IRP, Attributor &A) : AANoReturn(IRP, A) {}
5457
5458	/// See AbstractAttribute::initialize(...).
5459	void initialize(Attributor &A) override {
5460	bool IsKnown;
5461	assert(!AA::hasAssumedIRAttr<Attribute::NoReturn>(
5462	A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
5463	(void)IsKnown;
5464	}
5465
5466	/// See AbstractAttribute::getAsStr().
5467	const std::string getAsStr(Attributor *A) const override {
5468	return getAssumed() ? "noreturn" : "may-return";
5469	}
5470
5471	/// See AbstractAttribute::updateImpl(Attributor &A).
5472	ChangeStatus updateImpl(Attributor &A) override {
5473	auto CheckForNoReturn = [](Instruction &) { return false; };
5474	bool UsedAssumedInformation = false;
5475	if (!A.checkForAllInstructions(CheckForNoReturn, *this,
5476	{(unsigned)Instruction::Ret},
5477	UsedAssumedInformation))
5478	return indicatePessimisticFixpoint();
5479	return ChangeStatus::UNCHANGED;
5480	}
5481	};
5482
5483	struct AANoReturnFunction final : AANoReturnImpl {
5484	AANoReturnFunction(const IRPosition &IRP, Attributor &A)
5485	: AANoReturnImpl(IRP, A) {}
5486
5487	/// See AbstractAttribute::trackStatistics()
5488	void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(noreturn) }
5489	};
5490
5491	/// NoReturn attribute deduction for a call sites.
5492	struct AANoReturnCallSite final
5493	: AACalleeToCallSite<AANoReturn, AANoReturnImpl> {
5494	AANoReturnCallSite(const IRPosition &IRP, Attributor &A)
5495	: AACalleeToCallSite<AANoReturn, AANoReturnImpl>(IRP, A) {}
5496
5497	/// See AbstractAttribute::trackStatistics()
5498	void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(noreturn); }
5499	};
5500	} // namespace
5501
5502	/// ----------------------- Instance Info ---------------------------------
5503
5504	namespace {
5505	/// A class to hold the state of for no-capture attributes.
5506	struct AAInstanceInfoImpl : public AAInstanceInfo {
5507	AAInstanceInfoImpl(const IRPosition &IRP, Attributor &A)
5508	: AAInstanceInfo(IRP, A) {}
5509
5510	/// See AbstractAttribute::initialize(...).
5511	void initialize(Attributor &A) override {
5512	Value &V = getAssociatedValue();
5513	if (auto *C = dyn_cast<Constant>(Val: &V)) {
5514	if (C->isThreadDependent())
5515	indicatePessimisticFixpoint();
5516	else
5517	indicateOptimisticFixpoint();
5518	return;
5519	}
5520	if (auto *CB = dyn_cast<CallBase>(Val: &V))
5521	if (CB->arg_size() == 0 && !CB->mayHaveSideEffects() &&
5522	!CB->mayReadFromMemory()) {
5523	indicateOptimisticFixpoint();
5524	return;
5525	}
5526	if (auto *I = dyn_cast<Instruction>(Val: &V)) {
5527	const auto *CI =
5528	A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
5529	F: *I->getFunction());
5530	if (mayBeInCycle(CI, I, /* HeaderOnly */ false)) {
5531	indicatePessimisticFixpoint();
5532	return;
5533	}
5534	}
5535	}
5536
5537	/// See AbstractAttribute::updateImpl(...).
5538	ChangeStatus updateImpl(Attributor &A) override {
5539	ChangeStatus Changed = ChangeStatus::UNCHANGED;
5540
5541	Value &V = getAssociatedValue();
5542	const Function *Scope = nullptr;
5543	if (auto *I = dyn_cast<Instruction>(Val: &V))
5544	Scope = I->getFunction();
5545	if (auto *A = dyn_cast<Argument>(Val: &V)) {
5546	Scope = A->getParent();
5547	if (!Scope->hasLocalLinkage())
5548	return Changed;
5549	}
5550	if (!Scope)
5551	return indicateOptimisticFixpoint();
5552
5553	bool IsKnownNoRecurse;
5554	if (AA::hasAssumedIRAttr<Attribute::NoRecurse>(
5555	A, this, IRPosition::function(*Scope), DepClassTy::OPTIONAL,
5556	IsKnownNoRecurse))
5557	return Changed;
5558
5559	auto UsePred = [&](const Use &U, bool &Follow) {
5560	const Instruction *UserI = dyn_cast<Instruction>(Val: U.getUser());
5561	if (!UserI || isa<GetElementPtrInst>(Val: UserI) || isa<CastInst>(Val: UserI) ||
5562	isa<PHINode>(Val: UserI) || isa<SelectInst>(Val: UserI)) {
5563	Follow = true;
5564	return true;
5565	}
5566	if (isa<LoadInst>(Val: UserI) || isa<CmpInst>(Val: UserI) ||
5567	(isa<StoreInst>(Val: UserI) &&
5568	cast<StoreInst>(Val: UserI)->getValueOperand() != U.get()))
5569	return true;
5570	if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
5571	// This check is not guaranteeing uniqueness but for now that we cannot
5572	// end up with two versions of \p U thinking it was one.
5573	auto *Callee = dyn_cast_if_present<Function>(Val: CB->getCalledOperand());
5574	if (!Callee || !Callee->hasLocalLinkage())
5575	return true;
5576	if (!CB->isArgOperand(U: &U))
5577	return false;
5578	const auto *ArgInstanceInfoAA = A.getAAFor<AAInstanceInfo>(
5579	QueryingAA: *this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo: CB->getArgOperandNo(U: &U)),
5580	DepClass: DepClassTy::OPTIONAL);
5581	if (!ArgInstanceInfoAA ||
5582	!ArgInstanceInfoAA->isAssumedUniqueForAnalysis())
5583	return false;
5584	// If this call base might reach the scope again we might forward the
5585	// argument back here. This is very conservative.
5586	if (AA::isPotentiallyReachable(
5587	A, FromI: *CB, ToFn: *Scope, QueryingAA: *this, /* ExclusionSet */ nullptr,
5588	GoBackwardsCB: [Scope](const Function &Fn) { return &Fn != Scope; }))
5589	return false;
5590	return true;
5591	}
5592	return false;
5593	};
5594
5595	auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
5596	if (auto *SI = dyn_cast<StoreInst>(Val: OldU.getUser())) {
5597	auto *Ptr = SI->getPointerOperand()->stripPointerCasts();
5598	if ((isa<AllocaInst>(Val: Ptr) || isNoAliasCall(V: Ptr)) &&
5599	AA::isDynamicallyUnique(A, QueryingAA: *this, V: *Ptr))
5600	return true;
5601	}
5602	return false;
5603	};
5604
5605	if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V, /* CheckBBLivenessOnly */ true,
5606	LivenessDepClass: DepClassTy::OPTIONAL,
5607	/* IgnoreDroppableUses */ true, EquivalentUseCB))
5608	return indicatePessimisticFixpoint();
5609
5610	return Changed;
5611	}
5612
5613	/// See AbstractState::getAsStr().
5614	const std::string getAsStr(Attributor *A) const override {
5615	return isAssumedUniqueForAnalysis() ? "<unique [fAa]>" : "<unknown>";
5616	}
5617
5618	/// See AbstractAttribute::trackStatistics()
5619	void trackStatistics() const override {}
5620	};
5621
5622	/// InstanceInfo attribute for floating values.
5623	struct AAInstanceInfoFloating : AAInstanceInfoImpl {
5624	AAInstanceInfoFloating(const IRPosition &IRP, Attributor &A)
5625	: AAInstanceInfoImpl(IRP, A) {}
5626	};
5627
5628	/// NoCapture attribute for function arguments.
5629	struct AAInstanceInfoArgument final : AAInstanceInfoFloating {
5630	AAInstanceInfoArgument(const IRPosition &IRP, Attributor &A)
5631	: AAInstanceInfoFloating(IRP, A) {}
5632	};
5633
5634	/// InstanceInfo attribute for call site arguments.
5635	struct AAInstanceInfoCallSiteArgument final : AAInstanceInfoImpl {
5636	AAInstanceInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
5637	: AAInstanceInfoImpl(IRP, A) {}
5638
5639	/// See AbstractAttribute::updateImpl(...).
5640	ChangeStatus updateImpl(Attributor &A) override {
5641	// TODO: Once we have call site specific value information we can provide
5642	// call site specific liveness information and then it makes
5643	// sense to specialize attributes for call sites arguments instead of
5644	// redirecting requests to the callee argument.
5645	Argument *Arg = getAssociatedArgument();
5646	if (!Arg)
5647	return indicatePessimisticFixpoint();
5648	const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
5649	auto *ArgAA =
5650	A.getAAFor<AAInstanceInfo>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
5651	if (!ArgAA)
5652	return indicatePessimisticFixpoint();
5653	return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState());
5654	}
5655	};
5656
5657	/// InstanceInfo attribute for function return value.
5658	struct AAInstanceInfoReturned final : AAInstanceInfoImpl {
5659	AAInstanceInfoReturned(const IRPosition &IRP, Attributor &A)
5660	: AAInstanceInfoImpl(IRP, A) {
5661	llvm_unreachable("InstanceInfo is not applicable to function returns!");
5662	}
5663
5664	/// See AbstractAttribute::initialize(...).
5665	void initialize(Attributor &A) override {
5666	llvm_unreachable("InstanceInfo is not applicable to function returns!");
5667	}
5668
5669	/// See AbstractAttribute::updateImpl(...).
5670	ChangeStatus updateImpl(Attributor &A) override {
5671	llvm_unreachable("InstanceInfo is not applicable to function returns!");
5672	}
5673	};
5674
5675	/// InstanceInfo attribute deduction for a call site return value.
5676	struct AAInstanceInfoCallSiteReturned final : AAInstanceInfoFloating {
5677	AAInstanceInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
5678	: AAInstanceInfoFloating(IRP, A) {}
5679	};
5680	} // namespace
5681
5682	/// ----------------------- Variable Capturing ---------------------------------
5683	bool AANoCapture::isImpliedByIR(Attributor &A, const IRPosition &IRP,
5684	Attribute::AttrKind ImpliedAttributeKind,
5685	bool IgnoreSubsumingPositions) {
5686	assert(ImpliedAttributeKind == Attribute::NoCapture &&
5687	"Unexpected attribute kind");
5688	Value &V = IRP.getAssociatedValue();
5689	if (!IRP.isArgumentPosition())
5690	return V.use_empty();
5691
5692	// You cannot "capture" null in the default address space.
5693	if (isa<UndefValue>(Val: V) || (isa<ConstantPointerNull>(Val: V) &&
5694	V.getType()->getPointerAddressSpace() == 0)) {
5695	return true;
5696	}
5697
5698	if (A.hasAttr(IRP, {Attribute::NoCapture},
5699	/* IgnoreSubsumingPositions */ true, Attribute::NoCapture))
5700	return true;
5701
5702	if (IRP.getPositionKind() == IRP_CALL_SITE_ARGUMENT)
5703	if (Argument *Arg = IRP.getAssociatedArgument())
5704	if (A.hasAttr(IRPosition::argument(*Arg),
5705	{Attribute::NoCapture, Attribute::ByVal},
5706	/* IgnoreSubsumingPositions */ true)) {
5707	A.manifestAttrs(IRP,
5708	Attribute::get(V.getContext(), Attribute::NoCapture));
5709	return true;
5710	}
5711
5712	if (const Function *F = IRP.getAssociatedFunction()) {
5713	// Check what state the associated function can actually capture.
5714	AANoCapture::StateType State;
5715	determineFunctionCaptureCapabilities(IRP, *F, State);
5716	if (State.isKnown(NO_CAPTURE)) {
5717	A.manifestAttrs(IRP,
5718	Attribute::get(V.getContext(), Attribute::NoCapture));
5719	return true;
5720	}
5721	}
5722
5723	return false;
5724	}
5725
5726	/// Set the NOT_CAPTURED_IN_MEM and NOT_CAPTURED_IN_RET bits in \p Known
5727	/// depending on the ability of the function associated with \p IRP to capture
5728	/// state in memory and through "returning/throwing", respectively.
5729	void AANoCapture::determineFunctionCaptureCapabilities(const IRPosition &IRP,
5730	const Function &F,
5731	BitIntegerState &State) {
5732	// TODO: Once we have memory behavior attributes we should use them here.
5733
5734	// If we know we cannot communicate or write to memory, we do not care about
5735	// ptr2int anymore.
5736	bool ReadOnly = F.onlyReadsMemory();
5737	bool NoThrow = F.doesNotThrow();
5738	bool IsVoidReturn = F.getReturnType()->isVoidTy();
5739	if (ReadOnly && NoThrow && IsVoidReturn) {
5740	State.addKnownBits(NO_CAPTURE);
5741	return;
5742	}
5743
5744	// A function cannot capture state in memory if it only reads memory, it can
5745	// however return/throw state and the state might be influenced by the
5746	// pointer value, e.g., loading from a returned pointer might reveal a bit.
5747	if (ReadOnly)
5748	State.addKnownBits(NOT_CAPTURED_IN_MEM);
5749
5750	// A function cannot communicate state back if it does not through
5751	// exceptions and doesn not return values.
5752	if (NoThrow && IsVoidReturn)
5753	State.addKnownBits(NOT_CAPTURED_IN_RET);
5754
5755	// Check existing "returned" attributes.
5756	int ArgNo = IRP.getCalleeArgNo();
5757	if (!NoThrow || ArgNo < 0 ||
5758	!F.getAttributes().hasAttrSomewhere(Attribute::Returned))
5759	return;
5760
5761	for (unsigned U = 0, E = F.arg_size(); U < E; ++U)
5762	if (F.hasParamAttribute(U, Attribute::Returned)) {
5763	if (U == unsigned(ArgNo))
5764	State.removeAssumedBits(NOT_CAPTURED_IN_RET);
5765	else if (ReadOnly)
5766	State.addKnownBits(NO_CAPTURE);
5767	else
5768	State.addKnownBits(NOT_CAPTURED_IN_RET);
5769	break;
5770	}
5771	}
5772
5773	namespace {
5774	/// A class to hold the state of for no-capture attributes.
5775	struct AANoCaptureImpl : public AANoCapture {
5776	AANoCaptureImpl(const IRPosition &IRP, Attributor &A) : AANoCapture(IRP, A) {}
5777
5778	/// See AbstractAttribute::initialize(...).
5779	void initialize(Attributor &A) override {
5780	bool IsKnown;
5781	assert(!AA::hasAssumedIRAttr<Attribute::NoCapture>(
5782	A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
5783	(void)IsKnown;
5784	}
5785
5786	/// See AbstractAttribute::updateImpl(...).
5787	ChangeStatus updateImpl(Attributor &A) override;
5788
5789	/// see AbstractAttribute::isAssumedNoCaptureMaybeReturned(...).
5790	void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
5791	SmallVectorImpl<Attribute> &Attrs) const override {
5792	if (!isAssumedNoCaptureMaybeReturned())
5793	return;
5794
5795	if (isArgumentPosition()) {
5796	if (isAssumedNoCapture())
5797	Attrs.emplace_back(Attribute::get(Ctx, Attribute::NoCapture));
5798	else if (ManifestInternal)
5799	Attrs.emplace_back(Args: Attribute::get(Context&: Ctx, Kind: "no-capture-maybe-returned"));
5800	}
5801	}
5802
5803	/// See AbstractState::getAsStr().
5804	const std::string getAsStr(Attributor *A) const override {
5805	if (isKnownNoCapture())
5806	return "known not-captured";
5807	if (isAssumedNoCapture())
5808	return "assumed not-captured";
5809	if (isKnownNoCaptureMaybeReturned())
5810	return "known not-captured-maybe-returned";
5811	if (isAssumedNoCaptureMaybeReturned())
5812	return "assumed not-captured-maybe-returned";
5813	return "assumed-captured";
5814	}
5815
5816	/// Check the use \p U and update \p State accordingly. Return true if we
5817	/// should continue to update the state.
5818	bool checkUse(Attributor &A, AANoCapture::StateType &State, const Use &U,
5819	bool &Follow) {
5820	Instruction *UInst = cast<Instruction>(Val: U.getUser());
5821	LLVM_DEBUG(dbgs() << "[AANoCapture] Check use: " << *U.get() << " in "
5822	<< *UInst << "\n");
5823
5824	// Deal with ptr2int by following uses.
5825	if (isa<PtrToIntInst>(Val: UInst)) {
5826	LLVM_DEBUG(dbgs() << " - ptr2int assume the worst!\n");
5827	return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
5828	/* Return */ CapturedInRet: true);
5829	}
5830
5831	// For stores we already checked if we can follow them, if they make it
5832	// here we give up.
5833	if (isa<StoreInst>(Val: UInst))
5834	return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
5835	/* Return */ CapturedInRet: true);
5836
5837	// Explicitly catch return instructions.
5838	if (isa<ReturnInst>(Val: UInst)) {
5839	if (UInst->getFunction() == getAnchorScope())
5840	return isCapturedIn(State, /* Memory */ CapturedInMem: false, /* Integer */ CapturedInInt: false,
5841	/* Return */ CapturedInRet: true);
5842	return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
5843	/* Return */ CapturedInRet: true);
5844	}
5845
5846	// For now we only use special logic for call sites. However, the tracker
5847	// itself knows about a lot of other non-capturing cases already.
5848	auto *CB = dyn_cast<CallBase>(Val: UInst);
5849	if (!CB || !CB->isArgOperand(U: &U))
5850	return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
5851	/* Return */ CapturedInRet: true);
5852
5853	unsigned ArgNo = CB->getArgOperandNo(U: &U);
5854	const IRPosition &CSArgPos = IRPosition::callsite_argument(CB: *CB, ArgNo);
5855	// If we have a abstract no-capture attribute for the argument we can use
5856	// it to justify a non-capture attribute here. This allows recursion!
5857	bool IsKnownNoCapture;
5858	const AANoCapture *ArgNoCaptureAA = nullptr;
5859	bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
5860	A, this, CSArgPos, DepClassTy::REQUIRED, IsKnownNoCapture, false,
5861	&ArgNoCaptureAA);
5862	if (IsAssumedNoCapture)
5863	return isCapturedIn(State, /* Memory */ CapturedInMem: false, /* Integer */ CapturedInInt: false,
5864	/* Return */ CapturedInRet: false);
5865	if (ArgNoCaptureAA && ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned()) {
5866	Follow = true;
5867	return isCapturedIn(State, /* Memory */ CapturedInMem: false, /* Integer */ CapturedInInt: false,
5868	/* Return */ CapturedInRet: false);
5869	}
5870
5871	// Lastly, we could not find a reason no-capture can be assumed so we don't.
5872	return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
5873	/* Return */ CapturedInRet: true);
5874	}
5875
5876	/// Update \p State according to \p CapturedInMem, \p CapturedInInt, and
5877	/// \p CapturedInRet, then return true if we should continue updating the
5878	/// state.
5879	static bool isCapturedIn(AANoCapture::StateType &State, bool CapturedInMem,
5880	bool CapturedInInt, bool CapturedInRet) {
5881	LLVM_DEBUG(dbgs() << " - captures [Mem " << CapturedInMem << "|Int "
5882	<< CapturedInInt << "|Ret " << CapturedInRet << "]\n");
5883	if (CapturedInMem)
5884	State.removeAssumedBits(AANoCapture::NOT_CAPTURED_IN_MEM);
5885	if (CapturedInInt)
5886	State.removeAssumedBits(AANoCapture::NOT_CAPTURED_IN_INT);
5887	if (CapturedInRet)
5888	State.removeAssumedBits(AANoCapture::NOT_CAPTURED_IN_RET);
5889	return State.isAssumed(AANoCapture::NO_CAPTURE_MAYBE_RETURNED);
5890	}
5891	};
5892
5893	ChangeStatus AANoCaptureImpl::updateImpl(Attributor &A) {
5894	const IRPosition &IRP = getIRPosition();
5895	Value *V = isArgumentPosition() ? IRP.getAssociatedArgument()
5896	: &IRP.getAssociatedValue();
5897	if (!V)
5898	return indicatePessimisticFixpoint();
5899
5900	const Function *F =
5901	isArgumentPosition() ? IRP.getAssociatedFunction() : IRP.getAnchorScope();
5902	assert(F && "Expected a function!");
5903	const IRPosition &FnPos = IRPosition::function(F: *F);
5904
5905	AANoCapture::StateType T;
5906
5907	// Readonly means we cannot capture through memory.
5908	bool IsKnown;
5909	if (AA::isAssumedReadOnly(A, FnPos, *this, IsKnown)) {
5910	T.addKnownBits(NOT_CAPTURED_IN_MEM);
5911	if (IsKnown)
5912	addKnownBits(NOT_CAPTURED_IN_MEM);
5913	}
5914
5915	// Make sure all returned values are different than the underlying value.
5916	// TODO: we could do this in a more sophisticated way inside
5917	// AAReturnedValues, e.g., track all values that escape through returns
5918	// directly somehow.
5919	auto CheckReturnedArgs = [&](bool &UsedAssumedInformation) {
5920	SmallVector<AA::ValueAndContext> Values;
5921	if (!A.getAssumedSimplifiedValues(IRPosition::returned(F: *F), this, Values,
5922	AA::ValueScope::Intraprocedural,
5923	UsedAssumedInformation))
5924	return false;
5925	bool SeenConstant = false;
5926	for (const AA::ValueAndContext &VAC : Values) {
5927	if (isa<Constant>(Val: VAC.getValue())) {
5928	if (SeenConstant)
5929	return false;
5930	SeenConstant = true;
5931	} else if (!isa<Argument>(Val: VAC.getValue()) ||
5932	VAC.getValue() == getAssociatedArgument())
5933	return false;
5934	}
5935	return true;
5936	};
5937
5938	bool IsKnownNoUnwind;
5939	if (AA::hasAssumedIRAttr<Attribute::NoUnwind>(
5940	A, this, FnPos, DepClassTy::OPTIONAL, IsKnownNoUnwind)) {
5941	bool IsVoidTy = F->getReturnType()->isVoidTy();
5942	bool UsedAssumedInformation = false;
5943	if (IsVoidTy || CheckReturnedArgs(UsedAssumedInformation)) {
5944	T.addKnownBits(NOT_CAPTURED_IN_RET);
5945	if (T.isKnown(NOT_CAPTURED_IN_MEM))
5946	return ChangeStatus::UNCHANGED;
5947	if (IsKnownNoUnwind && (IsVoidTy || !UsedAssumedInformation)) {
5948	addKnownBits(NOT_CAPTURED_IN_RET);
5949	if (isKnown(NOT_CAPTURED_IN_MEM))
5950	return indicateOptimisticFixpoint();
5951	}
5952	}
5953	}
5954
5955	auto IsDereferenceableOrNull = [&](Value *O, const DataLayout &DL) {
5956	const auto *DerefAA = A.getAAFor<AADereferenceable>(
5957	*this, IRPosition::value(V: *O), DepClassTy::OPTIONAL);
5958	return DerefAA && DerefAA->getAssumedDereferenceableBytes();
5959	};
5960
5961	auto UseCheck = [&](const Use &U, bool &Follow) -> bool {
5962	switch (DetermineUseCaptureKind(U, IsDereferenceableOrNull)) {
5963	case UseCaptureKind::NO_CAPTURE:
5964	return true;
5965	case UseCaptureKind::MAY_CAPTURE:
5966	return checkUse(A, State&: T, U, Follow);
5967	case UseCaptureKind::PASSTHROUGH:
5968	Follow = true;
5969	return true;
5970	}
5971	llvm_unreachable("Unexpected use capture kind!");
5972	};
5973
5974	if (!A.checkForAllUses(UseCheck, *this, *V))
5975	return indicatePessimisticFixpoint();
5976
5977	AANoCapture::StateType &S = getState();
5978	auto Assumed = S.getAssumed();
5979	S.intersectAssumedBits(T.getAssumed());
5980	if (!isAssumedNoCaptureMaybeReturned())
5981	return indicatePessimisticFixpoint();
5982	return Assumed == S.getAssumed() ? ChangeStatus::UNCHANGED
5983	: ChangeStatus::CHANGED;
5984	}
5985
5986	/// NoCapture attribute for function arguments.
5987	struct AANoCaptureArgument final : AANoCaptureImpl {
5988	AANoCaptureArgument(const IRPosition &IRP, Attributor &A)
5989	: AANoCaptureImpl(IRP, A) {}
5990
5991	/// See AbstractAttribute::trackStatistics()
5992	void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nocapture) }
5993	};
5994
5995	/// NoCapture attribute for call site arguments.
5996	struct AANoCaptureCallSiteArgument final : AANoCaptureImpl {
5997	AANoCaptureCallSiteArgument(const IRPosition &IRP, Attributor &A)
5998	: AANoCaptureImpl(IRP, A) {}
5999
6000	/// See AbstractAttribute::updateImpl(...).
6001	ChangeStatus updateImpl(Attributor &A) override {
6002	// TODO: Once we have call site specific value information we can provide
6003	// call site specific liveness information and then it makes
6004	// sense to specialize attributes for call sites arguments instead of
6005	// redirecting requests to the callee argument.
6006	Argument *Arg = getAssociatedArgument();
6007	if (!Arg)
6008	return indicatePessimisticFixpoint();
6009	const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
6010	bool IsKnownNoCapture;
6011	const AANoCapture *ArgAA = nullptr;
6012	if (AA::hasAssumedIRAttr<Attribute::NoCapture>(
6013	A, this, ArgPos, DepClassTy::REQUIRED, IsKnownNoCapture, false,
6014	&ArgAA))
6015	return ChangeStatus::UNCHANGED;
6016	if (!ArgAA || !ArgAA->isAssumedNoCaptureMaybeReturned())
6017	return indicatePessimisticFixpoint();
6018	return clampStateAndIndicateChange(getState(), ArgAA->getState());
6019	}
6020
6021	/// See AbstractAttribute::trackStatistics()
6022	void trackStatistics() const override{STATS_DECLTRACK_CSARG_ATTR(nocapture)};
6023	};
6024
6025	/// NoCapture attribute for floating values.
6026	struct AANoCaptureFloating final : AANoCaptureImpl {
6027	AANoCaptureFloating(const IRPosition &IRP, Attributor &A)
6028	: AANoCaptureImpl(IRP, A) {}
6029
6030	/// See AbstractAttribute::trackStatistics()
6031	void trackStatistics() const override {
6032	STATS_DECLTRACK_FLOATING_ATTR(nocapture)
6033	}
6034	};
6035
6036	/// NoCapture attribute for function return value.
6037	struct AANoCaptureReturned final : AANoCaptureImpl {
6038	AANoCaptureReturned(const IRPosition &IRP, Attributor &A)
6039	: AANoCaptureImpl(IRP, A) {
6040	llvm_unreachable("NoCapture is not applicable to function returns!");
6041	}
6042
6043	/// See AbstractAttribute::initialize(...).
6044	void initialize(Attributor &A) override {
6045	llvm_unreachable("NoCapture is not applicable to function returns!");
6046	}
6047
6048	/// See AbstractAttribute::updateImpl(...).
6049	ChangeStatus updateImpl(Attributor &A) override {
6050	llvm_unreachable("NoCapture is not applicable to function returns!");
6051	}
6052
6053	/// See AbstractAttribute::trackStatistics()
6054	void trackStatistics() const override {}
6055	};
6056
6057	/// NoCapture attribute deduction for a call site return value.
6058	struct AANoCaptureCallSiteReturned final : AANoCaptureImpl {
6059	AANoCaptureCallSiteReturned(const IRPosition &IRP, Attributor &A)
6060	: AANoCaptureImpl(IRP, A) {}
6061
6062	/// See AbstractAttribute::initialize(...).
6063	void initialize(Attributor &A) override {
6064	const Function *F = getAnchorScope();
6065	// Check what state the associated function can actually capture.
6066	determineFunctionCaptureCapabilities(getIRPosition(), *F, *this);
6067	}
6068
6069	/// See AbstractAttribute::trackStatistics()
6070	void trackStatistics() const override {
6071	STATS_DECLTRACK_CSRET_ATTR(nocapture)
6072	}
6073	};
6074	} // namespace
6075
6076	/// ------------------ Value Simplify Attribute ----------------------------
6077
6078	bool ValueSimplifyStateType::unionAssumed(std::optional<Value *> Other) {
6079	// FIXME: Add a typecast support.
6080	SimplifiedAssociatedValue = AA::combineOptionalValuesInAAValueLatice(
6081	A: SimplifiedAssociatedValue, B: Other, Ty);
6082	if (SimplifiedAssociatedValue == std::optional<Value *>(nullptr))
6083	return false;
6084
6085	LLVM_DEBUG({
6086	if (SimplifiedAssociatedValue)
6087	dbgs() << "[ValueSimplify] is assumed to be "
6088	<< **SimplifiedAssociatedValue << "\n";
6089	else
6090	dbgs() << "[ValueSimplify] is assumed to be <none>\n";
6091	});
6092	return true;
6093	}
6094
6095	namespace {
6096	struct AAValueSimplifyImpl : AAValueSimplify {
6097	AAValueSimplifyImpl(const IRPosition &IRP, Attributor &A)
6098	: AAValueSimplify(IRP, A) {}
6099
6100	/// See AbstractAttribute::initialize(...).
6101	void initialize(Attributor &A) override {
6102	if (getAssociatedValue().getType()->isVoidTy())
6103	indicatePessimisticFixpoint();
6104	if (A.hasSimplificationCallback(IRP: getIRPosition()))
6105	indicatePessimisticFixpoint();
6106	}
6107
6108	/// See AbstractAttribute::getAsStr().
6109	const std::string getAsStr(Attributor *A) const override {
6110	LLVM_DEBUG({
6111	dbgs() << "SAV: " << (bool)SimplifiedAssociatedValue << " ";
6112	if (SimplifiedAssociatedValue && *SimplifiedAssociatedValue)
6113	dbgs() << "SAV: " << **SimplifiedAssociatedValue << " ";
6114	});
6115	return isValidState() ? (isAtFixpoint() ? "simplified" : "maybe-simple")
6116	: "not-simple";
6117	}
6118
6119	/// See AbstractAttribute::trackStatistics()
6120	void trackStatistics() const override {}
6121
6122	/// See AAValueSimplify::getAssumedSimplifiedValue()
6123	std::optional<Value *>
6124	getAssumedSimplifiedValue(Attributor &A) const override {
6125	return SimplifiedAssociatedValue;
6126	}
6127
6128	/// Ensure the return value is \p V with type \p Ty, if not possible return
6129	/// nullptr. If \p Check is true we will only verify such an operation would
6130	/// suceed and return a non-nullptr value if that is the case. No IR is
6131	/// generated or modified.
6132	static Value *ensureType(Attributor &A, Value &V, Type &Ty, Instruction *CtxI,
6133	bool Check) {
6134	if (auto *TypedV = AA::getWithType(V, Ty))
6135	return TypedV;
6136	if (CtxI && V.getType()->canLosslesslyBitCastTo(Ty: &Ty))
6137	return Check ? &V
6138	: BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
6139	S: &V, Ty: &Ty, Name: "", InsertBefore: CtxI->getIterator());
6140	return nullptr;
6141	}
6142
6143	/// Reproduce \p I with type \p Ty or return nullptr if that is not posisble.
6144	/// If \p Check is true we will only verify such an operation would suceed and
6145	/// return a non-nullptr value if that is the case. No IR is generated or
6146	/// modified.
6147	static Value *reproduceInst(Attributor &A,
6148	const AbstractAttribute &QueryingAA,
6149	Instruction &I, Type &Ty, Instruction *CtxI,
6150	bool Check, ValueToValueMapTy &VMap) {
6151	assert(CtxI && "Cannot reproduce an instruction without context!");
6152	if (Check && (I.mayReadFromMemory() ||
6153	!isSafeToSpeculativelyExecute(I: &I, CtxI, /* DT */ AC: nullptr,
6154	/* TLI */ DT: nullptr)))
6155	return nullptr;
6156	for (Value *Op : I.operands()) {
6157	Value *NewOp = reproduceValue(A, QueryingAA, V&: *Op, Ty, CtxI, Check, VMap);
6158	if (!NewOp) {
6159	assert(Check && "Manifest of new value unexpectedly failed!");
6160	return nullptr;
6161	}
6162	if (!Check)
6163	VMap[Op] = NewOp;
6164	}
6165	if (Check)
6166	return &I;
6167
6168	Instruction *CloneI = I.clone();
6169	// TODO: Try to salvage debug information here.
6170	CloneI->setDebugLoc(DebugLoc());
6171	VMap[&I] = CloneI;
6172	CloneI->insertBefore(InsertPos: CtxI);
6173	RemapInstruction(I: CloneI, VM&: VMap);
6174	return CloneI;
6175	}
6176
6177	/// Reproduce \p V with type \p Ty or return nullptr if that is not posisble.
6178	/// If \p Check is true we will only verify such an operation would suceed and
6179	/// return a non-nullptr value if that is the case. No IR is generated or
6180	/// modified.
6181	static Value *reproduceValue(Attributor &A,
6182	const AbstractAttribute &QueryingAA, Value &V,
6183	Type &Ty, Instruction *CtxI, bool Check,
6184	ValueToValueMapTy &VMap) {
6185	if (const auto &NewV = VMap.lookup(Val: &V))
6186	return NewV;
6187	bool UsedAssumedInformation = false;
6188	std::optional<Value *> SimpleV = A.getAssumedSimplified(
6189	V, AA: QueryingAA, UsedAssumedInformation, S: AA::Interprocedural);
6190	if (!SimpleV.has_value())
6191	return PoisonValue::get(T: &Ty);
6192	Value *EffectiveV = &V;
6193	if (*SimpleV)
6194	EffectiveV = *SimpleV;
6195	if (auto *C = dyn_cast<Constant>(Val: EffectiveV))
6196	return C;
6197	if (CtxI && AA::isValidAtPosition(VAC: AA::ValueAndContext(*EffectiveV, *CtxI),
6198	InfoCache&: A.getInfoCache()))
6199	return ensureType(A, V&: *EffectiveV, Ty, CtxI, Check);
6200	if (auto *I = dyn_cast<Instruction>(Val: EffectiveV))
6201	if (Value *NewV = reproduceInst(A, QueryingAA, I&: *I, Ty, CtxI, Check, VMap))
6202	return ensureType(A, V&: *NewV, Ty, CtxI, Check);
6203	return nullptr;
6204	}
6205
6206	/// Return a value we can use as replacement for the associated one, or
6207	/// nullptr if we don't have one that makes sense.
6208	Value *manifestReplacementValue(Attributor &A, Instruction *CtxI) const {
6209	Value *NewV = SimplifiedAssociatedValue
6210	? *SimplifiedAssociatedValue
6211	: UndefValue::get(T: getAssociatedType());
6212	if (NewV && NewV != &getAssociatedValue()) {
6213	ValueToValueMapTy VMap;
6214	// First verify we can reprduce the value with the required type at the
6215	// context location before we actually start modifying the IR.
6216	if (reproduceValue(A, QueryingAA: *this, V&: *NewV, Ty&: *getAssociatedType(), CtxI,
6217	/* CheckOnly */ Check: true, VMap))
6218	return reproduceValue(A, QueryingAA: *this, V&: *NewV, Ty&: *getAssociatedType(), CtxI,
6219	/* CheckOnly */ Check: false, VMap);
6220	}
6221	return nullptr;
6222	}
6223
6224	/// Helper function for querying AAValueSimplify and updating candidate.
6225	/// \param IRP The value position we are trying to unify with SimplifiedValue
6226	bool checkAndUpdate(Attributor &A, const AbstractAttribute &QueryingAA,
6227	const IRPosition &IRP, bool Simplify = true) {
6228	bool UsedAssumedInformation = false;
6229	std::optional<Value *> QueryingValueSimplified = &IRP.getAssociatedValue();
6230	if (Simplify)
6231	QueryingValueSimplified = A.getAssumedSimplified(
6232	IRP, AA: QueryingAA, UsedAssumedInformation, S: AA::Interprocedural);
6233	return unionAssumed(Other: QueryingValueSimplified);
6234	}
6235
6236	/// Returns a candidate is found or not
6237	template <typename AAType> bool askSimplifiedValueFor(Attributor &A) {
6238	if (!getAssociatedValue().getType()->isIntegerTy())
6239	return false;
6240
6241	// This will also pass the call base context.
6242	const auto *AA =
6243	A.getAAFor<AAType>(*this, getIRPosition(), DepClassTy::NONE);
6244	if (!AA)
6245	return false;
6246
6247	std::optional<Constant *> COpt = AA->getAssumedConstant(A);
6248
6249	if (!COpt) {
6250	SimplifiedAssociatedValue = std::nullopt;
6251	A.recordDependence(FromAA: *AA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
6252	return true;
6253	}
6254	if (auto *C = *COpt) {
6255	SimplifiedAssociatedValue = C;
6256	A.recordDependence(FromAA: *AA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
6257	return true;
6258	}
6259	return false;
6260	}
6261
6262	bool askSimplifiedValueForOtherAAs(Attributor &A) {
6263	if (askSimplifiedValueFor<AAValueConstantRange>(A))
6264	return true;
6265	if (askSimplifiedValueFor<AAPotentialConstantValues>(A))
6266	return true;
6267	return false;
6268	}
6269
6270	/// See AbstractAttribute::manifest(...).
6271	ChangeStatus manifest(Attributor &A) override {
6272	ChangeStatus Changed = ChangeStatus::UNCHANGED;
6273	for (auto &U : getAssociatedValue().uses()) {
6274	// Check if we need to adjust the insertion point to make sure the IR is
6275	// valid.
6276	Instruction *IP = dyn_cast<Instruction>(Val: U.getUser());
6277	if (auto *PHI = dyn_cast_or_null<PHINode>(Val: IP))
6278	IP = PHI->getIncomingBlock(U)->getTerminator();
6279	if (auto *NewV = manifestReplacementValue(A, CtxI: IP)) {
6280	LLVM_DEBUG(dbgs() << "[ValueSimplify] " << getAssociatedValue()
6281	<< " -> " << *NewV << " :: " << *this << "\n");
6282	if (A.changeUseAfterManifest(U, NV&: *NewV))
6283	Changed = ChangeStatus::CHANGED;
6284	}
6285	}
6286
6287	return Changed | AAValueSimplify::manifest(A);
6288	}
6289
6290	/// See AbstractState::indicatePessimisticFixpoint(...).
6291	ChangeStatus indicatePessimisticFixpoint() override {
6292	SimplifiedAssociatedValue = &getAssociatedValue();
6293	return AAValueSimplify::indicatePessimisticFixpoint();
6294	}
6295	};
6296
6297	struct AAValueSimplifyArgument final : AAValueSimplifyImpl {
6298	AAValueSimplifyArgument(const IRPosition &IRP, Attributor &A)
6299	: AAValueSimplifyImpl(IRP, A) {}
6300
6301	void initialize(Attributor &A) override {
6302	AAValueSimplifyImpl::initialize(A);
6303	if (A.hasAttr(getIRPosition(),
6304	{Attribute::InAlloca, Attribute::Preallocated,
6305	Attribute::StructRet, Attribute::Nest, Attribute::ByVal},
6306	/* IgnoreSubsumingPositions */ true))
6307	indicatePessimisticFixpoint();
6308	}
6309
6310	/// See AbstractAttribute::updateImpl(...).
6311	ChangeStatus updateImpl(Attributor &A) override {
6312	// Byval is only replacable if it is readonly otherwise we would write into
6313	// the replaced value and not the copy that byval creates implicitly.
6314	Argument *Arg = getAssociatedArgument();
6315	if (Arg->hasByValAttr()) {
6316	// TODO: We probably need to verify synchronization is not an issue, e.g.,
6317	// there is no race by not copying a constant byval.
6318	bool IsKnown;
6319	if (!AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown))
6320	return indicatePessimisticFixpoint();
6321	}
6322
6323	auto Before = SimplifiedAssociatedValue;
6324
6325	auto PredForCallSite = [&](AbstractCallSite ACS) {
6326	const IRPosition &ACSArgPos =
6327	IRPosition::callsite_argument(ACS, ArgNo: getCallSiteArgNo());
6328	// Check if a coresponding argument was found or if it is on not
6329	// associated (which can happen for callback calls).
6330	if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
6331	return false;
6332
6333	// Simplify the argument operand explicitly and check if the result is
6334	// valid in the current scope. This avoids refering to simplified values
6335	// in other functions, e.g., we don't want to say a an argument in a
6336	// static function is actually an argument in a different function.
6337	bool UsedAssumedInformation = false;
6338	std::optional<Constant *> SimpleArgOp =
6339	A.getAssumedConstant(IRP: ACSArgPos, AA: *this, UsedAssumedInformation);
6340	if (!SimpleArgOp)
6341	return true;
6342	if (!*SimpleArgOp)
6343	return false;
6344	if (!AA::isDynamicallyUnique(A, QueryingAA: *this, V: **SimpleArgOp))
6345	return false;
6346	return unionAssumed(Other: *SimpleArgOp);
6347	};
6348
6349	// Generate a answer specific to a call site context.
6350	bool Success;
6351	bool UsedAssumedInformation = false;
6352	if (hasCallBaseContext() &&
6353	getCallBaseContext()->getCalledOperand() == Arg->getParent())
6354	Success = PredForCallSite(
6355	AbstractCallSite(&getCallBaseContext()->getCalledOperandUse()));
6356	else
6357	Success = A.checkForAllCallSites(Pred: PredForCallSite, QueryingAA: *this, RequireAllCallSites: true,
6358	UsedAssumedInformation);
6359
6360	if (!Success)
6361	if (!askSimplifiedValueForOtherAAs(A))
6362	return indicatePessimisticFixpoint();
6363
6364	// If a candidate was found in this update, return CHANGED.
6365	return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6366	: ChangeStatus ::CHANGED;
6367	}
6368
6369	/// See AbstractAttribute::trackStatistics()
6370	void trackStatistics() const override {
6371	STATS_DECLTRACK_ARG_ATTR(value_simplify)
6372	}
6373	};
6374
6375	struct AAValueSimplifyReturned : AAValueSimplifyImpl {
6376	AAValueSimplifyReturned(const IRPosition &IRP, Attributor &A)
6377	: AAValueSimplifyImpl(IRP, A) {}
6378
6379	/// See AAValueSimplify::getAssumedSimplifiedValue()
6380	std::optional<Value *>
6381	getAssumedSimplifiedValue(Attributor &A) const override {
6382	if (!isValidState())
6383	return nullptr;
6384	return SimplifiedAssociatedValue;
6385	}
6386
6387	/// See AbstractAttribute::updateImpl(...).
6388	ChangeStatus updateImpl(Attributor &A) override {
6389	auto Before = SimplifiedAssociatedValue;
6390
6391	auto ReturnInstCB = [&](Instruction &I) {
6392	auto &RI = cast<ReturnInst>(Val&: I);
6393	return checkAndUpdate(
6394	A, QueryingAA: *this,
6395	IRP: IRPosition::value(V: *RI.getReturnValue(), CBContext: getCallBaseContext()));
6396	};
6397
6398	bool UsedAssumedInformation = false;
6399	if (!A.checkForAllInstructions(Pred: ReturnInstCB, QueryingAA: *this, Opcodes: {Instruction::Ret},
6400	UsedAssumedInformation))
6401	if (!askSimplifiedValueForOtherAAs(A))
6402	return indicatePessimisticFixpoint();
6403
6404	// If a candidate was found in this update, return CHANGED.
6405	return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6406	: ChangeStatus ::CHANGED;
6407	}
6408
6409	ChangeStatus manifest(Attributor &A) override {
6410	// We queried AAValueSimplify for the returned values so they will be
6411	// replaced if a simplified form was found. Nothing to do here.
6412	return ChangeStatus::UNCHANGED;
6413	}
6414
6415	/// See AbstractAttribute::trackStatistics()
6416	void trackStatistics() const override {
6417	STATS_DECLTRACK_FNRET_ATTR(value_simplify)
6418	}
6419	};
6420
6421	struct AAValueSimplifyFloating : AAValueSimplifyImpl {
6422	AAValueSimplifyFloating(const IRPosition &IRP, Attributor &A)
6423	: AAValueSimplifyImpl(IRP, A) {}
6424
6425	/// See AbstractAttribute::initialize(...).
6426	void initialize(Attributor &A) override {
6427	AAValueSimplifyImpl::initialize(A);
6428	Value &V = getAnchorValue();
6429
6430	// TODO: add other stuffs
6431	if (isa<Constant>(Val: V))
6432	indicatePessimisticFixpoint();
6433	}
6434
6435	/// See AbstractAttribute::updateImpl(...).
6436	ChangeStatus updateImpl(Attributor &A) override {
6437	auto Before = SimplifiedAssociatedValue;
6438	if (!askSimplifiedValueForOtherAAs(A))
6439	return indicatePessimisticFixpoint();
6440
6441	// If a candidate was found in this update, return CHANGED.
6442	return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6443	: ChangeStatus ::CHANGED;
6444	}
6445
6446	/// See AbstractAttribute::trackStatistics()
6447	void trackStatistics() const override {
6448	STATS_DECLTRACK_FLOATING_ATTR(value_simplify)
6449	}
6450	};
6451
6452	struct AAValueSimplifyFunction : AAValueSimplifyImpl {
6453	AAValueSimplifyFunction(const IRPosition &IRP, Attributor &A)
6454	: AAValueSimplifyImpl(IRP, A) {}
6455
6456	/// See AbstractAttribute::initialize(...).
6457	void initialize(Attributor &A) override {
6458	SimplifiedAssociatedValue = nullptr;
6459	indicateOptimisticFixpoint();
6460	}
6461	/// See AbstractAttribute::initialize(...).
6462	ChangeStatus updateImpl(Attributor &A) override {
6463	llvm_unreachable(
6464	"AAValueSimplify(Function|CallSite)::updateImpl will not be called");
6465	}
6466	/// See AbstractAttribute::trackStatistics()
6467	void trackStatistics() const override {
6468	STATS_DECLTRACK_FN_ATTR(value_simplify)
6469	}
6470	};
6471
6472	struct AAValueSimplifyCallSite : AAValueSimplifyFunction {
6473	AAValueSimplifyCallSite(const IRPosition &IRP, Attributor &A)
6474	: AAValueSimplifyFunction(IRP, A) {}
6475	/// See AbstractAttribute::trackStatistics()
6476	void trackStatistics() const override {
6477	STATS_DECLTRACK_CS_ATTR(value_simplify)
6478	}
6479	};
6480
6481	struct AAValueSimplifyCallSiteReturned : AAValueSimplifyImpl {
6482	AAValueSimplifyCallSiteReturned(const IRPosition &IRP, Attributor &A)
6483	: AAValueSimplifyImpl(IRP, A) {}
6484
6485	void initialize(Attributor &A) override {
6486	AAValueSimplifyImpl::initialize(A);
6487	Function *Fn = getAssociatedFunction();
6488	assert(Fn && "Did expect an associted function");
6489	for (Argument &Arg : Fn->args()) {
6490	if (Arg.hasReturnedAttr()) {
6491	auto IRP = IRPosition::callsite_argument(CB: *cast<CallBase>(Val: getCtxI()),
6492	ArgNo: Arg.getArgNo());
6493	if (IRP.getPositionKind() == IRPosition::IRP_CALL_SITE_ARGUMENT &&
6494	checkAndUpdate(A, QueryingAA: *this, IRP))
6495	indicateOptimisticFixpoint();
6496	else
6497	indicatePessimisticFixpoint();
6498	return;
6499	}
6500	}
6501	}
6502
6503	/// See AbstractAttribute::updateImpl(...).
6504	ChangeStatus updateImpl(Attributor &A) override {
6505	return indicatePessimisticFixpoint();
6506	}
6507
6508	void trackStatistics() const override {
6509	STATS_DECLTRACK_CSRET_ATTR(value_simplify)
6510	}
6511	};
6512
6513	struct AAValueSimplifyCallSiteArgument : AAValueSimplifyFloating {
6514	AAValueSimplifyCallSiteArgument(const IRPosition &IRP, Attributor &A)
6515	: AAValueSimplifyFloating(IRP, A) {}
6516
6517	/// See AbstractAttribute::manifest(...).
6518	ChangeStatus manifest(Attributor &A) override {
6519	ChangeStatus Changed = ChangeStatus::UNCHANGED;
6520	// TODO: We should avoid simplification duplication to begin with.
6521	auto *FloatAA = A.lookupAAFor<AAValueSimplify>(
6522	IRP: IRPosition::value(V: getAssociatedValue()), QueryingAA: this, DepClass: DepClassTy::NONE);
6523	if (FloatAA && FloatAA->getState().isValidState())
6524	return Changed;
6525
6526	if (auto *NewV = manifestReplacementValue(A, CtxI: getCtxI())) {
6527	Use &U = cast<CallBase>(Val: &getAnchorValue())
6528	->getArgOperandUse(i: getCallSiteArgNo());
6529	if (A.changeUseAfterManifest(U, NV&: *NewV))
6530	Changed = ChangeStatus::CHANGED;
6531	}
6532
6533	return Changed | AAValueSimplify::manifest(A);
6534	}
6535
6536	void trackStatistics() const override {
6537	STATS_DECLTRACK_CSARG_ATTR(value_simplify)
6538	}
6539	};
6540	} // namespace
6541
6542	/// ----------------------- Heap-To-Stack Conversion ---------------------------
6543	namespace {
6544	struct AAHeapToStackFunction final : public AAHeapToStack {
6545
6546	struct AllocationInfo {
6547	/// The call that allocates the memory.
6548	CallBase *const CB;
6549
6550	/// The library function id for the allocation.
6551	LibFunc LibraryFunctionId = NotLibFunc;
6552
6553	/// The status wrt. a rewrite.
6554	enum {
6555	STACK_DUE_TO_USE,
6556	STACK_DUE_TO_FREE,
6557	INVALID,
6558	} Status = STACK_DUE_TO_USE;
6559
6560	/// Flag to indicate if we encountered a use that might free this allocation
6561	/// but which is not in the deallocation infos.
6562	bool HasPotentiallyFreeingUnknownUses = false;
6563
6564	/// Flag to indicate that we should place the new alloca in the function
6565	/// entry block rather than where the call site (CB) is.
6566	bool MoveAllocaIntoEntry = true;
6567
6568	/// The set of free calls that use this allocation.
6569	SmallSetVector<CallBase *, 1> PotentialFreeCalls{};
6570	};
6571
6572	struct DeallocationInfo {
6573	/// The call that deallocates the memory.
6574	CallBase *const CB;
6575	/// The value freed by the call.
6576	Value *FreedOp;
6577
6578	/// Flag to indicate if we don't know all objects this deallocation might
6579	/// free.
6580	bool MightFreeUnknownObjects = false;
6581
6582	/// The set of allocation calls that are potentially freed.
6583	SmallSetVector<CallBase *, 1> PotentialAllocationCalls{};
6584	};
6585
6586	AAHeapToStackFunction(const IRPosition &IRP, Attributor &A)
6587	: AAHeapToStack(IRP, A) {}
6588
6589	~AAHeapToStackFunction() {
6590	// Ensure we call the destructor so we release any memory allocated in the
6591	// sets.
6592	for (auto &It : AllocationInfos)
6593	It.second->~AllocationInfo();
6594	for (auto &It : DeallocationInfos)
6595	It.second->~DeallocationInfo();
6596	}
6597
6598	void initialize(Attributor &A) override {
6599	AAHeapToStack::initialize(A);
6600
6601	const Function *F = getAnchorScope();
6602	const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
6603
6604	auto AllocationIdentifierCB = [&](Instruction &I) {
6605	CallBase *CB = dyn_cast<CallBase>(Val: &I);
6606	if (!CB)
6607	return true;
6608	if (Value *FreedOp = getFreedOperand(CB, TLI)) {
6609	DeallocationInfos[CB] = new (A.Allocator) DeallocationInfo{.CB: CB, .FreedOp: FreedOp};
6610	return true;
6611	}
6612	// To do heap to stack, we need to know that the allocation itself is
6613	// removable once uses are rewritten, and that we can initialize the
6614	// alloca to the same pattern as the original allocation result.
6615	if (isRemovableAlloc(V: CB, TLI)) {
6616	auto *I8Ty = Type::getInt8Ty(C&: CB->getParent()->getContext());
6617	if (nullptr != getInitialValueOfAllocation(V: CB, TLI, Ty: I8Ty)) {
6618	AllocationInfo *AI = new (A.Allocator) AllocationInfo{.CB: CB};
6619	AllocationInfos[CB] = AI;
6620	if (TLI)
6621	TLI->getLibFunc(CB: *CB, F&: AI->LibraryFunctionId);
6622	}
6623	}
6624	return true;
6625	};
6626
6627	bool UsedAssumedInformation = false;
6628	bool Success = A.checkForAllCallLikeInstructions(
6629	Pred: AllocationIdentifierCB, QueryingAA: *this, UsedAssumedInformation,
6630	/* CheckBBLivenessOnly */ false,
6631	/* CheckPotentiallyDead */ true);
6632	(void)Success;
6633	assert(Success && "Did not expect the call base visit callback to fail!");
6634
6635	Attributor::SimplifictionCallbackTy SCB =
6636	[](const IRPosition &, const AbstractAttribute *,
6637	bool &) -> std::optional<Value *> { return nullptr; };
6638	for (const auto &It : AllocationInfos)
6639	A.registerSimplificationCallback(IRP: IRPosition::callsite_returned(CB: *It.first),
6640	CB: SCB);
6641	for (const auto &It : DeallocationInfos)
6642	A.registerSimplificationCallback(IRP: IRPosition::callsite_returned(CB: *It.first),
6643	CB: SCB);
6644	}
6645
6646	const std::string getAsStr(Attributor *A) const override {
6647	unsigned NumH2SMallocs = 0, NumInvalidMallocs = 0;
6648	for (const auto &It : AllocationInfos) {
6649	if (It.second->Status == AllocationInfo::INVALID)
6650	++NumInvalidMallocs;
6651	else
6652	++NumH2SMallocs;
6653	}
6654	return "[H2S] Mallocs Good/Bad: " + std::to_string(val: NumH2SMallocs) + "/" +
6655	std::to_string(val: NumInvalidMallocs);
6656	}
6657
6658	/// See AbstractAttribute::trackStatistics().
6659	void trackStatistics() const override {
6660	STATS_DECL(
6661	MallocCalls, Function,
6662	"Number of malloc/calloc/aligned_alloc calls converted to allocas");
6663	for (const auto &It : AllocationInfos)
6664	if (It.second->Status != AllocationInfo::INVALID)
6665	++BUILD_STAT_NAME(MallocCalls, Function);
6666	}
6667
6668	bool isAssumedHeapToStack(const CallBase &CB) const override {
6669	if (isValidState())
6670	if (AllocationInfo *AI =
6671	AllocationInfos.lookup(Key: const_cast<CallBase *>(&CB)))
6672	return AI->Status != AllocationInfo::INVALID;
6673	return false;
6674	}
6675
6676	bool isAssumedHeapToStackRemovedFree(CallBase &CB) const override {
6677	if (!isValidState())
6678	return false;
6679
6680	for (const auto &It : AllocationInfos) {
6681	AllocationInfo &AI = *It.second;
6682	if (AI.Status == AllocationInfo::INVALID)
6683	continue;
6684
6685	if (AI.PotentialFreeCalls.count(key: &CB))
6686	return true;
6687	}
6688
6689	return false;
6690	}
6691
6692	ChangeStatus manifest(Attributor &A) override {
6693	assert(getState().isValidState() &&
6694	"Attempted to manifest an invalid state!");
6695
6696	ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
6697	Function *F = getAnchorScope();
6698	const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
6699
6700	for (auto &It : AllocationInfos) {
6701	AllocationInfo &AI = *It.second;
6702	if (AI.Status == AllocationInfo::INVALID)
6703	continue;
6704
6705	for (CallBase *FreeCall : AI.PotentialFreeCalls) {
6706	LLVM_DEBUG(dbgs() << "H2S: Removing free call: " << *FreeCall << "\n");
6707	A.deleteAfterManifest(I&: *FreeCall);
6708	HasChanged = ChangeStatus::CHANGED;
6709	}
6710
6711	LLVM_DEBUG(dbgs() << "H2S: Removing malloc-like call: " << *AI.CB
6712	<< "\n");
6713
6714	auto Remark = [&](OptimizationRemark OR) {
6715	LibFunc IsAllocShared;
6716	if (TLI->getLibFunc(CB: *AI.CB, F&: IsAllocShared))
6717	if (IsAllocShared == LibFunc___kmpc_alloc_shared)
6718	return OR << "Moving globalized variable to the stack.";
6719	return OR << "Moving memory allocation from the heap to the stack.";
6720	};
6721	if (AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared)
6722	A.emitRemark<OptimizationRemark>(I: AI.CB, RemarkName: "OMP110", RemarkCB&: Remark);
6723	else
6724	A.emitRemark<OptimizationRemark>(I: AI.CB, RemarkName: "HeapToStack", RemarkCB&: Remark);
6725
6726	const DataLayout &DL = A.getInfoCache().getDL();
6727	Value *Size;
6728	std::optional<APInt> SizeAPI = getSize(A, AA: *this, AI);
6729	if (SizeAPI) {
6730	Size = ConstantInt::get(Context&: AI.CB->getContext(), V: *SizeAPI);
6731	} else {
6732	LLVMContext &Ctx = AI.CB->getContext();
6733	ObjectSizeOpts Opts;
6734	ObjectSizeOffsetEvaluator Eval(DL, TLI, Ctx, Opts);
6735	SizeOffsetValue SizeOffsetPair = Eval.compute(V: AI.CB);
6736	assert(SizeOffsetPair != ObjectSizeOffsetEvaluator::unknown() &&
6737	cast<ConstantInt>(SizeOffsetPair.Offset)->isZero());
6738	Size = SizeOffsetPair.Size;
6739	}
6740
6741	BasicBlock::iterator IP = AI.MoveAllocaIntoEntry
6742	? F->getEntryBlock().begin()
6743	: AI.CB->getIterator();
6744
6745	Align Alignment(1);
6746	if (MaybeAlign RetAlign = AI.CB->getRetAlign())
6747	Alignment = std::max(a: Alignment, b: *RetAlign);
6748	if (Value *Align = getAllocAlignment(V: AI.CB, TLI)) {
6749	std::optional<APInt> AlignmentAPI = getAPInt(A, AA: *this, V&: *Align);
6750	assert(AlignmentAPI && AlignmentAPI->getZExtValue() > 0 &&
6751	"Expected an alignment during manifest!");
6752	Alignment =
6753	std::max(a: Alignment, b: assumeAligned(Value: AlignmentAPI->getZExtValue()));
6754	}
6755
6756	// TODO: Hoist the alloca towards the function entry.
6757	unsigned AS = DL.getAllocaAddrSpace();
6758	Instruction *Alloca =
6759	new AllocaInst(Type::getInt8Ty(C&: F->getContext()), AS, Size, Alignment,
6760	AI.CB->getName() + ".h2s", IP);
6761
6762	if (Alloca->getType() != AI.CB->getType())
6763	Alloca = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
6764	S: Alloca, Ty: AI.CB->getType(), Name: "malloc_cast", InsertBefore: AI.CB->getIterator());
6765
6766	auto *I8Ty = Type::getInt8Ty(C&: F->getContext());
6767	auto *InitVal = getInitialValueOfAllocation(V: AI.CB, TLI, Ty: I8Ty);
6768	assert(InitVal &&
6769	"Must be able to materialize initial memory state of allocation");
6770
6771	A.changeAfterManifest(IRP: IRPosition::inst(I: *AI.CB), NV&: *Alloca);
6772
6773	if (auto *II = dyn_cast<InvokeInst>(Val: AI.CB)) {
6774	auto *NBB = II->getNormalDest();
6775	BranchInst::Create(IfTrue: NBB, InsertAtEnd: AI.CB->getParent());
6776	A.deleteAfterManifest(I&: *AI.CB);
6777	} else {
6778	A.deleteAfterManifest(I&: *AI.CB);
6779	}
6780
6781	// Initialize the alloca with the same value as used by the allocation
6782	// function. We can skip undef as the initial value of an alloc is
6783	// undef, and the memset would simply end up being DSEd.
6784	if (!isa<UndefValue>(Val: InitVal)) {
6785	IRBuilder<> Builder(Alloca->getNextNode());
6786	// TODO: Use alignment above if align!=1
6787	Builder.CreateMemSet(Ptr: Alloca, Val: InitVal, Size, Align: std::nullopt);
6788	}
6789	HasChanged = ChangeStatus::CHANGED;
6790	}
6791
6792	return HasChanged;
6793	}
6794
6795	std::optional<APInt> getAPInt(Attributor &A, const AbstractAttribute &AA,
6796	Value &V) {
6797	bool UsedAssumedInformation = false;
6798	std::optional<Constant *> SimpleV =
6799	A.getAssumedConstant(V, AA, UsedAssumedInformation);
6800	if (!SimpleV)
6801	return APInt(64, 0);
6802	if (auto *CI = dyn_cast_or_null<ConstantInt>(Val: *SimpleV))
6803	return CI->getValue();
6804	return std::nullopt;
6805	}
6806
6807	std::optional<APInt> getSize(Attributor &A, const AbstractAttribute &AA,
6808	AllocationInfo &AI) {
6809	auto Mapper = [&](const Value *V) -> const Value * {
6810	bool UsedAssumedInformation = false;
6811	if (std::optional<Constant *> SimpleV =
6812	A.getAssumedConstant(V: *V, AA, UsedAssumedInformation))
6813	if (*SimpleV)
6814	return *SimpleV;
6815	return V;
6816	};
6817
6818	const Function *F = getAnchorScope();
6819	const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
6820	return getAllocSize(CB: AI.CB, TLI, Mapper);
6821	}
6822
6823	/// Collection of all malloc-like calls in a function with associated
6824	/// information.
6825	MapVector<CallBase *, AllocationInfo *> AllocationInfos;
6826
6827	/// Collection of all free-like calls in a function with associated
6828	/// information.
6829	MapVector<CallBase *, DeallocationInfo *> DeallocationInfos;
6830
6831	ChangeStatus updateImpl(Attributor &A) override;
6832	};
6833
6834	ChangeStatus AAHeapToStackFunction::updateImpl(Attributor &A) {
6835	ChangeStatus Changed = ChangeStatus::UNCHANGED;
6836	const Function *F = getAnchorScope();
6837	const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
6838
6839	const auto *LivenessAA =
6840	A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: IRPosition::function(F: *F), DepClass: DepClassTy::NONE);
6841
6842	MustBeExecutedContextExplorer *Explorer =
6843	A.getInfoCache().getMustBeExecutedContextExplorer();
6844
6845	bool StackIsAccessibleByOtherThreads =
6846	A.getInfoCache().stackIsAccessibleByOtherThreads();
6847
6848	LoopInfo *LI =
6849	A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(F: *F);
6850	std::optional<bool> MayContainIrreducibleControl;
6851	auto IsInLoop = [&](BasicBlock &BB) {
6852	if (&F->getEntryBlock() == &BB)
6853	return false;
6854	if (!MayContainIrreducibleControl.has_value())
6855	MayContainIrreducibleControl = mayContainIrreducibleControl(F: *F, LI);
6856	if (*MayContainIrreducibleControl)
6857	return true;
6858	if (!LI)
6859	return true;
6860	return LI->getLoopFor(BB: &BB) != nullptr;
6861	};
6862
6863	// Flag to ensure we update our deallocation information at most once per
6864	// updateImpl call and only if we use the free check reasoning.
6865	bool HasUpdatedFrees = false;
6866
6867	auto UpdateFrees = [&]() {
6868	HasUpdatedFrees = true;
6869
6870	for (auto &It : DeallocationInfos) {
6871	DeallocationInfo &DI = *It.second;
6872	// For now we cannot use deallocations that have unknown inputs, skip
6873	// them.
6874	if (DI.MightFreeUnknownObjects)
6875	continue;
6876
6877	// No need to analyze dead calls, ignore them instead.
6878	bool UsedAssumedInformation = false;
6879	if (A.isAssumedDead(I: *DI.CB, QueryingAA: this, LivenessAA, UsedAssumedInformation,
6880	/* CheckBBLivenessOnly */ true))
6881	continue;
6882
6883	// Use the non-optimistic version to get the freed object.
6884	Value *Obj = getUnderlyingObject(V: DI.FreedOp);
6885	if (!Obj) {
6886	LLVM_DEBUG(dbgs() << "[H2S] Unknown underlying object for free!\n");
6887	DI.MightFreeUnknownObjects = true;
6888	continue;
6889	}
6890
6891	// Free of null and undef can be ignored as no-ops (or UB in the latter
6892	// case).
6893	if (isa<ConstantPointerNull>(Val: Obj) || isa<UndefValue>(Val: Obj))
6894	continue;
6895
6896	CallBase *ObjCB = dyn_cast<CallBase>(Val: Obj);
6897	if (!ObjCB) {
6898	LLVM_DEBUG(dbgs() << "[H2S] Free of a non-call object: " << *Obj
6899	<< "\n");
6900	DI.MightFreeUnknownObjects = true;
6901	continue;
6902	}
6903
6904	AllocationInfo *AI = AllocationInfos.lookup(Key: ObjCB);
6905	if (!AI) {
6906	LLVM_DEBUG(dbgs() << "[H2S] Free of a non-allocation object: " << *Obj
6907	<< "\n");
6908	DI.MightFreeUnknownObjects = true;
6909	continue;
6910	}
6911
6912	DI.PotentialAllocationCalls.insert(X: ObjCB);
6913	}
6914	};
6915
6916	auto FreeCheck = [&](AllocationInfo &AI) {
6917	// If the stack is not accessible by other threads, the "must-free" logic
6918	// doesn't apply as the pointer could be shared and needs to be places in
6919	// "shareable" memory.
6920	if (!StackIsAccessibleByOtherThreads) {
6921	bool IsKnownNoSycn;
6922	if (!AA::hasAssumedIRAttr<Attribute::NoSync>(
6923	A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnownNoSycn)) {
6924	LLVM_DEBUG(
6925	dbgs() << "[H2S] found an escaping use, stack is not accessible by "
6926	"other threads and function is not nosync:\n");
6927	return false;
6928	}
6929	}
6930	if (!HasUpdatedFrees)
6931	UpdateFrees();
6932
6933	// TODO: Allow multi exit functions that have different free calls.
6934	if (AI.PotentialFreeCalls.size() != 1) {
6935	LLVM_DEBUG(dbgs() << "[H2S] did not find one free call but "
6936	<< AI.PotentialFreeCalls.size() << "\n");
6937	return false;
6938	}
6939	CallBase *UniqueFree = *AI.PotentialFreeCalls.begin();
6940	DeallocationInfo *DI = DeallocationInfos.lookup(Key: UniqueFree);
6941	if (!DI) {
6942	LLVM_DEBUG(
6943	dbgs() << "[H2S] unique free call was not known as deallocation call "
6944	<< *UniqueFree << "\n");
6945	return false;
6946	}
6947	if (DI->MightFreeUnknownObjects) {
6948	LLVM_DEBUG(
6949	dbgs() << "[H2S] unique free call might free unknown allocations\n");
6950	return false;
6951	}
6952	if (DI->PotentialAllocationCalls.empty())
6953	return true;
6954	if (DI->PotentialAllocationCalls.size() > 1) {
6955	LLVM_DEBUG(dbgs() << "[H2S] unique free call might free "
6956	<< DI->PotentialAllocationCalls.size()
6957	<< " different allocations\n");
6958	return false;
6959	}
6960	if (*DI->PotentialAllocationCalls.begin() != AI.CB) {
6961	LLVM_DEBUG(
6962	dbgs()
6963	<< "[H2S] unique free call not known to free this allocation but "
6964	<< **DI->PotentialAllocationCalls.begin() << "\n");
6965	return false;
6966	}
6967
6968	// __kmpc_alloc_shared and __kmpc_alloc_free are by construction matched.
6969	if (AI.LibraryFunctionId != LibFunc___kmpc_alloc_shared) {
6970	Instruction *CtxI = isa<InvokeInst>(Val: AI.CB) ? AI.CB : AI.CB->getNextNode();
6971	if (!Explorer || !Explorer->findInContextOf(I: UniqueFree, PP: CtxI)) {
6972	LLVM_DEBUG(
6973	dbgs()
6974	<< "[H2S] unique free call might not be executed with the allocation "
6975	<< *UniqueFree << "\n");
6976	return false;
6977	}
6978	}
6979	return true;
6980	};
6981
6982	auto UsesCheck = [&](AllocationInfo &AI) {
6983	bool ValidUsesOnly = true;
6984
6985	auto Pred = [&](const Use &U, bool &Follow) -> bool {
6986	Instruction *UserI = cast<Instruction>(Val: U.getUser());
6987	if (isa<LoadInst>(Val: UserI))
6988	return true;
6989	if (auto *SI = dyn_cast<StoreInst>(Val: UserI)) {
6990	if (SI->getValueOperand() == U.get()) {
6991	LLVM_DEBUG(dbgs()
6992	<< "[H2S] escaping store to memory: " << *UserI << "\n");
6993	ValidUsesOnly = false;
6994	} else {
6995	// A store into the malloc'ed memory is fine.
6996	}
6997	return true;
6998	}
6999	if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
7000	if (!CB->isArgOperand(U: &U) || CB->isLifetimeStartOrEnd())
7001	return true;
7002	if (DeallocationInfos.count(Key: CB)) {
7003	AI.PotentialFreeCalls.insert(X: CB);
7004	return true;
7005	}
7006
7007	unsigned ArgNo = CB->getArgOperandNo(U: &U);
7008	auto CBIRP = IRPosition::callsite_argument(CB: *CB, ArgNo);
7009
7010	bool IsKnownNoCapture;
7011	bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
7012	A, this, CBIRP, DepClassTy::OPTIONAL, IsKnownNoCapture);
7013
7014	// If a call site argument use is nofree, we are fine.
7015	bool IsKnownNoFree;
7016	bool IsAssumedNoFree = AA::hasAssumedIRAttr<Attribute::NoFree>(
7017	A, this, CBIRP, DepClassTy::OPTIONAL, IsKnownNoFree);
7018
7019	if (!IsAssumedNoCapture ||
7020	(AI.LibraryFunctionId != LibFunc___kmpc_alloc_shared &&
7021	!IsAssumedNoFree)) {
7022	AI.HasPotentiallyFreeingUnknownUses |= !IsAssumedNoFree;
7023
7024	// Emit a missed remark if this is missed OpenMP globalization.
7025	auto Remark = [&](OptimizationRemarkMissed ORM) {
7026	return ORM
7027	<< "Could not move globalized variable to the stack. "
7028	"Variable is potentially captured in call. Mark "
7029	"parameter as `__attribute__((noescape))` to override.";
7030	};
7031
7032	if (ValidUsesOnly &&
7033	AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared)
7034	A.emitRemark<OptimizationRemarkMissed>(I: CB, RemarkName: "OMP113", RemarkCB&: Remark);
7035
7036	LLVM_DEBUG(dbgs() << "[H2S] Bad user: " << *UserI << "\n");
7037	ValidUsesOnly = false;
7038	}
7039	return true;
7040	}
7041
7042	if (isa<GetElementPtrInst>(Val: UserI) || isa<BitCastInst>(Val: UserI) ||
7043	isa<PHINode>(Val: UserI) || isa<SelectInst>(Val: UserI)) {
7044	Follow = true;
7045	return true;
7046	}
7047	// Unknown user for which we can not track uses further (in a way that
7048	// makes sense).
7049	LLVM_DEBUG(dbgs() << "[H2S] Unknown user: " << *UserI << "\n");
7050	ValidUsesOnly = false;
7051	return true;
7052	};
7053	if (!A.checkForAllUses(Pred, QueryingAA: *this, V: *AI.CB, /* CheckBBLivenessOnly */ false,
7054	LivenessDepClass: DepClassTy::OPTIONAL, /* IgnoreDroppableUses */ true,
7055	EquivalentUseCB: [&](const Use &OldU, const Use &NewU) {
7056	auto *SI = dyn_cast<StoreInst>(Val: OldU.getUser());
7057	return !SI || StackIsAccessibleByOtherThreads ||
7058	AA::isAssumedThreadLocalObject(
7059	A, Obj&: *SI->getPointerOperand(), QueryingAA: *this);
7060	}))
7061	return false;
7062	return ValidUsesOnly;
7063	};
7064
7065	// The actual update starts here. We look at all allocations and depending on
7066	// their status perform the appropriate check(s).
7067	for (auto &It : AllocationInfos) {
7068	AllocationInfo &AI = *It.second;
7069	if (AI.Status == AllocationInfo::INVALID)
7070	continue;
7071
7072	if (Value *Align = getAllocAlignment(V: AI.CB, TLI)) {
7073	std::optional<APInt> APAlign = getAPInt(A, AA: *this, V&: *Align);
7074	if (!APAlign) {
7075	// Can't generate an alloca which respects the required alignment
7076	// on the allocation.
7077	LLVM_DEBUG(dbgs() << "[H2S] Unknown allocation alignment: " << *AI.CB
7078	<< "\n");
7079	AI.Status = AllocationInfo::INVALID;
7080	Changed = ChangeStatus::CHANGED;
7081	continue;
7082	}
7083	if (APAlign->ugt(RHS: llvm::Value::MaximumAlignment) ||
7084	!APAlign->isPowerOf2()) {
7085	LLVM_DEBUG(dbgs() << "[H2S] Invalid allocation alignment: " << APAlign
7086	<< "\n");
7087	AI.Status = AllocationInfo::INVALID;
7088	Changed = ChangeStatus::CHANGED;
7089	continue;
7090	}
7091	}
7092
7093	std::optional<APInt> Size = getSize(A, AA: *this, AI);
7094	if (AI.LibraryFunctionId != LibFunc___kmpc_alloc_shared &&
7095	MaxHeapToStackSize != -1) {
7096	if (!Size || Size->ugt(RHS: MaxHeapToStackSize)) {
7097	LLVM_DEBUG({
7098	if (!Size)
7099	dbgs() << "[H2S] Unknown allocation size: " << *AI.CB << "\n";
7100	else
7101	dbgs() << "[H2S] Allocation size too large: " << *AI.CB << " vs. "
7102	<< MaxHeapToStackSize << "\n";
7103	});
7104
7105	AI.Status = AllocationInfo::INVALID;
7106	Changed = ChangeStatus::CHANGED;
7107	continue;
7108	}
7109	}
7110
7111	switch (AI.Status) {
7112	case AllocationInfo::STACK_DUE_TO_USE:
7113	if (UsesCheck(AI))
7114	break;
7115	AI.Status = AllocationInfo::STACK_DUE_TO_FREE;
7116	[[fallthrough]];
7117	case AllocationInfo::STACK_DUE_TO_FREE:
7118	if (FreeCheck(AI))
7119	break;
7120	AI.Status = AllocationInfo::INVALID;
7121	Changed = ChangeStatus::CHANGED;
7122	break;
7123	case AllocationInfo::INVALID:
7124	llvm_unreachable("Invalid allocations should never reach this point!");
7125	};
7126
7127	// Check if we still think we can move it into the entry block. If the
7128	// alloca comes from a converted __kmpc_alloc_shared then we can usually
7129	// ignore the potential compilations associated with loops.
7130	bool IsGlobalizedLocal =
7131	AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared;
7132	if (AI.MoveAllocaIntoEntry &&
7133	(!Size.has_value() ||
7134	(!IsGlobalizedLocal && IsInLoop(*AI.CB->getParent()))))
7135	AI.MoveAllocaIntoEntry = false;
7136	}
7137
7138	return Changed;
7139	}
7140	} // namespace
7141
7142	/// ----------------------- Privatizable Pointers ------------------------------
7143	namespace {
7144	struct AAPrivatizablePtrImpl : public AAPrivatizablePtr {
7145	AAPrivatizablePtrImpl(const IRPosition &IRP, Attributor &A)
7146	: AAPrivatizablePtr(IRP, A), PrivatizableType(std::nullopt) {}
7147
7148	ChangeStatus indicatePessimisticFixpoint() override {
7149	AAPrivatizablePtr::indicatePessimisticFixpoint();
7150	PrivatizableType = nullptr;
7151	return ChangeStatus::CHANGED;
7152	}
7153
7154	/// Identify the type we can chose for a private copy of the underlying
7155	/// argument. std::nullopt means it is not clear yet, nullptr means there is
7156	/// none.
7157	virtual std::optional<Type *> identifyPrivatizableType(Attributor &A) = 0;
7158
7159	/// Return a privatizable type that encloses both T0 and T1.
7160	/// TODO: This is merely a stub for now as we should manage a mapping as well.
7161	std::optional<Type *> combineTypes(std::optional<Type *> T0,
7162	std::optional<Type *> T1) {
7163	if (!T0)
7164	return T1;
7165	if (!T1)
7166	return T0;
7167	if (T0 == T1)
7168	return T0;
7169	return nullptr;
7170	}
7171
7172	std::optional<Type *> getPrivatizableType() const override {
7173	return PrivatizableType;
7174	}
7175
7176	const std::string getAsStr(Attributor *A) const override {
7177	return isAssumedPrivatizablePtr() ? "[priv]" : "[no-priv]";
7178	}
7179
7180	protected:
7181	std::optional<Type *> PrivatizableType;
7182	};
7183
7184	// TODO: Do this for call site arguments (probably also other values) as well.
7185
7186	struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl {
7187	AAPrivatizablePtrArgument(const IRPosition &IRP, Attributor &A)
7188	: AAPrivatizablePtrImpl(IRP, A) {}
7189
7190	/// See AAPrivatizablePtrImpl::identifyPrivatizableType(...)
7191	std::optional<Type *> identifyPrivatizableType(Attributor &A) override {
7192	// If this is a byval argument and we know all the call sites (so we can
7193	// rewrite them), there is no need to check them explicitly.
7194	bool UsedAssumedInformation = false;
7195	SmallVector<Attribute, 1> Attrs;
7196	A.getAttrs(getIRPosition(), {Attribute::ByVal}, Attrs,
7197	/* IgnoreSubsumingPositions */ true);
7198	if (!Attrs.empty() &&
7199	A.checkForAllCallSites(Pred: [](AbstractCallSite ACS) { return true; }, QueryingAA: *this,
7200	RequireAllCallSites: true, UsedAssumedInformation))
7201	return Attrs[0].getValueAsType();
7202
7203	std::optional<Type *> Ty;
7204	unsigned ArgNo = getIRPosition().getCallSiteArgNo();
7205
7206	// Make sure the associated call site argument has the same type at all call
7207	// sites and it is an allocation we know is safe to privatize, for now that
7208	// means we only allow alloca instructions.
7209	// TODO: We can additionally analyze the accesses in the callee to create
7210	// the type from that information instead. That is a little more
7211	// involved and will be done in a follow up patch.
7212	auto CallSiteCheck = [&](AbstractCallSite ACS) {
7213	IRPosition ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo);
7214	// Check if a coresponding argument was found or if it is one not
7215	// associated (which can happen for callback calls).
7216	if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
7217	return false;
7218
7219	// Check that all call sites agree on a type.
7220	auto *PrivCSArgAA =
7221	A.getAAFor<AAPrivatizablePtr>(QueryingAA: *this, IRP: ACSArgPos, DepClass: DepClassTy::REQUIRED);
7222	if (!PrivCSArgAA)
7223	return false;
7224	std::optional<Type *> CSTy = PrivCSArgAA->getPrivatizableType();
7225
7226	LLVM_DEBUG({
7227	dbgs() << "[AAPrivatizablePtr] ACSPos: " << ACSArgPos << ", CSTy: ";
7228	if (CSTy && *CSTy)
7229	(*CSTy)->print(dbgs());
7230	else if (CSTy)
7231	dbgs() << "<nullptr>";
7232	else
7233	dbgs() << "<none>";
7234	});
7235
7236	Ty = combineTypes(T0: Ty, T1: CSTy);
7237
7238	LLVM_DEBUG({
7239	dbgs() << " : New Type: ";
7240	if (Ty && *Ty)
7241	(*Ty)->print(dbgs());
7242	else if (Ty)
7243	dbgs() << "<nullptr>";
7244	else
7245	dbgs() << "<none>";
7246	dbgs() << "\n";
7247	});
7248
7249	return !Ty || *Ty;
7250	};
7251
7252	if (!A.checkForAllCallSites(Pred: CallSiteCheck, QueryingAA: *this, RequireAllCallSites: true,
7253	UsedAssumedInformation))
7254	return nullptr;
7255	return Ty;
7256	}
7257
7258	/// See AbstractAttribute::updateImpl(...).
7259	ChangeStatus updateImpl(Attributor &A) override {
7260	PrivatizableType = identifyPrivatizableType(A);
7261	if (!PrivatizableType)
7262	return ChangeStatus::UNCHANGED;
7263	if (!*PrivatizableType)
7264	return indicatePessimisticFixpoint();
7265
7266	// The dependence is optional so we don't give up once we give up on the
7267	// alignment.
7268	A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: getAssociatedValue()),
7269	DepClass: DepClassTy::OPTIONAL);
7270
7271	// Avoid arguments with padding for now.
7272	if (!A.hasAttr(getIRPosition(), Attribute::ByVal) &&
7273	!isDenselyPacked(*PrivatizableType, A.getInfoCache().getDL())) {
7274	LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Padding detected\n");
7275	return indicatePessimisticFixpoint();
7276	}
7277
7278	// Collect the types that will replace the privatizable type in the function
7279	// signature.
7280	SmallVector<Type *, 16> ReplacementTypes;
7281	identifyReplacementTypes(PrivType: *PrivatizableType, ReplacementTypes);
7282
7283	// Verify callee and caller agree on how the promoted argument would be
7284	// passed.
7285	Function &Fn = *getIRPosition().getAnchorScope();
7286	const auto *TTI =
7287	A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(F: Fn);
7288	if (!TTI) {
7289	LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Missing TTI for function "
7290	<< Fn.getName() << "\n");
7291	return indicatePessimisticFixpoint();
7292	}
7293
7294	auto CallSiteCheck = [&](AbstractCallSite ACS) {
7295	CallBase *CB = ACS.getInstruction();
7296	return TTI->areTypesABICompatible(
7297	Caller: CB->getCaller(),
7298	Callee: dyn_cast_if_present<Function>(Val: CB->getCalledOperand()),
7299	Types: ReplacementTypes);
7300	};
7301	bool UsedAssumedInformation = false;
7302	if (!A.checkForAllCallSites(Pred: CallSiteCheck, QueryingAA: *this, RequireAllCallSites: true,
7303	UsedAssumedInformation)) {
7304	LLVM_DEBUG(
7305	dbgs() << "[AAPrivatizablePtr] ABI incompatibility detected for "
7306	<< Fn.getName() << "\n");
7307	return indicatePessimisticFixpoint();
7308	}
7309
7310	// Register a rewrite of the argument.
7311	Argument *Arg = getAssociatedArgument();
7312	if (!A.isValidFunctionSignatureRewrite(Arg&: *Arg, ReplacementTypes)) {
7313	LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Rewrite not valid\n");
7314	return indicatePessimisticFixpoint();
7315	}
7316
7317	unsigned ArgNo = Arg->getArgNo();
7318
7319	// Helper to check if for the given call site the associated argument is
7320	// passed to a callback where the privatization would be different.
7321	auto IsCompatiblePrivArgOfCallback = [&](CallBase &CB) {
7322	SmallVector<const Use *, 4> CallbackUses;
7323	AbstractCallSite::getCallbackUses(CB, CallbackUses);
7324	for (const Use *U : CallbackUses) {
7325	AbstractCallSite CBACS(U);
7326	assert(CBACS && CBACS.isCallbackCall());
7327	for (Argument &CBArg : CBACS.getCalledFunction()->args()) {
7328	int CBArgNo = CBACS.getCallArgOperandNo(Arg&: CBArg);
7329
7330	LLVM_DEBUG({
7331	dbgs()
7332	<< "[AAPrivatizablePtr] Argument " << *Arg
7333	<< "check if can be privatized in the context of its parent ("
7334	<< Arg->getParent()->getName()
7335	<< ")\n[AAPrivatizablePtr] because it is an argument in a "
7336	"callback ("
7337	<< CBArgNo << "@" << CBACS.getCalledFunction()->getName()
7338	<< ")\n[AAPrivatizablePtr] " << CBArg << " : "
7339	<< CBACS.getCallArgOperand(CBArg) << " vs "
7340	<< CB.getArgOperand(ArgNo) << "\n"
7341	<< "[AAPrivatizablePtr] " << CBArg << " : "
7342	<< CBACS.getCallArgOperandNo(CBArg) << " vs " << ArgNo << "\n";
7343	});
7344
7345	if (CBArgNo != int(ArgNo))
7346	continue;
7347	const auto *CBArgPrivAA = A.getAAFor<AAPrivatizablePtr>(
7348	QueryingAA: *this, IRP: IRPosition::argument(Arg: CBArg), DepClass: DepClassTy::REQUIRED);
7349	if (CBArgPrivAA && CBArgPrivAA->isValidState()) {
7350	auto CBArgPrivTy = CBArgPrivAA->getPrivatizableType();
7351	if (!CBArgPrivTy)
7352	continue;
7353	if (*CBArgPrivTy == PrivatizableType)
7354	continue;
7355	}
7356
7357	LLVM_DEBUG({
7358	dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7359	<< " cannot be privatized in the context of its parent ("
7360	<< Arg->getParent()->getName()
7361	<< ")\n[AAPrivatizablePtr] because it is an argument in a "
7362	"callback ("
7363	<< CBArgNo << "@" << CBACS.getCalledFunction()->getName()
7364	<< ").\n[AAPrivatizablePtr] for which the argument "
7365	"privatization is not compatible.\n";
7366	});
7367	return false;
7368	}
7369	}
7370	return true;
7371	};
7372
7373	// Helper to check if for the given call site the associated argument is
7374	// passed to a direct call where the privatization would be different.
7375	auto IsCompatiblePrivArgOfDirectCS = [&](AbstractCallSite ACS) {
7376	CallBase *DC = cast<CallBase>(Val: ACS.getInstruction());
7377	int DCArgNo = ACS.getCallArgOperandNo(ArgNo);
7378	assert(DCArgNo >= 0 && unsigned(DCArgNo) < DC->arg_size() &&
7379	"Expected a direct call operand for callback call operand");
7380
7381	Function *DCCallee =
7382	dyn_cast_if_present<Function>(Val: DC->getCalledOperand());
7383	LLVM_DEBUG({
7384	dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7385	<< " check if be privatized in the context of its parent ("
7386	<< Arg->getParent()->getName()
7387	<< ")\n[AAPrivatizablePtr] because it is an argument in a "
7388	"direct call of ("
7389	<< DCArgNo << "@" << DCCallee->getName() << ").\n";
7390	});
7391
7392	if (unsigned(DCArgNo) < DCCallee->arg_size()) {
7393	const auto *DCArgPrivAA = A.getAAFor<AAPrivatizablePtr>(
7394	QueryingAA: *this, IRP: IRPosition::argument(Arg: *DCCallee->getArg(i: DCArgNo)),
7395	DepClass: DepClassTy::REQUIRED);
7396	if (DCArgPrivAA && DCArgPrivAA->isValidState()) {
7397	auto DCArgPrivTy = DCArgPrivAA->getPrivatizableType();
7398	if (!DCArgPrivTy)
7399	return true;
7400	if (*DCArgPrivTy == PrivatizableType)
7401	return true;
7402	}
7403	}
7404
7405	LLVM_DEBUG({
7406	dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7407	<< " cannot be privatized in the context of its parent ("
7408	<< Arg->getParent()->getName()
7409	<< ")\n[AAPrivatizablePtr] because it is an argument in a "
7410	"direct call of ("
7411	<< ACS.getInstruction()->getCalledOperand()->getName()
7412	<< ").\n[AAPrivatizablePtr] for which the argument "
7413	"privatization is not compatible.\n";
7414	});
7415	return false;
7416	};
7417
7418	// Helper to check if the associated argument is used at the given abstract
7419	// call site in a way that is incompatible with the privatization assumed
7420	// here.
7421	auto IsCompatiblePrivArgOfOtherCallSite = [&](AbstractCallSite ACS) {
7422	if (ACS.isDirectCall())
7423	return IsCompatiblePrivArgOfCallback(*ACS.getInstruction());
7424	if (ACS.isCallbackCall())
7425	return IsCompatiblePrivArgOfDirectCS(ACS);
7426	return false;
7427	};
7428
7429	if (!A.checkForAllCallSites(Pred: IsCompatiblePrivArgOfOtherCallSite, QueryingAA: *this, RequireAllCallSites: true,
7430	UsedAssumedInformation))
7431	return indicatePessimisticFixpoint();
7432
7433	return ChangeStatus::UNCHANGED;
7434	}
7435
7436	/// Given a type to private \p PrivType, collect the constituates (which are
7437	/// used) in \p ReplacementTypes.
7438	static void
7439	identifyReplacementTypes(Type *PrivType,
7440	SmallVectorImpl<Type *> &ReplacementTypes) {
7441	// TODO: For now we expand the privatization type to the fullest which can
7442	// lead to dead arguments that need to be removed later.
7443	assert(PrivType && "Expected privatizable type!");
7444
7445	// Traverse the type, extract constituate types on the outermost level.
7446	if (auto *PrivStructType = dyn_cast<StructType>(Val: PrivType)) {
7447	for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++)
7448	ReplacementTypes.push_back(Elt: PrivStructType->getElementType(N: u));
7449	} else if (auto *PrivArrayType = dyn_cast<ArrayType>(Val: PrivType)) {
7450	ReplacementTypes.append(NumInputs: PrivArrayType->getNumElements(),
7451	Elt: PrivArrayType->getElementType());
7452	} else {
7453	ReplacementTypes.push_back(Elt: PrivType);
7454	}
7455	}
7456
7457	/// Initialize \p Base according to the type \p PrivType at position \p IP.
7458	/// The values needed are taken from the arguments of \p F starting at
7459	/// position \p ArgNo.
7460	static void createInitialization(Type *PrivType, Value &Base, Function &F,
7461	unsigned ArgNo, BasicBlock::iterator IP) {
7462	assert(PrivType && "Expected privatizable type!");
7463
7464	IRBuilder<NoFolder> IRB(IP->getParent(), IP);
7465	const DataLayout &DL = F.getParent()->getDataLayout();
7466
7467	// Traverse the type, build GEPs and stores.
7468	if (auto *PrivStructType = dyn_cast<StructType>(Val: PrivType)) {
7469	const StructLayout *PrivStructLayout = DL.getStructLayout(Ty: PrivStructType);
7470	for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {
7471	Value *Ptr =
7472	constructPointer(Ptr: &Base, Offset: PrivStructLayout->getElementOffset(Idx: u), IRB);
7473	new StoreInst(F.getArg(i: ArgNo + u), Ptr, IP);
7474	}
7475	} else if (auto *PrivArrayType = dyn_cast<ArrayType>(Val: PrivType)) {
7476	Type *PointeeTy = PrivArrayType->getElementType();
7477	uint64_t PointeeTySize = DL.getTypeStoreSize(Ty: PointeeTy);
7478	for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {
7479	Value *Ptr = constructPointer(Ptr: &Base, Offset: u * PointeeTySize, IRB);
7480	new StoreInst(F.getArg(i: ArgNo + u), Ptr, IP);
7481	}
7482	} else {
7483	new StoreInst(F.getArg(i: ArgNo), &Base, IP);
7484	}
7485	}
7486
7487	/// Extract values from \p Base according to the type \p PrivType at the
7488	/// call position \p ACS. The values are appended to \p ReplacementValues.
7489	void createReplacementValues(Align Alignment, Type *PrivType,
7490	AbstractCallSite ACS, Value *Base,
7491	SmallVectorImpl<Value *> &ReplacementValues) {
7492	assert(Base && "Expected base value!");
7493	assert(PrivType && "Expected privatizable type!");
7494	Instruction *IP = ACS.getInstruction();
7495
7496	IRBuilder<NoFolder> IRB(IP);
7497	const DataLayout &DL = IP->getModule()->getDataLayout();
7498
7499	// Traverse the type, build GEPs and loads.
7500	if (auto *PrivStructType = dyn_cast<StructType>(Val: PrivType)) {
7501	const StructLayout *PrivStructLayout = DL.getStructLayout(Ty: PrivStructType);
7502	for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {
7503	Type *PointeeTy = PrivStructType->getElementType(N: u);
7504	Value *Ptr =
7505	constructPointer(Ptr: Base, Offset: PrivStructLayout->getElementOffset(Idx: u), IRB);
7506	LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator());
7507	L->setAlignment(Alignment);
7508	ReplacementValues.push_back(Elt: L);
7509	}
7510	} else if (auto *PrivArrayType = dyn_cast<ArrayType>(Val: PrivType)) {
7511	Type *PointeeTy = PrivArrayType->getElementType();
7512	uint64_t PointeeTySize = DL.getTypeStoreSize(Ty: PointeeTy);
7513	for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {
7514	Value *Ptr = constructPointer(Ptr: Base, Offset: u * PointeeTySize, IRB);
7515	LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator());
7516	L->setAlignment(Alignment);
7517	ReplacementValues.push_back(Elt: L);
7518	}
7519	} else {
7520	LoadInst *L = new LoadInst(PrivType, Base, "", IP->getIterator());
7521	L->setAlignment(Alignment);
7522	ReplacementValues.push_back(Elt: L);
7523	}
7524	}
7525
7526	/// See AbstractAttribute::manifest(...)
7527	ChangeStatus manifest(Attributor &A) override {
7528	if (!PrivatizableType)
7529	return ChangeStatus::UNCHANGED;
7530	assert(*PrivatizableType && "Expected privatizable type!");
7531
7532	// Collect all tail calls in the function as we cannot allow new allocas to
7533	// escape into tail recursion.
7534	// TODO: Be smarter about new allocas escaping into tail calls.
7535	SmallVector<CallInst *, 16> TailCalls;
7536	bool UsedAssumedInformation = false;
7537	if (!A.checkForAllInstructions(
7538	Pred: [&](Instruction &I) {
7539	CallInst &CI = cast<CallInst>(Val&: I);
7540	if (CI.isTailCall())
7541	TailCalls.push_back(Elt: &CI);
7542	return true;
7543	},
7544	QueryingAA: *this, Opcodes: {Instruction::Call}, UsedAssumedInformation))
7545	return ChangeStatus::UNCHANGED;
7546
7547	Argument *Arg = getAssociatedArgument();
7548	// Query AAAlign attribute for alignment of associated argument to
7549	// determine the best alignment of loads.
7550	const auto *AlignAA =
7551	A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: *Arg), DepClass: DepClassTy::NONE);
7552
7553	// Callback to repair the associated function. A new alloca is placed at the
7554	// beginning and initialized with the values passed through arguments. The
7555	// new alloca replaces the use of the old pointer argument.
7556	Attributor::ArgumentReplacementInfo::CalleeRepairCBTy FnRepairCB =
7557	[=](const Attributor::ArgumentReplacementInfo &ARI,
7558	Function &ReplacementFn, Function::arg_iterator ArgIt) {
7559	BasicBlock &EntryBB = ReplacementFn.getEntryBlock();
7560	BasicBlock::iterator IP = EntryBB.getFirstInsertionPt();
7561	const DataLayout &DL = IP->getModule()->getDataLayout();
7562	unsigned AS = DL.getAllocaAddrSpace();
7563	Instruction *AI = new AllocaInst(*PrivatizableType, AS,
7564	Arg->getName() + ".priv", IP);
7565	createInitialization(PrivType: *PrivatizableType, Base&: *AI, F&: ReplacementFn,
7566	ArgNo: ArgIt->getArgNo(), IP);
7567
7568	if (AI->getType() != Arg->getType())
7569	AI = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
7570	S: AI, Ty: Arg->getType(), Name: "", InsertBefore: IP);
7571	Arg->replaceAllUsesWith(V: AI);
7572
7573	for (CallInst *CI : TailCalls)
7574	CI->setTailCall(false);
7575	};
7576
7577	// Callback to repair a call site of the associated function. The elements
7578	// of the privatizable type are loaded prior to the call and passed to the
7579	// new function version.
7580	Attributor::ArgumentReplacementInfo::ACSRepairCBTy ACSRepairCB =
7581	[=](const Attributor::ArgumentReplacementInfo &ARI,
7582	AbstractCallSite ACS, SmallVectorImpl<Value *> &NewArgOperands) {
7583	// When no alignment is specified for the load instruction,
7584	// natural alignment is assumed.
7585	createReplacementValues(
7586	Alignment: AlignAA ? AlignAA->getAssumedAlign() : Align(0),
7587	PrivType: *PrivatizableType, ACS,
7588	Base: ACS.getCallArgOperand(ArgNo: ARI.getReplacedArg().getArgNo()),
7589	ReplacementValues&: NewArgOperands);
7590	};
7591
7592	// Collect the types that will replace the privatizable type in the function
7593	// signature.
7594	SmallVector<Type *, 16> ReplacementTypes;
7595	identifyReplacementTypes(PrivType: *PrivatizableType, ReplacementTypes);
7596
7597	// Register a rewrite of the argument.
7598	if (A.registerFunctionSignatureRewrite(Arg&: *Arg, ReplacementTypes,
7599	CalleeRepairCB: std::move(FnRepairCB),
7600	ACSRepairCB: std::move(ACSRepairCB)))
7601	return ChangeStatus::CHANGED;
7602	return ChangeStatus::UNCHANGED;
7603	}
7604
7605	/// See AbstractAttribute::trackStatistics()
7606	void trackStatistics() const override {
7607	STATS_DECLTRACK_ARG_ATTR(privatizable_ptr);
7608	}
7609	};
7610
7611	struct AAPrivatizablePtrFloating : public AAPrivatizablePtrImpl {
7612	AAPrivatizablePtrFloating(const IRPosition &IRP, Attributor &A)
7613	: AAPrivatizablePtrImpl(IRP, A) {}
7614
7615	/// See AbstractAttribute::initialize(...).
7616	void initialize(Attributor &A) override {
7617	// TODO: We can privatize more than arguments.
7618	indicatePessimisticFixpoint();
7619	}
7620
7621	ChangeStatus updateImpl(Attributor &A) override {
7622	llvm_unreachable("AAPrivatizablePtr(Floating|Returned|CallSiteReturned)::"
7623	"updateImpl will not be called");
7624	}
7625
7626	/// See AAPrivatizablePtrImpl::identifyPrivatizableType(...)
7627	std::optional<Type *> identifyPrivatizableType(Attributor &A) override {
7628	Value *Obj = getUnderlyingObject(V: &getAssociatedValue());
7629	if (!Obj) {
7630	LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] No underlying object found!\n");
7631	return nullptr;
7632	}
7633
7634	if (auto *AI = dyn_cast<AllocaInst>(Val: Obj))
7635	if (auto *CI = dyn_cast<ConstantInt>(Val: AI->getArraySize()))
7636	if (CI->isOne())
7637	return AI->getAllocatedType();
7638	if (auto *Arg = dyn_cast<Argument>(Val: Obj)) {
7639	auto *PrivArgAA = A.getAAFor<AAPrivatizablePtr>(
7640	QueryingAA: *this, IRP: IRPosition::argument(Arg: *Arg), DepClass: DepClassTy::REQUIRED);
7641	if (PrivArgAA && PrivArgAA->isAssumedPrivatizablePtr())
7642	return PrivArgAA->getPrivatizableType();
7643	}
7644
7645	LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Underlying object neither valid "
7646	"alloca nor privatizable argument: "
7647	<< *Obj << "!\n");
7648	return nullptr;
7649	}
7650
7651	/// See AbstractAttribute::trackStatistics()
7652	void trackStatistics() const override {
7653	STATS_DECLTRACK_FLOATING_ATTR(privatizable_ptr);
7654	}
7655	};
7656
7657	struct AAPrivatizablePtrCallSiteArgument final
7658	: public AAPrivatizablePtrFloating {
7659	AAPrivatizablePtrCallSiteArgument(const IRPosition &IRP, Attributor &A)
7660	: AAPrivatizablePtrFloating(IRP, A) {}
7661
7662	/// See AbstractAttribute::initialize(...).
7663	void initialize(Attributor &A) override {
7664	if (A.hasAttr(getIRPosition(), Attribute::ByVal))
7665	indicateOptimisticFixpoint();
7666	}
7667
7668	/// See AbstractAttribute::updateImpl(...).
7669	ChangeStatus updateImpl(Attributor &A) override {
7670	PrivatizableType = identifyPrivatizableType(A);
7671	if (!PrivatizableType)
7672	return ChangeStatus::UNCHANGED;
7673	if (!*PrivatizableType)
7674	return indicatePessimisticFixpoint();
7675
7676	const IRPosition &IRP = getIRPosition();
7677	bool IsKnownNoCapture;
7678	bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
7679	A, this, IRP, DepClassTy::REQUIRED, IsKnownNoCapture);
7680	if (!IsAssumedNoCapture) {
7681	LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might be captured!\n");
7682	return indicatePessimisticFixpoint();
7683	}
7684
7685	bool IsKnownNoAlias;
7686	if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
7687	A, this, IRP, DepClassTy::REQUIRED, IsKnownNoAlias)) {
7688	LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might alias!\n");
7689	return indicatePessimisticFixpoint();
7690	}
7691
7692	bool IsKnown;
7693	if (!AA::isAssumedReadOnly(A, IRP, QueryingAA: *this, IsKnown)) {
7694	LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer is written!\n");
7695	return indicatePessimisticFixpoint();
7696	}
7697
7698	return ChangeStatus::UNCHANGED;
7699	}
7700
7701	/// See AbstractAttribute::trackStatistics()
7702	void trackStatistics() const override {
7703	STATS_DECLTRACK_CSARG_ATTR(privatizable_ptr);
7704	}
7705	};
7706
7707	struct AAPrivatizablePtrCallSiteReturned final
7708	: public AAPrivatizablePtrFloating {
7709	AAPrivatizablePtrCallSiteReturned(const IRPosition &IRP, Attributor &A)
7710	: AAPrivatizablePtrFloating(IRP, A) {}
7711
7712	/// See AbstractAttribute::initialize(...).
7713	void initialize(Attributor &A) override {
7714	// TODO: We can privatize more than arguments.
7715	indicatePessimisticFixpoint();
7716	}
7717
7718	/// See AbstractAttribute::trackStatistics()
7719	void trackStatistics() const override {
7720	STATS_DECLTRACK_CSRET_ATTR(privatizable_ptr);
7721	}
7722	};
7723
7724	struct AAPrivatizablePtrReturned final : public AAPrivatizablePtrFloating {
7725	AAPrivatizablePtrReturned(const IRPosition &IRP, Attributor &A)
7726	: AAPrivatizablePtrFloating(IRP, A) {}
7727
7728	/// See AbstractAttribute::initialize(...).
7729	void initialize(Attributor &A) override {
7730	// TODO: We can privatize more than arguments.
7731	indicatePessimisticFixpoint();
7732	}
7733
7734	/// See AbstractAttribute::trackStatistics()
7735	void trackStatistics() const override {
7736	STATS_DECLTRACK_FNRET_ATTR(privatizable_ptr);
7737	}
7738	};
7739	} // namespace
7740
7741	/// -------------------- Memory Behavior Attributes ----------------------------
7742	/// Includes read-none, read-only, and write-only.
7743	/// ----------------------------------------------------------------------------
7744	namespace {
7745	struct AAMemoryBehaviorImpl : public AAMemoryBehavior {
7746	AAMemoryBehaviorImpl(const IRPosition &IRP, Attributor &A)
7747	: AAMemoryBehavior(IRP, A) {}
7748
7749	/// See AbstractAttribute::initialize(...).
7750	void initialize(Attributor &A) override {
7751	intersectAssumedBits(BitsEncoding: BEST_STATE);
7752	getKnownStateFromValue(A, IRP: getIRPosition(), State&: getState());
7753	AAMemoryBehavior::initialize(A);
7754	}
7755
7756	/// Return the memory behavior information encoded in the IR for \p IRP.
7757	static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP,
7758	BitIntegerState &State,
7759	bool IgnoreSubsumingPositions = false) {
7760	SmallVector<Attribute, 2> Attrs;
7761	A.getAttrs(IRP, AKs: AttrKinds, Attrs, IgnoreSubsumingPositions);
7762	for (const Attribute &Attr : Attrs) {
7763	switch (Attr.getKindAsEnum()) {
7764	case Attribute::ReadNone:
7765	State.addKnownBits(Bits: NO_ACCESSES);
7766	break;
7767	case Attribute::ReadOnly:
7768	State.addKnownBits(Bits: NO_WRITES);
7769	break;
7770	case Attribute::WriteOnly:
7771	State.addKnownBits(Bits: NO_READS);
7772	break;
7773	default:
7774	llvm_unreachable("Unexpected attribute!");
7775	}
7776	}
7777
7778	if (auto *I = dyn_cast<Instruction>(Val: &IRP.getAnchorValue())) {
7779	if (!I->mayReadFromMemory())
7780	State.addKnownBits(Bits: NO_READS);
7781	if (!I->mayWriteToMemory())
7782	State.addKnownBits(Bits: NO_WRITES);
7783	}
7784	}
7785
7786	/// See AbstractAttribute::getDeducedAttributes(...).
7787	void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
7788	SmallVectorImpl<Attribute> &Attrs) const override {
7789	assert(Attrs.size() == 0);
7790	if (isAssumedReadNone())
7791	Attrs.push_back(Attribute::get(Ctx, Attribute::ReadNone));
7792	else if (isAssumedReadOnly())
7793	Attrs.push_back(Attribute::get(Ctx, Attribute::ReadOnly));
7794	else if (isAssumedWriteOnly())
7795	Attrs.push_back(Attribute::get(Ctx, Attribute::WriteOnly));
7796	assert(Attrs.size() <= 1);
7797	}
7798
7799	/// See AbstractAttribute::manifest(...).
7800	ChangeStatus manifest(Attributor &A) override {
7801	const IRPosition &IRP = getIRPosition();
7802
7803	if (A.hasAttr(IRP, Attribute::ReadNone,
7804	/* IgnoreSubsumingPositions */ true))
7805	return ChangeStatus::UNCHANGED;
7806
7807	// Check if we would improve the existing attributes first.
7808	SmallVector<Attribute, 4> DeducedAttrs;
7809	getDeducedAttributes(A, Ctx&: IRP.getAnchorValue().getContext(), Attrs&: DeducedAttrs);
7810	if (llvm::all_of(Range&: DeducedAttrs, P: [&](const Attribute &Attr) {
7811	return A.hasAttr(IRP, AKs: Attr.getKindAsEnum(),
7812	/* IgnoreSubsumingPositions */ true);
7813	}))
7814	return ChangeStatus::UNCHANGED;
7815
7816	// Clear existing attributes.
7817	A.removeAttrs(IRP, AttrKinds);
7818	// Clear conflicting writable attribute.
7819	if (isAssumedReadOnly())
7820	A.removeAttrs(IRP, Attribute::Writable);
7821
7822	// Use the generic manifest method.
7823	return IRAttribute::manifest(A);
7824	}
7825
7826	/// See AbstractState::getAsStr().
7827	const std::string getAsStr(Attributor *A) const override {
7828	if (isAssumedReadNone())
7829	return "readnone";
7830	if (isAssumedReadOnly())
7831	return "readonly";
7832	if (isAssumedWriteOnly())
7833	return "writeonly";
7834	return "may-read/write";
7835	}
7836
7837	/// The set of IR attributes AAMemoryBehavior deals with.
7838	static const Attribute::AttrKind AttrKinds[3];
7839	};
7840
7841	const Attribute::AttrKind AAMemoryBehaviorImpl::AttrKinds[] = {
7842	Attribute::ReadNone, Attribute::ReadOnly, Attribute::WriteOnly};
7843
7844	/// Memory behavior attribute for a floating value.
7845	struct AAMemoryBehaviorFloating : AAMemoryBehaviorImpl {
7846	AAMemoryBehaviorFloating(const IRPosition &IRP, Attributor &A)
7847	: AAMemoryBehaviorImpl(IRP, A) {}
7848
7849	/// See AbstractAttribute::updateImpl(...).
7850	ChangeStatus updateImpl(Attributor &A) override;
7851
7852	/// See AbstractAttribute::trackStatistics()
7853	void trackStatistics() const override {
7854	if (isAssumedReadNone())
7855	STATS_DECLTRACK_FLOATING_ATTR(readnone)
7856	else if (isAssumedReadOnly())
7857	STATS_DECLTRACK_FLOATING_ATTR(readonly)
7858	else if (isAssumedWriteOnly())
7859	STATS_DECLTRACK_FLOATING_ATTR(writeonly)
7860	}
7861
7862	private:
7863	/// Return true if users of \p UserI might access the underlying
7864	/// variable/location described by \p U and should therefore be analyzed.
7865	bool followUsersOfUseIn(Attributor &A, const Use &U,
7866	const Instruction *UserI);
7867
7868	/// Update the state according to the effect of use \p U in \p UserI.
7869	void analyzeUseIn(Attributor &A, const Use &U, const Instruction *UserI);
7870	};
7871
7872	/// Memory behavior attribute for function argument.
7873	struct AAMemoryBehaviorArgument : AAMemoryBehaviorFloating {
7874	AAMemoryBehaviorArgument(const IRPosition &IRP, Attributor &A)
7875	: AAMemoryBehaviorFloating(IRP, A) {}
7876
7877	/// See AbstractAttribute::initialize(...).
7878	void initialize(Attributor &A) override {
7879	intersectAssumedBits(BitsEncoding: BEST_STATE);
7880	const IRPosition &IRP = getIRPosition();
7881	// TODO: Make IgnoreSubsumingPositions a property of an IRAttribute so we
7882	// can query it when we use has/getAttr. That would allow us to reuse the
7883	// initialize of the base class here.
7884	bool HasByVal = A.hasAttr(IRP, {Attribute::ByVal},
7885	/* IgnoreSubsumingPositions */ true);
7886	getKnownStateFromValue(A, IRP, State&: getState(),
7887	/* IgnoreSubsumingPositions */ HasByVal);
7888	}
7889
7890	ChangeStatus manifest(Attributor &A) override {
7891	// TODO: Pointer arguments are not supported on vectors of pointers yet.
7892	if (!getAssociatedValue().getType()->isPointerTy())
7893	return ChangeStatus::UNCHANGED;
7894
7895	// TODO: From readattrs.ll: "inalloca parameters are always
7896	// considered written"
7897	if (A.hasAttr(getIRPosition(),
7898	{Attribute::InAlloca, Attribute::Preallocated})) {
7899	removeKnownBits(BitsEncoding: NO_WRITES);
7900	removeAssumedBits(BitsEncoding: NO_WRITES);
7901	}
7902	A.removeAttrs(IRP: getIRPosition(), AttrKinds);
7903	return AAMemoryBehaviorFloating::manifest(A);
7904	}
7905
7906	/// See AbstractAttribute::trackStatistics()
7907	void trackStatistics() const override {
7908	if (isAssumedReadNone())
7909	STATS_DECLTRACK_ARG_ATTR(readnone)
7910	else if (isAssumedReadOnly())
7911	STATS_DECLTRACK_ARG_ATTR(readonly)
7912	else if (isAssumedWriteOnly())
7913	STATS_DECLTRACK_ARG_ATTR(writeonly)
7914	}
7915	};
7916
7917	struct AAMemoryBehaviorCallSiteArgument final : AAMemoryBehaviorArgument {
7918	AAMemoryBehaviorCallSiteArgument(const IRPosition &IRP, Attributor &A)
7919	: AAMemoryBehaviorArgument(IRP, A) {}
7920
7921	/// See AbstractAttribute::initialize(...).
7922	void initialize(Attributor &A) override {
7923	// If we don't have an associated attribute this is either a variadic call
7924	// or an indirect call, either way, nothing to do here.
7925	Argument *Arg = getAssociatedArgument();
7926	if (!Arg) {
7927	indicatePessimisticFixpoint();
7928	return;
7929	}
7930	if (Arg->hasByValAttr()) {
7931	addKnownBits(Bits: NO_WRITES);
7932	removeKnownBits(BitsEncoding: NO_READS);
7933	removeAssumedBits(BitsEncoding: NO_READS);
7934	}
7935	AAMemoryBehaviorArgument::initialize(A);
7936	if (getAssociatedFunction()->isDeclaration())
7937	indicatePessimisticFixpoint();
7938	}
7939
7940	/// See AbstractAttribute::updateImpl(...).
7941	ChangeStatus updateImpl(Attributor &A) override {
7942	// TODO: Once we have call site specific value information we can provide
7943	// call site specific liveness liveness information and then it makes
7944	// sense to specialize attributes for call sites arguments instead of
7945	// redirecting requests to the callee argument.
7946	Argument *Arg = getAssociatedArgument();
7947	const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
7948	auto *ArgAA =
7949	A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
7950	if (!ArgAA)
7951	return indicatePessimisticFixpoint();
7952	return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState());
7953	}
7954
7955	/// See AbstractAttribute::trackStatistics()
7956	void trackStatistics() const override {
7957	if (isAssumedReadNone())
7958	STATS_DECLTRACK_CSARG_ATTR(readnone)
7959	else if (isAssumedReadOnly())
7960	STATS_DECLTRACK_CSARG_ATTR(readonly)
7961	else if (isAssumedWriteOnly())
7962	STATS_DECLTRACK_CSARG_ATTR(writeonly)
7963	}
7964	};
7965
7966	/// Memory behavior attribute for a call site return position.
7967	struct AAMemoryBehaviorCallSiteReturned final : AAMemoryBehaviorFloating {
7968	AAMemoryBehaviorCallSiteReturned(const IRPosition &IRP, Attributor &A)
7969	: AAMemoryBehaviorFloating(IRP, A) {}
7970
7971	/// See AbstractAttribute::initialize(...).
7972	void initialize(Attributor &A) override {
7973	AAMemoryBehaviorImpl::initialize(A);
7974	}
7975	/// See AbstractAttribute::manifest(...).
7976	ChangeStatus manifest(Attributor &A) override {
7977	// We do not annotate returned values.
7978	return ChangeStatus::UNCHANGED;
7979	}
7980
7981	/// See AbstractAttribute::trackStatistics()
7982	void trackStatistics() const override {}
7983	};
7984
7985	/// An AA to represent the memory behavior function attributes.
7986	struct AAMemoryBehaviorFunction final : public AAMemoryBehaviorImpl {
7987	AAMemoryBehaviorFunction(const IRPosition &IRP, Attributor &A)
7988	: AAMemoryBehaviorImpl(IRP, A) {}
7989
7990	/// See AbstractAttribute::updateImpl(Attributor &A).
7991	ChangeStatus updateImpl(Attributor &A) override;
7992
7993	/// See AbstractAttribute::manifest(...).
7994	ChangeStatus manifest(Attributor &A) override {
7995	// TODO: It would be better to merge this with AAMemoryLocation, so that
7996	// we could determine read/write per location. This would also have the
7997	// benefit of only one place trying to manifest the memory attribute.
7998	Function &F = cast<Function>(Val&: getAnchorValue());
7999	MemoryEffects ME = MemoryEffects::unknown();
8000	if (isAssumedReadNone())
8001	ME = MemoryEffects::none();
8002	else if (isAssumedReadOnly())
8003	ME = MemoryEffects::readOnly();
8004	else if (isAssumedWriteOnly())
8005	ME = MemoryEffects::writeOnly();
8006
8007	A.removeAttrs(IRP: getIRPosition(), AttrKinds);
8008	// Clear conflicting writable attribute.
8009	if (ME.onlyReadsMemory())
8010	for (Argument &Arg : F.args())
8011	A.removeAttrs(IRPosition::argument(Arg), Attribute::Writable);
8012	return A.manifestAttrs(IRP: getIRPosition(),
8013	DeducedAttrs: Attribute::getWithMemoryEffects(Context&: F.getContext(), ME));
8014	}
8015
8016	/// See AbstractAttribute::trackStatistics()
8017	void trackStatistics() const override {
8018	if (isAssumedReadNone())
8019	STATS_DECLTRACK_FN_ATTR(readnone)
8020	else if (isAssumedReadOnly())
8021	STATS_DECLTRACK_FN_ATTR(readonly)
8022	else if (isAssumedWriteOnly())
8023	STATS_DECLTRACK_FN_ATTR(writeonly)
8024	}
8025	};
8026
8027	/// AAMemoryBehavior attribute for call sites.
8028	struct AAMemoryBehaviorCallSite final
8029	: AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl> {
8030	AAMemoryBehaviorCallSite(const IRPosition &IRP, Attributor &A)
8031	: AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl>(IRP, A) {}
8032
8033	/// See AbstractAttribute::manifest(...).
8034	ChangeStatus manifest(Attributor &A) override {
8035	// TODO: Deduplicate this with AAMemoryBehaviorFunction.
8036	CallBase &CB = cast<CallBase>(Val&: getAnchorValue());
8037	MemoryEffects ME = MemoryEffects::unknown();
8038	if (isAssumedReadNone())
8039	ME = MemoryEffects::none();
8040	else if (isAssumedReadOnly())
8041	ME = MemoryEffects::readOnly();
8042	else if (isAssumedWriteOnly())
8043	ME = MemoryEffects::writeOnly();
8044
8045	A.removeAttrs(IRP: getIRPosition(), AttrKinds);
8046	// Clear conflicting writable attribute.
8047	if (ME.onlyReadsMemory())
8048	for (Use &U : CB.args())
8049	A.removeAttrs(IRPosition::callsite_argument(CB, U.getOperandNo()),
8050	Attribute::Writable);
8051	return A.manifestAttrs(
8052	IRP: getIRPosition(), DeducedAttrs: Attribute::getWithMemoryEffects(Context&: CB.getContext(), ME));
8053	}
8054
8055	/// See AbstractAttribute::trackStatistics()
8056	void trackStatistics() const override {
8057	if (isAssumedReadNone())
8058	STATS_DECLTRACK_CS_ATTR(readnone)
8059	else if (isAssumedReadOnly())
8060	STATS_DECLTRACK_CS_ATTR(readonly)
8061	else if (isAssumedWriteOnly())
8062	STATS_DECLTRACK_CS_ATTR(writeonly)
8063	}
8064	};
8065
8066	ChangeStatus AAMemoryBehaviorFunction::updateImpl(Attributor &A) {
8067
8068	// The current assumed state used to determine a change.
8069	auto AssumedState = getAssumed();
8070
8071	auto CheckRWInst = [&](Instruction &I) {
8072	// If the instruction has an own memory behavior state, use it to restrict
8073	// the local state. No further analysis is required as the other memory
8074	// state is as optimistic as it gets.
8075	if (const auto *CB = dyn_cast<CallBase>(Val: &I)) {
8076	const auto *MemBehaviorAA = A.getAAFor<AAMemoryBehavior>(
8077	QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::REQUIRED);
8078	if (MemBehaviorAA) {
8079	intersectAssumedBits(BitsEncoding: MemBehaviorAA->getAssumed());
8080	return !isAtFixpoint();
8081	}
8082	}
8083
8084	// Remove access kind modifiers if necessary.
8085	if (I.mayReadFromMemory())
8086	removeAssumedBits(BitsEncoding: NO_READS);
8087	if (I.mayWriteToMemory())
8088	removeAssumedBits(BitsEncoding: NO_WRITES);
8089	return !isAtFixpoint();
8090	};
8091
8092	bool UsedAssumedInformation = false;
8093	if (!A.checkForAllReadWriteInstructions(Pred: CheckRWInst, QueryingAA&: *this,
8094	UsedAssumedInformation))
8095	return indicatePessimisticFixpoint();
8096
8097	return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8098	: ChangeStatus::UNCHANGED;
8099	}
8100
8101	ChangeStatus AAMemoryBehaviorFloating::updateImpl(Attributor &A) {
8102
8103	const IRPosition &IRP = getIRPosition();
8104	const IRPosition &FnPos = IRPosition::function_scope(IRP);
8105	AAMemoryBehavior::StateType &S = getState();
8106
8107	// First, check the function scope. We take the known information and we avoid
8108	// work if the assumed information implies the current assumed information for
8109	// this attribute. This is a valid for all but byval arguments.
8110	Argument *Arg = IRP.getAssociatedArgument();
8111	AAMemoryBehavior::base_t FnMemAssumedState =
8112	AAMemoryBehavior::StateType::getWorstState();
8113	if (!Arg || !Arg->hasByValAttr()) {
8114	const auto *FnMemAA =
8115	A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::OPTIONAL);
8116	if (FnMemAA) {
8117	FnMemAssumedState = FnMemAA->getAssumed();
8118	S.addKnownBits(Bits: FnMemAA->getKnown());
8119	if ((S.getAssumed() & FnMemAA->getAssumed()) == S.getAssumed())
8120	return ChangeStatus::UNCHANGED;
8121	}
8122	}
8123
8124	// The current assumed state used to determine a change.
8125	auto AssumedState = S.getAssumed();
8126
8127	// Make sure the value is not captured (except through "return"), if
8128	// it is, any information derived would be irrelevant anyway as we cannot
8129	// check the potential aliases introduced by the capture. However, no need
8130	// to fall back to anythign less optimistic than the function state.
8131	bool IsKnownNoCapture;
8132	const AANoCapture *ArgNoCaptureAA = nullptr;
8133	bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
8134	A, this, IRP, DepClassTy::OPTIONAL, IsKnownNoCapture, false,
8135	&ArgNoCaptureAA);
8136
8137	if (!IsAssumedNoCapture &&
8138	(!ArgNoCaptureAA || !ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned())) {
8139	S.intersectAssumedBits(BitsEncoding: FnMemAssumedState);
8140	return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8141	: ChangeStatus::UNCHANGED;
8142	}
8143
8144	// Visit and expand uses until all are analyzed or a fixpoint is reached.
8145	auto UsePred = [&](const Use &U, bool &Follow) -> bool {
8146	Instruction *UserI = cast<Instruction>(Val: U.getUser());
8147	LLVM_DEBUG(dbgs() << "[AAMemoryBehavior] Use: " << *U << " in " << *UserI
8148	<< " \n");
8149
8150	// Droppable users, e.g., llvm::assume does not actually perform any action.
8151	if (UserI->isDroppable())
8152	return true;
8153
8154	// Check if the users of UserI should also be visited.
8155	Follow = followUsersOfUseIn(A, U, UserI);
8156
8157	// If UserI might touch memory we analyze the use in detail.
8158	if (UserI->mayReadOrWriteMemory())
8159	analyzeUseIn(A, U, UserI);
8160
8161	return !isAtFixpoint();
8162	};
8163
8164	if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: getAssociatedValue()))
8165	return indicatePessimisticFixpoint();
8166
8167	return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8168	: ChangeStatus::UNCHANGED;
8169	}
8170
8171	bool AAMemoryBehaviorFloating::followUsersOfUseIn(Attributor &A, const Use &U,
8172	const Instruction *UserI) {
8173	// The loaded value is unrelated to the pointer argument, no need to
8174	// follow the users of the load.
8175	if (isa<LoadInst>(Val: UserI) || isa<ReturnInst>(Val: UserI))
8176	return false;
8177
8178	// By default we follow all uses assuming UserI might leak information on U,
8179	// we have special handling for call sites operands though.
8180	const auto *CB = dyn_cast<CallBase>(Val: UserI);
8181	if (!CB || !CB->isArgOperand(U: &U))
8182	return true;
8183
8184	// If the use is a call argument known not to be captured, the users of
8185	// the call do not need to be visited because they have to be unrelated to
8186	// the input. Note that this check is not trivial even though we disallow
8187	// general capturing of the underlying argument. The reason is that the
8188	// call might the argument "through return", which we allow and for which we
8189	// need to check call users.
8190	if (U.get()->getType()->isPointerTy()) {
8191	unsigned ArgNo = CB->getArgOperandNo(U: &U);
8192	bool IsKnownNoCapture;
8193	return !AA::hasAssumedIRAttr<Attribute::NoCapture>(
8194	A, this, IRPosition::callsite_argument(*CB, ArgNo),
8195	DepClassTy::OPTIONAL, IsKnownNoCapture);
8196	}
8197
8198	return true;
8199	}
8200
8201	void AAMemoryBehaviorFloating::analyzeUseIn(Attributor &A, const Use &U,
8202	const Instruction *UserI) {
8203	assert(UserI->mayReadOrWriteMemory());
8204
8205	switch (UserI->getOpcode()) {
8206	default:
8207	// TODO: Handle all atomics and other side-effect operations we know of.
8208	break;
8209	case Instruction::Load:
8210	// Loads cause the NO_READS property to disappear.
8211	removeAssumedBits(BitsEncoding: NO_READS);
8212	return;
8213
8214	case Instruction::Store:
8215	// Stores cause the NO_WRITES property to disappear if the use is the
8216	// pointer operand. Note that while capturing was taken care of somewhere
8217	// else we need to deal with stores of the value that is not looked through.
8218	if (cast<StoreInst>(Val: UserI)->getPointerOperand() == U.get())
8219	removeAssumedBits(BitsEncoding: NO_WRITES);
8220	else
8221	indicatePessimisticFixpoint();
8222	return;
8223
8224	case Instruction::Call:
8225	case Instruction::CallBr:
8226	case Instruction::Invoke: {
8227	// For call sites we look at the argument memory behavior attribute (this
8228	// could be recursive!) in order to restrict our own state.
8229	const auto *CB = cast<CallBase>(Val: UserI);
8230
8231	// Give up on operand bundles.
8232	if (CB->isBundleOperand(U: &U)) {
8233	indicatePessimisticFixpoint();
8234	return;
8235	}
8236
8237	// Calling a function does read the function pointer, maybe write it if the
8238	// function is self-modifying.
8239	if (CB->isCallee(U: &U)) {
8240	removeAssumedBits(BitsEncoding: NO_READS);
8241	break;
8242	}
8243
8244	// Adjust the possible access behavior based on the information on the
8245	// argument.
8246	IRPosition Pos;
8247	if (U.get()->getType()->isPointerTy())
8248	Pos = IRPosition::callsite_argument(CB: *CB, ArgNo: CB->getArgOperandNo(U: &U));
8249	else
8250	Pos = IRPosition::callsite_function(CB: *CB);
8251	const auto *MemBehaviorAA =
8252	A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: Pos, DepClass: DepClassTy::OPTIONAL);
8253	if (!MemBehaviorAA)
8254	break;
8255	// "assumed" has at most the same bits as the MemBehaviorAA assumed
8256	// and at least "known".
8257	intersectAssumedBits(BitsEncoding: MemBehaviorAA->getAssumed());
8258	return;
8259	}
8260	};
8261
8262	// Generally, look at the "may-properties" and adjust the assumed state if we
8263	// did not trigger special handling before.
8264	if (UserI->mayReadFromMemory())
8265	removeAssumedBits(BitsEncoding: NO_READS);
8266	if (UserI->mayWriteToMemory())
8267	removeAssumedBits(BitsEncoding: NO_WRITES);
8268	}
8269	} // namespace
8270
8271	/// -------------------- Memory Locations Attributes ---------------------------
8272	/// Includes read-none, argmemonly, inaccessiblememonly,
8273	/// inaccessiblememorargmemonly
8274	/// ----------------------------------------------------------------------------
8275
8276	std::string AAMemoryLocation::getMemoryLocationsAsStr(
8277	AAMemoryLocation::MemoryLocationsKind MLK) {
8278	if (0 == (MLK & AAMemoryLocation::NO_LOCATIONS))
8279	return "all memory";
8280	if (MLK == AAMemoryLocation::NO_LOCATIONS)
8281	return "no memory";
8282	std::string S = "memory:";
8283	if (0 == (MLK & AAMemoryLocation::NO_LOCAL_MEM))
8284	S += "stack,";
8285	if (0 == (MLK & AAMemoryLocation::NO_CONST_MEM))
8286	S += "constant,";
8287	if (0 == (MLK & AAMemoryLocation::NO_GLOBAL_INTERNAL_MEM))
8288	S += "internal global,";
8289	if (0 == (MLK & AAMemoryLocation::NO_GLOBAL_EXTERNAL_MEM))
8290	S += "external global,";
8291	if (0 == (MLK & AAMemoryLocation::NO_ARGUMENT_MEM))
8292	S += "argument,";
8293	if (0 == (MLK & AAMemoryLocation::NO_INACCESSIBLE_MEM))
8294	S += "inaccessible,";
8295	if (0 == (MLK & AAMemoryLocation::NO_MALLOCED_MEM))
8296	S += "malloced,";
8297	if (0 == (MLK & AAMemoryLocation::NO_UNKOWN_MEM))
8298	S += "unknown,";
8299	S.pop_back();
8300	return S;
8301	}
8302
8303	namespace {
8304	struct AAMemoryLocationImpl : public AAMemoryLocation {
8305
8306	AAMemoryLocationImpl(const IRPosition &IRP, Attributor &A)
8307	: AAMemoryLocation(IRP, A), Allocator(A.Allocator) {
8308	AccessKind2Accesses.fill(u: nullptr);
8309	}
8310
8311	~AAMemoryLocationImpl() {
8312	// The AccessSets are allocated via a BumpPtrAllocator, we call
8313	// the destructor manually.
8314	for (AccessSet *AS : AccessKind2Accesses)
8315	if (AS)
8316	AS->~AccessSet();
8317	}
8318
8319	/// See AbstractAttribute::initialize(...).
8320	void initialize(Attributor &A) override {
8321	intersectAssumedBits(BitsEncoding: BEST_STATE);
8322	getKnownStateFromValue(A, IRP: getIRPosition(), State&: getState());
8323	AAMemoryLocation::initialize(A);
8324	}
8325
8326	/// Return the memory behavior information encoded in the IR for \p IRP.
8327	static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP,
8328	BitIntegerState &State,
8329	bool IgnoreSubsumingPositions = false) {
8330	// For internal functions we ignore `argmemonly` and
8331	// `inaccessiblememorargmemonly` as we might break it via interprocedural
8332	// constant propagation. It is unclear if this is the best way but it is
8333	// unlikely this will cause real performance problems. If we are deriving
8334	// attributes for the anchor function we even remove the attribute in
8335	// addition to ignoring it.
8336	// TODO: A better way to handle this would be to add ~NO_GLOBAL_MEM /
8337	// MemoryEffects::Other as a possible location.
8338	bool UseArgMemOnly = true;
8339	Function *AnchorFn = IRP.getAnchorScope();
8340	if (AnchorFn && A.isRunOn(Fn&: *AnchorFn))
8341	UseArgMemOnly = !AnchorFn->hasLocalLinkage();
8342
8343	SmallVector<Attribute, 2> Attrs;
8344	A.getAttrs(IRP, {Attribute::Memory}, Attrs, IgnoreSubsumingPositions);
8345	for (const Attribute &Attr : Attrs) {
8346	// TODO: We can map MemoryEffects to Attributor locations more precisely.
8347	MemoryEffects ME = Attr.getMemoryEffects();
8348	if (ME.doesNotAccessMemory()) {
8349	State.addKnownBits(Bits: NO_LOCAL_MEM | NO_CONST_MEM);
8350	continue;
8351	}
8352	if (ME.onlyAccessesInaccessibleMem()) {
8353	State.addKnownBits(Bits: inverseLocation(Loc: NO_INACCESSIBLE_MEM, AndLocalMem: true, AndConstMem: true));
8354	continue;
8355	}
8356	if (ME.onlyAccessesArgPointees()) {
8357	if (UseArgMemOnly)
8358	State.addKnownBits(Bits: inverseLocation(Loc: NO_ARGUMENT_MEM, AndLocalMem: true, AndConstMem: true));
8359	else {
8360	// Remove location information, only keep read/write info.
8361	ME = MemoryEffects(ME.getModRef());
8362	A.manifestAttrs(IRP,
8363	DeducedAttrs: Attribute::getWithMemoryEffects(
8364	Context&: IRP.getAnchorValue().getContext(), ME),
8365	/*ForceReplace*/ true);
8366	}
8367	continue;
8368	}
8369	if (ME.onlyAccessesInaccessibleOrArgMem()) {
8370	if (UseArgMemOnly)
8371	State.addKnownBits(Bits: inverseLocation(
8372	Loc: NO_INACCESSIBLE_MEM | NO_ARGUMENT_MEM, AndLocalMem: true, AndConstMem: true));
8373	else {
8374	// Remove location information, only keep read/write info.
8375	ME = MemoryEffects(ME.getModRef());
8376	A.manifestAttrs(IRP,
8377	DeducedAttrs: Attribute::getWithMemoryEffects(
8378	Context&: IRP.getAnchorValue().getContext(), ME),
8379	/*ForceReplace*/ true);
8380	}
8381	continue;
8382	}
8383	}
8384	}
8385
8386	/// See AbstractAttribute::getDeducedAttributes(...).
8387	void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
8388	SmallVectorImpl<Attribute> &Attrs) const override {
8389	// TODO: We can map Attributor locations to MemoryEffects more precisely.
8390	assert(Attrs.size() == 0);
8391	if (getIRPosition().getPositionKind() == IRPosition::IRP_FUNCTION) {
8392	if (isAssumedReadNone())
8393	Attrs.push_back(
8394	Elt: Attribute::getWithMemoryEffects(Context&: Ctx, ME: MemoryEffects::none()));
8395	else if (isAssumedInaccessibleMemOnly())
8396	Attrs.push_back(Elt: Attribute::getWithMemoryEffects(
8397	Context&: Ctx, ME: MemoryEffects::inaccessibleMemOnly()));
8398	else if (isAssumedArgMemOnly())
8399	Attrs.push_back(
8400	Elt: Attribute::getWithMemoryEffects(Context&: Ctx, ME: MemoryEffects::argMemOnly()));
8401	else if (isAssumedInaccessibleOrArgMemOnly())
8402	Attrs.push_back(Elt: Attribute::getWithMemoryEffects(
8403	Context&: Ctx, ME: MemoryEffects::inaccessibleOrArgMemOnly()));
8404	}
8405	assert(Attrs.size() <= 1);
8406	}
8407
8408	/// See AbstractAttribute::manifest(...).
8409	ChangeStatus manifest(Attributor &A) override {
8410	// TODO: If AAMemoryLocation and AAMemoryBehavior are merged, we could
8411	// provide per-location modref information here.
8412	const IRPosition &IRP = getIRPosition();
8413
8414	SmallVector<Attribute, 1> DeducedAttrs;
8415	getDeducedAttributes(A, Ctx&: IRP.getAnchorValue().getContext(), Attrs&: DeducedAttrs);
8416	if (DeducedAttrs.size() != 1)
8417	return ChangeStatus::UNCHANGED;
8418	MemoryEffects ME = DeducedAttrs[0].getMemoryEffects();
8419
8420	return A.manifestAttrs(IRP, DeducedAttrs: Attribute::getWithMemoryEffects(
8421	Context&: IRP.getAnchorValue().getContext(), ME));
8422	}
8423
8424	/// See AAMemoryLocation::checkForAllAccessesToMemoryKind(...).
8425	bool checkForAllAccessesToMemoryKind(
8426	function_ref<bool(const Instruction *, const Value *, AccessKind,
8427	MemoryLocationsKind)>
8428	Pred,
8429	MemoryLocationsKind RequestedMLK) const override {
8430	if (!isValidState())
8431	return false;
8432
8433	MemoryLocationsKind AssumedMLK = getAssumedNotAccessedLocation();
8434	if (AssumedMLK == NO_LOCATIONS)
8435	return true;
8436
8437	unsigned Idx = 0;
8438	for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS;
8439	CurMLK *= 2, ++Idx) {
8440	if (CurMLK & RequestedMLK)
8441	continue;
8442
8443	if (const AccessSet *Accesses = AccessKind2Accesses[Idx])
8444	for (const AccessInfo &AI : *Accesses)
8445	if (!Pred(AI.I, AI.Ptr, AI.Kind, CurMLK))
8446	return false;
8447	}
8448
8449	return true;
8450	}
8451
8452	ChangeStatus indicatePessimisticFixpoint() override {
8453	// If we give up and indicate a pessimistic fixpoint this instruction will
8454	// become an access for all potential access kinds:
8455	// TODO: Add pointers for argmemonly and globals to improve the results of
8456	// checkForAllAccessesToMemoryKind.
8457	bool Changed = false;
8458	MemoryLocationsKind KnownMLK = getKnown();
8459	Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
8460	for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2)
8461	if (!(CurMLK & KnownMLK))
8462	updateStateAndAccessesMap(State&: getState(), MLK: CurMLK, I, Ptr: nullptr, Changed,
8463	AK: getAccessKindFromInst(I));
8464	return AAMemoryLocation::indicatePessimisticFixpoint();
8465	}
8466
8467	protected:
8468	/// Helper struct to tie together an instruction that has a read or write
8469	/// effect with the pointer it accesses (if any).
8470	struct AccessInfo {
8471
8472	/// The instruction that caused the access.
8473	const Instruction *I;
8474
8475	/// The base pointer that is accessed, or null if unknown.
8476	const Value *Ptr;
8477
8478	/// The kind of access (read/write/read+write).
8479	AccessKind Kind;
8480
8481	bool operator==(const AccessInfo &RHS) const {
8482	return I == RHS.I && Ptr == RHS.Ptr && Kind == RHS.Kind;
8483	}
8484	bool operator()(const AccessInfo &LHS, const AccessInfo &RHS) const {
8485	if (LHS.I != RHS.I)
8486	return LHS.I < RHS.I;
8487	if (LHS.Ptr != RHS.Ptr)
8488	return LHS.Ptr < RHS.Ptr;
8489	if (LHS.Kind != RHS.Kind)
8490	return LHS.Kind < RHS.Kind;
8491	return false;
8492	}
8493	};
8494
8495	/// Mapping from *single* memory location kinds, e.g., LOCAL_MEM with the
8496	/// value of NO_LOCAL_MEM, to the accesses encountered for this memory kind.
8497	using AccessSet = SmallSet<AccessInfo, 2, AccessInfo>;
8498	std::array<AccessSet *, llvm::CTLog2<VALID_STATE>()> AccessKind2Accesses;
8499
8500	/// Categorize the pointer arguments of CB that might access memory in
8501	/// AccessedLoc and update the state and access map accordingly.
8502	void
8503	categorizeArgumentPointerLocations(Attributor &A, CallBase &CB,
8504	AAMemoryLocation::StateType &AccessedLocs,
8505	bool &Changed);
8506
8507	/// Return the kind(s) of location that may be accessed by \p V.
8508	AAMemoryLocation::MemoryLocationsKind
8509	categorizeAccessedLocations(Attributor &A, Instruction &I, bool &Changed);
8510
8511	/// Return the access kind as determined by \p I.
8512	AccessKind getAccessKindFromInst(const Instruction *I) {
8513	AccessKind AK = READ_WRITE;
8514	if (I) {
8515	AK = I->mayReadFromMemory() ? READ : NONE;
8516	AK = AccessKind(AK | (I->mayWriteToMemory() ? WRITE : NONE));
8517	}
8518	return AK;
8519	}
8520
8521	/// Update the state \p State and the AccessKind2Accesses given that \p I is
8522	/// an access of kind \p AK to a \p MLK memory location with the access
8523	/// pointer \p Ptr.
8524	void updateStateAndAccessesMap(AAMemoryLocation::StateType &State,
8525	MemoryLocationsKind MLK, const Instruction *I,
8526	const Value *Ptr, bool &Changed,
8527	AccessKind AK = READ_WRITE) {
8528
8529	assert(isPowerOf2_32(MLK) && "Expected a single location set!");
8530	auto *&Accesses = AccessKind2Accesses[llvm::Log2_32(Value: MLK)];
8531	if (!Accesses)
8532	Accesses = new (Allocator) AccessSet();
8533	Changed |= Accesses->insert(V: AccessInfo{.I: I, .Ptr: Ptr, .Kind: AK}).second;
8534	if (MLK == NO_UNKOWN_MEM)
8535	MLK = NO_LOCATIONS;
8536	State.removeAssumedBits(BitsEncoding: MLK);
8537	}
8538
8539	/// Determine the underlying locations kinds for \p Ptr, e.g., globals or
8540	/// arguments, and update the state and access map accordingly.
8541	void categorizePtrValue(Attributor &A, const Instruction &I, const Value &Ptr,
8542	AAMemoryLocation::StateType &State, bool &Changed,
8543	unsigned AccessAS = 0);
8544
8545	/// Used to allocate access sets.
8546	BumpPtrAllocator &Allocator;
8547	};
8548
8549	void AAMemoryLocationImpl::categorizePtrValue(
8550	Attributor &A, const Instruction &I, const Value &Ptr,
8551	AAMemoryLocation::StateType &State, bool &Changed, unsigned AccessAS) {
8552	LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize pointer locations for "
8553	<< Ptr << " ["
8554	<< getMemoryLocationsAsStr(State.getAssumed()) << "]\n");
8555
8556	auto Pred = [&](Value &Obj) {
8557	unsigned ObjectAS = Obj.getType()->getPointerAddressSpace();
8558	// TODO: recognize the TBAA used for constant accesses.
8559	MemoryLocationsKind MLK = NO_LOCATIONS;
8560
8561	// Filter accesses to constant (GPU) memory if we have an AS at the access
8562	// site or the object is known to actually have the associated AS.
8563	if ((AccessAS == (unsigned)AA::GPUAddressSpace::Constant ||
8564	(ObjectAS == (unsigned)AA::GPUAddressSpace::Constant &&
8565	isIdentifiedObject(V: &Obj))) &&
8566	AA::isGPU(M: *I.getModule()))
8567	return true;
8568
8569	if (isa<UndefValue>(Val: &Obj))
8570	return true;
8571	if (isa<Argument>(Val: &Obj)) {
8572	// TODO: For now we do not treat byval arguments as local copies performed
8573	// on the call edge, though, we should. To make that happen we need to
8574	// teach various passes, e.g., DSE, about the copy effect of a byval. That
8575	// would also allow us to mark functions only accessing byval arguments as
8576	// readnone again, arguably their accesses have no effect outside of the
8577	// function, like accesses to allocas.
8578	MLK = NO_ARGUMENT_MEM;
8579	} else if (auto *GV = dyn_cast<GlobalValue>(Val: &Obj)) {
8580	// Reading constant memory is not treated as a read "effect" by the
8581	// function attr pass so we won't neither. Constants defined by TBAA are
8582	// similar. (We know we do not write it because it is constant.)
8583	if (auto *GVar = dyn_cast<GlobalVariable>(Val: GV))
8584	if (GVar->isConstant())
8585	return true;
8586
8587	if (GV->hasLocalLinkage())
8588	MLK = NO_GLOBAL_INTERNAL_MEM;
8589	else
8590	MLK = NO_GLOBAL_EXTERNAL_MEM;
8591	} else if (isa<ConstantPointerNull>(Val: &Obj) &&
8592	(!NullPointerIsDefined(F: getAssociatedFunction(), AS: AccessAS) ||
8593	!NullPointerIsDefined(F: getAssociatedFunction(), AS: ObjectAS))) {
8594	return true;
8595	} else if (isa<AllocaInst>(Val: &Obj)) {
8596	MLK = NO_LOCAL_MEM;
8597	} else if (const auto *CB = dyn_cast<CallBase>(Val: &Obj)) {
8598	bool IsKnownNoAlias;
8599	if (AA::hasAssumedIRAttr<Attribute::NoAlias>(
8600	A, this, IRPosition::callsite_returned(*CB), DepClassTy::OPTIONAL,
8601	IsKnownNoAlias))
8602	MLK = NO_MALLOCED_MEM;
8603	else
8604	MLK = NO_UNKOWN_MEM;
8605	} else {
8606	MLK = NO_UNKOWN_MEM;
8607	}
8608
8609	assert(MLK != NO_LOCATIONS && "No location specified!");
8610	LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Ptr value can be categorized: "
8611	<< Obj << " -> " << getMemoryLocationsAsStr(MLK) << "\n");
8612	updateStateAndAccessesMap(State, MLK, I: &I, Ptr: &Obj, Changed,
8613	AK: getAccessKindFromInst(I: &I));
8614
8615	return true;
8616	};
8617
8618	const auto *AA = A.getAAFor<AAUnderlyingObjects>(
8619	QueryingAA: *this, IRP: IRPosition::value(V: Ptr), DepClass: DepClassTy::OPTIONAL);
8620	if (!AA || !AA->forallUnderlyingObjects(Pred, Scope: AA::Intraprocedural)) {
8621	LLVM_DEBUG(
8622	dbgs() << "[AAMemoryLocation] Pointer locations not categorized\n");
8623	updateStateAndAccessesMap(State, MLK: NO_UNKOWN_MEM, I: &I, Ptr: nullptr, Changed,
8624	AK: getAccessKindFromInst(I: &I));
8625	return;
8626	}
8627
8628	LLVM_DEBUG(
8629	dbgs() << "[AAMemoryLocation] Accessed locations with pointer locations: "
8630	<< getMemoryLocationsAsStr(State.getAssumed()) << "\n");
8631	}
8632
8633	void AAMemoryLocationImpl::categorizeArgumentPointerLocations(
8634	Attributor &A, CallBase &CB, AAMemoryLocation::StateType &AccessedLocs,
8635	bool &Changed) {
8636	for (unsigned ArgNo = 0, E = CB.arg_size(); ArgNo < E; ++ArgNo) {
8637
8638	// Skip non-pointer arguments.
8639	const Value *ArgOp = CB.getArgOperand(i: ArgNo);
8640	if (!ArgOp->getType()->isPtrOrPtrVectorTy())
8641	continue;
8642
8643	// Skip readnone arguments.
8644	const IRPosition &ArgOpIRP = IRPosition::callsite_argument(CB, ArgNo);
8645	const auto *ArgOpMemLocationAA =
8646	A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: ArgOpIRP, DepClass: DepClassTy::OPTIONAL);
8647
8648	if (ArgOpMemLocationAA && ArgOpMemLocationAA->isAssumedReadNone())
8649	continue;
8650
8651	// Categorize potentially accessed pointer arguments as if there was an
8652	// access instruction with them as pointer.
8653	categorizePtrValue(A, I: CB, Ptr: *ArgOp, State&: AccessedLocs, Changed);
8654	}
8655	}
8656
8657	AAMemoryLocation::MemoryLocationsKind
8658	AAMemoryLocationImpl::categorizeAccessedLocations(Attributor &A, Instruction &I,
8659	bool &Changed) {
8660	LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize accessed locations for "
8661	<< I << "\n");
8662
8663	AAMemoryLocation::StateType AccessedLocs;
8664	AccessedLocs.intersectAssumedBits(BitsEncoding: NO_LOCATIONS);
8665
8666	if (auto *CB = dyn_cast<CallBase>(Val: &I)) {
8667
8668	// First check if we assume any memory is access is visible.
8669	const auto *CBMemLocationAA = A.getAAFor<AAMemoryLocation>(
8670	QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::OPTIONAL);
8671	LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize call site: " << I
8672	<< " [" << CBMemLocationAA << "]\n");
8673	if (!CBMemLocationAA) {
8674	updateStateAndAccessesMap(State&: AccessedLocs, MLK: NO_UNKOWN_MEM, I: &I, Ptr: nullptr,
8675	Changed, AK: getAccessKindFromInst(I: &I));
8676	return NO_UNKOWN_MEM;
8677	}
8678
8679	if (CBMemLocationAA->isAssumedReadNone())
8680	return NO_LOCATIONS;
8681
8682	if (CBMemLocationAA->isAssumedInaccessibleMemOnly()) {
8683	updateStateAndAccessesMap(State&: AccessedLocs, MLK: NO_INACCESSIBLE_MEM, I: &I, Ptr: nullptr,
8684	Changed, AK: getAccessKindFromInst(I: &I));
8685	return AccessedLocs.getAssumed();
8686	}
8687
8688	uint32_t CBAssumedNotAccessedLocs =
8689	CBMemLocationAA->getAssumedNotAccessedLocation();
8690
8691	// Set the argmemonly and global bit as we handle them separately below.
8692	uint32_t CBAssumedNotAccessedLocsNoArgMem =
8693	CBAssumedNotAccessedLocs | NO_ARGUMENT_MEM | NO_GLOBAL_MEM;
8694
8695	for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2) {
8696	if (CBAssumedNotAccessedLocsNoArgMem & CurMLK)
8697	continue;
8698	updateStateAndAccessesMap(State&: AccessedLocs, MLK: CurMLK, I: &I, Ptr: nullptr, Changed,
8699	AK: getAccessKindFromInst(I: &I));
8700	}
8701
8702	// Now handle global memory if it might be accessed. This is slightly tricky
8703	// as NO_GLOBAL_MEM has multiple bits set.
8704	bool HasGlobalAccesses = ((~CBAssumedNotAccessedLocs) & NO_GLOBAL_MEM);
8705	if (HasGlobalAccesses) {
8706	auto AccessPred = [&](const Instruction *, const Value *Ptr,
8707	AccessKind Kind, MemoryLocationsKind MLK) {
8708	updateStateAndAccessesMap(State&: AccessedLocs, MLK, I: &I, Ptr, Changed,
8709	AK: getAccessKindFromInst(I: &I));
8710	return true;
8711	};
8712	if (!CBMemLocationAA->checkForAllAccessesToMemoryKind(
8713	Pred: AccessPred, MLK: inverseLocation(Loc: NO_GLOBAL_MEM, AndLocalMem: false, AndConstMem: false)))
8714	return AccessedLocs.getWorstState();
8715	}
8716
8717	LLVM_DEBUG(
8718	dbgs() << "[AAMemoryLocation] Accessed state before argument handling: "
8719	<< getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n");
8720
8721	// Now handle argument memory if it might be accessed.
8722	bool HasArgAccesses = ((~CBAssumedNotAccessedLocs) & NO_ARGUMENT_MEM);
8723	if (HasArgAccesses)
8724	categorizeArgumentPointerLocations(A, CB&: *CB, AccessedLocs, Changed);
8725
8726	LLVM_DEBUG(
8727	dbgs() << "[AAMemoryLocation] Accessed state after argument handling: "
8728	<< getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n");
8729
8730	return AccessedLocs.getAssumed();
8731	}
8732
8733	if (const Value *Ptr = getPointerOperand(I: &I, /* AllowVolatile */ true)) {
8734	LLVM_DEBUG(
8735	dbgs() << "[AAMemoryLocation] Categorize memory access with pointer: "
8736	<< I << " [" << *Ptr << "]\n");
8737	categorizePtrValue(A, I, Ptr: *Ptr, State&: AccessedLocs, Changed,
8738	AccessAS: Ptr->getType()->getPointerAddressSpace());
8739	return AccessedLocs.getAssumed();
8740	}
8741
8742	LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Failed to categorize instruction: "
8743	<< I << "\n");
8744	updateStateAndAccessesMap(State&: AccessedLocs, MLK: NO_UNKOWN_MEM, I: &I, Ptr: nullptr, Changed,
8745	AK: getAccessKindFromInst(I: &I));
8746	return AccessedLocs.getAssumed();
8747	}
8748
8749	/// An AA to represent the memory behavior function attributes.
8750	struct AAMemoryLocationFunction final : public AAMemoryLocationImpl {
8751	AAMemoryLocationFunction(const IRPosition &IRP, Attributor &A)
8752	: AAMemoryLocationImpl(IRP, A) {}
8753
8754	/// See AbstractAttribute::updateImpl(Attributor &A).
8755	ChangeStatus updateImpl(Attributor &A) override {
8756
8757	const auto *MemBehaviorAA =
8758	A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::NONE);
8759	if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) {
8760	if (MemBehaviorAA->isKnownReadNone())
8761	return indicateOptimisticFixpoint();
8762	assert(isAssumedReadNone() &&
8763	"AAMemoryLocation was not read-none but AAMemoryBehavior was!");
8764	A.recordDependence(FromAA: *MemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
8765	return ChangeStatus::UNCHANGED;
8766	}
8767
8768	// The current assumed state used to determine a change.
8769	auto AssumedState = getAssumed();
8770	bool Changed = false;
8771
8772	auto CheckRWInst = [&](Instruction &I) {
8773	MemoryLocationsKind MLK = categorizeAccessedLocations(A, I, Changed);
8774	LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Accessed locations for " << I
8775	<< ": " << getMemoryLocationsAsStr(MLK) << "\n");
8776	removeAssumedBits(BitsEncoding: inverseLocation(Loc: MLK, AndLocalMem: false, AndConstMem: false));
8777	// Stop once only the valid bit set in the *not assumed location*, thus
8778	// once we don't actually exclude any memory locations in the state.
8779	return getAssumedNotAccessedLocation() != VALID_STATE;
8780	};
8781
8782	bool UsedAssumedInformation = false;
8783	if (!A.checkForAllReadWriteInstructions(Pred: CheckRWInst, QueryingAA&: *this,
8784	UsedAssumedInformation))
8785	return indicatePessimisticFixpoint();
8786
8787	Changed |= AssumedState != getAssumed();
8788	return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
8789	}
8790
8791	/// See AbstractAttribute::trackStatistics()
8792	void trackStatistics() const override {
8793	if (isAssumedReadNone())
8794	STATS_DECLTRACK_FN_ATTR(readnone)
8795	else if (isAssumedArgMemOnly())
8796	STATS_DECLTRACK_FN_ATTR(argmemonly)
8797	else if (isAssumedInaccessibleMemOnly())
8798	STATS_DECLTRACK_FN_ATTR(inaccessiblememonly)
8799	else if (isAssumedInaccessibleOrArgMemOnly())
8800	STATS_DECLTRACK_FN_ATTR(inaccessiblememorargmemonly)
8801	}
8802	};
8803
8804	/// AAMemoryLocation attribute for call sites.
8805	struct AAMemoryLocationCallSite final : AAMemoryLocationImpl {
8806	AAMemoryLocationCallSite(const IRPosition &IRP, Attributor &A)
8807	: AAMemoryLocationImpl(IRP, A) {}
8808
8809	/// See AbstractAttribute::updateImpl(...).
8810	ChangeStatus updateImpl(Attributor &A) override {
8811	// TODO: Once we have call site specific value information we can provide
8812	// call site specific liveness liveness information and then it makes
8813	// sense to specialize attributes for call sites arguments instead of
8814	// redirecting requests to the callee argument.
8815	Function *F = getAssociatedFunction();
8816	const IRPosition &FnPos = IRPosition::function(F: *F);
8817	auto *FnAA =
8818	A.getAAFor<AAMemoryLocation>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::REQUIRED);
8819	if (!FnAA)
8820	return indicatePessimisticFixpoint();
8821	bool Changed = false;
8822	auto AccessPred = [&](const Instruction *I, const Value *Ptr,
8823	AccessKind Kind, MemoryLocationsKind MLK) {
8824	updateStateAndAccessesMap(State&: getState(), MLK, I, Ptr, Changed,
8825	AK: getAccessKindFromInst(I));
8826	return true;
8827	};
8828	if (!FnAA->checkForAllAccessesToMemoryKind(Pred: AccessPred, MLK: ALL_LOCATIONS))
8829	return indicatePessimisticFixpoint();
8830	return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
8831	}
8832
8833	/// See AbstractAttribute::trackStatistics()
8834	void trackStatistics() const override {
8835	if (isAssumedReadNone())
8836	STATS_DECLTRACK_CS_ATTR(readnone)
8837	}
8838	};
8839	} // namespace
8840
8841	/// ------------------ denormal-fp-math Attribute -------------------------
8842
8843	namespace {
8844	struct AADenormalFPMathImpl : public AADenormalFPMath {
8845	AADenormalFPMathImpl(const IRPosition &IRP, Attributor &A)
8846	: AADenormalFPMath(IRP, A) {}
8847
8848	const std::string getAsStr(Attributor *A) const override {
8849	std::string Str("AADenormalFPMath[");
8850	raw_string_ostream OS(Str);
8851
8852	DenormalState Known = getKnown();
8853	if (Known.Mode.isValid())
8854	OS << "denormal-fp-math=" << Known.Mode;
8855	else
8856	OS << "invalid";
8857
8858	if (Known.ModeF32.isValid())
8859	OS << " denormal-fp-math-f32=" << Known.ModeF32;
8860	OS << ']';
8861	return OS.str();
8862	}
8863	};
8864
8865	struct AADenormalFPMathFunction final : AADenormalFPMathImpl {
8866	AADenormalFPMathFunction(const IRPosition &IRP, Attributor &A)
8867	: AADenormalFPMathImpl(IRP, A) {}
8868
8869	void initialize(Attributor &A) override {
8870	const Function *F = getAnchorScope();
8871	DenormalMode Mode = F->getDenormalModeRaw();
8872	DenormalMode ModeF32 = F->getDenormalModeF32Raw();
8873
8874	// TODO: Handling this here prevents handling the case where a callee has a
8875	// fixed denormal-fp-math with dynamic denormal-fp-math-f32, but called from
8876	// a function with a fully fixed mode.
8877	if (ModeF32 == DenormalMode::getInvalid())
8878	ModeF32 = Mode;
8879	Known = DenormalState{.Mode: Mode, .ModeF32: ModeF32};
8880	if (isModeFixed())
8881	indicateFixpoint();
8882	}
8883
8884	ChangeStatus updateImpl(Attributor &A) override {
8885	ChangeStatus Change = ChangeStatus::UNCHANGED;
8886
8887	auto CheckCallSite = [=, &Change, &A](AbstractCallSite CS) {
8888	Function *Caller = CS.getInstruction()->getFunction();
8889	LLVM_DEBUG(dbgs() << "[AADenormalFPMath] Call " << Caller->getName()
8890	<< "->" << getAssociatedFunction()->getName() << '\n');
8891
8892	const auto *CallerInfo = A.getAAFor<AADenormalFPMath>(
8893	QueryingAA: *this, IRP: IRPosition::function(F: *Caller), DepClass: DepClassTy::REQUIRED);
8894	if (!CallerInfo)
8895	return false;
8896
8897	Change = Change | clampStateAndIndicateChange(S&: this->getState(),
8898	R: CallerInfo->getState());
8899	return true;
8900	};
8901
8902	bool AllCallSitesKnown = true;
8903	if (!A.checkForAllCallSites(Pred: CheckCallSite, QueryingAA: *this, RequireAllCallSites: true, UsedAssumedInformation&: AllCallSitesKnown))
8904	return indicatePessimisticFixpoint();
8905
8906	if (Change == ChangeStatus::CHANGED && isModeFixed())
8907	indicateFixpoint();
8908	return Change;
8909	}
8910
8911	ChangeStatus manifest(Attributor &A) override {
8912	LLVMContext &Ctx = getAssociatedFunction()->getContext();
8913
8914	SmallVector<Attribute, 2> AttrToAdd;
8915	SmallVector<StringRef, 2> AttrToRemove;
8916	if (Known.Mode == DenormalMode::getDefault()) {
8917	AttrToRemove.push_back(Elt: "denormal-fp-math");
8918	} else {
8919	AttrToAdd.push_back(
8920	Elt: Attribute::get(Context&: Ctx, Kind: "denormal-fp-math", Val: Known.Mode.str()));
8921	}
8922
8923	if (Known.ModeF32 != Known.Mode) {
8924	AttrToAdd.push_back(
8925	Elt: Attribute::get(Context&: Ctx, Kind: "denormal-fp-math-f32", Val: Known.ModeF32.str()));
8926	} else {
8927	AttrToRemove.push_back(Elt: "denormal-fp-math-f32");
8928	}
8929
8930	auto &IRP = getIRPosition();
8931
8932	// TODO: There should be a combined add and remove API.
8933	return A.removeAttrs(IRP, Attrs: AttrToRemove) |
8934	A.manifestAttrs(IRP, DeducedAttrs: AttrToAdd, /*ForceReplace=*/true);
8935	}
8936
8937	void trackStatistics() const override {
8938	STATS_DECLTRACK_FN_ATTR(denormal_fp_math)
8939	}
8940	};
8941	} // namespace
8942
8943	/// ------------------ Value Constant Range Attribute -------------------------
8944
8945	namespace {
8946	struct AAValueConstantRangeImpl : AAValueConstantRange {
8947	using StateType = IntegerRangeState;
8948	AAValueConstantRangeImpl(const IRPosition &IRP, Attributor &A)
8949	: AAValueConstantRange(IRP, A) {}
8950
8951	/// See AbstractAttribute::initialize(..).
8952	void initialize(Attributor &A) override {
8953	if (A.hasSimplificationCallback(IRP: getIRPosition())) {
8954	indicatePessimisticFixpoint();
8955	return;
8956	}
8957
8958	// Intersect a range given by SCEV.
8959	intersectKnown(R: getConstantRangeFromSCEV(A, I: getCtxI()));
8960
8961	// Intersect a range given by LVI.
8962	intersectKnown(R: getConstantRangeFromLVI(A, CtxI: getCtxI()));
8963	}
8964
8965	/// See AbstractAttribute::getAsStr().
8966	const std::string getAsStr(Attributor *A) const override {
8967	std::string Str;
8968	llvm::raw_string_ostream OS(Str);
8969	OS << "range(" << getBitWidth() << ")<";
8970	getKnown().print(OS);
8971	OS << " / ";
8972	getAssumed().print(OS);
8973	OS << ">";
8974	return OS.str();
8975	}
8976
8977	/// Helper function to get a SCEV expr for the associated value at program
8978	/// point \p I.
8979	const SCEV *getSCEV(Attributor &A, const Instruction *I = nullptr) const {
8980	if (!getAnchorScope())
8981	return nullptr;
8982
8983	ScalarEvolution *SE =
8984	A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(
8985	F: *getAnchorScope());
8986
8987	LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(
8988	F: *getAnchorScope());
8989
8990	if (!SE || !LI)
8991	return nullptr;
8992
8993	const SCEV *S = SE->getSCEV(V: &getAssociatedValue());
8994	if (!I)
8995	return S;
8996
8997	return SE->getSCEVAtScope(S, L: LI->getLoopFor(BB: I->getParent()));
8998	}
8999
9000	/// Helper function to get a range from SCEV for the associated value at
9001	/// program point \p I.
9002	ConstantRange getConstantRangeFromSCEV(Attributor &A,
9003	const Instruction *I = nullptr) const {
9004	if (!getAnchorScope())
9005	return getWorstState(BitWidth: getBitWidth());
9006
9007	ScalarEvolution *SE =
9008	A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(
9009	F: *getAnchorScope());
9010
9011	const SCEV *S = getSCEV(A, I);
9012	if (!SE || !S)
9013	return getWorstState(BitWidth: getBitWidth());
9014
9015	return SE->getUnsignedRange(S);
9016	}
9017
9018	/// Helper function to get a range from LVI for the associated value at
9019	/// program point \p I.
9020	ConstantRange
9021	getConstantRangeFromLVI(Attributor &A,
9022	const Instruction *CtxI = nullptr) const {
9023	if (!getAnchorScope())
9024	return getWorstState(BitWidth: getBitWidth());
9025
9026	LazyValueInfo *LVI =
9027	A.getInfoCache().getAnalysisResultForFunction<LazyValueAnalysis>(
9028	F: *getAnchorScope());
9029
9030	if (!LVI || !CtxI)
9031	return getWorstState(BitWidth: getBitWidth());
9032	return LVI->getConstantRange(V: &getAssociatedValue(),
9033	CxtI: const_cast<Instruction *>(CtxI),
9034	/*UndefAllowed*/ false);
9035	}
9036
9037	/// Return true if \p CtxI is valid for querying outside analyses.
9038	/// This basically makes sure we do not ask intra-procedural analysis
9039	/// about a context in the wrong function or a context that violates
9040	/// dominance assumptions they might have. The \p AllowAACtxI flag indicates
9041	/// if the original context of this AA is OK or should be considered invalid.
9042	bool isValidCtxInstructionForOutsideAnalysis(Attributor &A,
9043	const Instruction *CtxI,
9044	bool AllowAACtxI) const {
9045	if (!CtxI || (!AllowAACtxI && CtxI == getCtxI()))
9046	return false;
9047
9048	// Our context might be in a different function, neither intra-procedural
9049	// analysis (ScalarEvolution nor LazyValueInfo) can handle that.
9050	if (!AA::isValidInScope(V: getAssociatedValue(), Scope: CtxI->getFunction()))
9051	return false;
9052
9053	// If the context is not dominated by the value there are paths to the
9054	// context that do not define the value. This cannot be handled by
9055	// LazyValueInfo so we need to bail.
9056	if (auto *I = dyn_cast<Instruction>(Val: &getAssociatedValue())) {
9057	InformationCache &InfoCache = A.getInfoCache();
9058	const DominatorTree *DT =
9059	InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(
9060	F: *I->getFunction());
9061	return DT && DT->dominates(Def: I, User: CtxI);
9062	}
9063
9064	return true;
9065	}
9066
9067	/// See AAValueConstantRange::getKnownConstantRange(..).
9068	ConstantRange
9069	getKnownConstantRange(Attributor &A,
9070	const Instruction *CtxI = nullptr) const override {
9071	if (!isValidCtxInstructionForOutsideAnalysis(A, CtxI,
9072	/* AllowAACtxI */ false))
9073	return getKnown();
9074
9075	ConstantRange LVIR = getConstantRangeFromLVI(A, CtxI);
9076	ConstantRange SCEVR = getConstantRangeFromSCEV(A, I: CtxI);
9077	return getKnown().intersectWith(CR: SCEVR).intersectWith(CR: LVIR);
9078	}
9079
9080	/// See AAValueConstantRange::getAssumedConstantRange(..).
9081	ConstantRange
9082	getAssumedConstantRange(Attributor &A,
9083	const Instruction *CtxI = nullptr) const override {
9084	// TODO: Make SCEV use Attributor assumption.
9085	// We may be able to bound a variable range via assumptions in
9086	// Attributor. ex.) If x is assumed to be in [1, 3] and y is known to
9087	// evolve to x^2 + x, then we can say that y is in [2, 12].
9088	if (!isValidCtxInstructionForOutsideAnalysis(A, CtxI,
9089	/* AllowAACtxI */ false))
9090	return getAssumed();
9091
9092	ConstantRange LVIR = getConstantRangeFromLVI(A, CtxI);
9093	ConstantRange SCEVR = getConstantRangeFromSCEV(A, I: CtxI);
9094	return getAssumed().intersectWith(CR: SCEVR).intersectWith(CR: LVIR);
9095	}
9096
9097	/// Helper function to create MDNode for range metadata.
9098	static MDNode *
9099	getMDNodeForConstantRange(Type *Ty, LLVMContext &Ctx,
9100	const ConstantRange &AssumedConstantRange) {
9101	Metadata *LowAndHigh[] = {ConstantAsMetadata::get(C: ConstantInt::get(
9102	Ty, V: AssumedConstantRange.getLower())),
9103	ConstantAsMetadata::get(C: ConstantInt::get(
9104	Ty, V: AssumedConstantRange.getUpper()))};
9105	return MDNode::get(Context&: Ctx, MDs: LowAndHigh);
9106	}
9107
9108	/// Return true if \p Assumed is included in \p KnownRanges.
9109	static bool isBetterRange(const ConstantRange &Assumed, MDNode *KnownRanges) {
9110
9111	if (Assumed.isFullSet())
9112	return false;
9113
9114	if (!KnownRanges)
9115	return true;
9116
9117	// If multiple ranges are annotated in IR, we give up to annotate assumed
9118	// range for now.
9119
9120	// TODO: If there exists a known range which containts assumed range, we
9121	// can say assumed range is better.
9122	if (KnownRanges->getNumOperands() > 2)
9123	return false;
9124
9125	ConstantInt *Lower =
9126	mdconst::extract<ConstantInt>(MD: KnownRanges->getOperand(I: 0));
9127	ConstantInt *Upper =
9128	mdconst::extract<ConstantInt>(MD: KnownRanges->getOperand(I: 1));
9129
9130	ConstantRange Known(Lower->getValue(), Upper->getValue());
9131	return Known.contains(CR: Assumed) && Known != Assumed;
9132	}
9133
9134	/// Helper function to set range metadata.
9135	static bool
9136	setRangeMetadataIfisBetterRange(Instruction *I,
9137	const ConstantRange &AssumedConstantRange) {
9138	auto *OldRangeMD = I->getMetadata(KindID: LLVMContext::MD_range);
9139	if (isBetterRange(Assumed: AssumedConstantRange, KnownRanges: OldRangeMD)) {
9140	if (!AssumedConstantRange.isEmptySet()) {
9141	I->setMetadata(KindID: LLVMContext::MD_range,
9142	Node: getMDNodeForConstantRange(Ty: I->getType(), Ctx&: I->getContext(),
9143	AssumedConstantRange));
9144	return true;
9145	}
9146	}
9147	return false;
9148	}
9149
9150	/// See AbstractAttribute::manifest()
9151	ChangeStatus manifest(Attributor &A) override {
9152	ChangeStatus Changed = ChangeStatus::UNCHANGED;
9153	ConstantRange AssumedConstantRange = getAssumedConstantRange(A);
9154	assert(!AssumedConstantRange.isFullSet() && "Invalid state");
9155
9156	auto &V = getAssociatedValue();
9157	if (!AssumedConstantRange.isEmptySet() &&
9158	!AssumedConstantRange.isSingleElement()) {
9159	if (Instruction *I = dyn_cast<Instruction>(Val: &V)) {
9160	assert(I == getCtxI() && "Should not annotate an instruction which is "
9161	"not the context instruction");
9162	if (isa<CallInst>(Val: I) || isa<LoadInst>(Val: I))
9163	if (setRangeMetadataIfisBetterRange(I, AssumedConstantRange))
9164	Changed = ChangeStatus::CHANGED;
9165	}
9166	}
9167
9168	return Changed;
9169	}
9170	};
9171
9172	struct AAValueConstantRangeArgument final
9173	: AAArgumentFromCallSiteArguments<
9174	AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,
9175	true /* BridgeCallBaseContext */> {
9176	using Base = AAArgumentFromCallSiteArguments<
9177	AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,
9178	true /* BridgeCallBaseContext */>;
9179	AAValueConstantRangeArgument(const IRPosition &IRP, Attributor &A)
9180	: Base(IRP, A) {}
9181
9182	/// See AbstractAttribute::trackStatistics()
9183	void trackStatistics() const override {
9184	STATS_DECLTRACK_ARG_ATTR(value_range)
9185	}
9186	};
9187
9188	struct AAValueConstantRangeReturned
9189	: AAReturnedFromReturnedValues<AAValueConstantRange,
9190	AAValueConstantRangeImpl,
9191	AAValueConstantRangeImpl::StateType,
9192	/* PropogateCallBaseContext */ true> {
9193	using Base =
9194	AAReturnedFromReturnedValues<AAValueConstantRange,
9195	AAValueConstantRangeImpl,
9196	AAValueConstantRangeImpl::StateType,
9197	/* PropogateCallBaseContext */ true>;
9198	AAValueConstantRangeReturned(const IRPosition &IRP, Attributor &A)
9199	: Base(IRP, A) {}
9200
9201	/// See AbstractAttribute::initialize(...).
9202	void initialize(Attributor &A) override {
9203	if (!A.isFunctionIPOAmendable(F: *getAssociatedFunction()))
9204	indicatePessimisticFixpoint();
9205	}
9206
9207	/// See AbstractAttribute::trackStatistics()
9208	void trackStatistics() const override {
9209	STATS_DECLTRACK_FNRET_ATTR(value_range)
9210	}
9211	};
9212
9213	struct AAValueConstantRangeFloating : AAValueConstantRangeImpl {
9214	AAValueConstantRangeFloating(const IRPosition &IRP, Attributor &A)
9215	: AAValueConstantRangeImpl(IRP, A) {}
9216
9217	/// See AbstractAttribute::initialize(...).
9218	void initialize(Attributor &A) override {
9219	AAValueConstantRangeImpl::initialize(A);
9220	if (isAtFixpoint())
9221	return;
9222
9223	Value &V = getAssociatedValue();
9224
9225	if (auto *C = dyn_cast<ConstantInt>(Val: &V)) {
9226	unionAssumed(R: ConstantRange(C->getValue()));
9227	indicateOptimisticFixpoint();
9228	return;
9229	}
9230
9231	if (isa<UndefValue>(Val: &V)) {
9232	// Collapse the undef state to 0.
9233	unionAssumed(R: ConstantRange(APInt(getBitWidth(), 0)));
9234	indicateOptimisticFixpoint();
9235	return;
9236	}
9237
9238	if (isa<CallBase>(Val: &V))
9239	return;
9240
9241	if (isa<BinaryOperator>(Val: &V) || isa<CmpInst>(Val: &V) || isa<CastInst>(Val: &V))
9242	return;
9243
9244	// If it is a load instruction with range metadata, use it.
9245	if (LoadInst *LI = dyn_cast<LoadInst>(Val: &V))
9246	if (auto *RangeMD = LI->getMetadata(KindID: LLVMContext::MD_range)) {
9247	intersectKnown(R: getConstantRangeFromMetadata(RangeMD: *RangeMD));
9248	return;
9249	}
9250
9251	// We can work with PHI and select instruction as we traverse their operands
9252	// during update.
9253	if (isa<SelectInst>(Val: V) || isa<PHINode>(Val: V))
9254	return;
9255
9256	// Otherwise we give up.
9257	indicatePessimisticFixpoint();
9258
9259	LLVM_DEBUG(dbgs() << "[AAValueConstantRange] We give up: "
9260	<< getAssociatedValue() << "\n");
9261	}
9262
9263	bool calculateBinaryOperator(
9264	Attributor &A, BinaryOperator *BinOp, IntegerRangeState &T,
9265	const Instruction *CtxI,
9266	SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
9267	Value *LHS = BinOp->getOperand(i_nocapture: 0);
9268	Value *RHS = BinOp->getOperand(i_nocapture: 1);
9269
9270	// Simplify the operands first.
9271	bool UsedAssumedInformation = false;
9272	const auto &SimplifiedLHS = A.getAssumedSimplified(
9273	IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()), AA: *this,
9274	UsedAssumedInformation, S: AA::Interprocedural);
9275	if (!SimplifiedLHS.has_value())
9276	return true;
9277	if (!*SimplifiedLHS)
9278	return false;
9279	LHS = *SimplifiedLHS;
9280
9281	const auto &SimplifiedRHS = A.getAssumedSimplified(
9282	IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()), AA: *this,
9283	UsedAssumedInformation, S: AA::Interprocedural);
9284	if (!SimplifiedRHS.has_value())
9285	return true;
9286	if (!*SimplifiedRHS)
9287	return false;
9288	RHS = *SimplifiedRHS;
9289
9290	// TODO: Allow non integers as well.
9291	if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy())
9292	return false;
9293
9294	auto *LHSAA = A.getAAFor<AAValueConstantRange>(
9295	QueryingAA: *this, IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()),
9296	DepClass: DepClassTy::REQUIRED);
9297	if (!LHSAA)
9298	return false;
9299	QuerriedAAs.push_back(Elt: LHSAA);
9300	auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI);
9301
9302	auto *RHSAA = A.getAAFor<AAValueConstantRange>(
9303	QueryingAA: *this, IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()),
9304	DepClass: DepClassTy::REQUIRED);
9305	if (!RHSAA)
9306	return false;
9307	QuerriedAAs.push_back(Elt: RHSAA);
9308	auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI);
9309
9310	auto AssumedRange = LHSAARange.binaryOp(BinOp: BinOp->getOpcode(), Other: RHSAARange);
9311
9312	T.unionAssumed(R: AssumedRange);
9313
9314	// TODO: Track a known state too.
9315
9316	return T.isValidState();
9317	}
9318
9319	bool calculateCastInst(
9320	Attributor &A, CastInst *CastI, IntegerRangeState &T,
9321	const Instruction *CtxI,
9322	SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
9323	assert(CastI->getNumOperands() == 1 && "Expected cast to be unary!");
9324	// TODO: Allow non integers as well.
9325	Value *OpV = CastI->getOperand(i_nocapture: 0);
9326
9327	// Simplify the operand first.
9328	bool UsedAssumedInformation = false;
9329	const auto &SimplifiedOpV = A.getAssumedSimplified(
9330	IRP: IRPosition::value(V: *OpV, CBContext: getCallBaseContext()), AA: *this,
9331	UsedAssumedInformation, S: AA::Interprocedural);
9332	if (!SimplifiedOpV.has_value())
9333	return true;
9334	if (!*SimplifiedOpV)
9335	return false;
9336	OpV = *SimplifiedOpV;
9337
9338	if (!OpV->getType()->isIntegerTy())
9339	return false;
9340
9341	auto *OpAA = A.getAAFor<AAValueConstantRange>(
9342	QueryingAA: *this, IRP: IRPosition::value(V: *OpV, CBContext: getCallBaseContext()),
9343	DepClass: DepClassTy::REQUIRED);
9344	if (!OpAA)
9345	return false;
9346	QuerriedAAs.push_back(Elt: OpAA);
9347	T.unionAssumed(R: OpAA->getAssumed().castOp(CastOp: CastI->getOpcode(),
9348	BitWidth: getState().getBitWidth()));
9349	return T.isValidState();
9350	}
9351
9352	bool
9353	calculateCmpInst(Attributor &A, CmpInst *CmpI, IntegerRangeState &T,
9354	const Instruction *CtxI,
9355	SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
9356	Value *LHS = CmpI->getOperand(i_nocapture: 0);
9357	Value *RHS = CmpI->getOperand(i_nocapture: 1);
9358
9359	// Simplify the operands first.
9360	bool UsedAssumedInformation = false;
9361	const auto &SimplifiedLHS = A.getAssumedSimplified(
9362	IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()), AA: *this,
9363	UsedAssumedInformation, S: AA::Interprocedural);
9364	if (!SimplifiedLHS.has_value())
9365	return true;
9366	if (!*SimplifiedLHS)
9367	return false;
9368	LHS = *SimplifiedLHS;
9369
9370	const auto &SimplifiedRHS = A.getAssumedSimplified(
9371	IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()), AA: *this,
9372	UsedAssumedInformation, S: AA::Interprocedural);
9373	if (!SimplifiedRHS.has_value())
9374	return true;
9375	if (!*SimplifiedRHS)
9376	return false;
9377	RHS = *SimplifiedRHS;
9378
9379	// TODO: Allow non integers as well.
9380	if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy())
9381	return false;
9382
9383	auto *LHSAA = A.getAAFor<AAValueConstantRange>(
9384	QueryingAA: *this, IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()),
9385	DepClass: DepClassTy::REQUIRED);
9386	if (!LHSAA)
9387	return false;
9388	QuerriedAAs.push_back(Elt: LHSAA);
9389	auto *RHSAA = A.getAAFor<AAValueConstantRange>(
9390	QueryingAA: *this, IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()),
9391	DepClass: DepClassTy::REQUIRED);
9392	if (!RHSAA)
9393	return false;
9394	QuerriedAAs.push_back(Elt: RHSAA);
9395	auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI);
9396	auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI);
9397
9398	// If one of them is empty set, we can't decide.
9399	if (LHSAARange.isEmptySet() || RHSAARange.isEmptySet())
9400	return true;
9401
9402	bool MustTrue = false, MustFalse = false;
9403
9404	auto AllowedRegion =
9405	ConstantRange::makeAllowedICmpRegion(Pred: CmpI->getPredicate(), Other: RHSAARange);
9406
9407	if (AllowedRegion.intersectWith(CR: LHSAARange).isEmptySet())
9408	MustFalse = true;
9409
9410	if (LHSAARange.icmp(Pred: CmpI->getPredicate(), Other: RHSAARange))
9411	MustTrue = true;
9412
9413	assert((!MustTrue || !MustFalse) &&
9414	"Either MustTrue or MustFalse should be false!");
9415
9416	if (MustTrue)
9417	T.unionAssumed(R: ConstantRange(APInt(/* numBits */ 1, /* val */ 1)));
9418	else if (MustFalse)
9419	T.unionAssumed(R: ConstantRange(APInt(/* numBits */ 1, /* val */ 0)));
9420	else
9421	T.unionAssumed(R: ConstantRange(/* BitWidth */ 1, /* isFullSet */ true));
9422
9423	LLVM_DEBUG(dbgs() << "[AAValueConstantRange] " << *CmpI << " after "
9424	<< (MustTrue ? "true" : (MustFalse ? "false" : "unknown"))
9425	<< ": " << T << "\n\t" << *LHSAA << "\t<op>\n\t"
9426	<< *RHSAA);
9427
9428	// TODO: Track a known state too.
9429	return T.isValidState();
9430	}
9431
9432	/// See AbstractAttribute::updateImpl(...).
9433	ChangeStatus updateImpl(Attributor &A) override {
9434
9435	IntegerRangeState T(getBitWidth());
9436	auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool {
9437	Instruction *I = dyn_cast<Instruction>(Val: &V);
9438	if (!I || isa<CallBase>(Val: I)) {
9439
9440	// Simplify the operand first.
9441	bool UsedAssumedInformation = false;
9442	const auto &SimplifiedOpV = A.getAssumedSimplified(
9443	IRP: IRPosition::value(V, CBContext: getCallBaseContext()), AA: *this,
9444	UsedAssumedInformation, S: AA::Interprocedural);
9445	if (!SimplifiedOpV.has_value())
9446	return true;
9447	if (!*SimplifiedOpV)
9448	return false;
9449	Value *VPtr = *SimplifiedOpV;
9450
9451	// If the value is not instruction, we query AA to Attributor.
9452	const auto *AA = A.getAAFor<AAValueConstantRange>(
9453	QueryingAA: *this, IRP: IRPosition::value(V: *VPtr, CBContext: getCallBaseContext()),
9454	DepClass: DepClassTy::REQUIRED);
9455
9456	// Clamp operator is not used to utilize a program point CtxI.
9457	if (AA)
9458	T.unionAssumed(R: AA->getAssumedConstantRange(A, CtxI));
9459	else
9460	return false;
9461
9462	return T.isValidState();
9463	}
9464
9465	SmallVector<const AAValueConstantRange *, 4> QuerriedAAs;
9466	if (auto *BinOp = dyn_cast<BinaryOperator>(Val: I)) {
9467	if (!calculateBinaryOperator(A, BinOp, T, CtxI, QuerriedAAs))
9468	return false;
9469	} else if (auto *CmpI = dyn_cast<CmpInst>(Val: I)) {
9470	if (!calculateCmpInst(A, CmpI, T, CtxI, QuerriedAAs))
9471	return false;
9472	} else if (auto *CastI = dyn_cast<CastInst>(Val: I)) {
9473	if (!calculateCastInst(A, CastI, T, CtxI, QuerriedAAs))
9474	return false;
9475	} else {
9476	// Give up with other instructions.
9477	// TODO: Add other instructions
9478
9479	T.indicatePessimisticFixpoint();
9480	return false;
9481	}
9482
9483	// Catch circular reasoning in a pessimistic way for now.
9484	// TODO: Check how the range evolves and if we stripped anything, see also
9485	// AADereferenceable or AAAlign for similar situations.
9486	for (const AAValueConstantRange *QueriedAA : QuerriedAAs) {
9487	if (QueriedAA != this)
9488	continue;
9489	// If we are in a stady state we do not need to worry.
9490	if (T.getAssumed() == getState().getAssumed())
9491	continue;
9492	T.indicatePessimisticFixpoint();
9493	}
9494
9495	return T.isValidState();
9496	};
9497
9498	if (!VisitValueCB(getAssociatedValue(), getCtxI()))
9499	return indicatePessimisticFixpoint();
9500
9501	// Ensure that long def-use chains can't cause circular reasoning either by
9502	// introducing a cutoff below.
9503	if (clampStateAndIndicateChange(S&: getState(), R: T) == ChangeStatus::UNCHANGED)
9504	return ChangeStatus::UNCHANGED;
9505	if (++NumChanges > MaxNumChanges) {
9506	LLVM_DEBUG(dbgs() << "[AAValueConstantRange] performed " << NumChanges
9507	<< " but only " << MaxNumChanges
9508	<< " are allowed to avoid cyclic reasoning.");
9509	return indicatePessimisticFixpoint();
9510	}
9511	return ChangeStatus::CHANGED;
9512	}
9513
9514	/// See AbstractAttribute::trackStatistics()
9515	void trackStatistics() const override {
9516	STATS_DECLTRACK_FLOATING_ATTR(value_range)
9517	}
9518
9519	/// Tracker to bail after too many widening steps of the constant range.
9520	int NumChanges = 0;
9521
9522	/// Upper bound for the number of allowed changes (=widening steps) for the
9523	/// constant range before we give up.
9524	static constexpr int MaxNumChanges = 5;
9525	};
9526
9527	struct AAValueConstantRangeFunction : AAValueConstantRangeImpl {
9528	AAValueConstantRangeFunction(const IRPosition &IRP, Attributor &A)
9529	: AAValueConstantRangeImpl(IRP, A) {}
9530
9531	/// See AbstractAttribute::initialize(...).
9532	ChangeStatus updateImpl(Attributor &A) override {
9533	llvm_unreachable("AAValueConstantRange(Function|CallSite)::updateImpl will "
9534	"not be called");
9535	}
9536
9537	/// See AbstractAttribute::trackStatistics()
9538	void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(value_range) }
9539	};
9540
9541	struct AAValueConstantRangeCallSite : AAValueConstantRangeFunction {
9542	AAValueConstantRangeCallSite(const IRPosition &IRP, Attributor &A)
9543	: AAValueConstantRangeFunction(IRP, A) {}
9544
9545	/// See AbstractAttribute::trackStatistics()
9546	void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(value_range) }
9547	};
9548
9549	struct AAValueConstantRangeCallSiteReturned
9550	: AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,
9551	AAValueConstantRangeImpl::StateType,
9552	/* IntroduceCallBaseContext */ true> {
9553	AAValueConstantRangeCallSiteReturned(const IRPosition &IRP, Attributor &A)
9554	: AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,
9555	AAValueConstantRangeImpl::StateType,
9556	/* IntroduceCallBaseContext */ true>(IRP, A) {}
9557
9558	/// See AbstractAttribute::initialize(...).
9559	void initialize(Attributor &A) override {
9560	// If it is a load instruction with range metadata, use the metadata.
9561	if (CallInst *CI = dyn_cast<CallInst>(Val: &getAssociatedValue()))
9562	if (auto *RangeMD = CI->getMetadata(KindID: LLVMContext::MD_range))
9563	intersectKnown(R: getConstantRangeFromMetadata(RangeMD: *RangeMD));
9564
9565	AAValueConstantRangeImpl::initialize(A);
9566	}
9567
9568	/// See AbstractAttribute::trackStatistics()
9569	void trackStatistics() const override {
9570	STATS_DECLTRACK_CSRET_ATTR(value_range)
9571	}
9572	};
9573	struct AAValueConstantRangeCallSiteArgument : AAValueConstantRangeFloating {
9574	AAValueConstantRangeCallSiteArgument(const IRPosition &IRP, Attributor &A)
9575	: AAValueConstantRangeFloating(IRP, A) {}
9576
9577	/// See AbstractAttribute::manifest()
9578	ChangeStatus manifest(Attributor &A) override {
9579	return ChangeStatus::UNCHANGED;
9580	}
9581
9582	/// See AbstractAttribute::trackStatistics()
9583	void trackStatistics() const override {
9584	STATS_DECLTRACK_CSARG_ATTR(value_range)
9585	}
9586	};
9587	} // namespace
9588
9589	/// ------------------ Potential Values Attribute -------------------------
9590
9591	namespace {
9592	struct AAPotentialConstantValuesImpl : AAPotentialConstantValues {
9593	using StateType = PotentialConstantIntValuesState;
9594
9595	AAPotentialConstantValuesImpl(const IRPosition &IRP, Attributor &A)
9596	: AAPotentialConstantValues(IRP, A) {}
9597
9598	/// See AbstractAttribute::initialize(..).
9599	void initialize(Attributor &A) override {
9600	if (A.hasSimplificationCallback(IRP: getIRPosition()))
9601	indicatePessimisticFixpoint();
9602	else
9603	AAPotentialConstantValues::initialize(A);
9604	}
9605
9606	bool fillSetWithConstantValues(Attributor &A, const IRPosition &IRP, SetTy &S,
9607	bool &ContainsUndef, bool ForSelf) {
9608	SmallVector<AA::ValueAndContext> Values;
9609	bool UsedAssumedInformation = false;
9610	if (!A.getAssumedSimplifiedValues(IRP, AA: *this, Values, S: AA::Interprocedural,
9611	UsedAssumedInformation)) {
9612	// Avoid recursion when the caller is computing constant values for this
9613	// IRP itself.
9614	if (ForSelf)
9615	return false;
9616	if (!IRP.getAssociatedType()->isIntegerTy())
9617	return false;
9618	auto *PotentialValuesAA = A.getAAFor<AAPotentialConstantValues>(
9619	QueryingAA: *this, IRP, DepClass: DepClassTy::REQUIRED);
9620	if (!PotentialValuesAA || !PotentialValuesAA->getState().isValidState())
9621	return false;
9622	ContainsUndef = PotentialValuesAA->getState().undefIsContained();
9623	S = PotentialValuesAA->getState().getAssumedSet();
9624	return true;
9625	}
9626
9627	// Copy all the constant values, except UndefValue. ContainsUndef is true
9628	// iff Values contains only UndefValue instances. If there are other known
9629	// constants, then UndefValue is dropped.
9630	ContainsUndef = false;
9631	for (auto &It : Values) {
9632	if (isa<UndefValue>(Val: It.getValue())) {
9633	ContainsUndef = true;
9634	continue;
9635	}
9636	auto *CI = dyn_cast<ConstantInt>(Val: It.getValue());
9637	if (!CI)
9638	return false;
9639	S.insert(X: CI->getValue());
9640	}
9641	ContainsUndef &= S.empty();
9642
9643	return true;
9644	}
9645
9646	/// See AbstractAttribute::getAsStr().
9647	const std::string getAsStr(Attributor *A) const override {
9648	std::string Str;
9649	llvm::raw_string_ostream OS(Str);
9650	OS << getState();
9651	return OS.str();
9652	}
9653
9654	/// See AbstractAttribute::updateImpl(...).
9655	ChangeStatus updateImpl(Attributor &A) override {
9656	return indicatePessimisticFixpoint();
9657	}
9658	};
9659
9660	struct AAPotentialConstantValuesArgument final
9661	: AAArgumentFromCallSiteArguments<AAPotentialConstantValues,
9662	AAPotentialConstantValuesImpl,
9663	PotentialConstantIntValuesState> {
9664	using Base = AAArgumentFromCallSiteArguments<AAPotentialConstantValues,
9665	AAPotentialConstantValuesImpl,
9666	PotentialConstantIntValuesState>;
9667	AAPotentialConstantValuesArgument(const IRPosition &IRP, Attributor &A)
9668	: Base(IRP, A) {}
9669
9670	/// See AbstractAttribute::trackStatistics()
9671	void trackStatistics() const override {
9672	STATS_DECLTRACK_ARG_ATTR(potential_values)
9673	}
9674	};
9675
9676	struct AAPotentialConstantValuesReturned
9677	: AAReturnedFromReturnedValues<AAPotentialConstantValues,
9678	AAPotentialConstantValuesImpl> {
9679	using Base = AAReturnedFromReturnedValues<AAPotentialConstantValues,
9680	AAPotentialConstantValuesImpl>;
9681	AAPotentialConstantValuesReturned(const IRPosition &IRP, Attributor &A)
9682	: Base(IRP, A) {}
9683
9684	void initialize(Attributor &A) override {
9685	if (!A.isFunctionIPOAmendable(F: *getAssociatedFunction()))
9686	indicatePessimisticFixpoint();
9687	Base::initialize(A);
9688	}
9689
9690	/// See AbstractAttribute::trackStatistics()
9691	void trackStatistics() const override {
9692	STATS_DECLTRACK_FNRET_ATTR(potential_values)
9693	}
9694	};
9695
9696	struct AAPotentialConstantValuesFloating : AAPotentialConstantValuesImpl {
9697	AAPotentialConstantValuesFloating(const IRPosition &IRP, Attributor &A)
9698	: AAPotentialConstantValuesImpl(IRP, A) {}
9699
9700	/// See AbstractAttribute::initialize(..).
9701	void initialize(Attributor &A) override {
9702	AAPotentialConstantValuesImpl::initialize(A);
9703	if (isAtFixpoint())
9704	return;
9705
9706	Value &V = getAssociatedValue();
9707
9708	if (auto *C = dyn_cast<ConstantInt>(Val: &V)) {
9709	unionAssumed(C: C->getValue());
9710	indicateOptimisticFixpoint();
9711	return;
9712	}
9713
9714	if (isa<UndefValue>(Val: &V)) {
9715	unionAssumedWithUndef();
9716	indicateOptimisticFixpoint();
9717	return;
9718	}
9719
9720	if (isa<BinaryOperator>(Val: &V) || isa<ICmpInst>(Val: &V) || isa<CastInst>(Val: &V))
9721	return;
9722
9723	if (isa<SelectInst>(Val: V) || isa<PHINode>(Val: V) || isa<LoadInst>(Val: V))
9724	return;
9725
9726	indicatePessimisticFixpoint();
9727
9728	LLVM_DEBUG(dbgs() << "[AAPotentialConstantValues] We give up: "
9729	<< getAssociatedValue() << "\n");
9730	}
9731
9732	static bool calculateICmpInst(const ICmpInst *ICI, const APInt &LHS,
9733	const APInt &RHS) {
9734	return ICmpInst::compare(LHS, RHS, Pred: ICI->getPredicate());
9735	}
9736
9737	static APInt calculateCastInst(const CastInst *CI, const APInt &Src,
9738	uint32_t ResultBitWidth) {
9739	Instruction::CastOps CastOp = CI->getOpcode();
9740	switch (CastOp) {
9741	default:
9742	llvm_unreachable("unsupported or not integer cast");
9743	case Instruction::Trunc:
9744	return Src.trunc(width: ResultBitWidth);
9745	case Instruction::SExt:
9746	return Src.sext(width: ResultBitWidth);
9747	case Instruction::ZExt:
9748	return Src.zext(width: ResultBitWidth);
9749	case Instruction::BitCast:
9750	return Src;
9751	}
9752	}
9753
9754	static APInt calculateBinaryOperator(const BinaryOperator *BinOp,
9755	const APInt &LHS, const APInt &RHS,
9756	bool &SkipOperation, bool &Unsupported) {
9757	Instruction::BinaryOps BinOpcode = BinOp->getOpcode();
9758	// Unsupported is set to true when the binary operator is not supported.
9759	// SkipOperation is set to true when UB occur with the given operand pair
9760	// (LHS, RHS).
9761	// TODO: we should look at nsw and nuw keywords to handle operations
9762	// that create poison or undef value.
9763	switch (BinOpcode) {
9764	default:
9765	Unsupported = true;
9766	return LHS;
9767	case Instruction::Add:
9768	return LHS + RHS;
9769	case Instruction::Sub:
9770	return LHS - RHS;
9771	case Instruction::Mul:
9772	return LHS * RHS;
9773	case Instruction::UDiv:
9774	if (RHS.isZero()) {
9775	SkipOperation = true;
9776	return LHS;
9777	}
9778	return LHS.udiv(RHS);
9779	case Instruction::SDiv:
9780	if (RHS.isZero()) {
9781	SkipOperation = true;
9782	return LHS;
9783	}
9784	return LHS.sdiv(RHS);
9785	case Instruction::URem:
9786	if (RHS.isZero()) {
9787	SkipOperation = true;
9788	return LHS;
9789	}
9790	return LHS.urem(RHS);
9791	case Instruction::SRem:
9792	if (RHS.isZero()) {
9793	SkipOperation = true;
9794	return LHS;
9795	}
9796	return LHS.srem(RHS);
9797	case Instruction::Shl:
9798	return LHS.shl(ShiftAmt: RHS);
9799	case Instruction::LShr:
9800	return LHS.lshr(ShiftAmt: RHS);
9801	case Instruction::AShr:
9802	return LHS.ashr(ShiftAmt: RHS);
9803	case Instruction::And:
9804	return LHS & RHS;
9805	case Instruction::Or:
9806	return LHS | RHS;
9807	case Instruction::Xor:
9808	return LHS ^ RHS;
9809	}
9810	}
9811
9812	bool calculateBinaryOperatorAndTakeUnion(const BinaryOperator *BinOp,
9813	const APInt &LHS, const APInt &RHS) {
9814	bool SkipOperation = false;
9815	bool Unsupported = false;
9816	APInt Result =
9817	calculateBinaryOperator(BinOp, LHS, RHS, SkipOperation, Unsupported);
9818	if (Unsupported)
9819	return false;
9820	// If SkipOperation is true, we can ignore this operand pair (L, R).
9821	if (!SkipOperation)
9822	unionAssumed(C: Result);
9823	return isValidState();
9824	}
9825
9826	ChangeStatus updateWithICmpInst(Attributor &A, ICmpInst *ICI) {
9827	auto AssumedBefore = getAssumed();
9828	Value *LHS = ICI->getOperand(i_nocapture: 0);
9829	Value *RHS = ICI->getOperand(i_nocapture: 1);
9830
9831	bool LHSContainsUndef = false, RHSContainsUndef = false;
9832	SetTy LHSAAPVS, RHSAAPVS;
9833	if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *LHS), S&: LHSAAPVS,
9834	ContainsUndef&: LHSContainsUndef, /* ForSelf */ false) ||
9835	!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *RHS), S&: RHSAAPVS,
9836	ContainsUndef&: RHSContainsUndef, /* ForSelf */ false))
9837	return indicatePessimisticFixpoint();
9838
9839	// TODO: make use of undef flag to limit potential values aggressively.
9840	bool MaybeTrue = false, MaybeFalse = false;
9841	const APInt Zero(RHS->getType()->getIntegerBitWidth(), 0);
9842	if (LHSContainsUndef && RHSContainsUndef) {
9843	// The result of any comparison between undefs can be soundly replaced
9844	// with undef.
9845	unionAssumedWithUndef();
9846	} else if (LHSContainsUndef) {
9847	for (const APInt &R : RHSAAPVS) {
9848	bool CmpResult = calculateICmpInst(ICI, LHS: Zero, RHS: R);
9849	MaybeTrue |= CmpResult;
9850	MaybeFalse |= !CmpResult;
9851	if (MaybeTrue & MaybeFalse)
9852	return indicatePessimisticFixpoint();
9853	}
9854	} else if (RHSContainsUndef) {
9855	for (const APInt &L : LHSAAPVS) {
9856	bool CmpResult = calculateICmpInst(ICI, LHS: L, RHS: Zero);
9857	MaybeTrue |= CmpResult;
9858	MaybeFalse |= !CmpResult;
9859	if (MaybeTrue & MaybeFalse)
9860	return indicatePessimisticFixpoint();
9861	}
9862	} else {
9863	for (const APInt &L : LHSAAPVS) {
9864	for (const APInt &R : RHSAAPVS) {
9865	bool CmpResult = calculateICmpInst(ICI, LHS: L, RHS: R);
9866	MaybeTrue |= CmpResult;
9867	MaybeFalse |= !CmpResult;
9868	if (MaybeTrue & MaybeFalse)
9869	return indicatePessimisticFixpoint();
9870	}
9871	}
9872	}
9873	if (MaybeTrue)
9874	unionAssumed(C: APInt(/* numBits */ 1, /* val */ 1));
9875	if (MaybeFalse)
9876	unionAssumed(C: APInt(/* numBits */ 1, /* val */ 0));
9877	return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
9878	: ChangeStatus::CHANGED;
9879	}
9880
9881	ChangeStatus updateWithSelectInst(Attributor &A, SelectInst *SI) {
9882	auto AssumedBefore = getAssumed();
9883	Value *LHS = SI->getTrueValue();
9884	Value *RHS = SI->getFalseValue();
9885
9886	bool UsedAssumedInformation = false;
9887	std::optional<Constant *> C = A.getAssumedConstant(
9888	V: *SI->getCondition(), AA: *this, UsedAssumedInformation);
9889
9890	// Check if we only need one operand.
9891	bool OnlyLeft = false, OnlyRight = false;
9892	if (C && *C && (*C)->isOneValue())
9893	OnlyLeft = true;
9894	else if (C && *C && (*C)->isZeroValue())
9895	OnlyRight = true;
9896
9897	bool LHSContainsUndef = false, RHSContainsUndef = false;
9898	SetTy LHSAAPVS, RHSAAPVS;
9899	if (!OnlyRight &&
9900	!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *LHS), S&: LHSAAPVS,
9901	ContainsUndef&: LHSContainsUndef, /* ForSelf */ false))
9902	return indicatePessimisticFixpoint();
9903
9904	if (!OnlyLeft &&
9905	!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *RHS), S&: RHSAAPVS,
9906	ContainsUndef&: RHSContainsUndef, /* ForSelf */ false))
9907	return indicatePessimisticFixpoint();
9908
9909	if (OnlyLeft || OnlyRight) {
9910	// select (true/false), lhs, rhs
9911	auto *OpAA = OnlyLeft ? &LHSAAPVS : &RHSAAPVS;
9912	auto Undef = OnlyLeft ? LHSContainsUndef : RHSContainsUndef;
9913
9914	if (Undef)
9915	unionAssumedWithUndef();
9916	else {
9917	for (const auto &It : *OpAA)
9918	unionAssumed(C: It);
9919	}
9920
9921	} else if (LHSContainsUndef && RHSContainsUndef) {
9922	// select i1 *, undef , undef => undef
9923	unionAssumedWithUndef();
9924	} else {
9925	for (const auto &It : LHSAAPVS)
9926	unionAssumed(C: It);
9927	for (const auto &It : RHSAAPVS)
9928	unionAssumed(C: It);
9929	}
9930	return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
9931	: ChangeStatus::CHANGED;
9932	}
9933
9934	ChangeStatus updateWithCastInst(Attributor &A, CastInst *CI) {
9935	auto AssumedBefore = getAssumed();
9936	if (!CI->isIntegerCast())
9937	return indicatePessimisticFixpoint();
9938	assert(CI->getNumOperands() == 1 && "Expected cast to be unary!");
9939	uint32_t ResultBitWidth = CI->getDestTy()->getIntegerBitWidth();
9940	Value *Src = CI->getOperand(i_nocapture: 0);
9941
9942	bool SrcContainsUndef = false;
9943	SetTy SrcPVS;
9944	if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *Src), S&: SrcPVS,
9945	ContainsUndef&: SrcContainsUndef, /* ForSelf */ false))
9946	return indicatePessimisticFixpoint();
9947
9948	if (SrcContainsUndef)
9949	unionAssumedWithUndef();
9950	else {
9951	for (const APInt &S : SrcPVS) {
9952	APInt T = calculateCastInst(CI, Src: S, ResultBitWidth);
9953	unionAssumed(C: T);
9954	}
9955	}
9956	return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
9957	: ChangeStatus::CHANGED;
9958	}
9959
9960	ChangeStatus updateWithBinaryOperator(Attributor &A, BinaryOperator *BinOp) {
9961	auto AssumedBefore = getAssumed();
9962	Value *LHS = BinOp->getOperand(i_nocapture: 0);
9963	Value *RHS = BinOp->getOperand(i_nocapture: 1);
9964
9965	bool LHSContainsUndef = false, RHSContainsUndef = false;
9966	SetTy LHSAAPVS, RHSAAPVS;
9967	if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *LHS), S&: LHSAAPVS,
9968	ContainsUndef&: LHSContainsUndef, /* ForSelf */ false) ||
9969	!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *RHS), S&: RHSAAPVS,
9970	ContainsUndef&: RHSContainsUndef, /* ForSelf */ false))
9971	return indicatePessimisticFixpoint();
9972
9973	const APInt Zero = APInt(LHS->getType()->getIntegerBitWidth(), 0);
9974
9975	// TODO: make use of undef flag to limit potential values aggressively.
9976	if (LHSContainsUndef && RHSContainsUndef) {
9977	if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: Zero, RHS: Zero))
9978	return indicatePessimisticFixpoint();
9979	} else if (LHSContainsUndef) {
9980	for (const APInt &R : RHSAAPVS) {
9981	if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: Zero, RHS: R))
9982	return indicatePessimisticFixpoint();
9983	}
9984	} else if (RHSContainsUndef) {
9985	for (const APInt &L : LHSAAPVS) {
9986	if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: L, RHS: Zero))
9987	return indicatePessimisticFixpoint();
9988	}
9989	} else {
9990	for (const APInt &L : LHSAAPVS) {
9991	for (const APInt &R : RHSAAPVS) {
9992	if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: L, RHS: R))
9993	return indicatePessimisticFixpoint();
9994	}
9995	}
9996	}
9997	return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
9998	: ChangeStatus::CHANGED;
9999	}
10000
10001	ChangeStatus updateWithInstruction(Attributor &A, Instruction *Inst) {
10002	auto AssumedBefore = getAssumed();
10003	SetTy Incoming;
10004	bool ContainsUndef;
10005	if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *Inst), S&: Incoming,
10006	ContainsUndef, /* ForSelf */ true))
10007	return indicatePessimisticFixpoint();
10008	if (ContainsUndef) {
10009	unionAssumedWithUndef();
10010	} else {
10011	for (const auto &It : Incoming)
10012	unionAssumed(C: It);
10013	}
10014	return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10015	: ChangeStatus::CHANGED;
10016	}
10017
10018	/// See AbstractAttribute::updateImpl(...).
10019	ChangeStatus updateImpl(Attributor &A) override {
10020	Value &V = getAssociatedValue();
10021	Instruction *I = dyn_cast<Instruction>(Val: &V);
10022
10023	if (auto *ICI = dyn_cast<ICmpInst>(Val: I))
10024	return updateWithICmpInst(A, ICI);
10025
10026	if (auto *SI = dyn_cast<SelectInst>(Val: I))
10027	return updateWithSelectInst(A, SI);
10028
10029	if (auto *CI = dyn_cast<CastInst>(Val: I))
10030	return updateWithCastInst(A, CI);
10031
10032	if (auto *BinOp = dyn_cast<BinaryOperator>(Val: I))
10033	return updateWithBinaryOperator(A, BinOp);
10034
10035	if (isa<PHINode>(Val: I) || isa<LoadInst>(Val: I))
10036	return updateWithInstruction(A, Inst: I);
10037
10038	return indicatePessimisticFixpoint();
10039	}
10040
10041	/// See AbstractAttribute::trackStatistics()
10042	void trackStatistics() const override {
10043	STATS_DECLTRACK_FLOATING_ATTR(potential_values)
10044	}
10045	};
10046
10047	struct AAPotentialConstantValuesFunction : AAPotentialConstantValuesImpl {
10048	AAPotentialConstantValuesFunction(const IRPosition &IRP, Attributor &A)
10049	: AAPotentialConstantValuesImpl(IRP, A) {}
10050
10051	/// See AbstractAttribute::initialize(...).
10052	ChangeStatus updateImpl(Attributor &A) override {
10053	llvm_unreachable(
10054	"AAPotentialConstantValues(Function|CallSite)::updateImpl will "
10055	"not be called");
10056	}
10057
10058	/// See AbstractAttribute::trackStatistics()
10059	void trackStatistics() const override {
10060	STATS_DECLTRACK_FN_ATTR(potential_values)
10061	}
10062	};
10063
10064	struct AAPotentialConstantValuesCallSite : AAPotentialConstantValuesFunction {
10065	AAPotentialConstantValuesCallSite(const IRPosition &IRP, Attributor &A)
10066	: AAPotentialConstantValuesFunction(IRP, A) {}
10067
10068	/// See AbstractAttribute::trackStatistics()
10069	void trackStatistics() const override {
10070	STATS_DECLTRACK_CS_ATTR(potential_values)
10071	}
10072	};
10073
10074	struct AAPotentialConstantValuesCallSiteReturned
10075	: AACalleeToCallSite<AAPotentialConstantValues,
10076	AAPotentialConstantValuesImpl> {
10077	AAPotentialConstantValuesCallSiteReturned(const IRPosition &IRP,
10078	Attributor &A)
10079	: AACalleeToCallSite<AAPotentialConstantValues,
10080	AAPotentialConstantValuesImpl>(IRP, A) {}
10081
10082	/// See AbstractAttribute::trackStatistics()
10083	void trackStatistics() const override {
10084	STATS_DECLTRACK_CSRET_ATTR(potential_values)
10085	}
10086	};
10087
10088	struct AAPotentialConstantValuesCallSiteArgument
10089	: AAPotentialConstantValuesFloating {
10090	AAPotentialConstantValuesCallSiteArgument(const IRPosition &IRP,
10091	Attributor &A)
10092	: AAPotentialConstantValuesFloating(IRP, A) {}
10093
10094	/// See AbstractAttribute::initialize(..).
10095	void initialize(Attributor &A) override {
10096	AAPotentialConstantValuesImpl::initialize(A);
10097	if (isAtFixpoint())
10098	return;
10099
10100	Value &V = getAssociatedValue();
10101
10102	if (auto *C = dyn_cast<ConstantInt>(Val: &V)) {
10103	unionAssumed(C: C->getValue());
10104	indicateOptimisticFixpoint();
10105	return;
10106	}
10107
10108	if (isa<UndefValue>(Val: &V)) {
10109	unionAssumedWithUndef();
10110	indicateOptimisticFixpoint();
10111	return;
10112	}
10113	}
10114
10115	/// See AbstractAttribute::updateImpl(...).
10116	ChangeStatus updateImpl(Attributor &A) override {
10117	Value &V = getAssociatedValue();
10118	auto AssumedBefore = getAssumed();
10119	auto *AA = A.getAAFor<AAPotentialConstantValues>(
10120	QueryingAA: *this, IRP: IRPosition::value(V), DepClass: DepClassTy::REQUIRED);
10121	if (!AA)
10122	return indicatePessimisticFixpoint();
10123	const auto &S = AA->getAssumed();
10124	unionAssumed(PVS: S);
10125	return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10126	: ChangeStatus::CHANGED;
10127	}
10128
10129	/// See AbstractAttribute::trackStatistics()
10130	void trackStatistics() const override {
10131	STATS_DECLTRACK_CSARG_ATTR(potential_values)
10132	}
10133	};
10134	} // namespace
10135
10136	/// ------------------------ NoUndef Attribute ---------------------------------
10137	bool AANoUndef::isImpliedByIR(Attributor &A, const IRPosition &IRP,
10138	Attribute::AttrKind ImpliedAttributeKind,
10139	bool IgnoreSubsumingPositions) {
10140	assert(ImpliedAttributeKind == Attribute::NoUndef &&
10141	"Unexpected attribute kind");
10142	if (A.hasAttr(IRP, {Attribute::NoUndef}, IgnoreSubsumingPositions,
10143	Attribute::NoUndef))
10144	return true;
10145
10146	Value &Val = IRP.getAssociatedValue();
10147	if (IRP.getPositionKind() != IRPosition::IRP_RETURNED &&
10148	isGuaranteedNotToBeUndefOrPoison(V: &Val)) {
10149	LLVMContext &Ctx = Val.getContext();
10150	A.manifestAttrs(IRP, Attribute::get(Ctx, Attribute::NoUndef));
10151	return true;
10152	}
10153
10154	return false;
10155	}
10156
10157	namespace {
10158	struct AANoUndefImpl : AANoUndef {
10159	AANoUndefImpl(const IRPosition &IRP, Attributor &A) : AANoUndef(IRP, A) {}
10160
10161	/// See AbstractAttribute::initialize(...).
10162	void initialize(Attributor &A) override {
10163	Value &V = getAssociatedValue();
10164	if (isa<UndefValue>(Val: V))
10165	indicatePessimisticFixpoint();
10166	assert(!isImpliedByIR(A, getIRPosition(), Attribute::NoUndef));
10167	}
10168
10169	/// See followUsesInMBEC
10170	bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
10171	AANoUndef::StateType &State) {
10172	const Value *UseV = U->get();
10173	const DominatorTree *DT = nullptr;
10174	AssumptionCache *AC = nullptr;
10175	InformationCache &InfoCache = A.getInfoCache();
10176	if (Function *F = getAnchorScope()) {
10177	DT = InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F);
10178	AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F);
10179	}
10180	State.setKnown(isGuaranteedNotToBeUndefOrPoison(V: UseV, AC, CtxI: I, DT));
10181	bool TrackUse = false;
10182	// Track use for instructions which must produce undef or poison bits when
10183	// at least one operand contains such bits.
10184	if (isa<CastInst>(Val: *I) || isa<GetElementPtrInst>(Val: *I))
10185	TrackUse = true;
10186	return TrackUse;
10187	}
10188
10189	/// See AbstractAttribute::getAsStr().
10190	const std::string getAsStr(Attributor *A) const override {
10191	return getAssumed() ? "noundef" : "may-undef-or-poison";
10192	}
10193
10194	ChangeStatus manifest(Attributor &A) override {
10195	// We don't manifest noundef attribute for dead positions because the
10196	// associated values with dead positions would be replaced with undef
10197	// values.
10198	bool UsedAssumedInformation = false;
10199	if (A.isAssumedDead(getIRPosition(), nullptr, nullptr,
10200	UsedAssumedInformation))
10201	return ChangeStatus::UNCHANGED;
10202	// A position whose simplified value does not have any value is
10203	// considered to be dead. We don't manifest noundef in such positions for
10204	// the same reason above.
10205	if (!A.getAssumedSimplified(getIRPosition(), *this, UsedAssumedInformation,
10206	AA::Interprocedural)
10207	.has_value())
10208	return ChangeStatus::UNCHANGED;
10209	return AANoUndef::manifest(A);
10210	}
10211	};
10212
10213	struct AANoUndefFloating : public AANoUndefImpl {
10214	AANoUndefFloating(const IRPosition &IRP, Attributor &A)
10215	: AANoUndefImpl(IRP, A) {}
10216
10217	/// See AbstractAttribute::initialize(...).
10218	void initialize(Attributor &A) override {
10219	AANoUndefImpl::initialize(A);
10220	if (!getState().isAtFixpoint() && getAnchorScope() &&
10221	!getAnchorScope()->isDeclaration())
10222	if (Instruction *CtxI = getCtxI())
10223	followUsesInMBEC(*this, A, getState(), *CtxI);
10224	}
10225
10226	/// See AbstractAttribute::updateImpl(...).
10227	ChangeStatus updateImpl(Attributor &A) override {
10228	auto VisitValueCB = [&](const IRPosition &IRP) -> bool {
10229	bool IsKnownNoUndef;
10230	return AA::hasAssumedIRAttr<Attribute::NoUndef>(
10231	A, this, IRP, DepClassTy::REQUIRED, IsKnownNoUndef);
10232	};
10233
10234	bool Stripped;
10235	bool UsedAssumedInformation = false;
10236	Value *AssociatedValue = &getAssociatedValue();
10237	SmallVector<AA::ValueAndContext> Values;
10238	if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
10239	S: AA::AnyScope, UsedAssumedInformation))
10240	Stripped = false;
10241	else
10242	Stripped =
10243	Values.size() != 1 || Values.front().getValue() != AssociatedValue;
10244
10245	if (!Stripped) {
10246	// If we haven't stripped anything we might still be able to use a
10247	// different AA, but only if the IRP changes. Effectively when we
10248	// interpret this not as a call site value but as a floating/argument
10249	// value.
10250	const IRPosition AVIRP = IRPosition::value(V: *AssociatedValue);
10251	if (AVIRP == getIRPosition() || !VisitValueCB(AVIRP))
10252	return indicatePessimisticFixpoint();
10253	return ChangeStatus::UNCHANGED;
10254	}
10255
10256	for (const auto &VAC : Values)
10257	if (!VisitValueCB(IRPosition::value(V: *VAC.getValue())))
10258	return indicatePessimisticFixpoint();
10259
10260	return ChangeStatus::UNCHANGED;
10261	}
10262
10263	/// See AbstractAttribute::trackStatistics()
10264	void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) }
10265	};
10266
10267	struct AANoUndefReturned final
10268	: AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl> {
10269	AANoUndefReturned(const IRPosition &IRP, Attributor &A)
10270	: AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl>(IRP, A) {}
10271
10272	/// See AbstractAttribute::trackStatistics()
10273	void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) }
10274	};
10275
10276	struct AANoUndefArgument final
10277	: AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl> {
10278	AANoUndefArgument(const IRPosition &IRP, Attributor &A)
10279	: AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl>(IRP, A) {}
10280
10281	/// See AbstractAttribute::trackStatistics()
10282	void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noundef) }
10283	};
10284
10285	struct AANoUndefCallSiteArgument final : AANoUndefFloating {
10286	AANoUndefCallSiteArgument(const IRPosition &IRP, Attributor &A)
10287	: AANoUndefFloating(IRP, A) {}
10288
10289	/// See AbstractAttribute::trackStatistics()
10290	void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noundef) }
10291	};
10292
10293	struct AANoUndefCallSiteReturned final
10294	: AACalleeToCallSite<AANoUndef, AANoUndefImpl> {
10295	AANoUndefCallSiteReturned(const IRPosition &IRP, Attributor &A)
10296	: AACalleeToCallSite<AANoUndef, AANoUndefImpl>(IRP, A) {}
10297
10298	/// See AbstractAttribute::trackStatistics()
10299	void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noundef) }
10300	};
10301
10302	/// ------------------------ NoFPClass Attribute -------------------------------
10303
10304	struct AANoFPClassImpl : AANoFPClass {
10305	AANoFPClassImpl(const IRPosition &IRP, Attributor &A) : AANoFPClass(IRP, A) {}
10306
10307	void initialize(Attributor &A) override {
10308	const IRPosition &IRP = getIRPosition();
10309
10310	Value &V = IRP.getAssociatedValue();
10311	if (isa<UndefValue>(Val: V)) {
10312	indicateOptimisticFixpoint();
10313	return;
10314	}
10315
10316	SmallVector<Attribute> Attrs;
10317	A.getAttrs(getIRPosition(), {Attribute::NoFPClass}, Attrs, false);
10318	for (const auto &Attr : Attrs) {
10319	addKnownBits(Attr.getNoFPClass());
10320	}
10321
10322	const DataLayout &DL = A.getDataLayout();
10323	if (getPositionKind() != IRPosition::IRP_RETURNED) {
10324	KnownFPClass KnownFPClass = computeKnownFPClass(V: &V, DL);
10325	addKnownBits(~KnownFPClass.KnownFPClasses);
10326	}
10327
10328	if (Instruction *CtxI = getCtxI())
10329	followUsesInMBEC(*this, A, getState(), *CtxI);
10330	}
10331
10332	/// See followUsesInMBEC
10333	bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
10334	AANoFPClass::StateType &State) {
10335	const Value *UseV = U->get();
10336	const DominatorTree *DT = nullptr;
10337	AssumptionCache *AC = nullptr;
10338	const TargetLibraryInfo *TLI = nullptr;
10339	InformationCache &InfoCache = A.getInfoCache();
10340
10341	if (Function *F = getAnchorScope()) {
10342	DT = InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F);
10343	AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F);
10344	TLI = InfoCache.getTargetLibraryInfoForFunction(F: *F);
10345	}
10346
10347	const DataLayout &DL = A.getDataLayout();
10348
10349	KnownFPClass KnownFPClass =
10350	computeKnownFPClass(V: UseV, DL,
10351	/*InterestedClasses=*/fcAllFlags,
10352	/*Depth=*/0, TLI, AC, CxtI: I, DT);
10353	State.addKnownBits(~KnownFPClass.KnownFPClasses);
10354
10355	if (auto *CI = dyn_cast<CallInst>(Val: UseV)) {
10356	// Special case FP intrinsic with struct return type.
10357	switch (CI->getIntrinsicID()) {
10358	case Intrinsic::frexp:
10359	return true;
10360	case Intrinsic::not_intrinsic:
10361	// TODO: Could recognize math libcalls
10362	return false;
10363	default:
10364	break;
10365	}
10366	}
10367
10368	if (!UseV->getType()->isFPOrFPVectorTy())
10369	return false;
10370	return !isa<LoadInst, AtomicRMWInst>(Val: UseV);
10371	}
10372
10373	const std::string getAsStr(Attributor *A) const override {
10374	std::string Result = "nofpclass";
10375	raw_string_ostream OS(Result);
10376	OS << getAssumedNoFPClass();
10377	return Result;
10378	}
10379
10380	void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
10381	SmallVectorImpl<Attribute> &Attrs) const override {
10382	Attrs.emplace_back(Attribute::getWithNoFPClass(Context&: Ctx, Mask: getAssumedNoFPClass()));
10383	}
10384	};
10385
10386	struct AANoFPClassFloating : public AANoFPClassImpl {
10387	AANoFPClassFloating(const IRPosition &IRP, Attributor &A)
10388	: AANoFPClassImpl(IRP, A) {}
10389
10390	/// See AbstractAttribute::updateImpl(...).
10391	ChangeStatus updateImpl(Attributor &A) override {
10392	SmallVector<AA::ValueAndContext> Values;
10393	bool UsedAssumedInformation = false;
10394	if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
10395	S: AA::AnyScope, UsedAssumedInformation)) {
10396	Values.push_back(Elt: {getAssociatedValue(), getCtxI()});
10397	}
10398
10399	StateType T;
10400	auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool {
10401	const auto *AA = A.getAAFor<AANoFPClass>(*this, IRPosition::value(V),
10402	DepClassTy::REQUIRED);
10403	if (!AA || this == AA) {
10404	T.indicatePessimisticFixpoint();
10405	} else {
10406	const AANoFPClass::StateType &S =
10407	static_cast<const AANoFPClass::StateType &>(AA->getState());
10408	T ^= S;
10409	}
10410	return T.isValidState();
10411	};
10412
10413	for (const auto &VAC : Values)
10414	if (!VisitValueCB(*VAC.getValue(), VAC.getCtxI()))
10415	return indicatePessimisticFixpoint();
10416
10417	return clampStateAndIndicateChange(getState(), T);
10418	}
10419
10420	/// See AbstractAttribute::trackStatistics()
10421	void trackStatistics() const override {
10422	STATS_DECLTRACK_FNRET_ATTR(nofpclass)
10423	}
10424	};
10425
10426	struct AANoFPClassReturned final
10427	: AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,
10428	AANoFPClassImpl::StateType, false, Attribute::None, false> {
10429	AANoFPClassReturned(const IRPosition &IRP, Attributor &A)
10430	: AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,
10431	AANoFPClassImpl::StateType, false, Attribute::None, false>(
10432	IRP, A) {}
10433
10434	/// See AbstractAttribute::trackStatistics()
10435	void trackStatistics() const override {
10436	STATS_DECLTRACK_FNRET_ATTR(nofpclass)
10437	}
10438	};
10439
10440	struct AANoFPClassArgument final
10441	: AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl> {
10442	AANoFPClassArgument(const IRPosition &IRP, Attributor &A)
10443	: AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl>(IRP, A) {}
10444
10445	/// See AbstractAttribute::trackStatistics()
10446	void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofpclass) }
10447	};
10448
10449	struct AANoFPClassCallSiteArgument final : AANoFPClassFloating {
10450	AANoFPClassCallSiteArgument(const IRPosition &IRP, Attributor &A)
10451	: AANoFPClassFloating(IRP, A) {}
10452
10453	/// See AbstractAttribute::trackStatistics()
10454	void trackStatistics() const override {
10455	STATS_DECLTRACK_CSARG_ATTR(nofpclass)
10456	}
10457	};
10458
10459	struct AANoFPClassCallSiteReturned final
10460	: AACalleeToCallSite<AANoFPClass, AANoFPClassImpl> {
10461	AANoFPClassCallSiteReturned(const IRPosition &IRP, Attributor &A)
10462	: AACalleeToCallSite<AANoFPClass, AANoFPClassImpl>(IRP, A) {}
10463
10464	/// See AbstractAttribute::trackStatistics()
10465	void trackStatistics() const override {
10466	STATS_DECLTRACK_CSRET_ATTR(nofpclass)
10467	}
10468	};
10469
10470	struct AACallEdgesImpl : public AACallEdges {
10471	AACallEdgesImpl(const IRPosition &IRP, Attributor &A) : AACallEdges(IRP, A) {}
10472
10473	const SetVector<Function *> &getOptimisticEdges() const override {
10474	return CalledFunctions;
10475	}
10476
10477	bool hasUnknownCallee() const override { return HasUnknownCallee; }
10478
10479	bool hasNonAsmUnknownCallee() const override {
10480	return HasUnknownCalleeNonAsm;
10481	}
10482
10483	const std::string getAsStr(Attributor *A) const override {
10484	return "CallEdges[" + std::to_string(val: HasUnknownCallee) + "," +
10485	std::to_string(val: CalledFunctions.size()) + "]";
10486	}
10487
10488	void trackStatistics() const override {}
10489
10490	protected:
10491	void addCalledFunction(Function *Fn, ChangeStatus &Change) {
10492	if (CalledFunctions.insert(X: Fn)) {
10493	Change = ChangeStatus::CHANGED;
10494	LLVM_DEBUG(dbgs() << "[AACallEdges] New call edge: " << Fn->getName()
10495	<< "\n");
10496	}
10497	}
10498
10499	void setHasUnknownCallee(bool NonAsm, ChangeStatus &Change) {
10500	if (!HasUnknownCallee)
10501	Change = ChangeStatus::CHANGED;
10502	if (NonAsm && !HasUnknownCalleeNonAsm)
10503	Change = ChangeStatus::CHANGED;
10504	HasUnknownCalleeNonAsm |= NonAsm;
10505	HasUnknownCallee = true;
10506	}
10507
10508	private:
10509	/// Optimistic set of functions that might be called by this position.
10510	SetVector<Function *> CalledFunctions;
10511
10512	/// Is there any call with a unknown callee.
10513	bool HasUnknownCallee = false;
10514
10515	/// Is there any call with a unknown callee, excluding any inline asm.
10516	bool HasUnknownCalleeNonAsm = false;
10517	};
10518
10519	struct AACallEdgesCallSite : public AACallEdgesImpl {
10520	AACallEdgesCallSite(const IRPosition &IRP, Attributor &A)
10521	: AACallEdgesImpl(IRP, A) {}
10522	/// See AbstractAttribute::updateImpl(...).
10523	ChangeStatus updateImpl(Attributor &A) override {
10524	ChangeStatus Change = ChangeStatus::UNCHANGED;
10525
10526	auto VisitValue = [&](Value &V, const Instruction *CtxI) -> bool {
10527	if (Function *Fn = dyn_cast<Function>(Val: &V)) {
10528	addCalledFunction(Fn, Change);
10529	} else {
10530	LLVM_DEBUG(dbgs() << "[AACallEdges] Unrecognized value: " << V << "\n");
10531	setHasUnknownCallee(NonAsm: true, Change);
10532	}
10533
10534	// Explore all values.
10535	return true;
10536	};
10537
10538	SmallVector<AA::ValueAndContext> Values;
10539	// Process any value that we might call.
10540	auto ProcessCalledOperand = [&](Value *V, Instruction *CtxI) {
10541	if (isa<Constant>(Val: V)) {
10542	VisitValue(*V, CtxI);
10543	return;
10544	}
10545
10546	bool UsedAssumedInformation = false;
10547	Values.clear();
10548	if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: *V), AA: *this, Values,
10549	S: AA::AnyScope, UsedAssumedInformation)) {
10550	Values.push_back(Elt: {*V, CtxI});
10551	}
10552	for (auto &VAC : Values)
10553	VisitValue(*VAC.getValue(), VAC.getCtxI());
10554	};
10555
10556	CallBase *CB = cast<CallBase>(Val: getCtxI());
10557
10558	if (auto *IA = dyn_cast<InlineAsm>(Val: CB->getCalledOperand())) {
10559	if (IA->hasSideEffects() &&
10560	!hasAssumption(F: *CB->getCaller(), AssumptionStr: "ompx_no_call_asm") &&
10561	!hasAssumption(CB: *CB, AssumptionStr: "ompx_no_call_asm")) {
10562	setHasUnknownCallee(NonAsm: false, Change);
10563	}
10564	return Change;
10565	}
10566
10567	if (CB->isIndirectCall())
10568	if (auto *IndirectCallAA = A.getAAFor<AAIndirectCallInfo>(
10569	QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL))
10570	if (IndirectCallAA->foreachCallee(
10571	CB: [&](Function *Fn) { return VisitValue(*Fn, CB); }))
10572	return Change;
10573
10574	// The most simple case.
10575	ProcessCalledOperand(CB->getCalledOperand(), CB);
10576
10577	// Process callback functions.
10578	SmallVector<const Use *, 4u> CallbackUses;
10579	AbstractCallSite::getCallbackUses(CB: *CB, CallbackUses);
10580	for (const Use *U : CallbackUses)
10581	ProcessCalledOperand(U->get(), CB);
10582
10583	return Change;
10584	}
10585	};
10586
10587	struct AACallEdgesFunction : public AACallEdgesImpl {
10588	AACallEdgesFunction(const IRPosition &IRP, Attributor &A)
10589	: AACallEdgesImpl(IRP, A) {}
10590
10591	/// See AbstractAttribute::updateImpl(...).
10592	ChangeStatus updateImpl(Attributor &A) override {
10593	ChangeStatus Change = ChangeStatus::UNCHANGED;
10594
10595	auto ProcessCallInst = [&](Instruction &Inst) {
10596	CallBase &CB = cast<CallBase>(Val&: Inst);
10597
10598	auto *CBEdges = A.getAAFor<AACallEdges>(
10599	QueryingAA: *this, IRP: IRPosition::callsite_function(CB), DepClass: DepClassTy::REQUIRED);
10600	if (!CBEdges)
10601	return false;
10602	if (CBEdges->hasNonAsmUnknownCallee())
10603	setHasUnknownCallee(NonAsm: true, Change);
10604	if (CBEdges->hasUnknownCallee())
10605	setHasUnknownCallee(NonAsm: false, Change);
10606
10607	for (Function *F : CBEdges->getOptimisticEdges())
10608	addCalledFunction(Fn: F, Change);
10609
10610	return true;
10611	};
10612
10613	// Visit all callable instructions.
10614	bool UsedAssumedInformation = false;
10615	if (!A.checkForAllCallLikeInstructions(Pred: ProcessCallInst, QueryingAA: *this,
10616	UsedAssumedInformation,
10617	/* CheckBBLivenessOnly */ true)) {
10618	// If we haven't looked at all call like instructions, assume that there
10619	// are unknown callees.
10620	setHasUnknownCallee(NonAsm: true, Change);
10621	}
10622
10623	return Change;
10624	}
10625	};
10626
10627	/// -------------------AAInterFnReachability Attribute--------------------------
10628
10629	struct AAInterFnReachabilityFunction
10630	: public CachedReachabilityAA<AAInterFnReachability, Function> {
10631	using Base = CachedReachabilityAA<AAInterFnReachability, Function>;
10632	AAInterFnReachabilityFunction(const IRPosition &IRP, Attributor &A)
10633	: Base(IRP, A) {}
10634
10635	bool instructionCanReach(
10636	Attributor &A, const Instruction &From, const Function &To,
10637	const AA::InstExclusionSetTy *ExclusionSet) const override {
10638	assert(From.getFunction() == getAnchorScope() && "Queried the wrong AA!");
10639	auto *NonConstThis = const_cast<AAInterFnReachabilityFunction *>(this);
10640
10641	RQITy StackRQI(A, From, To, ExclusionSet, false);
10642	typename RQITy::Reachable Result;
10643	if (!NonConstThis->checkQueryCache(A, StackRQI, Result))
10644	return NonConstThis->isReachableImpl(A, RQI&: StackRQI,
10645	/*IsTemporaryRQI=*/true);
10646	return Result == RQITy::Reachable::Yes;
10647	}
10648
10649	bool isReachableImpl(Attributor &A, RQITy &RQI,
10650	bool IsTemporaryRQI) override {
10651	const Instruction *EntryI =
10652	&RQI.From->getFunction()->getEntryBlock().front();
10653	if (EntryI != RQI.From &&
10654	!instructionCanReach(A, From: *EntryI, To: *RQI.To, ExclusionSet: nullptr))
10655	return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet: false,
10656	IsTemporaryRQI);
10657
10658	auto CheckReachableCallBase = [&](CallBase *CB) {
10659	auto *CBEdges = A.getAAFor<AACallEdges>(
10660	QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::OPTIONAL);
10661	if (!CBEdges || !CBEdges->getState().isValidState())
10662	return false;
10663	// TODO Check To backwards in this case.
10664	if (CBEdges->hasUnknownCallee())
10665	return false;
10666
10667	for (Function *Fn : CBEdges->getOptimisticEdges()) {
10668	if (Fn == RQI.To)
10669	return false;
10670
10671	if (Fn->isDeclaration()) {
10672	if (Fn->hasFnAttribute(Attribute::NoCallback))
10673	continue;
10674	// TODO Check To backwards in this case.
10675	return false;
10676	}
10677
10678	if (Fn == getAnchorScope()) {
10679	if (EntryI == RQI.From)
10680	continue;
10681	return false;
10682	}
10683
10684	const AAInterFnReachability *InterFnReachability =
10685	A.getAAFor<AAInterFnReachability>(QueryingAA: *this, IRP: IRPosition::function(F: *Fn),
10686	DepClass: DepClassTy::OPTIONAL);
10687
10688	const Instruction &FnFirstInst = Fn->getEntryBlock().front();
10689	if (!InterFnReachability ||
10690	InterFnReachability->instructionCanReach(A, Inst: FnFirstInst, Fn: *RQI.To,
10691	ExclusionSet: RQI.ExclusionSet))
10692	return false;
10693	}
10694	return true;
10695	};
10696
10697	const auto *IntraFnReachability = A.getAAFor<AAIntraFnReachability>(
10698	QueryingAA: *this, IRP: IRPosition::function(F: *RQI.From->getFunction()),
10699	DepClass: DepClassTy::OPTIONAL);
10700
10701	// Determine call like instructions that we can reach from the inst.
10702	auto CheckCallBase = [&](Instruction &CBInst) {
10703	// There are usually less nodes in the call graph, check inter function
10704	// reachability first.
10705	if (CheckReachableCallBase(cast<CallBase>(Val: &CBInst)))
10706	return true;
10707	return IntraFnReachability && !IntraFnReachability->isAssumedReachable(
10708	A, From: *RQI.From, To: CBInst, ExclusionSet: RQI.ExclusionSet);
10709	};
10710
10711	bool UsedExclusionSet = /* conservative */ true;
10712	bool UsedAssumedInformation = false;
10713	if (!A.checkForAllCallLikeInstructions(Pred: CheckCallBase, QueryingAA: *this,
10714	UsedAssumedInformation,
10715	/* CheckBBLivenessOnly */ true))
10716	return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
10717	IsTemporaryRQI);
10718
10719	return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
10720	IsTemporaryRQI);
10721	}
10722
10723	void trackStatistics() const override {}
10724	};
10725	} // namespace
10726
10727	template <typename AAType>
10728	static std::optional<Constant *>
10729	askForAssumedConstant(Attributor &A, const AbstractAttribute &QueryingAA,
10730	const IRPosition &IRP, Type &Ty) {
10731	if (!Ty.isIntegerTy())
10732	return nullptr;
10733
10734	// This will also pass the call base context.
10735	const auto *AA = A.getAAFor<AAType>(QueryingAA, IRP, DepClassTy::NONE);
10736	if (!AA)
10737	return nullptr;
10738
10739	std::optional<Constant *> COpt = AA->getAssumedConstant(A);
10740
10741	if (!COpt.has_value()) {
10742	A.recordDependence(FromAA: *AA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
10743	return std::nullopt;
10744	}
10745	if (auto *C = *COpt) {
10746	A.recordDependence(FromAA: *AA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
10747	return C;
10748	}
10749	return nullptr;
10750	}
10751
10752	Value *AAPotentialValues::getSingleValue(
10753	Attributor &A, const AbstractAttribute &AA, const IRPosition &IRP,
10754	SmallVectorImpl<AA::ValueAndContext> &Values) {
10755	Type &Ty = *IRP.getAssociatedType();
10756	std::optional<Value *> V;
10757	for (auto &It : Values) {
10758	V = AA::combineOptionalValuesInAAValueLatice(A: V, B: It.getValue(), Ty: &Ty);
10759	if (V.has_value() && !*V)
10760	break;
10761	}
10762	if (!V.has_value())
10763	return UndefValue::get(T: &Ty);
10764	return *V;
10765	}
10766
10767	namespace {
10768	struct AAPotentialValuesImpl : AAPotentialValues {
10769	using StateType = PotentialLLVMValuesState;
10770
10771	AAPotentialValuesImpl(const IRPosition &IRP, Attributor &A)
10772	: AAPotentialValues(IRP, A) {}
10773
10774	/// See AbstractAttribute::initialize(..).
10775	void initialize(Attributor &A) override {
10776	if (A.hasSimplificationCallback(IRP: getIRPosition())) {
10777	indicatePessimisticFixpoint();
10778	return;
10779	}
10780	Value *Stripped = getAssociatedValue().stripPointerCasts();
10781	auto *CE = dyn_cast<ConstantExpr>(Val: Stripped);
10782	if (isa<Constant>(Val: Stripped) &&
10783	(!CE || CE->getOpcode() != Instruction::ICmp)) {
10784	addValue(A, State&: getState(), V&: *Stripped, CtxI: getCtxI(), S: AA::AnyScope,
10785	AnchorScope: getAnchorScope());
10786	indicateOptimisticFixpoint();
10787	return;
10788	}
10789	AAPotentialValues::initialize(A);
10790	}
10791
10792	/// See AbstractAttribute::getAsStr().
10793	const std::string getAsStr(Attributor *A) const override {
10794	std::string Str;
10795	llvm::raw_string_ostream OS(Str);
10796	OS << getState();
10797	return OS.str();
10798	}
10799
10800	template <typename AAType>
10801	static std::optional<Value *> askOtherAA(Attributor &A,
10802	const AbstractAttribute &AA,
10803	const IRPosition &IRP, Type &Ty) {
10804	if (isa<Constant>(Val: IRP.getAssociatedValue()))
10805	return &IRP.getAssociatedValue();
10806	std::optional<Constant *> C = askForAssumedConstant<AAType>(A, AA, IRP, Ty);
10807	if (!C)
10808	return std::nullopt;
10809	if (*C)
10810	if (auto *CC = AA::getWithType(V&: **C, Ty))
10811	return CC;
10812	return nullptr;
10813	}
10814
10815	virtual void addValue(Attributor &A, StateType &State, Value &V,
10816	const Instruction *CtxI, AA::ValueScope S,
10817	Function *AnchorScope) const {
10818
10819	IRPosition ValIRP = IRPosition::value(V);
10820	if (auto *CB = dyn_cast_or_null<CallBase>(Val: CtxI)) {
10821	for (const auto &U : CB->args()) {
10822	if (U.get() != &V)
10823	continue;
10824	ValIRP = IRPosition::callsite_argument(CB: *CB, ArgNo: CB->getArgOperandNo(U: &U));
10825	break;
10826	}
10827	}
10828
10829	Value *VPtr = &V;
10830	if (ValIRP.getAssociatedType()->isIntegerTy()) {
10831	Type &Ty = *getAssociatedType();
10832	std::optional<Value *> SimpleV =
10833	askOtherAA<AAValueConstantRange>(A, AA: *this, IRP: ValIRP, Ty);
10834	if (SimpleV.has_value() && !*SimpleV) {
10835	auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>(
10836	QueryingAA: *this, IRP: ValIRP, DepClass: DepClassTy::OPTIONAL);
10837	if (PotentialConstantsAA && PotentialConstantsAA->isValidState()) {
10838	for (const auto &It : PotentialConstantsAA->getAssumedSet())
10839	State.unionAssumed(C: {{*ConstantInt::get(Ty: &Ty, V: It), nullptr}, S});
10840	if (PotentialConstantsAA->undefIsContained())
10841	State.unionAssumed(C: {{*UndefValue::get(T: &Ty), nullptr}, S});
10842	return;
10843	}
10844	}
10845	if (!SimpleV.has_value())
10846	return;
10847
10848	if (*SimpleV)
10849	VPtr = *SimpleV;
10850	}
10851
10852	if (isa<ConstantInt>(Val: VPtr))
10853	CtxI = nullptr;
10854	if (!AA::isValidInScope(V: *VPtr, Scope: AnchorScope))
10855	S = AA::ValueScope(S | AA::Interprocedural);
10856
10857	State.unionAssumed(C: {{*VPtr, CtxI}, S});
10858	}
10859
10860	/// Helper struct to tie a value+context pair together with the scope for
10861	/// which this is the simplified version.
10862	struct ItemInfo {
10863	AA::ValueAndContext I;
10864	AA::ValueScope S;
10865
10866	bool operator==(const ItemInfo &II) const {
10867	return II.I == I && II.S == S;
10868	};
10869	bool operator<(const ItemInfo &II) const {
10870	if (I == II.I)
10871	return S < II.S;
10872	return I < II.I;
10873	};
10874	};
10875
10876	bool recurseForValue(Attributor &A, const IRPosition &IRP, AA::ValueScope S) {
10877	SmallMapVector<AA::ValueAndContext, int, 8> ValueScopeMap;
10878	for (auto CS : {AA::Intraprocedural, AA::Interprocedural}) {
10879	if (!(CS & S))
10880	continue;
10881
10882	bool UsedAssumedInformation = false;
10883	SmallVector<AA::ValueAndContext> Values;
10884	if (!A.getAssumedSimplifiedValues(IRP, AA: this, Values, S: CS,
10885	UsedAssumedInformation))
10886	return false;
10887
10888	for (auto &It : Values)
10889	ValueScopeMap[It] += CS;
10890	}
10891	for (auto &It : ValueScopeMap)
10892	addValue(A, State&: getState(), V&: *It.first.getValue(), CtxI: It.first.getCtxI(),
10893	S: AA::ValueScope(It.second), AnchorScope: getAnchorScope());
10894
10895	return true;
10896	}
10897
10898	void giveUpOnIntraprocedural(Attributor &A) {
10899	auto NewS = StateType::getBestState(PVS: getState());
10900	for (const auto &It : getAssumedSet()) {
10901	if (It.second == AA::Intraprocedural)
10902	continue;
10903	addValue(A, State&: NewS, V&: *It.first.getValue(), CtxI: It.first.getCtxI(),
10904	S: AA::Interprocedural, AnchorScope: getAnchorScope());
10905	}
10906	assert(!undefIsContained() && "Undef should be an explicit value!");
10907	addValue(A, State&: NewS, V&: getAssociatedValue(), CtxI: getCtxI(), S: AA::Intraprocedural,
10908	AnchorScope: getAnchorScope());
10909	getState() = NewS;
10910	}
10911
10912	/// See AbstractState::indicatePessimisticFixpoint(...).
10913	ChangeStatus indicatePessimisticFixpoint() override {
10914	getState() = StateType::getBestState(PVS: getState());
10915	getState().unionAssumed(C: {{getAssociatedValue(), getCtxI()}, AA::AnyScope});
10916	AAPotentialValues::indicateOptimisticFixpoint();
10917	return ChangeStatus::CHANGED;
10918	}
10919
10920	/// See AbstractAttribute::updateImpl(...).
10921	ChangeStatus updateImpl(Attributor &A) override {
10922	return indicatePessimisticFixpoint();
10923	}
10924
10925	/// See AbstractAttribute::manifest(...).
10926	ChangeStatus manifest(Attributor &A) override {
10927	SmallVector<AA::ValueAndContext> Values;
10928	for (AA::ValueScope S : {AA::Interprocedural, AA::Intraprocedural}) {
10929	Values.clear();
10930	if (!getAssumedSimplifiedValues(A, Values, S))
10931	continue;
10932	Value &OldV = getAssociatedValue();
10933	if (isa<UndefValue>(Val: OldV))
10934	continue;
10935	Value *NewV = getSingleValue(A, AA: *this, IRP: getIRPosition(), Values);
10936	if (!NewV || NewV == &OldV)
10937	continue;
10938	if (getCtxI() &&
10939	!AA::isValidAtPosition(VAC: {*NewV, *getCtxI()}, InfoCache&: A.getInfoCache()))
10940	continue;
10941	if (A.changeAfterManifest(IRP: getIRPosition(), NV&: *NewV))
10942	return ChangeStatus::CHANGED;
10943	}
10944	return ChangeStatus::UNCHANGED;
10945	}
10946
10947	bool getAssumedSimplifiedValues(
10948	Attributor &A, SmallVectorImpl<AA::ValueAndContext> &Values,
10949	AA::ValueScope S, bool RecurseForSelectAndPHI = false) const override {
10950	if (!isValidState())
10951	return false;
10952	bool UsedAssumedInformation = false;
10953	for (const auto &It : getAssumedSet())
10954	if (It.second & S) {
10955	if (RecurseForSelectAndPHI && (isa<PHINode>(Val: It.first.getValue()) ||
10956	isa<SelectInst>(Val: It.first.getValue()))) {
10957	if (A.getAssumedSimplifiedValues(
10958	IRP: IRPosition::inst(I: *cast<Instruction>(Val: It.first.getValue())),
10959	AA: this, Values, S, UsedAssumedInformation))
10960	continue;
10961	}
10962	Values.push_back(Elt: It.first);
10963	}
10964	assert(!undefIsContained() && "Undef should be an explicit value!");
10965	return true;
10966	}
10967	};
10968
10969	struct AAPotentialValuesFloating : AAPotentialValuesImpl {
10970	AAPotentialValuesFloating(const IRPosition &IRP, Attributor &A)
10971	: AAPotentialValuesImpl(IRP, A) {}
10972
10973	/// See AbstractAttribute::updateImpl(...).
10974	ChangeStatus updateImpl(Attributor &A) override {
10975	auto AssumedBefore = getAssumed();
10976
10977	genericValueTraversal(A, InitialV: &getAssociatedValue());
10978
10979	return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
10980	: ChangeStatus::CHANGED;
10981	}
10982
10983	/// Helper struct to remember which AAIsDead instances we actually used.
10984	struct LivenessInfo {
10985	const AAIsDead *LivenessAA = nullptr;
10986	bool AnyDead = false;
10987	};
10988
10989	/// Check if \p Cmp is a comparison we can simplify.
10990	///
10991	/// We handle multiple cases, one in which at least one operand is an
10992	/// (assumed) nullptr. If so, try to simplify it using AANonNull on the other
10993	/// operand. Return true if successful, in that case Worklist will be updated.
10994	bool handleCmp(Attributor &A, Value &Cmp, Value *LHS, Value *RHS,
10995	CmpInst::Predicate Pred, ItemInfo II,
10996	SmallVectorImpl<ItemInfo> &Worklist) {
10997
10998	// Simplify the operands first.
10999	bool UsedAssumedInformation = false;
11000	SmallVector<AA::ValueAndContext> LHSValues, RHSValues;
11001	auto GetSimplifiedValues = [&](Value &V,
11002	SmallVector<AA::ValueAndContext> &Values) {
11003	if (!A.getAssumedSimplifiedValues(
11004	IRP: IRPosition::value(V, CBContext: getCallBaseContext()), AA: this, Values,
11005	S: AA::Intraprocedural, UsedAssumedInformation)) {
11006	Values.clear();
11007	Values.push_back(Elt: AA::ValueAndContext{V, II.I.getCtxI()});
11008	}
11009	return Values.empty();
11010	};
11011	if (GetSimplifiedValues(*LHS, LHSValues))
11012	return true;
11013	if (GetSimplifiedValues(*RHS, RHSValues))
11014	return true;
11015
11016	LLVMContext &Ctx = LHS->getContext();
11017
11018	InformationCache &InfoCache = A.getInfoCache();
11019	Instruction *CmpI = dyn_cast<Instruction>(Val: &Cmp);
11020	Function *F = CmpI ? CmpI->getFunction() : nullptr;
11021	const auto *DT =
11022	F ? InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F)
11023	: nullptr;
11024	const auto *TLI =
11025	F ? A.getInfoCache().getTargetLibraryInfoForFunction(F: *F) : nullptr;
11026	auto *AC =
11027	F ? InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F)
11028	: nullptr;
11029
11030	const DataLayout &DL = A.getDataLayout();
11031	SimplifyQuery Q(DL, TLI, DT, AC, CmpI);
11032
11033	auto CheckPair = [&](Value &LHSV, Value &RHSV) {
11034	if (isa<UndefValue>(Val: LHSV) || isa<UndefValue>(Val: RHSV)) {
11035	addValue(A, State&: getState(), V&: *UndefValue::get(T: Cmp.getType()),
11036	/* CtxI */ nullptr, S: II.S, AnchorScope: getAnchorScope());
11037	return true;
11038	}
11039
11040	// Handle the trivial case first in which we don't even need to think
11041	// about null or non-null.
11042	if (&LHSV == &RHSV &&
11043	(CmpInst::isTrueWhenEqual(predicate: Pred) || CmpInst::isFalseWhenEqual(predicate: Pred))) {
11044	Constant *NewV = ConstantInt::get(Ty: Type::getInt1Ty(C&: Ctx),
11045	V: CmpInst::isTrueWhenEqual(predicate: Pred));
11046	addValue(A, State&: getState(), V&: *NewV, /* CtxI */ nullptr, S: II.S,
11047	AnchorScope: getAnchorScope());
11048	return true;
11049	}
11050
11051	auto *TypedLHS = AA::getWithType(V&: LHSV, Ty&: *LHS->getType());
11052	auto *TypedRHS = AA::getWithType(V&: RHSV, Ty&: *RHS->getType());
11053	if (TypedLHS && TypedRHS) {
11054	Value *NewV = simplifyCmpInst(Predicate: Pred, LHS: TypedLHS, RHS: TypedRHS, Q);
11055	if (NewV && NewV != &Cmp) {
11056	addValue(A, State&: getState(), V&: *NewV, /* CtxI */ nullptr, S: II.S,
11057	AnchorScope: getAnchorScope());
11058	return true;
11059	}
11060	}
11061
11062	// From now on we only handle equalities (==, !=).
11063	if (!CmpInst::isEquality(pred: Pred))
11064	return false;
11065
11066	bool LHSIsNull = isa<ConstantPointerNull>(Val: LHSV);
11067	bool RHSIsNull = isa<ConstantPointerNull>(Val: RHSV);
11068	if (!LHSIsNull && !RHSIsNull)
11069	return false;
11070
11071	// Left is the nullptr ==/!= non-nullptr case. We'll use AANonNull on the
11072	// non-nullptr operand and if we assume it's non-null we can conclude the
11073	// result of the comparison.
11074	assert((LHSIsNull || RHSIsNull) &&
11075	"Expected nullptr versus non-nullptr comparison at this point");
11076
11077	// The index is the operand that we assume is not null.
11078	unsigned PtrIdx = LHSIsNull;
11079	bool IsKnownNonNull;
11080	bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
11081	A, this, IRPosition::value(*(PtrIdx ? &RHSV : &LHSV)),
11082	DepClassTy::REQUIRED, IsKnownNonNull);
11083	if (!IsAssumedNonNull)
11084	return false;
11085
11086	// The new value depends on the predicate, true for != and false for ==.
11087	Constant *NewV =
11088	ConstantInt::get(Ty: Type::getInt1Ty(C&: Ctx), V: Pred == CmpInst::ICMP_NE);
11089	addValue(A, State&: getState(), V&: *NewV, /* CtxI */ nullptr, S: II.S,
11090	AnchorScope: getAnchorScope());
11091	return true;
11092	};
11093
11094	for (auto &LHSValue : LHSValues)
11095	for (auto &RHSValue : RHSValues)
11096	if (!CheckPair(*LHSValue.getValue(), *RHSValue.getValue()))
11097	return false;
11098	return true;
11099	}
11100
11101	bool handleSelectInst(Attributor &A, SelectInst &SI, ItemInfo II,
11102	SmallVectorImpl<ItemInfo> &Worklist) {
11103	const Instruction *CtxI = II.I.getCtxI();
11104	bool UsedAssumedInformation = false;
11105
11106	std::optional<Constant *> C =
11107	A.getAssumedConstant(V: *SI.getCondition(), AA: *this, UsedAssumedInformation);
11108	bool NoValueYet = !C.has_value();
11109	if (NoValueYet || isa_and_nonnull<UndefValue>(Val: *C))
11110	return true;
11111	if (auto *CI = dyn_cast_or_null<ConstantInt>(Val: *C)) {
11112	if (CI->isZero())
11113	Worklist.push_back(Elt: {.I: {*SI.getFalseValue(), CtxI}, .S: II.S});
11114	else
11115	Worklist.push_back(Elt: {.I: {*SI.getTrueValue(), CtxI}, .S: II.S});
11116	} else if (&SI == &getAssociatedValue()) {
11117	// We could not simplify the condition, assume both values.
11118	Worklist.push_back(Elt: {.I: {*SI.getTrueValue(), CtxI}, .S: II.S});
11119	Worklist.push_back(Elt: {.I: {*SI.getFalseValue(), CtxI}, .S: II.S});
11120	} else {
11121	std::optional<Value *> SimpleV = A.getAssumedSimplified(
11122	IRP: IRPosition::inst(I: SI), AA: *this, UsedAssumedInformation, S: II.S);
11123	if (!SimpleV.has_value())
11124	return true;
11125	if (*SimpleV) {
11126	addValue(A, State&: getState(), V&: **SimpleV, CtxI, S: II.S, AnchorScope: getAnchorScope());
11127	return true;
11128	}
11129	return false;
11130	}
11131	return true;
11132	}
11133
11134	bool handleLoadInst(Attributor &A, LoadInst &LI, ItemInfo II,
11135	SmallVectorImpl<ItemInfo> &Worklist) {
11136	SmallSetVector<Value *, 4> PotentialCopies;
11137	SmallSetVector<Instruction *, 4> PotentialValueOrigins;
11138	bool UsedAssumedInformation = false;
11139	if (!AA::getPotentiallyLoadedValues(A, LI, PotentialValues&: PotentialCopies,
11140	PotentialValueOrigins, QueryingAA: *this,
11141	UsedAssumedInformation,
11142	/* OnlyExact */ true)) {
11143	LLVM_DEBUG(dbgs() << "[AAPotentialValues] Failed to get potentially "
11144	"loaded values for load instruction "
11145	<< LI << "\n");
11146	return false;
11147	}
11148
11149	// Do not simplify loads that are only used in llvm.assume if we cannot also
11150	// remove all stores that may feed into the load. The reason is that the
11151	// assume is probably worth something as long as the stores are around.
11152	InformationCache &InfoCache = A.getInfoCache();
11153	if (InfoCache.isOnlyUsedByAssume(I: LI)) {
11154	if (!llvm::all_of(Range&: PotentialValueOrigins, P: [&](Instruction *I) {
11155	if (!I || isa<AssumeInst>(Val: I))
11156	return true;
11157	if (auto *SI = dyn_cast<StoreInst>(Val: I))
11158	return A.isAssumedDead(U: SI->getOperandUse(i: 0), QueryingAA: this,
11159	/* LivenessAA */ FnLivenessAA: nullptr,
11160	UsedAssumedInformation,
11161	/* CheckBBLivenessOnly */ false);
11162	return A.isAssumedDead(I: *I, QueryingAA: this, /* LivenessAA */ nullptr,
11163	UsedAssumedInformation,
11164	/* CheckBBLivenessOnly */ false);
11165	})) {
11166	LLVM_DEBUG(dbgs() << "[AAPotentialValues] Load is onl used by assumes "
11167	"and we cannot delete all the stores: "
11168	<< LI << "\n");
11169	return false;
11170	}
11171	}
11172
11173	// Values have to be dynamically unique or we loose the fact that a
11174	// single llvm::Value might represent two runtime values (e.g.,
11175	// stack locations in different recursive calls).
11176	const Instruction *CtxI = II.I.getCtxI();
11177	bool ScopeIsLocal = (II.S & AA::Intraprocedural);
11178	bool AllLocal = ScopeIsLocal;
11179	bool DynamicallyUnique = llvm::all_of(Range&: PotentialCopies, P: [&](Value *PC) {
11180	AllLocal &= AA::isValidInScope(V: *PC, Scope: getAnchorScope());
11181	return AA::isDynamicallyUnique(A, QueryingAA: *this, V: *PC);
11182	});
11183	if (!DynamicallyUnique) {
11184	LLVM_DEBUG(dbgs() << "[AAPotentialValues] Not all potentially loaded "
11185	"values are dynamically unique: "
11186	<< LI << "\n");
11187	return false;
11188	}
11189
11190	for (auto *PotentialCopy : PotentialCopies) {
11191	if (AllLocal) {
11192	Worklist.push_back(Elt: {.I: {*PotentialCopy, CtxI}, .S: II.S});
11193	} else {
11194	Worklist.push_back(Elt: {.I: {*PotentialCopy, CtxI}, .S: AA::Interprocedural});
11195	}
11196	}
11197	if (!AllLocal && ScopeIsLocal)
11198	addValue(A, State&: getState(), V&: LI, CtxI, S: AA::Intraprocedural, AnchorScope: getAnchorScope());
11199	return true;
11200	}
11201
11202	bool handlePHINode(
11203	Attributor &A, PHINode &PHI, ItemInfo II,
11204	SmallVectorImpl<ItemInfo> &Worklist,
11205	SmallMapVector<const Function *, LivenessInfo, 4> &LivenessAAs) {
11206	auto GetLivenessInfo = [&](const Function &F) -> LivenessInfo & {
11207	LivenessInfo &LI = LivenessAAs[&F];
11208	if (!LI.LivenessAA)
11209	LI.LivenessAA = A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: IRPosition::function(F),
11210	DepClass: DepClassTy::NONE);
11211	return LI;
11212	};
11213
11214	if (&PHI == &getAssociatedValue()) {
11215	LivenessInfo &LI = GetLivenessInfo(*PHI.getFunction());
11216	const auto *CI =
11217	A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
11218	F: *PHI.getFunction());
11219
11220	Cycle *C = nullptr;
11221	bool CyclePHI = mayBeInCycle(CI, I: &PHI, /* HeaderOnly */ true, CPtr: &C);
11222	for (unsigned u = 0, e = PHI.getNumIncomingValues(); u < e; u++) {
11223	BasicBlock *IncomingBB = PHI.getIncomingBlock(i: u);
11224	if (LI.LivenessAA &&
11225	LI.LivenessAA->isEdgeDead(From: IncomingBB, To: PHI.getParent())) {
11226	LI.AnyDead = true;
11227	continue;
11228	}
11229	Value *V = PHI.getIncomingValue(i: u);
11230	if (V == &PHI)
11231	continue;
11232
11233	// If the incoming value is not the PHI but an instruction in the same
11234	// cycle we might have multiple versions of it flying around.
11235	if (CyclePHI && isa<Instruction>(Val: V) &&
11236	(!C || C->contains(Block: cast<Instruction>(Val: V)->getParent())))
11237	return false;
11238
11239	Worklist.push_back(Elt: {.I: {*V, IncomingBB->getTerminator()}, .S: II.S});
11240	}
11241	return true;
11242	}
11243
11244	bool UsedAssumedInformation = false;
11245	std::optional<Value *> SimpleV = A.getAssumedSimplified(
11246	IRP: IRPosition::inst(I: PHI), AA: *this, UsedAssumedInformation, S: II.S);
11247	if (!SimpleV.has_value())
11248	return true;
11249	if (!(*SimpleV))
11250	return false;
11251	addValue(A, State&: getState(), V&: **SimpleV, CtxI: &PHI, S: II.S, AnchorScope: getAnchorScope());
11252	return true;
11253	}
11254
11255	/// Use the generic, non-optimistic InstSimplfy functionality if we managed to
11256	/// simplify any operand of the instruction \p I. Return true if successful,
11257	/// in that case Worklist will be updated.
11258	bool handleGenericInst(Attributor &A, Instruction &I, ItemInfo II,
11259	SmallVectorImpl<ItemInfo> &Worklist) {
11260	bool SomeSimplified = false;
11261	bool UsedAssumedInformation = false;
11262
11263	SmallVector<Value *, 8> NewOps(I.getNumOperands());
11264	int Idx = 0;
11265	for (Value *Op : I.operands()) {
11266	const auto &SimplifiedOp = A.getAssumedSimplified(
11267	IRP: IRPosition::value(V: *Op, CBContext: getCallBaseContext()), AA: *this,
11268	UsedAssumedInformation, S: AA::Intraprocedural);
11269	// If we are not sure about any operand we are not sure about the entire
11270	// instruction, we'll wait.
11271	if (!SimplifiedOp.has_value())
11272	return true;
11273
11274	if (*SimplifiedOp)
11275	NewOps[Idx] = *SimplifiedOp;
11276	else
11277	NewOps[Idx] = Op;
11278
11279	SomeSimplified |= (NewOps[Idx] != Op);
11280	++Idx;
11281	}
11282
11283	// We won't bother with the InstSimplify interface if we didn't simplify any
11284	// operand ourselves.
11285	if (!SomeSimplified)
11286	return false;
11287
11288	InformationCache &InfoCache = A.getInfoCache();
11289	Function *F = I.getFunction();
11290	const auto *DT =
11291	InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F);
11292	const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
11293	auto *AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F);
11294
11295	const DataLayout &DL = I.getModule()->getDataLayout();
11296	SimplifyQuery Q(DL, TLI, DT, AC, &I);
11297	Value *NewV = simplifyInstructionWithOperands(I: &I, NewOps, Q);
11298	if (!NewV || NewV == &I)
11299	return false;
11300
11301	LLVM_DEBUG(dbgs() << "Generic inst " << I << " assumed simplified to "
11302	<< *NewV << "\n");
11303	Worklist.push_back(Elt: {.I: {*NewV, II.I.getCtxI()}, .S: II.S});
11304	return true;
11305	}
11306
11307	bool simplifyInstruction(
11308	Attributor &A, Instruction &I, ItemInfo II,
11309	SmallVectorImpl<ItemInfo> &Worklist,
11310	SmallMapVector<const Function *, LivenessInfo, 4> &LivenessAAs) {
11311	if (auto *CI = dyn_cast<CmpInst>(Val: &I))
11312	return handleCmp(A, Cmp&: *CI, LHS: CI->getOperand(i_nocapture: 0), RHS: CI->getOperand(i_nocapture: 1),
11313	Pred: CI->getPredicate(), II, Worklist);
11314
11315	switch (I.getOpcode()) {
11316	case Instruction::Select:
11317	return handleSelectInst(A, SI&: cast<SelectInst>(Val&: I), II, Worklist);
11318	case Instruction::PHI:
11319	return handlePHINode(A, PHI&: cast<PHINode>(Val&: I), II, Worklist, LivenessAAs);
11320	case Instruction::Load:
11321	return handleLoadInst(A, LI&: cast<LoadInst>(Val&: I), II, Worklist);
11322	default:
11323	return handleGenericInst(A, I, II, Worklist);
11324	};
11325	return false;
11326	}
11327
11328	void genericValueTraversal(Attributor &A, Value *InitialV) {
11329	SmallMapVector<const Function *, LivenessInfo, 4> LivenessAAs;
11330
11331	SmallSet<ItemInfo, 16> Visited;
11332	SmallVector<ItemInfo, 16> Worklist;
11333	Worklist.push_back(Elt: {.I: {*InitialV, getCtxI()}, .S: AA::AnyScope});
11334
11335	int Iteration = 0;
11336	do {
11337	ItemInfo II = Worklist.pop_back_val();
11338	Value *V = II.I.getValue();
11339	assert(V);
11340	const Instruction *CtxI = II.I.getCtxI();
11341	AA::ValueScope S = II.S;
11342
11343	// Check if we should process the current value. To prevent endless
11344	// recursion keep a record of the values we followed!
11345	if (!Visited.insert(V: II).second)
11346	continue;
11347
11348	// Make sure we limit the compile time for complex expressions.
11349	if (Iteration++ >= MaxPotentialValuesIterations) {
11350	LLVM_DEBUG(dbgs() << "Generic value traversal reached iteration limit: "
11351	<< Iteration << "!\n");
11352	addValue(A, State&: getState(), V&: *V, CtxI, S, AnchorScope: getAnchorScope());
11353	continue;
11354	}
11355
11356	// Explicitly look through calls with a "returned" attribute if we do
11357	// not have a pointer as stripPointerCasts only works on them.
11358	Value *NewV = nullptr;
11359	if (V->getType()->isPointerTy()) {
11360	NewV = AA::getWithType(V&: *V->stripPointerCasts(), Ty&: *V->getType());
11361	} else {
11362	if (auto *CB = dyn_cast<CallBase>(Val: V))
11363	if (auto *Callee =
11364	dyn_cast_if_present<Function>(Val: CB->getCalledOperand())) {
11365	for (Argument &Arg : Callee->args())
11366	if (Arg.hasReturnedAttr()) {
11367	NewV = CB->getArgOperand(i: Arg.getArgNo());
11368	break;
11369	}
11370	}
11371	}
11372	if (NewV && NewV != V) {
11373	Worklist.push_back(Elt: {.I: {*NewV, CtxI}, .S: S});
11374	continue;
11375	}
11376
11377	if (auto *CE = dyn_cast<ConstantExpr>(Val: V)) {
11378	if (CE->getOpcode() == Instruction::ICmp)
11379	if (handleCmp(A, Cmp&: *CE, LHS: CE->getOperand(i_nocapture: 0), RHS: CE->getOperand(i_nocapture: 1),
11380	Pred: CmpInst::Predicate(CE->getPredicate()), II, Worklist))
11381	continue;
11382	}
11383
11384	if (auto *I = dyn_cast<Instruction>(Val: V)) {
11385	if (simplifyInstruction(A, I&: *I, II, Worklist, LivenessAAs))
11386	continue;
11387	}
11388
11389	if (V != InitialV || isa<Argument>(Val: V))
11390	if (recurseForValue(A, IRP: IRPosition::value(V: *V), S: II.S))
11391	continue;
11392
11393	// If we haven't stripped anything we give up.
11394	if (V == InitialV && CtxI == getCtxI()) {
11395	indicatePessimisticFixpoint();
11396	return;
11397	}
11398
11399	addValue(A, State&: getState(), V&: *V, CtxI, S, AnchorScope: getAnchorScope());
11400	} while (!Worklist.empty());
11401
11402	// If we actually used liveness information so we have to record a
11403	// dependence.
11404	for (auto &It : LivenessAAs)
11405	if (It.second.AnyDead)
11406	A.recordDependence(FromAA: *It.second.LivenessAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
11407	}
11408
11409	/// See AbstractAttribute::trackStatistics()
11410	void trackStatistics() const override {
11411	STATS_DECLTRACK_FLOATING_ATTR(potential_values)
11412	}
11413	};
11414
11415	struct AAPotentialValuesArgument final : AAPotentialValuesImpl {
11416	using Base = AAPotentialValuesImpl;
11417	AAPotentialValuesArgument(const IRPosition &IRP, Attributor &A)
11418	: Base(IRP, A) {}
11419
11420	/// See AbstractAttribute::initialize(..).
11421	void initialize(Attributor &A) override {
11422	auto &Arg = cast<Argument>(Val&: getAssociatedValue());
11423	if (Arg.hasPointeeInMemoryValueAttr())
11424	indicatePessimisticFixpoint();
11425	}
11426
11427	/// See AbstractAttribute::updateImpl(...).
11428	ChangeStatus updateImpl(Attributor &A) override {
11429	auto AssumedBefore = getAssumed();
11430
11431	unsigned ArgNo = getCalleeArgNo();
11432
11433	bool UsedAssumedInformation = false;
11434	SmallVector<AA::ValueAndContext> Values;
11435	auto CallSitePred = [&](AbstractCallSite ACS) {
11436	const auto CSArgIRP = IRPosition::callsite_argument(ACS, ArgNo);
11437	if (CSArgIRP.getPositionKind() == IRP_INVALID)
11438	return false;
11439
11440	if (!A.getAssumedSimplifiedValues(IRP: CSArgIRP, AA: this, Values,
11441	S: AA::Interprocedural,
11442	UsedAssumedInformation))
11443	return false;
11444
11445	return isValidState();
11446	};
11447
11448	if (!A.checkForAllCallSites(Pred: CallSitePred, QueryingAA: *this,
11449	/* RequireAllCallSites */ true,
11450	UsedAssumedInformation))
11451	return indicatePessimisticFixpoint();
11452
11453	Function *Fn = getAssociatedFunction();
11454	bool AnyNonLocal = false;
11455	for (auto &It : Values) {
11456	if (isa<Constant>(Val: It.getValue())) {
11457	addValue(A, State&: getState(), V&: *It.getValue(), CtxI: It.getCtxI(), S: AA::AnyScope,
11458	AnchorScope: getAnchorScope());
11459	continue;
11460	}
11461	if (!AA::isDynamicallyUnique(A, QueryingAA: *this, V: *It.getValue()))
11462	return indicatePessimisticFixpoint();
11463
11464	if (auto *Arg = dyn_cast<Argument>(Val: It.getValue()))
11465	if (Arg->getParent() == Fn) {
11466	addValue(A, State&: getState(), V&: *It.getValue(), CtxI: It.getCtxI(), S: AA::AnyScope,
11467	AnchorScope: getAnchorScope());
11468	continue;
11469	}
11470	addValue(A, State&: getState(), V&: *It.getValue(), CtxI: It.getCtxI(), S: AA::Interprocedural,
11471	AnchorScope: getAnchorScope());
11472	AnyNonLocal = true;
11473	}
11474	assert(!undefIsContained() && "Undef should be an explicit value!");
11475	if (AnyNonLocal)
11476	giveUpOnIntraprocedural(A);
11477
11478	return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11479	: ChangeStatus::CHANGED;
11480	}
11481
11482	/// See AbstractAttribute::trackStatistics()
11483	void trackStatistics() const override {
11484	STATS_DECLTRACK_ARG_ATTR(potential_values)
11485	}
11486	};
11487
11488	struct AAPotentialValuesReturned : public AAPotentialValuesFloating {
11489	using Base = AAPotentialValuesFloating;
11490	AAPotentialValuesReturned(const IRPosition &IRP, Attributor &A)
11491	: Base(IRP, A) {}
11492
11493	/// See AbstractAttribute::initialize(..).
11494	void initialize(Attributor &A) override {
11495	Function *F = getAssociatedFunction();
11496	if (!F || F->isDeclaration() || F->getReturnType()->isVoidTy()) {
11497	indicatePessimisticFixpoint();
11498	return;
11499	}
11500
11501	for (Argument &Arg : F->args())
11502	if (Arg.hasReturnedAttr()) {
11503	addValue(A, State&: getState(), V&: Arg, CtxI: nullptr, S: AA::AnyScope, AnchorScope: F);
11504	ReturnedArg = &Arg;
11505	break;
11506	}
11507	if (!A.isFunctionIPOAmendable(F: *F) ||
11508	A.hasSimplificationCallback(IRP: getIRPosition())) {
11509	if (!ReturnedArg)
11510	indicatePessimisticFixpoint();
11511	else
11512	indicateOptimisticFixpoint();
11513	}
11514	}
11515
11516	/// See AbstractAttribute::updateImpl(...).
11517	ChangeStatus updateImpl(Attributor &A) override {
11518	auto AssumedBefore = getAssumed();
11519	bool UsedAssumedInformation = false;
11520
11521	SmallVector<AA::ValueAndContext> Values;
11522	Function *AnchorScope = getAnchorScope();
11523	auto HandleReturnedValue = [&](Value &V, Instruction *CtxI,
11524	bool AddValues) {
11525	for (AA::ValueScope S : {AA::Interprocedural, AA::Intraprocedural}) {
11526	Values.clear();
11527	if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V), AA: this, Values, S,
11528	UsedAssumedInformation,
11529	/* RecurseForSelectAndPHI */ true))
11530	return false;
11531	if (!AddValues)
11532	continue;
11533	for (const AA::ValueAndContext &VAC : Values)
11534	addValue(A, State&: getState(), V&: *VAC.getValue(),
11535	CtxI: VAC.getCtxI() ? VAC.getCtxI() : CtxI, S, AnchorScope);
11536	}
11537	return true;
11538	};
11539
11540	if (ReturnedArg) {
11541	HandleReturnedValue(*ReturnedArg, nullptr, true);
11542	} else {
11543	auto RetInstPred = [&](Instruction &RetI) {
11544	bool AddValues = true;
11545	if (isa<PHINode>(Val: RetI.getOperand(i: 0)) ||
11546	isa<SelectInst>(Val: RetI.getOperand(i: 0))) {
11547	addValue(A, State&: getState(), V&: *RetI.getOperand(i: 0), CtxI: &RetI, S: AA::AnyScope,
11548	AnchorScope);
11549	AddValues = false;
11550	}
11551	return HandleReturnedValue(*RetI.getOperand(i: 0), &RetI, AddValues);
11552	};
11553
11554	if (!A.checkForAllInstructions(Pred: RetInstPred, QueryingAA: *this, Opcodes: {Instruction::Ret},
11555	UsedAssumedInformation,
11556	/* CheckBBLivenessOnly */ true))
11557	return indicatePessimisticFixpoint();
11558	}
11559
11560	return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11561	: ChangeStatus::CHANGED;
11562	}
11563
11564	void addValue(Attributor &A, StateType &State, Value &V,
11565	const Instruction *CtxI, AA::ValueScope S,
11566	Function *AnchorScope) const override {
11567	Function *F = getAssociatedFunction();
11568	if (auto *CB = dyn_cast<CallBase>(Val: &V))
11569	if (CB->getCalledOperand() == F)
11570	return;
11571	Base::addValue(A, State, V, CtxI, S, AnchorScope);
11572	}
11573
11574	ChangeStatus manifest(Attributor &A) override {
11575	if (ReturnedArg)
11576	return ChangeStatus::UNCHANGED;
11577	SmallVector<AA::ValueAndContext> Values;
11578	if (!getAssumedSimplifiedValues(A, Values, S: AA::ValueScope::Intraprocedural,
11579	/* RecurseForSelectAndPHI */ true))
11580	return ChangeStatus::UNCHANGED;
11581	Value *NewVal = getSingleValue(A, AA: *this, IRP: getIRPosition(), Values);
11582	if (!NewVal)
11583	return ChangeStatus::UNCHANGED;
11584
11585	ChangeStatus Changed = ChangeStatus::UNCHANGED;
11586	if (auto *Arg = dyn_cast<Argument>(Val: NewVal)) {
11587	STATS_DECLTRACK(UniqueReturnValue, FunctionReturn,
11588	"Number of function with unique return");
11589	Changed |= A.manifestAttrs(
11590	IRPosition::argument(*Arg),
11591	{Attribute::get(Arg->getContext(), Attribute::Returned)});
11592	STATS_DECLTRACK_ARG_ATTR(returned);
11593	}
11594
11595	auto RetInstPred = [&](Instruction &RetI) {
11596	Value *RetOp = RetI.getOperand(i: 0);
11597	if (isa<UndefValue>(Val: RetOp) || RetOp == NewVal)
11598	return true;
11599	if (AA::isValidAtPosition(VAC: {*NewVal, RetI}, InfoCache&: A.getInfoCache()))
11600	if (A.changeUseAfterManifest(U&: RetI.getOperandUse(i: 0), NV&: *NewVal))
11601	Changed = ChangeStatus::CHANGED;
11602	return true;
11603	};
11604	bool UsedAssumedInformation = false;
11605	(void)A.checkForAllInstructions(Pred: RetInstPred, QueryingAA: *this, Opcodes: {Instruction::Ret},
11606	UsedAssumedInformation,
11607	/* CheckBBLivenessOnly */ true);
11608	return Changed;
11609	}
11610
11611	ChangeStatus indicatePessimisticFixpoint() override {
11612	return AAPotentialValues::indicatePessimisticFixpoint();
11613	}
11614
11615	/// See AbstractAttribute::trackStatistics()
11616	void trackStatistics() const override{
11617	STATS_DECLTRACK_FNRET_ATTR(potential_values)}
11618
11619	/// The argumented with an existing `returned` attribute.
11620	Argument *ReturnedArg = nullptr;
11621	};
11622
11623	struct AAPotentialValuesFunction : AAPotentialValuesImpl {
11624	AAPotentialValuesFunction(const IRPosition &IRP, Attributor &A)
11625	: AAPotentialValuesImpl(IRP, A) {}
11626
11627	/// See AbstractAttribute::updateImpl(...).
11628	ChangeStatus updateImpl(Attributor &A) override {
11629	llvm_unreachable("AAPotentialValues(Function|CallSite)::updateImpl will "
11630	"not be called");
11631	}
11632
11633	/// See AbstractAttribute::trackStatistics()
11634	void trackStatistics() const override {
11635	STATS_DECLTRACK_FN_ATTR(potential_values)
11636	}
11637	};
11638
11639	struct AAPotentialValuesCallSite : AAPotentialValuesFunction {
11640	AAPotentialValuesCallSite(const IRPosition &IRP, Attributor &A)
11641	: AAPotentialValuesFunction(IRP, A) {}
11642
11643	/// See AbstractAttribute::trackStatistics()
11644	void trackStatistics() const override {
11645	STATS_DECLTRACK_CS_ATTR(potential_values)
11646	}
11647	};
11648
11649	struct AAPotentialValuesCallSiteReturned : AAPotentialValuesImpl {
11650	AAPotentialValuesCallSiteReturned(const IRPosition &IRP, Attributor &A)
11651	: AAPotentialValuesImpl(IRP, A) {}
11652
11653	/// See AbstractAttribute::updateImpl(...).
11654	ChangeStatus updateImpl(Attributor &A) override {
11655	auto AssumedBefore = getAssumed();
11656
11657	Function *Callee = getAssociatedFunction();
11658	if (!Callee)
11659	return indicatePessimisticFixpoint();
11660
11661	bool UsedAssumedInformation = false;
11662	auto *CB = cast<CallBase>(Val: getCtxI());
11663	if (CB->isMustTailCall() &&
11664	!A.isAssumedDead(IRP: IRPosition::inst(I: *CB), QueryingAA: this, FnLivenessAA: nullptr,
11665	UsedAssumedInformation))
11666	return indicatePessimisticFixpoint();
11667
11668	SmallVector<AA::ValueAndContext> Values;
11669	if (!A.getAssumedSimplifiedValues(IRP: IRPosition::returned(F: *Callee), AA: this,
11670	Values, S: AA::Intraprocedural,
11671	UsedAssumedInformation))
11672	return indicatePessimisticFixpoint();
11673
11674	Function *Caller = CB->getCaller();
11675
11676	bool AnyNonLocal = false;
11677	for (auto &It : Values) {
11678	Value *V = It.getValue();
11679	std::optional<Value *> CallerV = A.translateArgumentToCallSiteContent(
11680	V, CB&: *CB, AA: *this, UsedAssumedInformation);
11681	if (!CallerV.has_value()) {
11682	// Nothing to do as long as no value was determined.
11683	continue;
11684	}
11685	V = *CallerV ? *CallerV : V;
11686	if (AA::isDynamicallyUnique(A, QueryingAA: *this, V: *V) &&
11687	AA::isValidInScope(V: *V, Scope: Caller)) {
11688	if (*CallerV) {
11689	SmallVector<AA::ValueAndContext> ArgValues;
11690	IRPosition IRP = IRPosition::value(V: *V);
11691	if (auto *Arg = dyn_cast<Argument>(Val: V))
11692	if (Arg->getParent() == CB->getCalledOperand())
11693	IRP = IRPosition::callsite_argument(CB: *CB, ArgNo: Arg->getArgNo());
11694	if (recurseForValue(A, IRP, S: AA::AnyScope))
11695	continue;
11696	}
11697	addValue(A, State&: getState(), V&: *V, CtxI: CB, S: AA::AnyScope, AnchorScope: getAnchorScope());
11698	} else {
11699	AnyNonLocal = true;
11700	break;
11701	}
11702	}
11703	if (AnyNonLocal) {
11704	Values.clear();
11705	if (!A.getAssumedSimplifiedValues(IRP: IRPosition::returned(F: *Callee), AA: this,
11706	Values, S: AA::Interprocedural,
11707	UsedAssumedInformation))
11708	return indicatePessimisticFixpoint();
11709	AnyNonLocal = false;
11710	getState() = PotentialLLVMValuesState::getBestState();
11711	for (auto &It : Values) {
11712	Value *V = It.getValue();
11713	if (!AA::isDynamicallyUnique(A, QueryingAA: *this, V: *V))
11714	return indicatePessimisticFixpoint();
11715	if (AA::isValidInScope(V: *V, Scope: Caller)) {
11716	addValue(A, State&: getState(), V&: *V, CtxI: CB, S: AA::AnyScope, AnchorScope: getAnchorScope());
11717	} else {
11718	AnyNonLocal = true;
11719	addValue(A, State&: getState(), V&: *V, CtxI: CB, S: AA::Interprocedural,
11720	AnchorScope: getAnchorScope());
11721	}
11722	}
11723	if (AnyNonLocal)
11724	giveUpOnIntraprocedural(A);
11725	}
11726	return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11727	: ChangeStatus::CHANGED;
11728	}
11729
11730	ChangeStatus indicatePessimisticFixpoint() override {
11731	return AAPotentialValues::indicatePessimisticFixpoint();
11732	}
11733
11734	/// See AbstractAttribute::trackStatistics()
11735	void trackStatistics() const override {
11736	STATS_DECLTRACK_CSRET_ATTR(potential_values)
11737	}
11738	};
11739
11740	struct AAPotentialValuesCallSiteArgument : AAPotentialValuesFloating {
11741	AAPotentialValuesCallSiteArgument(const IRPosition &IRP, Attributor &A)
11742	: AAPotentialValuesFloating(IRP, A) {}
11743
11744	/// See AbstractAttribute::trackStatistics()
11745	void trackStatistics() const override {
11746	STATS_DECLTRACK_CSARG_ATTR(potential_values)
11747	}
11748	};
11749	} // namespace
11750
11751	/// ---------------------- Assumption Propagation ------------------------------
11752	namespace {
11753	struct AAAssumptionInfoImpl : public AAAssumptionInfo {
11754	AAAssumptionInfoImpl(const IRPosition &IRP, Attributor &A,
11755	const DenseSet<StringRef> &Known)
11756	: AAAssumptionInfo(IRP, A, Known) {}
11757
11758	/// See AbstractAttribute::manifest(...).
11759	ChangeStatus manifest(Attributor &A) override {
11760	// Don't manifest a universal set if it somehow made it here.
11761	if (getKnown().isUniversal())
11762	return ChangeStatus::UNCHANGED;
11763
11764	const IRPosition &IRP = getIRPosition();
11765	return A.manifestAttrs(
11766	IRP,
11767	DeducedAttrs: Attribute::get(Context&: IRP.getAnchorValue().getContext(), Kind: AssumptionAttrKey,
11768	Val: llvm::join(R: getAssumed().getSet(), Separator: ",")),
11769	/* ForceReplace */ true);
11770	}
11771
11772	bool hasAssumption(const StringRef Assumption) const override {
11773	return isValidState() && setContains(Assumption);
11774	}
11775
11776	/// See AbstractAttribute::getAsStr()
11777	const std::string getAsStr(Attributor *A) const override {
11778	const SetContents &Known = getKnown();
11779	const SetContents &Assumed = getAssumed();
11780
11781	const std::string KnownStr =
11782	llvm::join(Begin: Known.getSet().begin(), End: Known.getSet().end(), Separator: ",");
11783	const std::string AssumedStr =
11784	(Assumed.isUniversal())
11785	? "Universal"
11786	: llvm::join(Begin: Assumed.getSet().begin(), End: Assumed.getSet().end(), Separator: ",");
11787
11788	return "Known [" + KnownStr + "]," + " Assumed [" + AssumedStr + "]";
11789	}
11790	};
11791
11792	/// Propagates assumption information from parent functions to all of their
11793	/// successors. An assumption can be propagated if the containing function
11794	/// dominates the called function.
11795	///
11796	/// We start with a "known" set of assumptions already valid for the associated
11797	/// function and an "assumed" set that initially contains all possible
11798	/// assumptions. The assumed set is inter-procedurally updated by narrowing its
11799	/// contents as concrete values are known. The concrete values are seeded by the
11800	/// first nodes that are either entries into the call graph, or contains no
11801	/// assumptions. Each node is updated as the intersection of the assumed state
11802	/// with all of its predecessors.
11803	struct AAAssumptionInfoFunction final : AAAssumptionInfoImpl {
11804	AAAssumptionInfoFunction(const IRPosition &IRP, Attributor &A)
11805	: AAAssumptionInfoImpl(IRP, A,
11806	getAssumptions(F: *IRP.getAssociatedFunction())) {}
11807
11808	/// See AbstractAttribute::updateImpl(...).
11809	ChangeStatus updateImpl(Attributor &A) override {
11810	bool Changed = false;
11811
11812	auto CallSitePred = [&](AbstractCallSite ACS) {
11813	const auto *AssumptionAA = A.getAAFor<AAAssumptionInfo>(
11814	QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *ACS.getInstruction()),
11815	DepClass: DepClassTy::REQUIRED);
11816	if (!AssumptionAA)
11817	return false;
11818	// Get the set of assumptions shared by all of this function's callers.
11819	Changed |= getIntersection(RHS: AssumptionAA->getAssumed());
11820	return !getAssumed().empty() || !getKnown().empty();
11821	};
11822
11823	bool UsedAssumedInformation = false;
11824	// Get the intersection of all assumptions held by this node's predecessors.
11825	// If we don't know all the call sites then this is either an entry into the
11826	// call graph or an empty node. This node is known to only contain its own
11827	// assumptions and can be propagated to its successors.
11828	if (!A.checkForAllCallSites(Pred: CallSitePred, QueryingAA: *this, RequireAllCallSites: true,
11829	UsedAssumedInformation))
11830	return indicatePessimisticFixpoint();
11831
11832	return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
11833	}
11834
11835	void trackStatistics() const override {}
11836	};
11837
11838	/// Assumption Info defined for call sites.
11839	struct AAAssumptionInfoCallSite final : AAAssumptionInfoImpl {
11840
11841	AAAssumptionInfoCallSite(const IRPosition &IRP, Attributor &A)
11842	: AAAssumptionInfoImpl(IRP, A, getInitialAssumptions(IRP)) {}
11843
11844	/// See AbstractAttribute::initialize(...).
11845	void initialize(Attributor &A) override {
11846	const IRPosition &FnPos = IRPosition::function(F: *getAnchorScope());
11847	A.getAAFor<AAAssumptionInfo>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::REQUIRED);
11848	}
11849
11850	/// See AbstractAttribute::updateImpl(...).
11851	ChangeStatus updateImpl(Attributor &A) override {
11852	const IRPosition &FnPos = IRPosition::function(F: *getAnchorScope());
11853	auto *AssumptionAA =
11854	A.getAAFor<AAAssumptionInfo>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::REQUIRED);
11855	if (!AssumptionAA)
11856	return indicatePessimisticFixpoint();
11857	bool Changed = getIntersection(RHS: AssumptionAA->getAssumed());
11858	return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
11859	}
11860
11861	/// See AbstractAttribute::trackStatistics()
11862	void trackStatistics() const override {}
11863
11864	private:
11865	/// Helper to initialized the known set as all the assumptions this call and
11866	/// the callee contain.
11867	DenseSet<StringRef> getInitialAssumptions(const IRPosition &IRP) {
11868	const CallBase &CB = cast<CallBase>(Val&: IRP.getAssociatedValue());
11869	auto Assumptions = getAssumptions(CB);
11870	if (const Function *F = CB.getCaller())
11871	set_union(S1&: Assumptions, S2: getAssumptions(F: *F));
11872	if (Function *F = IRP.getAssociatedFunction())
11873	set_union(S1&: Assumptions, S2: getAssumptions(F: *F));
11874	return Assumptions;
11875	}
11876	};
11877	} // namespace
11878
11879	AACallGraphNode *AACallEdgeIterator::operator*() const {
11880	return static_cast<AACallGraphNode *>(const_cast<AACallEdges *>(
11881	A.getOrCreateAAFor<AACallEdges>(IRP: IRPosition::function(F: **I))));
11882	}
11883
11884	void AttributorCallGraph::print() { llvm::WriteGraph(O&: outs(), G: this); }
11885
11886	/// ------------------------ UnderlyingObjects ---------------------------------
11887
11888	namespace {
11889	struct AAUnderlyingObjectsImpl
11890	: StateWrapper<BooleanState, AAUnderlyingObjects> {
11891	using BaseTy = StateWrapper<BooleanState, AAUnderlyingObjects>;
11892	AAUnderlyingObjectsImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {}
11893
11894	/// See AbstractAttribute::getAsStr().
11895	const std::string getAsStr(Attributor *A) const override {
11896	return std::string("UnderlyingObjects ") +
11897	(isValidState()
11898	? (std::string("inter #") +
11899	std::to_string(val: InterAssumedUnderlyingObjects.size()) +
11900	" objs" + std::string(", intra #") +
11901	std::to_string(val: IntraAssumedUnderlyingObjects.size()) +
11902	" objs")
11903	: "<invalid>");
11904	}
11905
11906	/// See AbstractAttribute::trackStatistics()
11907	void trackStatistics() const override {}
11908
11909	/// See AbstractAttribute::updateImpl(...).
11910	ChangeStatus updateImpl(Attributor &A) override {
11911	auto &Ptr = getAssociatedValue();
11912
11913	auto DoUpdate = [&](SmallSetVector<Value *, 8> &UnderlyingObjects,
11914	AA::ValueScope Scope) {
11915	bool UsedAssumedInformation = false;
11916	SmallPtrSet<Value *, 8> SeenObjects;
11917	SmallVector<AA::ValueAndContext> Values;
11918
11919	if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: Ptr), AA: *this, Values,
11920	S: Scope, UsedAssumedInformation))
11921	return UnderlyingObjects.insert(X: &Ptr);
11922
11923	bool Changed = false;
11924
11925	for (unsigned I = 0; I < Values.size(); ++I) {
11926	auto &VAC = Values[I];
11927	auto *Obj = VAC.getValue();
11928	Value *UO = getUnderlyingObject(V: Obj);
11929	if (UO && UO != VAC.getValue() && SeenObjects.insert(Ptr: UO).second) {
11930	const auto *OtherAA = A.getAAFor<AAUnderlyingObjects>(
11931	QueryingAA: *this, IRP: IRPosition::value(V: *UO), DepClass: DepClassTy::OPTIONAL);
11932	auto Pred = [&Values](Value &V) {
11933	Values.emplace_back(Args&: V, Args: nullptr);
11934	return true;
11935	};
11936
11937	if (!OtherAA || !OtherAA->forallUnderlyingObjects(Pred, Scope))
11938	llvm_unreachable(
11939	"The forall call should not return false at this position");
11940
11941	continue;
11942	}
11943
11944	if (isa<SelectInst>(Val: Obj)) {
11945	Changed |= handleIndirect(A, V&: *Obj, UnderlyingObjects, Scope);
11946	continue;
11947	}
11948	if (auto *PHI = dyn_cast<PHINode>(Val: Obj)) {
11949	// Explicitly look through PHIs as we do not care about dynamically
11950	// uniqueness.
11951	for (unsigned u = 0, e = PHI->getNumIncomingValues(); u < e; u++) {
11952	Changed |= handleIndirect(A, V&: *PHI->getIncomingValue(i: u),
11953	UnderlyingObjects, Scope);
11954	}
11955	continue;
11956	}
11957
11958	Changed |= UnderlyingObjects.insert(X: Obj);
11959	}
11960
11961	return Changed;
11962	};
11963
11964	bool Changed = false;
11965	Changed |= DoUpdate(IntraAssumedUnderlyingObjects, AA::Intraprocedural);
11966	Changed |= DoUpdate(InterAssumedUnderlyingObjects, AA::Interprocedural);
11967
11968	return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
11969	}
11970
11971	bool forallUnderlyingObjects(
11972	function_ref<bool(Value &)> Pred,
11973	AA::ValueScope Scope = AA::Interprocedural) const override {
11974	if (!isValidState())
11975	return Pred(getAssociatedValue());
11976
11977	auto &AssumedUnderlyingObjects = Scope == AA::Intraprocedural
11978	? IntraAssumedUnderlyingObjects
11979	: InterAssumedUnderlyingObjects;
11980	for (Value *Obj : AssumedUnderlyingObjects)
11981	if (!Pred(*Obj))
11982	return false;
11983
11984	return true;
11985	}
11986
11987	private:
11988	/// Handle the case where the value is not the actual underlying value, such
11989	/// as a phi node or a select instruction.
11990	bool handleIndirect(Attributor &A, Value &V,
11991	SmallSetVector<Value *, 8> &UnderlyingObjects,
11992	AA::ValueScope Scope) {
11993	bool Changed = false;
11994	const auto *AA = A.getAAFor<AAUnderlyingObjects>(
11995	QueryingAA: *this, IRP: IRPosition::value(V), DepClass: DepClassTy::OPTIONAL);
11996	auto Pred = [&](Value &V) {
11997	Changed |= UnderlyingObjects.insert(X: &V);
11998	return true;
11999	};
12000	if (!AA || !AA->forallUnderlyingObjects(Pred, Scope))
12001	llvm_unreachable(
12002	"The forall call should not return false at this position");
12003	return Changed;
12004	}
12005
12006	/// All the underlying objects collected so far via intra procedural scope.
12007	SmallSetVector<Value *, 8> IntraAssumedUnderlyingObjects;
12008	/// All the underlying objects collected so far via inter procedural scope.
12009	SmallSetVector<Value *, 8> InterAssumedUnderlyingObjects;
12010	};
12011
12012	struct AAUnderlyingObjectsFloating final : AAUnderlyingObjectsImpl {
12013	AAUnderlyingObjectsFloating(const IRPosition &IRP, Attributor &A)
12014	: AAUnderlyingObjectsImpl(IRP, A) {}
12015	};
12016
12017	struct AAUnderlyingObjectsArgument final : AAUnderlyingObjectsImpl {
12018	AAUnderlyingObjectsArgument(const IRPosition &IRP, Attributor &A)
12019	: AAUnderlyingObjectsImpl(IRP, A) {}
12020	};
12021
12022	struct AAUnderlyingObjectsCallSite final : AAUnderlyingObjectsImpl {
12023	AAUnderlyingObjectsCallSite(const IRPosition &IRP, Attributor &A)
12024	: AAUnderlyingObjectsImpl(IRP, A) {}
12025	};
12026
12027	struct AAUnderlyingObjectsCallSiteArgument final : AAUnderlyingObjectsImpl {
12028	AAUnderlyingObjectsCallSiteArgument(const IRPosition &IRP, Attributor &A)
12029	: AAUnderlyingObjectsImpl(IRP, A) {}
12030	};
12031
12032	struct AAUnderlyingObjectsReturned final : AAUnderlyingObjectsImpl {
12033	AAUnderlyingObjectsReturned(const IRPosition &IRP, Attributor &A)
12034	: AAUnderlyingObjectsImpl(IRP, A) {}
12035	};
12036
12037	struct AAUnderlyingObjectsCallSiteReturned final : AAUnderlyingObjectsImpl {
12038	AAUnderlyingObjectsCallSiteReturned(const IRPosition &IRP, Attributor &A)
12039	: AAUnderlyingObjectsImpl(IRP, A) {}
12040	};
12041
12042	struct AAUnderlyingObjectsFunction final : AAUnderlyingObjectsImpl {
12043	AAUnderlyingObjectsFunction(const IRPosition &IRP, Attributor &A)
12044	: AAUnderlyingObjectsImpl(IRP, A) {}
12045	};
12046	} // namespace
12047
12048	/// ------------------------ Global Value Info -------------------------------
12049	namespace {
12050	struct AAGlobalValueInfoFloating : public AAGlobalValueInfo {
12051	AAGlobalValueInfoFloating(const IRPosition &IRP, Attributor &A)
12052	: AAGlobalValueInfo(IRP, A) {}
12053
12054	/// See AbstractAttribute::initialize(...).
12055	void initialize(Attributor &A) override {}
12056
12057	bool checkUse(Attributor &A, const Use &U, bool &Follow,
12058	SmallVectorImpl<const Value *> &Worklist) {
12059	Instruction *UInst = dyn_cast<Instruction>(Val: U.getUser());
12060	if (!UInst) {
12061	Follow = true;
12062	return true;
12063	}
12064
12065	LLVM_DEBUG(dbgs() << "[AAGlobalValueInfo] Check use: " << *U.get() << " in "
12066	<< *UInst << "\n");
12067
12068	if (auto *Cmp = dyn_cast<ICmpInst>(Val: U.getUser())) {
12069	int Idx = &Cmp->getOperandUse(i: 0) == &U;
12070	if (isa<Constant>(Val: Cmp->getOperand(i_nocapture: Idx)))
12071	return true;
12072	return U == &getAnchorValue();
12073	}
12074
12075	// Explicitly catch return instructions.
12076	if (isa<ReturnInst>(Val: UInst)) {
12077	auto CallSitePred = [&](AbstractCallSite ACS) {
12078	Worklist.push_back(Elt: ACS.getInstruction());
12079	return true;
12080	};
12081	bool UsedAssumedInformation = false;
12082	// TODO: We should traverse the uses or add a "non-call-site" CB.
12083	if (!A.checkForAllCallSites(Pred: CallSitePred, Fn: *UInst->getFunction(),
12084	/*RequireAllCallSites=*/true, QueryingAA: this,
12085	UsedAssumedInformation))
12086	return false;
12087	return true;
12088	}
12089
12090	// For now we only use special logic for call sites. However, the tracker
12091	// itself knows about a lot of other non-capturing cases already.
12092	auto *CB = dyn_cast<CallBase>(Val: UInst);
12093	if (!CB)
12094	return false;
12095	// Direct calls are OK uses.
12096	if (CB->isCallee(U: &U))
12097	return true;
12098	// Non-argument uses are scary.
12099	if (!CB->isArgOperand(U: &U))
12100	return false;
12101	// TODO: Iterate callees.
12102	auto *Fn = dyn_cast<Function>(Val: CB->getCalledOperand());
12103	if (!Fn || !A.isFunctionIPOAmendable(F: *Fn))
12104	return false;
12105
12106	unsigned ArgNo = CB->getArgOperandNo(U: &U);
12107	Worklist.push_back(Elt: Fn->getArg(i: ArgNo));
12108	return true;
12109	}
12110
12111	ChangeStatus updateImpl(Attributor &A) override {
12112	unsigned NumUsesBefore = Uses.size();
12113
12114	SmallPtrSet<const Value *, 8> Visited;
12115	SmallVector<const Value *> Worklist;
12116	Worklist.push_back(Elt: &getAnchorValue());
12117
12118	auto UsePred = [&](const Use &U, bool &Follow) -> bool {
12119	Uses.insert(Ptr: &U);
12120	switch (DetermineUseCaptureKind(U, IsDereferenceableOrNull: nullptr)) {
12121	case UseCaptureKind::NO_CAPTURE:
12122	return checkUse(A, U, Follow, Worklist);
12123	case UseCaptureKind::MAY_CAPTURE:
12124	return checkUse(A, U, Follow, Worklist);
12125	case UseCaptureKind::PASSTHROUGH:
12126	Follow = true;
12127	return true;
12128	}
12129	return true;
12130	};
12131	auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
12132	Uses.insert(Ptr: &OldU);
12133	return true;
12134	};
12135
12136	while (!Worklist.empty()) {
12137	const Value *V = Worklist.pop_back_val();
12138	if (!Visited.insert(Ptr: V).second)
12139	continue;
12140	if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: *V,
12141	/* CheckBBLivenessOnly */ true,
12142	LivenessDepClass: DepClassTy::OPTIONAL,
12143	/* IgnoreDroppableUses */ true, EquivalentUseCB)) {
12144	return indicatePessimisticFixpoint();
12145	}
12146	}
12147
12148	return Uses.size() == NumUsesBefore ? ChangeStatus::UNCHANGED
12149	: ChangeStatus::CHANGED;
12150	}
12151
12152	bool isPotentialUse(const Use &U) const override {
12153	return !isValidState() || Uses.contains(Ptr: &U);
12154	}
12155
12156	/// See AbstractAttribute::manifest(...).
12157	ChangeStatus manifest(Attributor &A) override {
12158	return ChangeStatus::UNCHANGED;
12159	}
12160
12161	/// See AbstractAttribute::getAsStr().
12162	const std::string getAsStr(Attributor *A) const override {
12163	return "[" + std::to_string(val: Uses.size()) + " uses]";
12164	}
12165
12166	void trackStatistics() const override {
12167	STATS_DECLTRACK_FLOATING_ATTR(GlobalValuesTracked);
12168	}
12169
12170	private:
12171	/// Set of (transitive) uses of this GlobalValue.
12172	SmallPtrSet<const Use *, 8> Uses;
12173	};
12174	} // namespace
12175
12176	/// ------------------------ Indirect Call Info -------------------------------
12177	namespace {
12178	struct AAIndirectCallInfoCallSite : public AAIndirectCallInfo {
12179	AAIndirectCallInfoCallSite(const IRPosition &IRP, Attributor &A)
12180	: AAIndirectCallInfo(IRP, A) {}
12181
12182	/// See AbstractAttribute::initialize(...).
12183	void initialize(Attributor &A) override {
12184	auto *MD = getCtxI()->getMetadata(KindID: LLVMContext::MD_callees);
12185	if (!MD && !A.isClosedWorldModule())
12186	return;
12187
12188	if (MD) {
12189	for (const auto &Op : MD->operands())
12190	if (Function *Callee = mdconst::dyn_extract_or_null<Function>(MD: Op))
12191	PotentialCallees.insert(X: Callee);
12192	} else if (A.isClosedWorldModule()) {
12193	ArrayRef<Function *> IndirectlyCallableFunctions =
12194	A.getInfoCache().getIndirectlyCallableFunctions(A);
12195	PotentialCallees.insert(Start: IndirectlyCallableFunctions.begin(),
12196	End: IndirectlyCallableFunctions.end());
12197	}
12198
12199	if (PotentialCallees.empty())
12200	indicateOptimisticFixpoint();
12201	}
12202
12203	ChangeStatus updateImpl(Attributor &A) override {
12204	CallBase *CB = cast<CallBase>(Val: getCtxI());
12205	const Use &CalleeUse = CB->getCalledOperandUse();
12206	Value *FP = CB->getCalledOperand();
12207
12208	SmallSetVector<Function *, 4> AssumedCalleesNow;
12209	bool AllCalleesKnownNow = AllCalleesKnown;
12210
12211	auto CheckPotentialCalleeUse = [&](Function &PotentialCallee,
12212	bool &UsedAssumedInformation) {
12213	const auto *GIAA = A.getAAFor<AAGlobalValueInfo>(
12214	QueryingAA: *this, IRP: IRPosition::value(V: PotentialCallee), DepClass: DepClassTy::OPTIONAL);
12215	if (!GIAA || GIAA->isPotentialUse(U: CalleeUse))
12216	return true;
12217	UsedAssumedInformation = !GIAA->isAtFixpoint();
12218	return false;
12219	};
12220
12221	auto AddPotentialCallees = [&]() {
12222	for (auto *PotentialCallee : PotentialCallees) {
12223	bool UsedAssumedInformation = false;
12224	if (CheckPotentialCalleeUse(*PotentialCallee, UsedAssumedInformation))
12225	AssumedCalleesNow.insert(X: PotentialCallee);
12226	}
12227	};
12228
12229	// Use simplification to find potential callees, if !callees was present,
12230	// fallback to that set if necessary.
12231	bool UsedAssumedInformation = false;
12232	SmallVector<AA::ValueAndContext> Values;
12233	if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: *FP), AA: this, Values,
12234	S: AA::ValueScope::AnyScope,
12235	UsedAssumedInformation)) {
12236	if (PotentialCallees.empty())
12237	return indicatePessimisticFixpoint();
12238	AddPotentialCallees();
12239	}
12240
12241	// Try to find a reason for \p Fn not to be a potential callee. If none was
12242	// found, add it to the assumed callees set.
12243	auto CheckPotentialCallee = [&](Function &Fn) {
12244	if (!PotentialCallees.empty() && !PotentialCallees.count(key: &Fn))
12245	return false;
12246
12247	auto &CachedResult = FilterResults[&Fn];
12248	if (CachedResult.has_value())
12249	return CachedResult.value();
12250
12251	bool UsedAssumedInformation = false;
12252	if (!CheckPotentialCalleeUse(Fn, UsedAssumedInformation)) {
12253	if (!UsedAssumedInformation)
12254	CachedResult = false;
12255	return false;
12256	}
12257
12258	int NumFnArgs = Fn.arg_size();
12259	int NumCBArgs = CB->arg_size();
12260
12261	// Check if any excess argument (which we fill up with poison) is known to
12262	// be UB on undef.
12263	for (int I = NumCBArgs; I < NumFnArgs; ++I) {
12264	bool IsKnown = false;
12265	if (AA::hasAssumedIRAttr<Attribute::NoUndef>(
12266	A, this, IRPosition::argument(*Fn.getArg(I)),
12267	DepClassTy::OPTIONAL, IsKnown)) {
12268	if (IsKnown)
12269	CachedResult = false;
12270	return false;
12271	}
12272	}
12273
12274	CachedResult = true;
12275	return true;
12276	};
12277
12278	// Check simplification result, prune known UB callees, also restrict it to
12279	// the !callees set, if present.
12280	for (auto &VAC : Values) {
12281	if (isa<UndefValue>(Val: VAC.getValue()))
12282	continue;
12283	if (isa<ConstantPointerNull>(Val: VAC.getValue()) &&
12284	VAC.getValue()->getType()->getPointerAddressSpace() == 0)
12285	continue;
12286	// TODO: Check for known UB, e.g., poison + noundef.
12287	if (auto *VACFn = dyn_cast<Function>(Val: VAC.getValue())) {
12288	if (CheckPotentialCallee(*VACFn))
12289	AssumedCalleesNow.insert(X: VACFn);
12290	continue;
12291	}
12292	if (!PotentialCallees.empty()) {
12293	AddPotentialCallees();
12294	break;
12295	}
12296	AllCalleesKnownNow = false;
12297	}
12298
12299	if (AssumedCalleesNow == AssumedCallees &&
12300	AllCalleesKnown == AllCalleesKnownNow)
12301	return ChangeStatus::UNCHANGED;
12302
12303	std::swap(LHS&: AssumedCallees, RHS&: AssumedCalleesNow);
12304	AllCalleesKnown = AllCalleesKnownNow;
12305	return ChangeStatus::CHANGED;
12306	}
12307
12308	/// See AbstractAttribute::manifest(...).
12309	ChangeStatus manifest(Attributor &A) override {
12310	// If we can't specialize at all, give up now.
12311	if (!AllCalleesKnown && AssumedCallees.empty())
12312	return ChangeStatus::UNCHANGED;
12313
12314	CallBase *CB = cast<CallBase>(Val: getCtxI());
12315	bool UsedAssumedInformation = false;
12316	if (A.isAssumedDead(I: *CB, QueryingAA: this, /*LivenessAA=*/nullptr,
12317	UsedAssumedInformation))
12318	return ChangeStatus::UNCHANGED;
12319
12320	ChangeStatus Changed = ChangeStatus::UNCHANGED;
12321	Value *FP = CB->getCalledOperand();
12322	if (FP->getType()->getPointerAddressSpace())
12323	FP = new AddrSpaceCastInst(FP, PointerType::get(ElementType: FP->getType(), AddressSpace: 0),
12324	FP->getName() + ".as0", CB->getIterator());
12325
12326	bool CBIsVoid = CB->getType()->isVoidTy();
12327	BasicBlock::iterator IP = CB->getIterator();
12328	FunctionType *CSFT = CB->getFunctionType();
12329	SmallVector<Value *> CSArgs(CB->arg_begin(), CB->arg_end());
12330
12331	// If we know all callees and there are none, the call site is (effectively)
12332	// dead (or UB).
12333	if (AssumedCallees.empty()) {
12334	assert(AllCalleesKnown &&
12335	"Expected all callees to be known if there are none.");
12336	A.changeToUnreachableAfterManifest(I: CB);
12337	return ChangeStatus::CHANGED;
12338	}
12339
12340	// Special handling for the single callee case.
12341	if (AllCalleesKnown && AssumedCallees.size() == 1) {
12342	auto *NewCallee = AssumedCallees.front();
12343	if (isLegalToPromote(CB: *CB, Callee: NewCallee)) {
12344	promoteCall(CB&: *CB, Callee: NewCallee, RetBitCast: nullptr);
12345	return ChangeStatus::CHANGED;
12346	}
12347	Instruction *NewCall =
12348	CallInst::Create(Func: FunctionCallee(CSFT, NewCallee), Args: CSArgs,
12349	NameStr: CB->getName(), InsertBefore: CB->getIterator());
12350	if (!CBIsVoid)
12351	A.changeAfterManifest(IRP: IRPosition::callsite_returned(CB: *CB), NV&: *NewCall);
12352	A.deleteAfterManifest(I&: *CB);
12353	return ChangeStatus::CHANGED;
12354	}
12355
12356	// For each potential value we create a conditional
12357	//
12358	// ```
12359	// if (ptr == value) value(args);
12360	// else ...
12361	// ```
12362	//
12363	bool SpecializedForAnyCallees = false;
12364	bool SpecializedForAllCallees = AllCalleesKnown;
12365	ICmpInst *LastCmp = nullptr;
12366	SmallVector<Function *, 8> SkippedAssumedCallees;
12367	SmallVector<std::pair<CallInst *, Instruction *>> NewCalls;
12368	for (Function *NewCallee : AssumedCallees) {
12369	if (!A.shouldSpecializeCallSiteForCallee(AA: *this, CB&: *CB, Callee&: *NewCallee)) {
12370	SkippedAssumedCallees.push_back(Elt: NewCallee);
12371	SpecializedForAllCallees = false;
12372	continue;
12373	}
12374	SpecializedForAnyCallees = true;
12375
12376	LastCmp = new ICmpInst(IP, llvm::CmpInst::ICMP_EQ, FP, NewCallee);
12377	Instruction *ThenTI =
12378	SplitBlockAndInsertIfThen(Cond: LastCmp, SplitBefore: IP, /* Unreachable */ false);
12379	BasicBlock *CBBB = CB->getParent();
12380	A.registerManifestAddedBasicBlock(BB&: *ThenTI->getParent());
12381	A.registerManifestAddedBasicBlock(BB&: *IP->getParent());
12382	auto *SplitTI = cast<BranchInst>(Val: LastCmp->getNextNode());
12383	BasicBlock *ElseBB;
12384	if (&*IP == CB) {
12385	ElseBB = BasicBlock::Create(Context&: ThenTI->getContext(), Name: "",
12386	Parent: ThenTI->getFunction(), InsertBefore: CBBB);
12387	A.registerManifestAddedBasicBlock(BB&: *ElseBB);
12388	IP = BranchInst::Create(IfTrue: CBBB, InsertAtEnd: ElseBB)->getIterator();
12389	SplitTI->replaceUsesOfWith(From: CBBB, To: ElseBB);
12390	} else {
12391	ElseBB = IP->getParent();
12392	ThenTI->replaceUsesOfWith(From: ElseBB, To: CBBB);
12393	}
12394	CastInst *RetBC = nullptr;
12395	CallInst *NewCall = nullptr;
12396	if (isLegalToPromote(CB: *CB, Callee: NewCallee)) {
12397	auto *CBClone = cast<CallBase>(Val: CB->clone());
12398	CBClone->insertBefore(InsertPos: ThenTI);
12399	NewCall = &cast<CallInst>(Val&: promoteCall(CB&: *CBClone, Callee: NewCallee, RetBitCast: &RetBC));
12400	} else {
12401	NewCall = CallInst::Create(Func: FunctionCallee(CSFT, NewCallee), Args: CSArgs,
12402	NameStr: CB->getName(), InsertBefore: ThenTI->getIterator());
12403	}
12404	NewCalls.push_back(Elt: {NewCall, RetBC});
12405	}
12406
12407	auto AttachCalleeMetadata = [&](CallBase &IndirectCB) {
12408	if (!AllCalleesKnown)
12409	return ChangeStatus::UNCHANGED;
12410	MDBuilder MDB(IndirectCB.getContext());
12411	MDNode *Callees = MDB.createCallees(Callees: SkippedAssumedCallees);
12412	IndirectCB.setMetadata(KindID: LLVMContext::MD_callees, Node: Callees);
12413	return ChangeStatus::CHANGED;
12414	};
12415
12416	if (!SpecializedForAnyCallees)
12417	return AttachCalleeMetadata(*CB);
12418
12419	// Check if we need the fallback indirect call still.
12420	if (SpecializedForAllCallees) {
12421	LastCmp->replaceAllUsesWith(V: ConstantInt::getTrue(Context&: LastCmp->getContext()));
12422	LastCmp->eraseFromParent();
12423	new UnreachableInst(IP->getContext(), IP);
12424	IP->eraseFromParent();
12425	} else {
12426	auto *CBClone = cast<CallInst>(Val: CB->clone());
12427	CBClone->setName(CB->getName());
12428	CBClone->insertBefore(BB&: *IP->getParent(), InsertPos: IP);
12429	NewCalls.push_back(Elt: {CBClone, nullptr});
12430	AttachCalleeMetadata(*CBClone);
12431	}
12432
12433	// Check if we need a PHI to merge the results.
12434	if (!CBIsVoid) {
12435	auto *PHI = PHINode::Create(Ty: CB->getType(), NumReservedValues: NewCalls.size(),
12436	NameStr: CB->getName() + ".phi",
12437	InsertBefore: CB->getParent()->getFirstInsertionPt());
12438	for (auto &It : NewCalls) {
12439	CallBase *NewCall = It.first;
12440	Instruction *CallRet = It.second ? It.second : It.first;
12441	if (CallRet->getType() == CB->getType())
12442	PHI->addIncoming(V: CallRet, BB: CallRet->getParent());
12443	else if (NewCall->getType()->isVoidTy())
12444	PHI->addIncoming(V: PoisonValue::get(T: CB->getType()),
12445	BB: NewCall->getParent());
12446	else
12447	llvm_unreachable("Call return should match or be void!");
12448	}
12449	A.changeAfterManifest(IRP: IRPosition::callsite_returned(CB: *CB), NV&: *PHI);
12450	}
12451
12452	A.deleteAfterManifest(I&: *CB);
12453	Changed = ChangeStatus::CHANGED;
12454
12455	return Changed;
12456	}
12457
12458	/// See AbstractAttribute::getAsStr().
12459	const std::string getAsStr(Attributor *A) const override {
12460	return std::string(AllCalleesKnown ? "eliminate" : "specialize") +
12461	" indirect call site with " + std::to_string(val: AssumedCallees.size()) +
12462	" functions";
12463	}
12464
12465	void trackStatistics() const override {
12466	if (AllCalleesKnown) {
12467	STATS_DECLTRACK(
12468	Eliminated, CallSites,
12469	"Number of indirect call sites eliminated via specialization")
12470	} else {
12471	STATS_DECLTRACK(Specialized, CallSites,
12472	"Number of indirect call sites specialized")
12473	}
12474	}
12475
12476	bool foreachCallee(function_ref<bool(Function *)> CB) const override {
12477	return isValidState() && AllCalleesKnown && all_of(Range: AssumedCallees, P: CB);
12478	}
12479
12480	private:
12481	/// Map to remember filter results.
12482	DenseMap<Function *, std::optional<bool>> FilterResults;
12483
12484	/// If the !callee metadata was present, this set will contain all potential
12485	/// callees (superset).
12486	SmallSetVector<Function *, 4> PotentialCallees;
12487
12488	/// This set contains all currently assumed calllees, which might grow over
12489	/// time.
12490	SmallSetVector<Function *, 4> AssumedCallees;
12491
12492	/// Flag to indicate if all possible callees are in the AssumedCallees set or
12493	/// if there could be others.
12494	bool AllCalleesKnown = true;
12495	};
12496	} // namespace
12497
12498	/// ------------------------ Address Space ------------------------------------
12499	namespace {
12500	struct AAAddressSpaceImpl : public AAAddressSpace {
12501	AAAddressSpaceImpl(const IRPosition &IRP, Attributor &A)
12502	: AAAddressSpace(IRP, A) {}
12503
12504	int32_t getAddressSpace() const override {
12505	assert(isValidState() && "the AA is invalid");
12506	return AssumedAddressSpace;
12507	}
12508
12509	/// See AbstractAttribute::initialize(...).
12510	void initialize(Attributor &A) override {
12511	assert(getAssociatedType()->isPtrOrPtrVectorTy() &&
12512	"Associated value is not a pointer");
12513	}
12514
12515	ChangeStatus updateImpl(Attributor &A) override {
12516	int32_t OldAddressSpace = AssumedAddressSpace;
12517	auto *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(IRP: getIRPosition(), QueryingAA: this,
12518	DepClass: DepClassTy::REQUIRED);
12519	auto Pred = [&](Value &Obj) {
12520	if (isa<UndefValue>(Val: &Obj))
12521	return true;
12522	return takeAddressSpace(AS: Obj.getType()->getPointerAddressSpace());
12523	};
12524
12525	if (!AUO->forallUnderlyingObjects(Pred))
12526	return indicatePessimisticFixpoint();
12527
12528	return OldAddressSpace == AssumedAddressSpace ? ChangeStatus::UNCHANGED
12529	: ChangeStatus::CHANGED;
12530	}
12531
12532	/// See AbstractAttribute::manifest(...).
12533	ChangeStatus manifest(Attributor &A) override {
12534	Value *AssociatedValue = &getAssociatedValue();
12535	Value *OriginalValue = peelAddrspacecast(V: AssociatedValue);
12536	if (getAddressSpace() == NoAddressSpace ||
12537	static_cast<uint32_t>(getAddressSpace()) ==
12538	getAssociatedType()->getPointerAddressSpace())
12539	return ChangeStatus::UNCHANGED;
12540
12541	Type *NewPtrTy = PointerType::get(C&: getAssociatedType()->getContext(),
12542	AddressSpace: static_cast<uint32_t>(getAddressSpace()));
12543	bool UseOriginalValue =
12544	OriginalValue->getType()->getPointerAddressSpace() ==
12545	static_cast<uint32_t>(getAddressSpace());
12546
12547	bool Changed = false;
12548
12549	auto MakeChange = [&](Instruction *I, Use &U) {
12550	Changed = true;
12551	if (UseOriginalValue) {
12552	A.changeUseAfterManifest(U, NV&: *OriginalValue);
12553	return;
12554	}
12555	Instruction *CastInst = new AddrSpaceCastInst(OriginalValue, NewPtrTy);
12556	CastInst->insertBefore(InsertPos: cast<Instruction>(Val: I));
12557	A.changeUseAfterManifest(U, NV&: *CastInst);
12558	};
12559
12560	auto Pred = [&](const Use &U, bool &) {
12561	if (U.get() != AssociatedValue)
12562	return true;
12563	auto *Inst = dyn_cast<Instruction>(Val: U.getUser());
12564	if (!Inst)
12565	return true;
12566	// This is a WA to make sure we only change uses from the corresponding
12567	// CGSCC if the AA is run on CGSCC instead of the entire module.
12568	if (!A.isRunOn(Fn: Inst->getFunction()))
12569	return true;
12570	if (isa<LoadInst>(Val: Inst))
12571	MakeChange(Inst, const_cast<Use &>(U));
12572	if (isa<StoreInst>(Val: Inst)) {
12573	// We only make changes if the use is the pointer operand.
12574	if (U.getOperandNo() == 1)
12575	MakeChange(Inst, const_cast<Use &>(U));
12576	}
12577	return true;
12578	};
12579
12580	// It doesn't matter if we can't check all uses as we can simply
12581	// conservatively ignore those that can not be visited.
12582	(void)A.checkForAllUses(Pred, QueryingAA: *this, V: getAssociatedValue(),
12583	/* CheckBBLivenessOnly */ true);
12584
12585	return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
12586	}
12587
12588	/// See AbstractAttribute::getAsStr().
12589	const std::string getAsStr(Attributor *A) const override {
12590	if (!isValidState())
12591	return "addrspace(<invalid>)";
12592	return "addrspace(" +
12593	(AssumedAddressSpace == NoAddressSpace
12594	? "none"
12595	: std::to_string(val: AssumedAddressSpace)) +
12596	")";
12597	}
12598
12599	private:
12600	int32_t AssumedAddressSpace = NoAddressSpace;
12601
12602	bool takeAddressSpace(int32_t AS) {
12603	if (AssumedAddressSpace == NoAddressSpace) {
12604	AssumedAddressSpace = AS;
12605	return true;
12606	}
12607	return AssumedAddressSpace == AS;
12608	}
12609
12610	static Value *peelAddrspacecast(Value *V) {
12611	if (auto *I = dyn_cast<AddrSpaceCastInst>(Val: V))
12612	return peelAddrspacecast(V: I->getPointerOperand());
12613	if (auto *C = dyn_cast<ConstantExpr>(Val: V))
12614	if (C->getOpcode() == Instruction::AddrSpaceCast)
12615	return peelAddrspacecast(V: C->getOperand(i_nocapture: 0));
12616	return V;
12617	}
12618	};
12619
12620	struct AAAddressSpaceFloating final : AAAddressSpaceImpl {
12621	AAAddressSpaceFloating(const IRPosition &IRP, Attributor &A)
12622	: AAAddressSpaceImpl(IRP, A) {}
12623
12624	void trackStatistics() const override {
12625	STATS_DECLTRACK_FLOATING_ATTR(addrspace);
12626	}
12627	};
12628
12629	struct AAAddressSpaceReturned final : AAAddressSpaceImpl {
12630	AAAddressSpaceReturned(const IRPosition &IRP, Attributor &A)
12631	: AAAddressSpaceImpl(IRP, A) {}
12632
12633	/// See AbstractAttribute::initialize(...).
12634	void initialize(Attributor &A) override {
12635	// TODO: we don't rewrite function argument for now because it will need to
12636	// rewrite the function signature and all call sites.
12637	(void)indicatePessimisticFixpoint();
12638	}
12639
12640	void trackStatistics() const override {
12641	STATS_DECLTRACK_FNRET_ATTR(addrspace);
12642	}
12643	};
12644
12645	struct AAAddressSpaceCallSiteReturned final : AAAddressSpaceImpl {
12646	AAAddressSpaceCallSiteReturned(const IRPosition &IRP, Attributor &A)
12647	: AAAddressSpaceImpl(IRP, A) {}
12648
12649	void trackStatistics() const override {
12650	STATS_DECLTRACK_CSRET_ATTR(addrspace);
12651	}
12652	};
12653
12654	struct AAAddressSpaceArgument final : AAAddressSpaceImpl {
12655	AAAddressSpaceArgument(const IRPosition &IRP, Attributor &A)
12656	: AAAddressSpaceImpl(IRP, A) {}
12657
12658	void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(addrspace); }
12659	};
12660
12661	struct AAAddressSpaceCallSiteArgument final : AAAddressSpaceImpl {
12662	AAAddressSpaceCallSiteArgument(const IRPosition &IRP, Attributor &A)
12663	: AAAddressSpaceImpl(IRP, A) {}
12664
12665	/// See AbstractAttribute::initialize(...).
12666	void initialize(Attributor &A) override {
12667	// TODO: we don't rewrite call site argument for now because it will need to
12668	// rewrite the function signature of the callee.
12669	(void)indicatePessimisticFixpoint();
12670	}
12671
12672	void trackStatistics() const override {
12673	STATS_DECLTRACK_CSARG_ATTR(addrspace);
12674	}
12675	};
12676	} // namespace
12677
12678	/// ----------- Allocation Info ----------
12679	namespace {
12680	struct AAAllocationInfoImpl : public AAAllocationInfo {
12681	AAAllocationInfoImpl(const IRPosition &IRP, Attributor &A)
12682	: AAAllocationInfo(IRP, A) {}
12683
12684	std::optional<TypeSize> getAllocatedSize() const override {
12685	assert(isValidState() && "the AA is invalid");
12686	return AssumedAllocatedSize;
12687	}
12688
12689	std::optional<TypeSize> findInitialAllocationSize(Instruction *I,
12690	const DataLayout &DL) {
12691
12692	// TODO: implement case for malloc like instructions
12693	switch (I->getOpcode()) {
12694	case Instruction::Alloca: {
12695	AllocaInst *AI = cast<AllocaInst>(Val: I);
12696	return AI->getAllocationSize(DL);
12697	}
12698	default:
12699	return std::nullopt;
12700	}
12701	}
12702
12703	ChangeStatus updateImpl(Attributor &A) override {
12704
12705	const IRPosition &IRP = getIRPosition();
12706	Instruction *I = IRP.getCtxI();
12707
12708	// TODO: update check for malloc like calls
12709	if (!isa<AllocaInst>(Val: I))
12710	return indicatePessimisticFixpoint();
12711
12712	bool IsKnownNoCapture;
12713	if (!AA::hasAssumedIRAttr<Attribute::NoCapture>(
12714	A, this, IRP, DepClassTy::OPTIONAL, IsKnownNoCapture))
12715	return indicatePessimisticFixpoint();
12716
12717	const AAPointerInfo *PI =
12718	A.getOrCreateAAFor<AAPointerInfo>(IRP, QueryingAA: *this, DepClass: DepClassTy::REQUIRED);
12719
12720	if (!PI)
12721	return indicatePessimisticFixpoint();
12722
12723	if (!PI->getState().isValidState())
12724	return indicatePessimisticFixpoint();
12725
12726	const DataLayout &DL = A.getDataLayout();
12727	const auto AllocationSize = findInitialAllocationSize(I, DL);
12728
12729	// If allocation size is nullopt, we give up.
12730	if (!AllocationSize)
12731	return indicatePessimisticFixpoint();
12732
12733	// For zero sized allocations, we give up.
12734	// Since we can't reduce further
12735	if (*AllocationSize == 0)
12736	return indicatePessimisticFixpoint();
12737
12738	int64_t BinSize = PI->numOffsetBins();
12739
12740	// TODO: implement for multiple bins
12741	if (BinSize > 1)
12742	return indicatePessimisticFixpoint();
12743
12744	if (BinSize == 0) {
12745	auto NewAllocationSize = std::optional<TypeSize>(TypeSize(0, false));
12746	if (!changeAllocationSize(Size: NewAllocationSize))
12747	return ChangeStatus::UNCHANGED;
12748	return ChangeStatus::CHANGED;
12749	}
12750
12751	// TODO: refactor this to be part of multiple bin case
12752	const auto &It = PI->begin();
12753
12754	// TODO: handle if Offset is not zero
12755	if (It->first.Offset != 0)
12756	return indicatePessimisticFixpoint();
12757
12758	uint64_t SizeOfBin = It->first.Offset + It->first.Size;
12759
12760	if (SizeOfBin >= *AllocationSize)
12761	return indicatePessimisticFixpoint();
12762
12763	auto NewAllocationSize =
12764	std::optional<TypeSize>(TypeSize(SizeOfBin * 8, false));
12765
12766	if (!changeAllocationSize(Size: NewAllocationSize))
12767	return ChangeStatus::UNCHANGED;
12768
12769	return ChangeStatus::CHANGED;
12770	}
12771
12772	/// See AbstractAttribute::manifest(...).
12773	ChangeStatus manifest(Attributor &A) override {
12774
12775	assert(isValidState() &&
12776	"Manifest should only be called if the state is valid.");
12777
12778	Instruction *I = getIRPosition().getCtxI();
12779
12780	auto FixedAllocatedSizeInBits = getAllocatedSize()->getFixedValue();
12781
12782	unsigned long NumBytesToAllocate = (FixedAllocatedSizeInBits + 7) / 8;
12783
12784	switch (I->getOpcode()) {
12785	// TODO: add case for malloc like calls
12786	case Instruction::Alloca: {
12787
12788	AllocaInst *AI = cast<AllocaInst>(Val: I);
12789
12790	Type *CharType = Type::getInt8Ty(C&: I->getContext());
12791
12792	auto *NumBytesToValue =
12793	ConstantInt::get(Context&: I->getContext(), V: APInt(32, NumBytesToAllocate));
12794
12795	BasicBlock::iterator insertPt = AI->getIterator();
12796	insertPt = std::next(x: insertPt);
12797	AllocaInst *NewAllocaInst =
12798	new AllocaInst(CharType, AI->getAddressSpace(), NumBytesToValue,
12799	AI->getAlign(), AI->getName(), insertPt);
12800
12801	if (A.changeAfterManifest(IRP: IRPosition::inst(I: *AI), NV&: *NewAllocaInst))
12802	return ChangeStatus::CHANGED;
12803
12804	break;
12805	}
12806	default:
12807	break;
12808	}
12809
12810	return ChangeStatus::UNCHANGED;
12811	}
12812
12813	/// See AbstractAttribute::getAsStr().
12814	const std::string getAsStr(Attributor *A) const override {
12815	if (!isValidState())
12816	return "allocationinfo(<invalid>)";
12817	return "allocationinfo(" +
12818	(AssumedAllocatedSize == HasNoAllocationSize
12819	? "none"
12820	: std::to_string(val: AssumedAllocatedSize->getFixedValue())) +
12821	")";
12822	}
12823
12824	private:
12825	std::optional<TypeSize> AssumedAllocatedSize = HasNoAllocationSize;
12826
12827	// Maintain the computed allocation size of the object.
12828	// Returns (bool) weather the size of the allocation was modified or not.
12829	bool changeAllocationSize(std::optional<TypeSize> Size) {
12830	if (AssumedAllocatedSize == HasNoAllocationSize ||
12831	AssumedAllocatedSize != Size) {
12832	AssumedAllocatedSize = Size;
12833	return true;
12834	}
12835	return false;
12836	}
12837	};
12838
12839	struct AAAllocationInfoFloating : AAAllocationInfoImpl {
12840	AAAllocationInfoFloating(const IRPosition &IRP, Attributor &A)
12841	: AAAllocationInfoImpl(IRP, A) {}
12842
12843	void trackStatistics() const override {
12844	STATS_DECLTRACK_FLOATING_ATTR(allocationinfo);
12845	}
12846	};
12847
12848	struct AAAllocationInfoReturned : AAAllocationInfoImpl {
12849	AAAllocationInfoReturned(const IRPosition &IRP, Attributor &A)
12850	: AAAllocationInfoImpl(IRP, A) {}
12851
12852	/// See AbstractAttribute::initialize(...).
12853	void initialize(Attributor &A) override {
12854	// TODO: we don't rewrite function argument for now because it will need to
12855	// rewrite the function signature and all call sites
12856	(void)indicatePessimisticFixpoint();
12857	}
12858
12859	void trackStatistics() const override {
12860	STATS_DECLTRACK_FNRET_ATTR(allocationinfo);
12861	}
12862	};
12863
12864	struct AAAllocationInfoCallSiteReturned : AAAllocationInfoImpl {
12865	AAAllocationInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
12866	: AAAllocationInfoImpl(IRP, A) {}
12867
12868	void trackStatistics() const override {
12869	STATS_DECLTRACK_CSRET_ATTR(allocationinfo);
12870	}
12871	};
12872
12873	struct AAAllocationInfoArgument : AAAllocationInfoImpl {
12874	AAAllocationInfoArgument(const IRPosition &IRP, Attributor &A)
12875	: AAAllocationInfoImpl(IRP, A) {}
12876
12877	void trackStatistics() const override {
12878	STATS_DECLTRACK_ARG_ATTR(allocationinfo);
12879	}
12880	};
12881
12882	struct AAAllocationInfoCallSiteArgument : AAAllocationInfoImpl {
12883	AAAllocationInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
12884	: AAAllocationInfoImpl(IRP, A) {}
12885
12886	/// See AbstractAttribute::initialize(...).
12887	void initialize(Attributor &A) override {
12888
12889	(void)indicatePessimisticFixpoint();
12890	}
12891
12892	void trackStatistics() const override {
12893	STATS_DECLTRACK_CSARG_ATTR(allocationinfo);
12894	}
12895	};
12896	} // namespace
12897
12898	const char AANoUnwind::ID = 0;
12899	const char AANoSync::ID = 0;
12900	const char AANoFree::ID = 0;
12901	const char AANonNull::ID = 0;
12902	const char AAMustProgress::ID = 0;
12903	const char AANoRecurse::ID = 0;
12904	const char AANonConvergent::ID = 0;
12905	const char AAWillReturn::ID = 0;
12906	const char AAUndefinedBehavior::ID = 0;
12907	const char AANoAlias::ID = 0;
12908	const char AAIntraFnReachability::ID = 0;
12909	const char AANoReturn::ID = 0;
12910	const char AAIsDead::ID = 0;
12911	const char AADereferenceable::ID = 0;
12912	const char AAAlign::ID = 0;
12913	const char AAInstanceInfo::ID = 0;
12914	const char AANoCapture::ID = 0;
12915	const char AAValueSimplify::ID = 0;
12916	const char AAHeapToStack::ID = 0;
12917	const char AAPrivatizablePtr::ID = 0;
12918	const char AAMemoryBehavior::ID = 0;
12919	const char AAMemoryLocation::ID = 0;
12920	const char AAValueConstantRange::ID = 0;
12921	const char AAPotentialConstantValues::ID = 0;
12922	const char AAPotentialValues::ID = 0;
12923	const char AANoUndef::ID = 0;
12924	const char AANoFPClass::ID = 0;
12925	const char AACallEdges::ID = 0;
12926	const char AAInterFnReachability::ID = 0;
12927	const char AAPointerInfo::ID = 0;
12928	const char AAAssumptionInfo::ID = 0;
12929	const char AAUnderlyingObjects::ID = 0;
12930	const char AAAddressSpace::ID = 0;
12931	const char AAAllocationInfo::ID = 0;
12932	const char AAIndirectCallInfo::ID = 0;
12933	const char AAGlobalValueInfo::ID = 0;
12934	const char AADenormalFPMath::ID = 0;
12935
12936	// Macro magic to create the static generator function for attributes that
12937	// follow the naming scheme.
12938
12939	#define SWITCH_PK_INV(CLASS, PK, POS_NAME) \
12940	case IRPosition::PK: \
12941	llvm_unreachable("Cannot create " #CLASS " for a " POS_NAME " position!");
12942
12943	#define SWITCH_PK_CREATE(CLASS, IRP, PK, SUFFIX) \
12944	case IRPosition::PK: \
12945	AA = new (A.Allocator) CLASS##SUFFIX(IRP, A); \
12946	++NumAAs; \
12947	break;
12948
12949	#define CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
12950	CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
12951	CLASS *AA = nullptr; \
12952	switch (IRP.getPositionKind()) { \
12953	SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
12954	SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \
12955	SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \
12956	SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
12957	SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \
12958	SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \
12959	SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
12960	SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
12961	} \
12962	return *AA; \
12963	}
12964
12965	#define CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
12966	CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
12967	CLASS *AA = nullptr; \
12968	switch (IRP.getPositionKind()) { \
12969	SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
12970	SWITCH_PK_INV(CLASS, IRP_FUNCTION, "function") \
12971	SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \
12972	SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
12973	SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
12974	SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \
12975	SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
12976	SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
12977	} \
12978	return *AA; \
12979	}
12980
12981	#define CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(POS, SUFFIX, CLASS) \
12982	CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
12983	CLASS *AA = nullptr; \
12984	switch (IRP.getPositionKind()) { \
12985	SWITCH_PK_CREATE(CLASS, IRP, POS, SUFFIX) \
12986	default: \
12987	llvm_unreachable("Cannot create " #CLASS " for position otherthan " #POS \
12988	" position!"); \
12989	} \
12990	return *AA; \
12991	}
12992
12993	#define CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
12994	CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
12995	CLASS *AA = nullptr; \
12996	switch (IRP.getPositionKind()) { \
12997	SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
12998	SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
12999	SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13000	SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13001	SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13002	SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \
13003	SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13004	SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13005	} \
13006	return *AA; \
13007	}
13008
13009	#define CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13010	CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13011	CLASS *AA = nullptr; \
13012	switch (IRP.getPositionKind()) { \
13013	SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13014	SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \
13015	SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \
13016	SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13017	SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \
13018	SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \
13019	SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \
13020	SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13021	} \
13022	return *AA; \
13023	}
13024
13025	#define CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13026	CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13027	CLASS *AA = nullptr; \
13028	switch (IRP.getPositionKind()) { \
13029	SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13030	SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13031	SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13032	SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13033	SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13034	SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13035	SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13036	SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13037	} \
13038	return *AA; \
13039	}
13040
13041	CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoUnwind)
13042	CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoSync)
13043	CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoRecurse)
13044	CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAWillReturn)
13045	CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoReturn)
13046	CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMemoryLocation)
13047	CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AACallEdges)
13048	CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAssumptionInfo)
13049	CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMustProgress)
13050
13051	CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANonNull)
13052	CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoAlias)
13053	CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPrivatizablePtr)
13054	CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AADereferenceable)
13055	CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAlign)
13056	CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInstanceInfo)
13057	CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoCapture)
13058	CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueConstantRange)
13059	CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialConstantValues)
13060	CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialValues)
13061	CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoUndef)
13062	CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoFPClass)
13063	CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPointerInfo)
13064	CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAddressSpace)
13065	CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAllocationInfo)
13066
13067	CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueSimplify)
13068	CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAIsDead)
13069	CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoFree)
13070	CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAUnderlyingObjects)
13071
13072	CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(IRP_CALL_SITE, CallSite,
13073	AAIndirectCallInfo)
13074	CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(IRP_FLOAT, Floating,
13075	AAGlobalValueInfo)
13076
13077	CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAHeapToStack)
13078	CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAUndefinedBehavior)
13079	CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANonConvergent)
13080	CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAIntraFnReachability)
13081	CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInterFnReachability)
13082	CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AADenormalFPMath)
13083
13084	CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMemoryBehavior)
13085
13086	#undef CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION
13087	#undef CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION
13088	#undef CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION
13089	#undef CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION
13090	#undef CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION
13091	#undef CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION
13092	#undef SWITCH_PK_CREATE
13093	#undef SWITCH_PK_INV
13094