1	//===-- ConstantFolding.cpp - Fold instructions into constants ------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file defines routines for folding instructions into constants.
10	//
11	// Also, to supplement the basic IR ConstantExpr simplifications,
12	// this file defines some additional folding routines that can make use of
13	// DataLayout information. These functions cannot go in IR due to library
14	// dependency issues.
15	//
16	//===----------------------------------------------------------------------===//
17
18	#include "llvm/Analysis/ConstantFolding.h"
19	#include "llvm/ADT/APFloat.h"
20	#include "llvm/ADT/APInt.h"
21	#include "llvm/ADT/APSInt.h"
22	#include "llvm/ADT/ArrayRef.h"
23	#include "llvm/ADT/DenseMap.h"
24	#include "llvm/ADT/STLExtras.h"
25	#include "llvm/ADT/SmallVector.h"
26	#include "llvm/ADT/StringRef.h"
27	#include "llvm/Analysis/TargetFolder.h"
28	#include "llvm/Analysis/TargetLibraryInfo.h"
29	#include "llvm/Analysis/ValueTracking.h"
30	#include "llvm/Analysis/VectorUtils.h"
31	#include "llvm/Config/config.h"
32	#include "llvm/IR/Constant.h"
33	#include "llvm/IR/ConstantFold.h"
34	#include "llvm/IR/Constants.h"
35	#include "llvm/IR/DataLayout.h"
36	#include "llvm/IR/DerivedTypes.h"
37	#include "llvm/IR/Function.h"
38	#include "llvm/IR/GlobalValue.h"
39	#include "llvm/IR/GlobalVariable.h"
40	#include "llvm/IR/InstrTypes.h"
41	#include "llvm/IR/Instruction.h"
42	#include "llvm/IR/Instructions.h"
43	#include "llvm/IR/IntrinsicInst.h"
44	#include "llvm/IR/Intrinsics.h"
45	#include "llvm/IR/IntrinsicsAArch64.h"
46	#include "llvm/IR/IntrinsicsAMDGPU.h"
47	#include "llvm/IR/IntrinsicsARM.h"
48	#include "llvm/IR/IntrinsicsWebAssembly.h"
49	#include "llvm/IR/IntrinsicsX86.h"
50	#include "llvm/IR/Operator.h"
51	#include "llvm/IR/Type.h"
52	#include "llvm/IR/Value.h"
53	#include "llvm/Support/Casting.h"
54	#include "llvm/Support/ErrorHandling.h"
55	#include "llvm/Support/KnownBits.h"
56	#include "llvm/Support/MathExtras.h"
57	#include <cassert>
58	#include <cerrno>
59	#include <cfenv>
60	#include <cmath>
61	#include <cstdint>
62
63	using namespace llvm;
64
65	namespace {
66
67	//===----------------------------------------------------------------------===//
68	// Constant Folding internal helper functions
69	//===----------------------------------------------------------------------===//
70
71	static Constant *foldConstVectorToAPInt(APInt &Result, Type *DestTy,
72	Constant *C, Type *SrcEltTy,
73	unsigned NumSrcElts,
74	const DataLayout &DL) {
75	// Now that we know that the input value is a vector of integers, just shift
76	// and insert them into our result.
77	unsigned BitShift = DL.getTypeSizeInBits(Ty: SrcEltTy);
78	for (unsigned i = 0; i != NumSrcElts; ++i) {
79	Constant *Element;
80	if (DL.isLittleEndian())
81	Element = C->getAggregateElement(Elt: NumSrcElts - i - 1);
82	else
83	Element = C->getAggregateElement(Elt: i);
84
85	if (Element && isa<UndefValue>(Val: Element)) {
86	Result <<= BitShift;
87	continue;
88	}
89
90	auto *ElementCI = dyn_cast_or_null<ConstantInt>(Val: Element);
91	if (!ElementCI)
92	return ConstantExpr::getBitCast(C, Ty: DestTy);
93
94	Result <<= BitShift;
95	Result |= ElementCI->getValue().zext(width: Result.getBitWidth());
96	}
97
98	return nullptr;
99	}
100
101	/// Constant fold bitcast, symbolically evaluating it with DataLayout.
102	/// This always returns a non-null constant, but it may be a
103	/// ConstantExpr if unfoldable.
104	Constant *FoldBitCast(Constant *C, Type *DestTy, const DataLayout &DL) {
105	assert(CastInst::castIsValid(Instruction::BitCast, C, DestTy) &&
106	"Invalid constantexpr bitcast!");
107
108	// Catch the obvious splat cases.
109	if (Constant *Res = ConstantFoldLoadFromUniformValue(C, Ty: DestTy))
110	return Res;
111
112	if (auto *VTy = dyn_cast<VectorType>(Val: C->getType())) {
113	// Handle a vector->scalar integer/fp cast.
114	if (isa<IntegerType>(Val: DestTy) || DestTy->isFloatingPointTy()) {
115	unsigned NumSrcElts = cast<FixedVectorType>(Val: VTy)->getNumElements();
116	Type *SrcEltTy = VTy->getElementType();
117
118	// If the vector is a vector of floating point, convert it to vector of int
119	// to simplify things.
120	if (SrcEltTy->isFloatingPointTy()) {
121	unsigned FPWidth = SrcEltTy->getPrimitiveSizeInBits();
122	auto *SrcIVTy = FixedVectorType::get(
123	ElementType: IntegerType::get(C&: C->getContext(), NumBits: FPWidth), NumElts: NumSrcElts);
124	// Ask IR to do the conversion now that #elts line up.
125	C = ConstantExpr::getBitCast(C, Ty: SrcIVTy);
126	}
127
128	APInt Result(DL.getTypeSizeInBits(Ty: DestTy), 0);
129	if (Constant *CE = foldConstVectorToAPInt(Result, DestTy, C,
130	SrcEltTy, NumSrcElts, DL))
131	return CE;
132
133	if (isa<IntegerType>(Val: DestTy))
134	return ConstantInt::get(Ty: DestTy, V: Result);
135
136	APFloat FP(DestTy->getFltSemantics(), Result);
137	return ConstantFP::get(Context&: DestTy->getContext(), V: FP);
138	}
139	}
140
141	// The code below only handles casts to vectors currently.
142	auto *DestVTy = dyn_cast<VectorType>(Val: DestTy);
143	if (!DestVTy)
144	return ConstantExpr::getBitCast(C, Ty: DestTy);
145
146	// If this is a scalar -> vector cast, convert the input into a <1 x scalar>
147	// vector so the code below can handle it uniformly.
148	if (isa<ConstantFP>(Val: C) || isa<ConstantInt>(Val: C)) {
149	Constant *Ops = C; // don't take the address of C!
150	return FoldBitCast(C: ConstantVector::get(V: Ops), DestTy, DL);
151	}
152
153	// If this is a bitcast from constant vector -> vector, fold it.
154	if (!isa<ConstantDataVector>(Val: C) && !isa<ConstantVector>(Val: C))
155	return ConstantExpr::getBitCast(C, Ty: DestTy);
156
157	// If the element types match, IR can fold it.
158	unsigned NumDstElt = cast<FixedVectorType>(Val: DestVTy)->getNumElements();
159	unsigned NumSrcElt = cast<FixedVectorType>(Val: C->getType())->getNumElements();
160	if (NumDstElt == NumSrcElt)
161	return ConstantExpr::getBitCast(C, Ty: DestTy);
162
163	Type *SrcEltTy = cast<VectorType>(Val: C->getType())->getElementType();
164	Type *DstEltTy = DestVTy->getElementType();
165
166	// Otherwise, we're changing the number of elements in a vector, which
167	// requires endianness information to do the right thing. For example,
168	// bitcast (<2 x i64> <i64 0, i64 1> to <4 x i32>)
169	// folds to (little endian):
170	// <4 x i32> <i32 0, i32 0, i32 1, i32 0>
171	// and to (big endian):
172	// <4 x i32> <i32 0, i32 0, i32 0, i32 1>
173
174	// First thing is first. We only want to think about integer here, so if
175	// we have something in FP form, recast it as integer.
176	if (DstEltTy->isFloatingPointTy()) {
177	// Fold to an vector of integers with same size as our FP type.
178	unsigned FPWidth = DstEltTy->getPrimitiveSizeInBits();
179	auto *DestIVTy = FixedVectorType::get(
180	ElementType: IntegerType::get(C&: C->getContext(), NumBits: FPWidth), NumElts: NumDstElt);
181	// Recursively handle this integer conversion, if possible.
182	C = FoldBitCast(C, DestTy: DestIVTy, DL);
183
184	// Finally, IR can handle this now that #elts line up.
185	return ConstantExpr::getBitCast(C, Ty: DestTy);
186	}
187
188	// Okay, we know the destination is integer, if the input is FP, convert
189	// it to integer first.
190	if (SrcEltTy->isFloatingPointTy()) {
191	unsigned FPWidth = SrcEltTy->getPrimitiveSizeInBits();
192	auto *SrcIVTy = FixedVectorType::get(
193	ElementType: IntegerType::get(C&: C->getContext(), NumBits: FPWidth), NumElts: NumSrcElt);
194	// Ask IR to do the conversion now that #elts line up.
195	C = ConstantExpr::getBitCast(C, Ty: SrcIVTy);
196	// If IR wasn't able to fold it, bail out.
197	if (!isa<ConstantVector>(Val: C) && // FIXME: Remove ConstantVector.
198	!isa<ConstantDataVector>(Val: C))
199	return C;
200	}
201
202	// Now we know that the input and output vectors are both integer vectors
203	// of the same size, and that their #elements is not the same. Do the
204	// conversion here, which depends on whether the input or output has
205	// more elements.
206	bool isLittleEndian = DL.isLittleEndian();
207
208	SmallVector<Constant*, 32> Result;
209	if (NumDstElt < NumSrcElt) {
210	// Handle: bitcast (<4 x i32> <i32 0, i32 1, i32 2, i32 3> to <2 x i64>)
211	Constant *Zero = Constant::getNullValue(Ty: DstEltTy);
212	unsigned Ratio = NumSrcElt/NumDstElt;
213	unsigned SrcBitSize = SrcEltTy->getPrimitiveSizeInBits();
214	unsigned SrcElt = 0;
215	for (unsigned i = 0; i != NumDstElt; ++i) {
216	// Build each element of the result.
217	Constant *Elt = Zero;
218	unsigned ShiftAmt = isLittleEndian ? 0 : SrcBitSize*(Ratio-1);
219	for (unsigned j = 0; j != Ratio; ++j) {
220	Constant *Src = C->getAggregateElement(Elt: SrcElt++);
221	if (Src && isa<UndefValue>(Val: Src))
222	Src = Constant::getNullValue(
223	Ty: cast<VectorType>(Val: C->getType())->getElementType());
224	else
225	Src = dyn_cast_or_null<ConstantInt>(Val: Src);
226	if (!Src) // Reject constantexpr elements.
227	return ConstantExpr::getBitCast(C, Ty: DestTy);
228
229	// Zero extend the element to the right size.
230	Src = ConstantFoldCastOperand(Opcode: Instruction::ZExt, C: Src, DestTy: Elt->getType(),
231	DL);
232	assert(Src && "Constant folding cannot fail on plain integers");
233
234	// Shift it to the right place, depending on endianness.
235	Src = ConstantFoldBinaryOpOperands(
236	Opcode: Instruction::Shl, LHS: Src, RHS: ConstantInt::get(Ty: Src->getType(), V: ShiftAmt),
237	DL);
238	assert(Src && "Constant folding cannot fail on plain integers");
239
240	ShiftAmt += isLittleEndian ? SrcBitSize : -SrcBitSize;
241
242	// Mix it in.
243	Elt = ConstantFoldBinaryOpOperands(Opcode: Instruction::Or, LHS: Elt, RHS: Src, DL);
244	assert(Elt && "Constant folding cannot fail on plain integers");
245	}
246	Result.push_back(Elt);
247	}
248	return ConstantVector::get(V: Result);
249	}
250
251	// Handle: bitcast (<2 x i64> <i64 0, i64 1> to <4 x i32>)
252	unsigned Ratio = NumDstElt/NumSrcElt;
253	unsigned DstBitSize = DL.getTypeSizeInBits(Ty: DstEltTy);
254
255	// Loop over each source value, expanding into multiple results.
256	for (unsigned i = 0; i != NumSrcElt; ++i) {
257	auto *Element = C->getAggregateElement(Elt: i);
258
259	if (!Element) // Reject constantexpr elements.
260	return ConstantExpr::getBitCast(C, Ty: DestTy);
261
262	if (isa<UndefValue>(Val: Element)) {
263	// Correctly Propagate undef values.
264	Result.append(NumInputs: Ratio, Elt: UndefValue::get(T: DstEltTy));
265	continue;
266	}
267
268	auto *Src = dyn_cast<ConstantInt>(Val: Element);
269	if (!Src)
270	return ConstantExpr::getBitCast(C, Ty: DestTy);
271
272	unsigned ShiftAmt = isLittleEndian ? 0 : DstBitSize*(Ratio-1);
273	for (unsigned j = 0; j != Ratio; ++j) {
274	// Shift the piece of the value into the right place, depending on
275	// endianness.
276	APInt Elt = Src->getValue().lshr(shiftAmt: ShiftAmt);
277	ShiftAmt += isLittleEndian ? DstBitSize : -DstBitSize;
278
279	// Truncate and remember this piece.
280	Result.push_back(Elt: ConstantInt::get(Ty: DstEltTy, V: Elt.trunc(width: DstBitSize)));
281	}
282	}
283
284	return ConstantVector::get(V: Result);
285	}
286
287	} // end anonymous namespace
288
289	/// If this constant is a constant offset from a global, return the global and
290	/// the constant. Because of constantexprs, this function is recursive.
291	bool llvm::IsConstantOffsetFromGlobal(Constant *C, GlobalValue *&GV,
292	APInt &Offset, const DataLayout &DL,
293	DSOLocalEquivalent **DSOEquiv) {
294	if (DSOEquiv)
295	*DSOEquiv = nullptr;
296
297	// Trivial case, constant is the global.
298	if ((GV = dyn_cast<GlobalValue>(Val: C))) {
299	unsigned BitWidth = DL.getIndexTypeSizeInBits(Ty: GV->getType());
300	Offset = APInt(BitWidth, 0);
301	return true;
302	}
303
304	if (auto *FoundDSOEquiv = dyn_cast<DSOLocalEquivalent>(Val: C)) {
305	if (DSOEquiv)
306	*DSOEquiv = FoundDSOEquiv;
307	GV = FoundDSOEquiv->getGlobalValue();
308	unsigned BitWidth = DL.getIndexTypeSizeInBits(Ty: GV->getType());
309	Offset = APInt(BitWidth, 0);
310	return true;
311	}
312
313	// Otherwise, if this isn't a constant expr, bail out.
314	auto *CE = dyn_cast<ConstantExpr>(Val: C);
315	if (!CE) return false;
316
317	// Look through ptr->int and ptr->ptr casts.
318	if (CE->getOpcode() == Instruction::PtrToInt ||
319	CE->getOpcode() == Instruction::BitCast)
320	return IsConstantOffsetFromGlobal(C: CE->getOperand(i_nocapture: 0), GV, Offset, DL,
321	DSOEquiv);
322
323	// i32* getelementptr ([5 x i32]* @a, i32 0, i32 5)
324	auto *GEP = dyn_cast<GEPOperator>(Val: CE);
325	if (!GEP)
326	return false;
327
328	unsigned BitWidth = DL.getIndexTypeSizeInBits(Ty: GEP->getType());
329	APInt TmpOffset(BitWidth, 0);
330
331	// If the base isn't a global+constant, we aren't either.
332	if (!IsConstantOffsetFromGlobal(C: CE->getOperand(i_nocapture: 0), GV, Offset&: TmpOffset, DL,
333	DSOEquiv))
334	return false;
335
336	// Otherwise, add any offset that our operands provide.
337	if (!GEP->accumulateConstantOffset(DL, Offset&: TmpOffset))
338	return false;
339
340	Offset = TmpOffset;
341	return true;
342	}
343
344	Constant *llvm::ConstantFoldLoadThroughBitcast(Constant *C, Type *DestTy,
345	const DataLayout &DL) {
346	do {
347	Type *SrcTy = C->getType();
348	if (SrcTy == DestTy)
349	return C;
350
351	TypeSize DestSize = DL.getTypeSizeInBits(Ty: DestTy);
352	TypeSize SrcSize = DL.getTypeSizeInBits(Ty: SrcTy);
353	if (!TypeSize::isKnownGE(LHS: SrcSize, RHS: DestSize))
354	return nullptr;
355
356	// Catch the obvious splat cases (since all-zeros can coerce non-integral
357	// pointers legally).
358	if (Constant *Res = ConstantFoldLoadFromUniformValue(C, Ty: DestTy))
359	return Res;
360
361	// If the type sizes are the same and a cast is legal, just directly
362	// cast the constant.
363	// But be careful not to coerce non-integral pointers illegally.
364	if (SrcSize == DestSize &&
365	DL.isNonIntegralPointerType(Ty: SrcTy->getScalarType()) ==
366	DL.isNonIntegralPointerType(Ty: DestTy->getScalarType())) {
367	Instruction::CastOps Cast = Instruction::BitCast;
368	// If we are going from a pointer to int or vice versa, we spell the cast
369	// differently.
370	if (SrcTy->isIntegerTy() && DestTy->isPointerTy())
371	Cast = Instruction::IntToPtr;
372	else if (SrcTy->isPointerTy() && DestTy->isIntegerTy())
373	Cast = Instruction::PtrToInt;
374
375	if (CastInst::castIsValid(op: Cast, S: C, DstTy: DestTy))
376	return ConstantFoldCastOperand(Opcode: Cast, C, DestTy, DL);
377	}
378
379	// If this isn't an aggregate type, there is nothing we can do to drill down
380	// and find a bitcastable constant.
381	if (!SrcTy->isAggregateType() && !SrcTy->isVectorTy())
382	return nullptr;
383
384	// We're simulating a load through a pointer that was bitcast to point to
385	// a different type, so we can try to walk down through the initial
386	// elements of an aggregate to see if some part of the aggregate is
387	// castable to implement the "load" semantic model.
388	if (SrcTy->isStructTy()) {
389	// Struct types might have leading zero-length elements like [0 x i32],
390	// which are certainly not what we are looking for, so skip them.
391	unsigned Elem = 0;
392	Constant *ElemC;
393	do {
394	ElemC = C->getAggregateElement(Elt: Elem++);
395	} while (ElemC && DL.getTypeSizeInBits(Ty: ElemC->getType()).isZero());
396	C = ElemC;
397	} else {
398	// For non-byte-sized vector elements, the first element is not
399	// necessarily located at the vector base address.
400	if (auto *VT = dyn_cast<VectorType>(Val: SrcTy))
401	if (!DL.typeSizeEqualsStoreSize(Ty: VT->getElementType()))
402	return nullptr;
403
404	C = C->getAggregateElement(Elt: 0u);
405	}
406	} while (C);
407
408	return nullptr;
409	}
410
411	namespace {
412
413	/// Recursive helper to read bits out of global. C is the constant being copied
414	/// out of. ByteOffset is an offset into C. CurPtr is the pointer to copy
415	/// results into and BytesLeft is the number of bytes left in
416	/// the CurPtr buffer. DL is the DataLayout.
417	bool ReadDataFromGlobal(Constant *C, uint64_t ByteOffset, unsigned char *CurPtr,
418	unsigned BytesLeft, const DataLayout &DL) {
419	assert(ByteOffset <= DL.getTypeAllocSize(C->getType()) &&
420	"Out of range access");
421
422	// If this element is zero or undefined, we can just return since *CurPtr is
423	// zero initialized.
424	if (isa<ConstantAggregateZero>(Val: C) || isa<UndefValue>(Val: C))
425	return true;
426
427	if (auto *CI = dyn_cast<ConstantInt>(Val: C)) {
428	if ((CI->getBitWidth() & 7) != 0)
429	return false;
430	const APInt &Val = CI->getValue();
431	unsigned IntBytes = unsigned(CI->getBitWidth()/8);
432
433	for (unsigned i = 0; i != BytesLeft && ByteOffset != IntBytes; ++i) {
434	unsigned n = ByteOffset;
435	if (!DL.isLittleEndian())
436	n = IntBytes - n - 1;
437	CurPtr[i] = Val.extractBits(numBits: 8, bitPosition: n * 8).getZExtValue();
438	++ByteOffset;
439	}
440	return true;
441	}
442
443	if (auto *CFP = dyn_cast<ConstantFP>(Val: C)) {
444	if (CFP->getType()->isDoubleTy()) {
445	C = FoldBitCast(C, DestTy: Type::getInt64Ty(C&: C->getContext()), DL);
446	return ReadDataFromGlobal(C, ByteOffset, CurPtr, BytesLeft, DL);
447	}
448	if (CFP->getType()->isFloatTy()){
449	C = FoldBitCast(C, DestTy: Type::getInt32Ty(C&: C->getContext()), DL);
450	return ReadDataFromGlobal(C, ByteOffset, CurPtr, BytesLeft, DL);
451	}
452	if (CFP->getType()->isHalfTy()){
453	C = FoldBitCast(C, DestTy: Type::getInt16Ty(C&: C->getContext()), DL);
454	return ReadDataFromGlobal(C, ByteOffset, CurPtr, BytesLeft, DL);
455	}
456	return false;
457	}
458
459	if (auto *CS = dyn_cast<ConstantStruct>(Val: C)) {
460	const StructLayout *SL = DL.getStructLayout(Ty: CS->getType());
461	unsigned Index = SL->getElementContainingOffset(FixedOffset: ByteOffset);
462	uint64_t CurEltOffset = SL->getElementOffset(Idx: Index);
463	ByteOffset -= CurEltOffset;
464
465	while (true) {
466	// If the element access is to the element itself and not to tail padding,
467	// read the bytes from the element.
468	uint64_t EltSize = DL.getTypeAllocSize(Ty: CS->getOperand(i_nocapture: Index)->getType());
469
470	if (ByteOffset < EltSize &&
471	!ReadDataFromGlobal(C: CS->getOperand(i_nocapture: Index), ByteOffset, CurPtr,
472	BytesLeft, DL))
473	return false;
474
475	++Index;
476
477	// Check to see if we read from the last struct element, if so we're done.
478	if (Index == CS->getType()->getNumElements())
479	return true;
480
481	// If we read all of the bytes we needed from this element we're done.
482	uint64_t NextEltOffset = SL->getElementOffset(Idx: Index);
483
484	if (BytesLeft <= NextEltOffset - CurEltOffset - ByteOffset)
485	return true;
486
487	// Move to the next element of the struct.
488	CurPtr += NextEltOffset - CurEltOffset - ByteOffset;
489	BytesLeft -= NextEltOffset - CurEltOffset - ByteOffset;
490	ByteOffset = 0;
491	CurEltOffset = NextEltOffset;
492	}
493	// not reached.
494	}
495
496	if (isa<ConstantArray>(Val: C) || isa<ConstantVector>(Val: C) ||
497	isa<ConstantDataSequential>(Val: C)) {
498	uint64_t NumElts, EltSize;
499	Type *EltTy;
500	if (auto *AT = dyn_cast<ArrayType>(Val: C->getType())) {
501	NumElts = AT->getNumElements();
502	EltTy = AT->getElementType();
503	EltSize = DL.getTypeAllocSize(Ty: EltTy);
504	} else {
505	NumElts = cast<FixedVectorType>(Val: C->getType())->getNumElements();
506	EltTy = cast<FixedVectorType>(Val: C->getType())->getElementType();
507	// TODO: For non-byte-sized vectors, current implementation assumes there is
508	// padding to the next byte boundary between elements.
509	if (!DL.typeSizeEqualsStoreSize(Ty: EltTy))
510	return false;
511
512	EltSize = DL.getTypeStoreSize(Ty: EltTy);
513	}
514	uint64_t Index = ByteOffset / EltSize;
515	uint64_t Offset = ByteOffset - Index * EltSize;
516
517	for (; Index != NumElts; ++Index) {
518	if (!ReadDataFromGlobal(C: C->getAggregateElement(Elt: Index), ByteOffset: Offset, CurPtr,
519	BytesLeft, DL))
520	return false;
521
522	uint64_t BytesWritten = EltSize - Offset;
523	assert(BytesWritten <= EltSize && "Not indexing into this element?");
524	if (BytesWritten >= BytesLeft)
525	return true;
526
527	Offset = 0;
528	BytesLeft -= BytesWritten;
529	CurPtr += BytesWritten;
530	}
531	return true;
532	}
533
534	if (auto *CE = dyn_cast<ConstantExpr>(Val: C)) {
535	if (CE->getOpcode() == Instruction::IntToPtr &&
536	CE->getOperand(i_nocapture: 0)->getType() == DL.getIntPtrType(CE->getType())) {
537	return ReadDataFromGlobal(C: CE->getOperand(i_nocapture: 0), ByteOffset, CurPtr,
538	BytesLeft, DL);
539	}
540	}
541
542	// Otherwise, unknown initializer type.
543	return false;
544	}
545
546	Constant *FoldReinterpretLoadFromConst(Constant *C, Type *LoadTy,
547	int64_t Offset, const DataLayout &DL) {
548	// Bail out early. Not expect to load from scalable global variable.
549	if (isa<ScalableVectorType>(Val: LoadTy))
550	return nullptr;
551
552	auto *IntType = dyn_cast<IntegerType>(Val: LoadTy);
553
554	// If this isn't an integer load we can't fold it directly.
555	if (!IntType) {
556	// If this is a non-integer load, we can try folding it as an int load and
557	// then bitcast the result. This can be useful for union cases. Note
558	// that address spaces don't matter here since we're not going to result in
559	// an actual new load.
560	if (!LoadTy->isFloatingPointTy() && !LoadTy->isPointerTy() &&
561	!LoadTy->isVectorTy())
562	return nullptr;
563
564	Type *MapTy = Type::getIntNTy(C&: C->getContext(),
565	N: DL.getTypeSizeInBits(Ty: LoadTy).getFixedValue());
566	if (Constant *Res = FoldReinterpretLoadFromConst(C, LoadTy: MapTy, Offset, DL)) {
567	if (Res->isNullValue() && !LoadTy->isX86_MMXTy() &&
568	!LoadTy->isX86_AMXTy())
569	// Materializing a zero can be done trivially without a bitcast
570	return Constant::getNullValue(Ty: LoadTy);
571	Type *CastTy = LoadTy->isPtrOrPtrVectorTy() ? DL.getIntPtrType(LoadTy) : LoadTy;
572	Res = FoldBitCast(C: Res, DestTy: CastTy, DL);
573	if (LoadTy->isPtrOrPtrVectorTy()) {
574	// For vector of pointer, we needed to first convert to a vector of integer, then do vector inttoptr
575	if (Res->isNullValue() && !LoadTy->isX86_MMXTy() &&
576	!LoadTy->isX86_AMXTy())
577	return Constant::getNullValue(Ty: LoadTy);
578	if (DL.isNonIntegralPointerType(Ty: LoadTy->getScalarType()))
579	// Be careful not to replace a load of an addrspace value with an inttoptr here
580	return nullptr;
581	Res = ConstantExpr::getIntToPtr(C: Res, Ty: LoadTy);
582	}
583	return Res;
584	}
585	return nullptr;
586	}
587
588	unsigned BytesLoaded = (IntType->getBitWidth() + 7) / 8;
589	if (BytesLoaded > 32 || BytesLoaded == 0)
590	return nullptr;
591
592	// If we're not accessing anything in this constant, the result is undefined.
593	if (Offset <= -1 * static_cast<int64_t>(BytesLoaded))
594	return PoisonValue::get(T: IntType);
595
596	// TODO: We should be able to support scalable types.
597	TypeSize InitializerSize = DL.getTypeAllocSize(Ty: C->getType());
598	if (InitializerSize.isScalable())
599	return nullptr;
600
601	// If we're not accessing anything in this constant, the result is undefined.
602	if (Offset >= (int64_t)InitializerSize.getFixedValue())
603	return PoisonValue::get(T: IntType);
604
605	unsigned char RawBytes[32] = {0};
606	unsigned char *CurPtr = RawBytes;
607	unsigned BytesLeft = BytesLoaded;
608
609	// If we're loading off the beginning of the global, some bytes may be valid.
610	if (Offset < 0) {
611	CurPtr += -Offset;
612	BytesLeft += Offset;
613	Offset = 0;
614	}
615
616	if (!ReadDataFromGlobal(C, ByteOffset: Offset, CurPtr, BytesLeft, DL))
617	return nullptr;
618
619	APInt ResultVal = APInt(IntType->getBitWidth(), 0);
620	if (DL.isLittleEndian()) {
621	ResultVal = RawBytes[BytesLoaded - 1];
622	for (unsigned i = 1; i != BytesLoaded; ++i) {
623	ResultVal <<= 8;
624	ResultVal |= RawBytes[BytesLoaded - 1 - i];
625	}
626	} else {
627	ResultVal = RawBytes[0];
628	for (unsigned i = 1; i != BytesLoaded; ++i) {
629	ResultVal <<= 8;
630	ResultVal |= RawBytes[i];
631	}
632	}
633
634	return ConstantInt::get(Context&: IntType->getContext(), V: ResultVal);
635	}
636
637	} // anonymous namespace
638
639	// If GV is a constant with an initializer read its representation starting
640	// at Offset and return it as a constant array of unsigned char. Otherwise
641	// return null.
642	Constant *llvm::ReadByteArrayFromGlobal(const GlobalVariable *GV,
643	uint64_t Offset) {
644	if (!GV->isConstant() || !GV->hasDefinitiveInitializer())
645	return nullptr;
646
647	const DataLayout &DL = GV->getParent()->getDataLayout();
648	Constant *Init = const_cast<Constant *>(GV->getInitializer());
649	TypeSize InitSize = DL.getTypeAllocSize(Ty: Init->getType());
650	if (InitSize < Offset)
651	return nullptr;
652
653	uint64_t NBytes = InitSize - Offset;
654	if (NBytes > UINT16_MAX)
655	// Bail for large initializers in excess of 64K to avoid allocating
656	// too much memory.
657	// Offset is assumed to be less than or equal than InitSize (this
658	// is enforced in ReadDataFromGlobal).
659	return nullptr;
660
661	SmallVector<unsigned char, 256> RawBytes(static_cast<size_t>(NBytes));
662	unsigned char *CurPtr = RawBytes.data();
663
664	if (!ReadDataFromGlobal(C: Init, ByteOffset: Offset, CurPtr, BytesLeft: NBytes, DL))
665	return nullptr;
666
667	return ConstantDataArray::get(Context&: GV->getContext(), Elts&: RawBytes);
668	}
669
670	/// If this Offset points exactly to the start of an aggregate element, return
671	/// that element, otherwise return nullptr.
672	Constant *getConstantAtOffset(Constant *Base, APInt Offset,
673	const DataLayout &DL) {
674	if (Offset.isZero())
675	return Base;
676
677	if (!isa<ConstantAggregate>(Val: Base) && !isa<ConstantDataSequential>(Val: Base))
678	return nullptr;
679
680	Type *ElemTy = Base->getType();
681	SmallVector<APInt> Indices = DL.getGEPIndicesForOffset(ElemTy, Offset);
682	if (!Offset.isZero() || !Indices[0].isZero())
683	return nullptr;
684
685	Constant *C = Base;
686	for (const APInt &Index : drop_begin(RangeOrContainer&: Indices)) {
687	if (Index.isNegative() || Index.getActiveBits() >= 32)
688	return nullptr;
689
690	C = C->getAggregateElement(Elt: Index.getZExtValue());
691	if (!C)
692	return nullptr;
693	}
694
695	return C;
696	}
697
698	Constant *llvm::ConstantFoldLoadFromConst(Constant *C, Type *Ty,
699	const APInt &Offset,
700	const DataLayout &DL) {
701	if (Constant *AtOffset = getConstantAtOffset(Base: C, Offset, DL))
702	if (Constant *Result = ConstantFoldLoadThroughBitcast(C: AtOffset, DestTy: Ty, DL))
703	return Result;
704
705	// Explicitly check for out-of-bounds access, so we return poison even if the
706	// constant is a uniform value.
707	TypeSize Size = DL.getTypeAllocSize(Ty: C->getType());
708	if (!Size.isScalable() && Offset.sge(RHS: Size.getFixedValue()))
709	return PoisonValue::get(T: Ty);
710
711	// Try an offset-independent fold of a uniform value.
712	if (Constant *Result = ConstantFoldLoadFromUniformValue(C, Ty))
713	return Result;
714
715	// Try hard to fold loads from bitcasted strange and non-type-safe things.
716	if (Offset.getSignificantBits() <= 64)
717	if (Constant *Result =
718	FoldReinterpretLoadFromConst(C, LoadTy: Ty, Offset: Offset.getSExtValue(), DL))
719	return Result;
720
721	return nullptr;
722	}
723
724	Constant *llvm::ConstantFoldLoadFromConst(Constant *C, Type *Ty,
725	const DataLayout &DL) {
726	return ConstantFoldLoadFromConst(C, Ty, Offset: APInt(64, 0), DL);
727	}
728
729	Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty,
730	APInt Offset,
731	const DataLayout &DL) {
732	// We can only fold loads from constant globals with a definitive initializer.
733	// Check this upfront, to skip expensive offset calculations.
734	auto *GV = dyn_cast<GlobalVariable>(Val: getUnderlyingObject(V: C));
735	if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer())
736	return nullptr;
737
738	C = cast<Constant>(Val: C->stripAndAccumulateConstantOffsets(
739	DL, Offset, /* AllowNonInbounds */ true));
740
741	if (C == GV)
742	if (Constant *Result = ConstantFoldLoadFromConst(C: GV->getInitializer(), Ty,
743	Offset, DL))
744	return Result;
745
746	// If this load comes from anywhere in a uniform constant global, the value
747	// is always the same, regardless of the loaded offset.
748	return ConstantFoldLoadFromUniformValue(C: GV->getInitializer(), Ty);
749	}
750
751	Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty,
752	const DataLayout &DL) {
753	APInt Offset(DL.getIndexTypeSizeInBits(Ty: C->getType()), 0);
754	return ConstantFoldLoadFromConstPtr(C, Ty, Offset, DL);
755	}
756
757	Constant *llvm::ConstantFoldLoadFromUniformValue(Constant *C, Type *Ty) {
758	if (isa<PoisonValue>(Val: C))
759	return PoisonValue::get(T: Ty);
760	if (isa<UndefValue>(Val: C))
761	return UndefValue::get(T: Ty);
762	if (C->isNullValue() && !Ty->isX86_MMXTy() && !Ty->isX86_AMXTy())
763	return Constant::getNullValue(Ty);
764	if (C->isAllOnesValue() &&
765	(Ty->isIntOrIntVectorTy() || Ty->isFPOrFPVectorTy()))
766	return Constant::getAllOnesValue(Ty);
767	return nullptr;
768	}
769
770	namespace {
771
772	/// One of Op0/Op1 is a constant expression.
773	/// Attempt to symbolically evaluate the result of a binary operator merging
774	/// these together. If target data info is available, it is provided as DL,
775	/// otherwise DL is null.
776	Constant *SymbolicallyEvaluateBinop(unsigned Opc, Constant *Op0, Constant *Op1,
777	const DataLayout &DL) {
778	// SROA
779
780	// Fold (and 0xffffffff00000000, (shl x, 32)) -> shl.
781	// Fold (lshr (or X, Y), 32) -> (lshr [X/Y], 32) if one doesn't contribute
782	// bits.
783
784	if (Opc == Instruction::And) {
785	KnownBits Known0 = computeKnownBits(V: Op0, DL);
786	KnownBits Known1 = computeKnownBits(V: Op1, DL);
787	if ((Known1.One | Known0.Zero).isAllOnes()) {
788	// All the bits of Op0 that the 'and' could be masking are already zero.
789	return Op0;
790	}
791	if ((Known0.One | Known1.Zero).isAllOnes()) {
792	// All the bits of Op1 that the 'and' could be masking are already zero.
793	return Op1;
794	}
795
796	Known0 &= Known1;
797	if (Known0.isConstant())
798	return ConstantInt::get(Ty: Op0->getType(), V: Known0.getConstant());
799	}
800
801	// If the constant expr is something like &A[123] - &A[4].f, fold this into a
802	// constant. This happens frequently when iterating over a global array.
803	if (Opc == Instruction::Sub) {
804	GlobalValue *GV1, *GV2;
805	APInt Offs1, Offs2;
806
807	if (IsConstantOffsetFromGlobal(C: Op0, GV&: GV1, Offset&: Offs1, DL))
808	if (IsConstantOffsetFromGlobal(C: Op1, GV&: GV2, Offset&: Offs2, DL) && GV1 == GV2) {
809	unsigned OpSize = DL.getTypeSizeInBits(Ty: Op0->getType());
810
811	// (&GV+C1) - (&GV+C2) -> C1-C2, pointer arithmetic cannot overflow.
812	// PtrToInt may change the bitwidth so we have convert to the right size
813	// first.
814	return ConstantInt::get(Ty: Op0->getType(), V: Offs1.zextOrTrunc(width: OpSize) -
815	Offs2.zextOrTrunc(width: OpSize));
816	}
817	}
818
819	return nullptr;
820	}
821
822	/// If array indices are not pointer-sized integers, explicitly cast them so
823	/// that they aren't implicitly casted by the getelementptr.
824	Constant *CastGEPIndices(Type *SrcElemTy, ArrayRef<Constant *> Ops,
825	Type *ResultTy, bool InBounds,
826	std::optional<unsigned> InRangeIndex,
827	const DataLayout &DL, const TargetLibraryInfo *TLI) {
828	Type *IntIdxTy = DL.getIndexType(PtrTy: ResultTy);
829	Type *IntIdxScalarTy = IntIdxTy->getScalarType();
830
831	bool Any = false;
832	SmallVector<Constant*, 32> NewIdxs;
833	for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
834	if ((i == 1 ||
835	!isa<StructType>(Val: GetElementPtrInst::getIndexedType(
836	Ty: SrcElemTy, IdxList: Ops.slice(N: 1, M: i - 1)))) &&
837	Ops[i]->getType()->getScalarType() != IntIdxScalarTy) {
838	Any = true;
839	Type *NewType =
840	Ops[i]->getType()->isVectorTy() ? IntIdxTy : IntIdxScalarTy;
841	Constant *NewIdx = ConstantFoldCastOperand(
842	Opcode: CastInst::getCastOpcode(Val: Ops[i], SrcIsSigned: true, Ty: NewType, DstIsSigned: true), C: Ops[i], DestTy: NewType,
843	DL);
844	if (!NewIdx)
845	return nullptr;
846	NewIdxs.push_back(Elt: NewIdx);
847	} else
848	NewIdxs.push_back(Elt: Ops[i]);
849	}
850
851	if (!Any)
852	return nullptr;
853
854	Constant *C = ConstantExpr::getGetElementPtr(
855	Ty: SrcElemTy, C: Ops[0], IdxList: NewIdxs, InBounds, InRangeIndex);
856	return ConstantFoldConstant(C, DL, TLI);
857	}
858
859	/// If we can symbolically evaluate the GEP constant expression, do so.
860	Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP,
861	ArrayRef<Constant *> Ops,
862	const DataLayout &DL,
863	const TargetLibraryInfo *TLI) {
864	const GEPOperator *InnermostGEP = GEP;
865	bool InBounds = GEP->isInBounds();
866
867	Type *SrcElemTy = GEP->getSourceElementType();
868	Type *ResElemTy = GEP->getResultElementType();
869	Type *ResTy = GEP->getType();
870	if (!SrcElemTy->isSized() || isa<ScalableVectorType>(Val: SrcElemTy))
871	return nullptr;
872
873	if (Constant *C = CastGEPIndices(SrcElemTy, Ops, ResultTy: ResTy,
874	InBounds: GEP->isInBounds(), InRangeIndex: GEP->getInRangeIndex(),
875	DL, TLI))
876	return C;
877
878	Constant *Ptr = Ops[0];
879	if (!Ptr->getType()->isPointerTy())
880	return nullptr;
881
882	Type *IntIdxTy = DL.getIndexType(PtrTy: Ptr->getType());
883
884	for (unsigned i = 1, e = Ops.size(); i != e; ++i)
885	if (!isa<ConstantInt>(Val: Ops[i]))
886	return nullptr;
887
888	unsigned BitWidth = DL.getTypeSizeInBits(Ty: IntIdxTy);
889	APInt Offset = APInt(
890	BitWidth,
891	DL.getIndexedOffsetInType(
892	ElemTy: SrcElemTy, Indices: ArrayRef((Value *const *)Ops.data() + 1, Ops.size() - 1)));
893
894	// If this is a GEP of a GEP, fold it all into a single GEP.
895	while (auto *GEP = dyn_cast<GEPOperator>(Val: Ptr)) {
896	InnermostGEP = GEP;
897	InBounds &= GEP->isInBounds();
898
899	SmallVector<Value *, 4> NestedOps(llvm::drop_begin(RangeOrContainer: GEP->operands()));
900
901	// Do not try the incorporate the sub-GEP if some index is not a number.
902	bool AllConstantInt = true;
903	for (Value *NestedOp : NestedOps)
904	if (!isa<ConstantInt>(Val: NestedOp)) {
905	AllConstantInt = false;
906	break;
907	}
908	if (!AllConstantInt)
909	break;
910
911	Ptr = cast<Constant>(Val: GEP->getOperand(i_nocapture: 0));
912	SrcElemTy = GEP->getSourceElementType();
913	Offset += APInt(BitWidth, DL.getIndexedOffsetInType(ElemTy: SrcElemTy, Indices: NestedOps));
914	}
915
916	// If the base value for this address is a literal integer value, fold the
917	// getelementptr to the resulting integer value casted to the pointer type.
918	APInt BasePtr(BitWidth, 0);
919	if (auto *CE = dyn_cast<ConstantExpr>(Val: Ptr)) {
920	if (CE->getOpcode() == Instruction::IntToPtr) {
921	if (auto *Base = dyn_cast<ConstantInt>(Val: CE->getOperand(i_nocapture: 0)))
922	BasePtr = Base->getValue().zextOrTrunc(width: BitWidth);
923	}
924	}
925
926	auto *PTy = cast<PointerType>(Val: Ptr->getType());
927	if ((Ptr->isNullValue() || BasePtr != 0) &&
928	!DL.isNonIntegralPointerType(PT: PTy)) {
929	Constant *C = ConstantInt::get(Context&: Ptr->getContext(), V: Offset + BasePtr);
930	return ConstantExpr::getIntToPtr(C, Ty: ResTy);
931	}
932
933	// Otherwise form a regular getelementptr. Recompute the indices so that
934	// we eliminate over-indexing of the notional static type array bounds.
935	// This makes it easy to determine if the getelementptr is "inbounds".
936
937	// For GEPs of GlobalValues, use the value type, otherwise use an i8 GEP.
938	if (auto *GV = dyn_cast<GlobalValue>(Val: Ptr))
939	SrcElemTy = GV->getValueType();
940	else
941	SrcElemTy = Type::getInt8Ty(C&: Ptr->getContext());
942
943	if (!SrcElemTy->isSized())
944	return nullptr;
945
946	Type *ElemTy = SrcElemTy;
947	SmallVector<APInt> Indices = DL.getGEPIndicesForOffset(ElemTy, Offset);
948	if (Offset != 0)
949	return nullptr;
950
951	// Try to add additional zero indices to reach the desired result element
952	// type.
953	// TODO: Should we avoid extra zero indices if ResElemTy can't be reached and
954	// we'll have to insert a bitcast anyway?
955	while (ElemTy != ResElemTy) {
956	Type *NextTy = GetElementPtrInst::getTypeAtIndex(Ty: ElemTy, Idx: (uint64_t)0);
957	if (!NextTy)
958	break;
959
960	Indices.push_back(Elt: APInt::getZero(numBits: isa<StructType>(Val: ElemTy) ? 32 : BitWidth));
961	ElemTy = NextTy;
962	}
963
964	SmallVector<Constant *, 32> NewIdxs;
965	for (const APInt &Index : Indices)
966	NewIdxs.push_back(Elt: ConstantInt::get(
967	Ty: Type::getIntNTy(C&: Ptr->getContext(), N: Index.getBitWidth()), V: Index));
968
969	// Preserve the inrange index from the innermost GEP if possible. We must
970	// have calculated the same indices up to and including the inrange index.
971	std::optional<unsigned> InRangeIndex;
972	if (std::optional<unsigned> LastIRIndex = InnermostGEP->getInRangeIndex())
973	if (SrcElemTy == InnermostGEP->getSourceElementType() &&
974	NewIdxs.size() > *LastIRIndex) {
975	InRangeIndex = LastIRIndex;
976	for (unsigned I = 0; I <= *LastIRIndex; ++I)
977	if (NewIdxs[I] != InnermostGEP->getOperand(i_nocapture: I + 1))
978	return nullptr;
979	}
980
981	// Create a GEP.
982	return ConstantExpr::getGetElementPtr(Ty: SrcElemTy, C: Ptr, IdxList: NewIdxs, InBounds,
983	InRangeIndex);
984	}
985
986	/// Attempt to constant fold an instruction with the
987	/// specified opcode and operands. If successful, the constant result is
988	/// returned, if not, null is returned. Note that this function can fail when
989	/// attempting to fold instructions like loads and stores, which have no
990	/// constant expression form.
991	Constant *ConstantFoldInstOperandsImpl(const Value *InstOrCE, unsigned Opcode,
992	ArrayRef<Constant *> Ops,
993	const DataLayout &DL,
994	const TargetLibraryInfo *TLI) {
995	Type *DestTy = InstOrCE->getType();
996
997	if (Instruction::isUnaryOp(Opcode))
998	return ConstantFoldUnaryOpOperand(Opcode, Op: Ops[0], DL);
999
1000	if (Instruction::isBinaryOp(Opcode)) {
1001	switch (Opcode) {
1002	default:
1003	break;
1004	case Instruction::FAdd:
1005	case Instruction::FSub:
1006	case Instruction::FMul:
1007	case Instruction::FDiv:
1008	case Instruction::FRem:
1009	// Handle floating point instructions separately to account for denormals
1010	// TODO: If a constant expression is being folded rather than an
1011	// instruction, denormals will not be flushed/treated as zero
1012	if (const auto *I = dyn_cast<Instruction>(Val: InstOrCE)) {
1013	return ConstantFoldFPInstOperands(Opcode, LHS: Ops[0], RHS: Ops[1], DL, I);
1014	}
1015	}
1016	return ConstantFoldBinaryOpOperands(Opcode, LHS: Ops[0], RHS: Ops[1], DL);
1017	}
1018
1019	if (Instruction::isCast(Opcode))
1020	return ConstantFoldCastOperand(Opcode, C: Ops[0], DestTy, DL);
1021
1022	if (auto *GEP = dyn_cast<GEPOperator>(Val: InstOrCE)) {
1023	Type *SrcElemTy = GEP->getSourceElementType();
1024	if (!ConstantExpr::isSupportedGetElementPtr(SrcElemTy))
1025	return nullptr;
1026
1027	if (Constant *C = SymbolicallyEvaluateGEP(GEP, Ops, DL, TLI))
1028	return C;
1029
1030	return ConstantExpr::getGetElementPtr(Ty: SrcElemTy, C: Ops[0], IdxList: Ops.slice(N: 1),
1031	InBounds: GEP->isInBounds(),
1032	InRangeIndex: GEP->getInRangeIndex());
1033	}
1034
1035	if (auto *CE = dyn_cast<ConstantExpr>(Val: InstOrCE)) {
1036	if (CE->isCompare())
1037	return ConstantFoldCompareInstOperands(Predicate: CE->getPredicate(), LHS: Ops[0], RHS: Ops[1],
1038	DL, TLI);
1039	return CE->getWithOperands(Ops);
1040	}
1041
1042	switch (Opcode) {
1043	default: return nullptr;
1044	case Instruction::ICmp:
1045	case Instruction::FCmp: {
1046	auto *C = cast<CmpInst>(Val: InstOrCE);
1047	return ConstantFoldCompareInstOperands(Predicate: C->getPredicate(), LHS: Ops[0], RHS: Ops[1],
1048	DL, TLI, I: C);
1049	}
1050	case Instruction::Freeze:
1051	return isGuaranteedNotToBeUndefOrPoison(V: Ops[0]) ? Ops[0] : nullptr;
1052	case Instruction::Call:
1053	if (auto *F = dyn_cast<Function>(Val: Ops.back())) {
1054	const auto *Call = cast<CallBase>(Val: InstOrCE);
1055	if (canConstantFoldCallTo(Call, F))
1056	return ConstantFoldCall(Call, F, Operands: Ops.slice(N: 0, M: Ops.size() - 1), TLI);
1057	}
1058	return nullptr;
1059	case Instruction::Select:
1060	return ConstantFoldSelectInstruction(Cond: Ops[0], V1: Ops[1], V2: Ops[2]);
1061	case Instruction::ExtractElement:
1062	return ConstantExpr::getExtractElement(Vec: Ops[0], Idx: Ops[1]);
1063	case Instruction::ExtractValue:
1064	return ConstantFoldExtractValueInstruction(
1065	Agg: Ops[0], Idxs: cast<ExtractValueInst>(Val: InstOrCE)->getIndices());
1066	case Instruction::InsertElement:
1067	return ConstantExpr::getInsertElement(Vec: Ops[0], Elt: Ops[1], Idx: Ops[2]);
1068	case Instruction::InsertValue:
1069	return ConstantFoldInsertValueInstruction(
1070	Agg: Ops[0], Val: Ops[1], Idxs: cast<InsertValueInst>(Val: InstOrCE)->getIndices());
1071	case Instruction::ShuffleVector:
1072	return ConstantExpr::getShuffleVector(
1073	V1: Ops[0], V2: Ops[1], Mask: cast<ShuffleVectorInst>(Val: InstOrCE)->getShuffleMask());
1074	case Instruction::Load: {
1075	const auto *LI = dyn_cast<LoadInst>(Val: InstOrCE);
1076	if (LI->isVolatile())
1077	return nullptr;
1078	return ConstantFoldLoadFromConstPtr(C: Ops[0], Ty: LI->getType(), DL);
1079	}
1080	}
1081	}
1082
1083	} // end anonymous namespace
1084
1085	//===----------------------------------------------------------------------===//
1086	// Constant Folding public APIs
1087	//===----------------------------------------------------------------------===//
1088
1089	namespace {
1090
1091	Constant *
1092	ConstantFoldConstantImpl(const Constant *C, const DataLayout &DL,
1093	const TargetLibraryInfo *TLI,
1094	SmallDenseMap<Constant *, Constant *> &FoldedOps) {
1095	if (!isa<ConstantVector>(Val: C) && !isa<ConstantExpr>(Val: C))
1096	return const_cast<Constant *>(C);
1097
1098	SmallVector<Constant *, 8> Ops;
1099	for (const Use &OldU : C->operands()) {
1100	Constant *OldC = cast<Constant>(Val: &OldU);
1101	Constant *NewC = OldC;
1102	// Recursively fold the ConstantExpr's operands. If we have already folded
1103	// a ConstantExpr, we don't have to process it again.
1104	if (isa<ConstantVector>(Val: OldC) || isa<ConstantExpr>(Val: OldC)) {
1105	auto It = FoldedOps.find(Val: OldC);
1106	if (It == FoldedOps.end()) {
1107	NewC = ConstantFoldConstantImpl(C: OldC, DL, TLI, FoldedOps);
1108	FoldedOps.insert(KV: {OldC, NewC});
1109	} else {
1110	NewC = It->second;
1111	}
1112	}
1113	Ops.push_back(Elt: NewC);
1114	}
1115
1116	if (auto *CE = dyn_cast<ConstantExpr>(Val: C)) {
1117	if (Constant *Res =
1118	ConstantFoldInstOperandsImpl(InstOrCE: CE, Opcode: CE->getOpcode(), Ops, DL, TLI))
1119	return Res;
1120	return const_cast<Constant *>(C);
1121	}
1122
1123	assert(isa<ConstantVector>(C));
1124	return ConstantVector::get(V: Ops);
1125	}
1126
1127	} // end anonymous namespace
1128
1129	Constant *llvm::ConstantFoldInstruction(Instruction *I, const DataLayout &DL,
1130	const TargetLibraryInfo *TLI) {
1131	// Handle PHI nodes quickly here...
1132	if (auto *PN = dyn_cast<PHINode>(Val: I)) {
1133	Constant *CommonValue = nullptr;
1134
1135	SmallDenseMap<Constant *, Constant *> FoldedOps;
1136	for (Value *Incoming : PN->incoming_values()) {
1137	// If the incoming value is undef then skip it. Note that while we could
1138	// skip the value if it is equal to the phi node itself we choose not to
1139	// because that would break the rule that constant folding only applies if
1140	// all operands are constants.
1141	if (isa<UndefValue>(Val: Incoming))
1142	continue;
1143	// If the incoming value is not a constant, then give up.
1144	auto *C = dyn_cast<Constant>(Val: Incoming);
1145	if (!C)
1146	return nullptr;
1147	// Fold the PHI's operands.
1148	C = ConstantFoldConstantImpl(C, DL, TLI, FoldedOps);
1149	// If the incoming value is a different constant to
1150	// the one we saw previously, then give up.
1151	if (CommonValue && C != CommonValue)
1152	return nullptr;
1153	CommonValue = C;
1154	}
1155
1156	// If we reach here, all incoming values are the same constant or undef.
1157	return CommonValue ? CommonValue : UndefValue::get(T: PN->getType());
1158	}
1159
1160	// Scan the operand list, checking to see if they are all constants, if so,
1161	// hand off to ConstantFoldInstOperandsImpl.
1162	if (!all_of(Range: I->operands(), P: [](Use &U) { return isa<Constant>(Val: U); }))
1163	return nullptr;
1164
1165	SmallDenseMap<Constant *, Constant *> FoldedOps;
1166	SmallVector<Constant *, 8> Ops;
1167	for (const Use &OpU : I->operands()) {
1168	auto *Op = cast<Constant>(Val: &OpU);
1169	// Fold the Instruction's operands.
1170	Op = ConstantFoldConstantImpl(C: Op, DL, TLI, FoldedOps);
1171	Ops.push_back(Elt: Op);
1172	}
1173
1174	return ConstantFoldInstOperands(I, Ops, DL, TLI);
1175	}
1176
1177	Constant *llvm::ConstantFoldConstant(const Constant *C, const DataLayout &DL,
1178	const TargetLibraryInfo *TLI) {
1179	SmallDenseMap<Constant *, Constant *> FoldedOps;
1180	return ConstantFoldConstantImpl(C, DL, TLI, FoldedOps);
1181	}
1182
1183	Constant *llvm::ConstantFoldInstOperands(Instruction *I,
1184	ArrayRef<Constant *> Ops,
1185	const DataLayout &DL,
1186	const TargetLibraryInfo *TLI) {
1187	return ConstantFoldInstOperandsImpl(InstOrCE: I, Opcode: I->getOpcode(), Ops, DL, TLI);
1188	}
1189
1190	Constant *llvm::ConstantFoldCompareInstOperands(
1191	unsigned IntPredicate, Constant *Ops0, Constant *Ops1, const DataLayout &DL,
1192	const TargetLibraryInfo *TLI, const Instruction *I) {
1193	CmpInst::Predicate Predicate = (CmpInst::Predicate)IntPredicate;
1194	// fold: icmp (inttoptr x), null -> icmp x, 0
1195	// fold: icmp null, (inttoptr x) -> icmp 0, x
1196	// fold: icmp (ptrtoint x), 0 -> icmp x, null
1197	// fold: icmp 0, (ptrtoint x) -> icmp null, x
1198	// fold: icmp (inttoptr x), (inttoptr y) -> icmp trunc/zext x, trunc/zext y
1199	// fold: icmp (ptrtoint x), (ptrtoint y) -> icmp x, y
1200	//
1201	// FIXME: The following comment is out of data and the DataLayout is here now.
1202	// ConstantExpr::getCompare cannot do this, because it doesn't have DL
1203	// around to know if bit truncation is happening.
1204	if (auto *CE0 = dyn_cast<ConstantExpr>(Val: Ops0)) {
1205	if (Ops1->isNullValue()) {
1206	if (CE0->getOpcode() == Instruction::IntToPtr) {
1207	Type *IntPtrTy = DL.getIntPtrType(CE0->getType());
1208	// Convert the integer value to the right size to ensure we get the
1209	// proper extension or truncation.
1210	if (Constant *C = ConstantFoldIntegerCast(C: CE0->getOperand(i_nocapture: 0), DestTy: IntPtrTy,
1211	/*IsSigned*/ false, DL)) {
1212	Constant *Null = Constant::getNullValue(Ty: C->getType());
1213	return ConstantFoldCompareInstOperands(IntPredicate: Predicate, Ops0: C, Ops1: Null, DL, TLI);
1214	}
1215	}
1216
1217	// Only do this transformation if the int is intptrty in size, otherwise
1218	// there is a truncation or extension that we aren't modeling.
1219	if (CE0->getOpcode() == Instruction::PtrToInt) {
1220	Type *IntPtrTy = DL.getIntPtrType(CE0->getOperand(i_nocapture: 0)->getType());
1221	if (CE0->getType() == IntPtrTy) {
1222	Constant *C = CE0->getOperand(i_nocapture: 0);
1223	Constant *Null = Constant::getNullValue(Ty: C->getType());
1224	return ConstantFoldCompareInstOperands(IntPredicate: Predicate, Ops0: C, Ops1: Null, DL, TLI);
1225	}
1226	}
1227	}
1228
1229	if (auto *CE1 = dyn_cast<ConstantExpr>(Val: Ops1)) {
1230	if (CE0->getOpcode() == CE1->getOpcode()) {
1231	if (CE0->getOpcode() == Instruction::IntToPtr) {
1232	Type *IntPtrTy = DL.getIntPtrType(CE0->getType());
1233
1234	// Convert the integer value to the right size to ensure we get the
1235	// proper extension or truncation.
1236	Constant *C0 = ConstantFoldIntegerCast(C: CE0->getOperand(i_nocapture: 0), DestTy: IntPtrTy,
1237	/*IsSigned*/ false, DL);
1238	Constant *C1 = ConstantFoldIntegerCast(C: CE1->getOperand(i_nocapture: 0), DestTy: IntPtrTy,
1239	/*IsSigned*/ false, DL);
1240	if (C0 && C1)
1241	return ConstantFoldCompareInstOperands(IntPredicate: Predicate, Ops0: C0, Ops1: C1, DL, TLI);
1242	}
1243
1244	// Only do this transformation if the int is intptrty in size, otherwise
1245	// there is a truncation or extension that we aren't modeling.
1246	if (CE0->getOpcode() == Instruction::PtrToInt) {
1247	Type *IntPtrTy = DL.getIntPtrType(CE0->getOperand(i_nocapture: 0)->getType());
1248	if (CE0->getType() == IntPtrTy &&
1249	CE0->getOperand(i_nocapture: 0)->getType() == CE1->getOperand(i_nocapture: 0)->getType()) {
1250	return ConstantFoldCompareInstOperands(
1251	IntPredicate: Predicate, Ops0: CE0->getOperand(i_nocapture: 0), Ops1: CE1->getOperand(i_nocapture: 0), DL, TLI);
1252	}
1253	}
1254	}
1255	}
1256
1257	// Convert pointer comparison (base+offset1) pred (base+offset2) into
1258	// offset1 pred offset2, for the case where the offset is inbounds. This
1259	// only works for equality and unsigned comparison, as inbounds permits
1260	// crossing the sign boundary. However, the offset comparison itself is
1261	// signed.
1262	if (Ops0->getType()->isPointerTy() && !ICmpInst::isSigned(predicate: Predicate)) {
1263	unsigned IndexWidth = DL.getIndexTypeSizeInBits(Ty: Ops0->getType());
1264	APInt Offset0(IndexWidth, 0);
1265	Value *Stripped0 =
1266	Ops0->stripAndAccumulateInBoundsConstantOffsets(DL, Offset&: Offset0);
1267	APInt Offset1(IndexWidth, 0);
1268	Value *Stripped1 =
1269	Ops1->stripAndAccumulateInBoundsConstantOffsets(DL, Offset&: Offset1);
1270	if (Stripped0 == Stripped1)
1271	return ConstantExpr::getCompare(
1272	pred: ICmpInst::getSignedPredicate(pred: Predicate),
1273	C1: ConstantInt::get(Context&: CE0->getContext(), V: Offset0),
1274	C2: ConstantInt::get(Context&: CE0->getContext(), V: Offset1));
1275	}
1276	} else if (isa<ConstantExpr>(Val: Ops1)) {
1277	// If RHS is a constant expression, but the left side isn't, swap the
1278	// operands and try again.
1279	Predicate = ICmpInst::getSwappedPredicate(pred: Predicate);
1280	return ConstantFoldCompareInstOperands(IntPredicate: Predicate, Ops0: Ops1, Ops1: Ops0, DL, TLI);
1281	}
1282
1283	// Flush any denormal constant float input according to denormal handling
1284	// mode.
1285	Ops0 = FlushFPConstant(Operand: Ops0, I, /* IsOutput */ false);
1286	if (!Ops0)
1287	return nullptr;
1288	Ops1 = FlushFPConstant(Operand: Ops1, I, /* IsOutput */ false);
1289	if (!Ops1)
1290	return nullptr;
1291
1292	return ConstantExpr::getCompare(pred: Predicate, C1: Ops0, C2: Ops1);
1293	}
1294
1295	Constant *llvm::ConstantFoldUnaryOpOperand(unsigned Opcode, Constant *Op,
1296	const DataLayout &DL) {
1297	assert(Instruction::isUnaryOp(Opcode));
1298
1299	return ConstantFoldUnaryInstruction(Opcode, V: Op);
1300	}
1301
1302	Constant *llvm::ConstantFoldBinaryOpOperands(unsigned Opcode, Constant *LHS,
1303	Constant *RHS,
1304	const DataLayout &DL) {
1305	assert(Instruction::isBinaryOp(Opcode));
1306	if (isa<ConstantExpr>(Val: LHS) || isa<ConstantExpr>(Val: RHS))
1307	if (Constant *C = SymbolicallyEvaluateBinop(Opc: Opcode, Op0: LHS, Op1: RHS, DL))
1308	return C;
1309
1310	if (ConstantExpr::isDesirableBinOp(Opcode))
1311	return ConstantExpr::get(Opcode, C1: LHS, C2: RHS);
1312	return ConstantFoldBinaryInstruction(Opcode, V1: LHS, V2: RHS);
1313	}
1314
1315	Constant *llvm::FlushFPConstant(Constant *Operand, const Instruction *I,
1316	bool IsOutput) {
1317	if (!I || !I->getParent() || !I->getFunction())
1318	return Operand;
1319
1320	ConstantFP *CFP = dyn_cast<ConstantFP>(Val: Operand);
1321	if (!CFP)
1322	return Operand;
1323
1324	const APFloat &APF = CFP->getValueAPF();
1325	// TODO: Should this canonicalize nans?
1326	if (!APF.isDenormal())
1327	return Operand;
1328
1329	Type *Ty = CFP->getType();
1330	DenormalMode DenormMode =
1331	I->getFunction()->getDenormalMode(FPType: Ty->getFltSemantics());
1332	DenormalMode::DenormalModeKind Mode =
1333	IsOutput ? DenormMode.Output : DenormMode.Input;
1334	switch (Mode) {
1335	default:
1336	llvm_unreachable("unknown denormal mode");
1337	case DenormalMode::Dynamic:
1338	return nullptr;
1339	case DenormalMode::IEEE:
1340	return Operand;
1341	case DenormalMode::PreserveSign:
1342	if (APF.isDenormal()) {
1343	return ConstantFP::get(
1344	Context&: Ty->getContext(),
1345	V: APFloat::getZero(Sem: Ty->getFltSemantics(), Negative: APF.isNegative()));
1346	}
1347	return Operand;
1348	case DenormalMode::PositiveZero:
1349	if (APF.isDenormal()) {
1350	return ConstantFP::get(Context&: Ty->getContext(),
1351	V: APFloat::getZero(Sem: Ty->getFltSemantics(), Negative: false));
1352	}
1353	return Operand;
1354	}
1355	return Operand;
1356	}
1357
1358	Constant *llvm::ConstantFoldFPInstOperands(unsigned Opcode, Constant *LHS,
1359	Constant *RHS, const DataLayout &DL,
1360	const Instruction *I) {
1361	if (Instruction::isBinaryOp(Opcode)) {
1362	// Flush denormal inputs if needed.
1363	Constant *Op0 = FlushFPConstant(Operand: LHS, I, /* IsOutput */ false);
1364	if (!Op0)
1365	return nullptr;
1366	Constant *Op1 = FlushFPConstant(Operand: RHS, I, /* IsOutput */ false);
1367	if (!Op1)
1368	return nullptr;
1369
1370	// Calculate constant result.
1371	Constant *C = ConstantFoldBinaryOpOperands(Opcode, LHS: Op0, RHS: Op1, DL);
1372	if (!C)
1373	return nullptr;
1374
1375	// Flush denormal output if needed.
1376	return FlushFPConstant(Operand: C, I, /* IsOutput */ true);
1377	}
1378	// If instruction lacks a parent/function and the denormal mode cannot be
1379	// determined, use the default (IEEE).
1380	return ConstantFoldBinaryOpOperands(Opcode, LHS, RHS, DL);
1381	}
1382
1383	Constant *llvm::ConstantFoldCastOperand(unsigned Opcode, Constant *C,
1384	Type *DestTy, const DataLayout &DL) {
1385	assert(Instruction::isCast(Opcode));
1386	switch (Opcode) {
1387	default:
1388	llvm_unreachable("Missing case");
1389	case Instruction::PtrToInt:
1390	if (auto *CE = dyn_cast<ConstantExpr>(Val: C)) {
1391	Constant *FoldedValue = nullptr;
1392	// If the input is a inttoptr, eliminate the pair. This requires knowing
1393	// the width of a pointer, so it can't be done in ConstantExpr::getCast.
1394	if (CE->getOpcode() == Instruction::IntToPtr) {
1395	// zext/trunc the inttoptr to pointer size.
1396	FoldedValue = ConstantFoldIntegerCast(C: CE->getOperand(i_nocapture: 0),
1397	DestTy: DL.getIntPtrType(CE->getType()),
1398	/*IsSigned=*/false, DL);
1399	} else if (auto *GEP = dyn_cast<GEPOperator>(Val: CE)) {
1400	// If we have GEP, we can perform the following folds:
1401	// (ptrtoint (gep null, x)) -> x
1402	// (ptrtoint (gep (gep null, x), y) -> x + y, etc.
1403	unsigned BitWidth = DL.getIndexTypeSizeInBits(Ty: GEP->getType());
1404	APInt BaseOffset(BitWidth, 0);
1405	auto *Base = cast<Constant>(Val: GEP->stripAndAccumulateConstantOffsets(
1406	DL, Offset&: BaseOffset, /*AllowNonInbounds=*/true));
1407	if (Base->isNullValue()) {
1408	FoldedValue = ConstantInt::get(Context&: CE->getContext(), V: BaseOffset);
1409	} else {
1410	// ptrtoint (gep i8, Ptr, (sub 0, V)) -> sub (ptrtoint Ptr), V
1411	if (GEP->getNumIndices() == 1 &&
1412	GEP->getSourceElementType()->isIntegerTy(Bitwidth: 8)) {
1413	auto *Ptr = cast<Constant>(Val: GEP->getPointerOperand());
1414	auto *Sub = dyn_cast<ConstantExpr>(Val: GEP->getOperand(i_nocapture: 1));
1415	Type *IntIdxTy = DL.getIndexType(PtrTy: Ptr->getType());
1416	if (Sub && Sub->getType() == IntIdxTy &&
1417	Sub->getOpcode() == Instruction::Sub &&
1418	Sub->getOperand(i_nocapture: 0)->isNullValue())
1419	FoldedValue = ConstantExpr::getSub(
1420	C1: ConstantExpr::getPtrToInt(C: Ptr, Ty: IntIdxTy), C2: Sub->getOperand(i_nocapture: 1));
1421	}
1422	}
1423	}
1424	if (FoldedValue) {
1425	// Do a zext or trunc to get to the ptrtoint dest size.
1426	return ConstantFoldIntegerCast(C: FoldedValue, DestTy, /*IsSigned=*/false,
1427	DL);
1428	}
1429	}
1430	break;
1431	case Instruction::IntToPtr:
1432	// If the input is a ptrtoint, turn the pair into a ptr to ptr bitcast if
1433	// the int size is >= the ptr size and the address spaces are the same.
1434	// This requires knowing the width of a pointer, so it can't be done in
1435	// ConstantExpr::getCast.
1436	if (auto *CE = dyn_cast<ConstantExpr>(Val: C)) {
1437	if (CE->getOpcode() == Instruction::PtrToInt) {
1438	Constant *SrcPtr = CE->getOperand(i_nocapture: 0);
1439	unsigned SrcPtrSize = DL.getPointerTypeSizeInBits(SrcPtr->getType());
1440	unsigned MidIntSize = CE->getType()->getScalarSizeInBits();
1441
1442	if (MidIntSize >= SrcPtrSize) {
1443	unsigned SrcAS = SrcPtr->getType()->getPointerAddressSpace();
1444	if (SrcAS == DestTy->getPointerAddressSpace())
1445	return FoldBitCast(C: CE->getOperand(i_nocapture: 0), DestTy, DL);
1446	}
1447	}
1448	}
1449	break;
1450	case Instruction::Trunc:
1451	case Instruction::ZExt:
1452	case Instruction::SExt:
1453	case Instruction::FPTrunc:
1454	case Instruction::FPExt:
1455	case Instruction::UIToFP:
1456	case Instruction::SIToFP:
1457	case Instruction::FPToUI:
1458	case Instruction::FPToSI:
1459	case Instruction::AddrSpaceCast:
1460	break;
1461	case Instruction::BitCast:
1462	return FoldBitCast(C, DestTy, DL);
1463	}
1464
1465	if (ConstantExpr::isDesirableCastOp(Opcode))
1466	return ConstantExpr::getCast(ops: Opcode, C, Ty: DestTy);
1467	return ConstantFoldCastInstruction(opcode: Opcode, V: C, DestTy);
1468	}
1469
1470	Constant *llvm::ConstantFoldIntegerCast(Constant *C, Type *DestTy,
1471	bool IsSigned, const DataLayout &DL) {
1472	Type *SrcTy = C->getType();
1473	if (SrcTy == DestTy)
1474	return C;
1475	if (SrcTy->getScalarSizeInBits() > DestTy->getScalarSizeInBits())
1476	return ConstantFoldCastOperand(Opcode: Instruction::Trunc, C, DestTy, DL);
1477	if (IsSigned)
1478	return ConstantFoldCastOperand(Opcode: Instruction::SExt, C, DestTy, DL);
1479	return ConstantFoldCastOperand(Opcode: Instruction::ZExt, C, DestTy, DL);
1480	}
1481
1482	//===----------------------------------------------------------------------===//
1483	// Constant Folding for Calls
1484	//
1485
1486	bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
1487	if (Call->isNoBuiltin())
1488	return false;
1489	if (Call->getFunctionType() != F->getFunctionType())
1490	return false;
1491	switch (F->getIntrinsicID()) {
1492	// Operations that do not operate floating-point numbers and do not depend on
1493	// FP environment can be folded even in strictfp functions.
1494	case Intrinsic::bswap:
1495	case Intrinsic::ctpop:
1496	case Intrinsic::ctlz:
1497	case Intrinsic::cttz:
1498	case Intrinsic::fshl:
1499	case Intrinsic::fshr:
1500	case Intrinsic::launder_invariant_group:
1501	case Intrinsic::strip_invariant_group:
1502	case Intrinsic::masked_load:
1503	case Intrinsic::get_active_lane_mask:
1504	case Intrinsic::abs:
1505	case Intrinsic::smax:
1506	case Intrinsic::smin:
1507	case Intrinsic::umax:
1508	case Intrinsic::umin:
1509	case Intrinsic::sadd_with_overflow:
1510	case Intrinsic::uadd_with_overflow:
1511	case Intrinsic::ssub_with_overflow:
1512	case Intrinsic::usub_with_overflow:
1513	case Intrinsic::smul_with_overflow:
1514	case Intrinsic::umul_with_overflow:
1515	case Intrinsic::sadd_sat:
1516	case Intrinsic::uadd_sat:
1517	case Intrinsic::ssub_sat:
1518	case Intrinsic::usub_sat:
1519	case Intrinsic::smul_fix:
1520	case Intrinsic::smul_fix_sat:
1521	case Intrinsic::bitreverse:
1522	case Intrinsic::is_constant:
1523	case Intrinsic::vector_reduce_add:
1524	case Intrinsic::vector_reduce_mul:
1525	case Intrinsic::vector_reduce_and:
1526	case Intrinsic::vector_reduce_or:
1527	case Intrinsic::vector_reduce_xor:
1528	case Intrinsic::vector_reduce_smin:
1529	case Intrinsic::vector_reduce_smax:
1530	case Intrinsic::vector_reduce_umin:
1531	case Intrinsic::vector_reduce_umax:
1532	// Target intrinsics
1533	case Intrinsic::amdgcn_perm:
1534	case Intrinsic::amdgcn_wave_reduce_umin:
1535	case Intrinsic::amdgcn_wave_reduce_umax:
1536	case Intrinsic::amdgcn_s_wqm:
1537	case Intrinsic::amdgcn_s_quadmask:
1538	case Intrinsic::amdgcn_s_bitreplicate:
1539	case Intrinsic::arm_mve_vctp8:
1540	case Intrinsic::arm_mve_vctp16:
1541	case Intrinsic::arm_mve_vctp32:
1542	case Intrinsic::arm_mve_vctp64:
1543	case Intrinsic::aarch64_sve_convert_from_svbool:
1544	// WebAssembly float semantics are always known
1545	case Intrinsic::wasm_trunc_signed:
1546	case Intrinsic::wasm_trunc_unsigned:
1547	return true;
1548
1549	// Floating point operations cannot be folded in strictfp functions in
1550	// general case. They can be folded if FP environment is known to compiler.
1551	case Intrinsic::minnum:
1552	case Intrinsic::maxnum:
1553	case Intrinsic::minimum:
1554	case Intrinsic::maximum:
1555	case Intrinsic::log:
1556	case Intrinsic::log2:
1557	case Intrinsic::log10:
1558	case Intrinsic::exp:
1559	case Intrinsic::exp2:
1560	case Intrinsic::exp10:
1561	case Intrinsic::sqrt:
1562	case Intrinsic::sin:
1563	case Intrinsic::cos:
1564	case Intrinsic::pow:
1565	case Intrinsic::powi:
1566	case Intrinsic::ldexp:
1567	case Intrinsic::fma:
1568	case Intrinsic::fmuladd:
1569	case Intrinsic::frexp:
1570	case Intrinsic::fptoui_sat:
1571	case Intrinsic::fptosi_sat:
1572	case Intrinsic::convert_from_fp16:
1573	case Intrinsic::convert_to_fp16:
1574	case Intrinsic::amdgcn_cos:
1575	case Intrinsic::amdgcn_cubeid:
1576	case Intrinsic::amdgcn_cubema:
1577	case Intrinsic::amdgcn_cubesc:
1578	case Intrinsic::amdgcn_cubetc:
1579	case Intrinsic::amdgcn_fmul_legacy:
1580	case Intrinsic::amdgcn_fma_legacy:
1581	case Intrinsic::amdgcn_fract:
1582	case Intrinsic::amdgcn_sin:
1583	// The intrinsics below depend on rounding mode in MXCSR.
1584	case Intrinsic::x86_sse_cvtss2si:
1585	case Intrinsic::x86_sse_cvtss2si64:
1586	case Intrinsic::x86_sse_cvttss2si:
1587	case Intrinsic::x86_sse_cvttss2si64:
1588	case Intrinsic::x86_sse2_cvtsd2si:
1589	case Intrinsic::x86_sse2_cvtsd2si64:
1590	case Intrinsic::x86_sse2_cvttsd2si:
1591	case Intrinsic::x86_sse2_cvttsd2si64:
1592	case Intrinsic::x86_avx512_vcvtss2si32:
1593	case Intrinsic::x86_avx512_vcvtss2si64:
1594	case Intrinsic::x86_avx512_cvttss2si:
1595	case Intrinsic::x86_avx512_cvttss2si64:
1596	case Intrinsic::x86_avx512_vcvtsd2si32:
1597	case Intrinsic::x86_avx512_vcvtsd2si64:
1598	case Intrinsic::x86_avx512_cvttsd2si:
1599	case Intrinsic::x86_avx512_cvttsd2si64:
1600	case Intrinsic::x86_avx512_vcvtss2usi32:
1601	case Intrinsic::x86_avx512_vcvtss2usi64:
1602	case Intrinsic::x86_avx512_cvttss2usi:
1603	case Intrinsic::x86_avx512_cvttss2usi64:
1604	case Intrinsic::x86_avx512_vcvtsd2usi32:
1605	case Intrinsic::x86_avx512_vcvtsd2usi64:
1606	case Intrinsic::x86_avx512_cvttsd2usi:
1607	case Intrinsic::x86_avx512_cvttsd2usi64:
1608	return !Call->isStrictFP();
1609
1610	// Sign operations are actually bitwise operations, they do not raise
1611	// exceptions even for SNANs.
1612	case Intrinsic::fabs:
1613	case Intrinsic::copysign:
1614	case Intrinsic::is_fpclass:
1615	// Non-constrained variants of rounding operations means default FP
1616	// environment, they can be folded in any case.
1617	case Intrinsic::ceil:
1618	case Intrinsic::floor:
1619	case Intrinsic::round:
1620	case Intrinsic::roundeven:
1621	case Intrinsic::trunc:
1622	case Intrinsic::nearbyint:
1623	case Intrinsic::rint:
1624	case Intrinsic::canonicalize:
1625	// Constrained intrinsics can be folded if FP environment is known
1626	// to compiler.
1627	case Intrinsic::experimental_constrained_fma:
1628	case Intrinsic::experimental_constrained_fmuladd:
1629	case Intrinsic::experimental_constrained_fadd:
1630	case Intrinsic::experimental_constrained_fsub:
1631	case Intrinsic::experimental_constrained_fmul:
1632	case Intrinsic::experimental_constrained_fdiv:
1633	case Intrinsic::experimental_constrained_frem:
1634	case Intrinsic::experimental_constrained_ceil:
1635	case Intrinsic::experimental_constrained_floor:
1636	case Intrinsic::experimental_constrained_round:
1637	case Intrinsic::experimental_constrained_roundeven:
1638	case Intrinsic::experimental_constrained_trunc:
1639	case Intrinsic::experimental_constrained_nearbyint:
1640	case Intrinsic::experimental_constrained_rint:
1641	case Intrinsic::experimental_constrained_fcmp:
1642	case Intrinsic::experimental_constrained_fcmps:
1643	return true;
1644	default:
1645	return false;
1646	case Intrinsic::not_intrinsic: break;
1647	}
1648
1649	if (!F->hasName() || Call->isStrictFP())
1650	return false;
1651
1652	// In these cases, the check of the length is required. We don't want to
1653	// return true for a name like "cos\0blah" which strcmp would return equal to
1654	// "cos", but has length 8.
1655	StringRef Name = F->getName();
1656	switch (Name[0]) {
1657	default:
1658	return false;
1659	case 'a':
1660	return Name == "acos" || Name == "acosf" ||
1661	Name == "asin" || Name == "asinf" ||
1662	Name == "atan" || Name == "atanf" ||
1663	Name == "atan2" || Name == "atan2f";
1664	case 'c':
1665	return Name == "ceil" || Name == "ceilf" ||
1666	Name == "cos" || Name == "cosf" ||
1667	Name == "cosh" || Name == "coshf";
1668	case 'e':
1669	return Name == "exp" || Name == "expf" ||
1670	Name == "exp2" || Name == "exp2f";
1671	case 'f':
1672	return Name == "fabs" || Name == "fabsf" ||
1673	Name == "floor" || Name == "floorf" ||
1674	Name == "fmod" || Name == "fmodf";
1675	case 'l':
1676	return Name == "log" || Name == "logf" ||
1677	Name == "log2" || Name == "log2f" ||
1678	Name == "log10" || Name == "log10f";
1679	case 'n':
1680	return Name == "nearbyint" || Name == "nearbyintf";
1681	case 'p':
1682	return Name == "pow" || Name == "powf";
1683	case 'r':
1684	return Name == "remainder" || Name == "remainderf" ||
1685	Name == "rint" || Name == "rintf" ||
1686	Name == "round" || Name == "roundf";
1687	case 's':
1688	return Name == "sin" || Name == "sinf" ||
1689	Name == "sinh" || Name == "sinhf" ||
1690	Name == "sqrt" || Name == "sqrtf";
1691	case 't':
1692	return Name == "tan" || Name == "tanf" ||
1693	Name == "tanh" || Name == "tanhf" ||
1694	Name == "trunc" || Name == "truncf";
1695	case '_':
1696	// Check for various function names that get used for the math functions
1697	// when the header files are preprocessed with the macro
1698	// __FINITE_MATH_ONLY__ enabled.
1699	// The '12' here is the length of the shortest name that can match.
1700	// We need to check the size before looking at Name[1] and Name[2]
1701	// so we may as well check a limit that will eliminate mismatches.
1702	if (Name.size() < 12 || Name[1] != '_')
1703	return false;
1704	switch (Name[2]) {
1705	default:
1706	return false;
1707	case 'a':
1708	return Name == "__acos_finite" || Name == "__acosf_finite" ||
1709	Name == "__asin_finite" || Name == "__asinf_finite" ||
1710	Name == "__atan2_finite" || Name == "__atan2f_finite";
1711	case 'c':
1712	return Name == "__cosh_finite" || Name == "__coshf_finite";
1713	case 'e':
1714	return Name == "__exp_finite" || Name == "__expf_finite" ||
1715	Name == "__exp2_finite" || Name == "__exp2f_finite";
1716	case 'l':
1717	return Name == "__log_finite" || Name == "__logf_finite" ||
1718	Name == "__log10_finite" || Name == "__log10f_finite";
1719	case 'p':
1720	return Name == "__pow_finite" || Name == "__powf_finite";
1721	case 's':
1722	return Name == "__sinh_finite" || Name == "__sinhf_finite";
1723	}
1724	}
1725	}
1726
1727	namespace {
1728
1729	Constant *GetConstantFoldFPValue(double V, Type *Ty) {
1730	if (Ty->isHalfTy() || Ty->isFloatTy()) {
1731	APFloat APF(V);
1732	bool unused;
1733	APF.convert(ToSemantics: Ty->getFltSemantics(), RM: APFloat::rmNearestTiesToEven, losesInfo: &unused);
1734	return ConstantFP::get(Context&: Ty->getContext(), V: APF);
1735	}
1736	if (Ty->isDoubleTy())
1737	return ConstantFP::get(Context&: Ty->getContext(), V: APFloat(V));
1738	llvm_unreachable("Can only constant fold half/float/double");
1739	}
1740
1741	/// Clear the floating-point exception state.
1742	inline void llvm_fenv_clearexcept() {
1743	#if defined(HAVE_FENV_H) && HAVE_DECL_FE_ALL_EXCEPT
1744	feclearexcept(FE_ALL_EXCEPT);
1745	#endif
1746	errno = 0;
1747	}
1748
1749	/// Test if a floating-point exception was raised.
1750	inline bool llvm_fenv_testexcept() {
1751	int errno_val = errno;
1752	if (errno_val == ERANGE || errno_val == EDOM)
1753	return true;
1754	#if defined(HAVE_FENV_H) && HAVE_DECL_FE_ALL_EXCEPT && HAVE_DECL_FE_INEXACT
1755	if (fetestexcept(FE_ALL_EXCEPT & ~FE_INEXACT))
1756	return true;
1757	#endif
1758	return false;
1759	}
1760
1761	Constant *ConstantFoldFP(double (*NativeFP)(double), const APFloat &V,
1762	Type *Ty) {
1763	llvm_fenv_clearexcept();
1764	double Result = NativeFP(V.convertToDouble());
1765	if (llvm_fenv_testexcept()) {
1766	llvm_fenv_clearexcept();
1767	return nullptr;
1768	}
1769
1770	return GetConstantFoldFPValue(V: Result, Ty);
1771	}
1772
1773	Constant *ConstantFoldBinaryFP(double (*NativeFP)(double, double),
1774	const APFloat &V, const APFloat &W, Type *Ty) {
1775	llvm_fenv_clearexcept();
1776	double Result = NativeFP(V.convertToDouble(), W.convertToDouble());
1777	if (llvm_fenv_testexcept()) {
1778	llvm_fenv_clearexcept();
1779	return nullptr;
1780	}
1781
1782	return GetConstantFoldFPValue(V: Result, Ty);
1783	}
1784
1785	Constant *constantFoldVectorReduce(Intrinsic::ID IID, Constant *Op) {
1786	FixedVectorType *VT = dyn_cast<FixedVectorType>(Val: Op->getType());
1787	if (!VT)
1788	return nullptr;
1789
1790	// This isn't strictly necessary, but handle the special/common case of zero:
1791	// all integer reductions of a zero input produce zero.
1792	if (isa<ConstantAggregateZero>(Val: Op))
1793	return ConstantInt::get(Ty: VT->getElementType(), V: 0);
1794
1795	// This is the same as the underlying binops - poison propagates.
1796	if (isa<PoisonValue>(Val: Op) || Op->containsPoisonElement())
1797	return PoisonValue::get(T: VT->getElementType());
1798
1799	// TODO: Handle undef.
1800	if (!isa<ConstantVector>(Val: Op) && !isa<ConstantDataVector>(Val: Op))
1801	return nullptr;
1802
1803	auto *EltC = dyn_cast<ConstantInt>(Val: Op->getAggregateElement(Elt: 0U));
1804	if (!EltC)
1805	return nullptr;
1806
1807	APInt Acc = EltC->getValue();
1808	for (unsigned I = 1, E = VT->getNumElements(); I != E; I++) {
1809	if (!(EltC = dyn_cast<ConstantInt>(Val: Op->getAggregateElement(Elt: I))))
1810	return nullptr;
1811	const APInt &X = EltC->getValue();
1812	switch (IID) {
1813	case Intrinsic::vector_reduce_add:
1814	Acc = Acc + X;
1815	break;
1816	case Intrinsic::vector_reduce_mul:
1817	Acc = Acc * X;
1818	break;
1819	case Intrinsic::vector_reduce_and:
1820	Acc = Acc & X;
1821	break;
1822	case Intrinsic::vector_reduce_or:
1823	Acc = Acc | X;
1824	break;
1825	case Intrinsic::vector_reduce_xor:
1826	Acc = Acc ^ X;
1827	break;
1828	case Intrinsic::vector_reduce_smin:
1829	Acc = APIntOps::smin(A: Acc, B: X);
1830	break;
1831	case Intrinsic::vector_reduce_smax:
1832	Acc = APIntOps::smax(A: Acc, B: X);
1833	break;
1834	case Intrinsic::vector_reduce_umin:
1835	Acc = APIntOps::umin(A: Acc, B: X);
1836	break;
1837	case Intrinsic::vector_reduce_umax:
1838	Acc = APIntOps::umax(A: Acc, B: X);
1839	break;
1840	}
1841	}
1842
1843	return ConstantInt::get(Context&: Op->getContext(), V: Acc);
1844	}
1845
1846	/// Attempt to fold an SSE floating point to integer conversion of a constant
1847	/// floating point. If roundTowardZero is false, the default IEEE rounding is
1848	/// used (toward nearest, ties to even). This matches the behavior of the
1849	/// non-truncating SSE instructions in the default rounding mode. The desired
1850	/// integer type Ty is used to select how many bits are available for the
1851	/// result. Returns null if the conversion cannot be performed, otherwise
1852	/// returns the Constant value resulting from the conversion.
1853	Constant *ConstantFoldSSEConvertToInt(const APFloat &Val, bool roundTowardZero,
1854	Type *Ty, bool IsSigned) {
1855	// All of these conversion intrinsics form an integer of at most 64bits.
1856	unsigned ResultWidth = Ty->getIntegerBitWidth();
1857	assert(ResultWidth <= 64 &&
1858	"Can only constant fold conversions to 64 and 32 bit ints");
1859
1860	uint64_t UIntVal;
1861	bool isExact = false;
1862	APFloat::roundingMode mode = roundTowardZero? APFloat::rmTowardZero
1863	: APFloat::rmNearestTiesToEven;
1864	APFloat::opStatus status =
1865	Val.convertToInteger(Input: MutableArrayRef(UIntVal), Width: ResultWidth,
1866	IsSigned, RM: mode, IsExact: &isExact);
1867	if (status != APFloat::opOK &&
1868	(!roundTowardZero || status != APFloat::opInexact))
1869	return nullptr;
1870	return ConstantInt::get(Ty, V: UIntVal, IsSigned);
1871	}
1872
1873	double getValueAsDouble(ConstantFP *Op) {
1874	Type *Ty = Op->getType();
1875
1876	if (Ty->isBFloatTy() || Ty->isHalfTy() || Ty->isFloatTy() || Ty->isDoubleTy())
1877	return Op->getValueAPF().convertToDouble();
1878
1879	bool unused;
1880	APFloat APF = Op->getValueAPF();
1881	APF.convert(ToSemantics: APFloat::IEEEdouble(), RM: APFloat::rmNearestTiesToEven, losesInfo: &unused);
1882	return APF.convertToDouble();
1883	}
1884
1885	static bool getConstIntOrUndef(Value *Op, const APInt *&C) {
1886	if (auto *CI = dyn_cast<ConstantInt>(Val: Op)) {
1887	C = &CI->getValue();
1888	return true;
1889	}
1890	if (isa<UndefValue>(Val: Op)) {
1891	C = nullptr;
1892	return true;
1893	}
1894	return false;
1895	}
1896
1897	/// Checks if the given intrinsic call, which evaluates to constant, is allowed
1898	/// to be folded.
1899	///
1900	/// \param CI Constrained intrinsic call.
1901	/// \param St Exception flags raised during constant evaluation.
1902	static bool mayFoldConstrained(ConstrainedFPIntrinsic *CI,
1903	APFloat::opStatus St) {
1904	std::optional<RoundingMode> ORM = CI->getRoundingMode();
1905	std::optional<fp::ExceptionBehavior> EB = CI->getExceptionBehavior();
1906
1907	// If the operation does not change exception status flags, it is safe
1908	// to fold.
1909	if (St == APFloat::opStatus::opOK)
1910	return true;
1911
1912	// If evaluation raised FP exception, the result can depend on rounding
1913	// mode. If the latter is unknown, folding is not possible.
1914	if (ORM && *ORM == RoundingMode::Dynamic)
1915	return false;
1916
1917	// If FP exceptions are ignored, fold the call, even if such exception is
1918	// raised.
1919	if (EB && *EB != fp::ExceptionBehavior::ebStrict)
1920	return true;
1921
1922	// Leave the calculation for runtime so that exception flags be correctly set
1923	// in hardware.
1924	return false;
1925	}
1926
1927	/// Returns the rounding mode that should be used for constant evaluation.
1928	static RoundingMode
1929	getEvaluationRoundingMode(const ConstrainedFPIntrinsic *CI) {
1930	std::optional<RoundingMode> ORM = CI->getRoundingMode();
1931	if (!ORM || *ORM == RoundingMode::Dynamic)
1932	// Even if the rounding mode is unknown, try evaluating the operation.
1933	// If it does not raise inexact exception, rounding was not applied,
1934	// so the result is exact and does not depend on rounding mode. Whether
1935	// other FP exceptions are raised, it does not depend on rounding mode.
1936	return RoundingMode::NearestTiesToEven;
1937	return *ORM;
1938	}
1939
1940	/// Try to constant fold llvm.canonicalize for the given caller and value.
1941	static Constant *constantFoldCanonicalize(const Type *Ty, const CallBase *CI,
1942	const APFloat &Src) {
1943	// Zero, positive and negative, is always OK to fold.
1944	if (Src.isZero()) {
1945	// Get a fresh 0, since ppc_fp128 does have non-canonical zeros.
1946	return ConstantFP::get(
1947	Context&: CI->getContext(),
1948	V: APFloat::getZero(Sem: Src.getSemantics(), Negative: Src.isNegative()));
1949	}
1950
1951	if (!Ty->isIEEELikeFPTy())
1952	return nullptr;
1953
1954	// Zero is always canonical and the sign must be preserved.
1955	//
1956	// Denorms and nans may have special encodings, but it should be OK to fold a
1957	// totally average number.
1958	if (Src.isNormal() || Src.isInfinity())
1959	return ConstantFP::get(Context&: CI->getContext(), V: Src);
1960
1961	if (Src.isDenormal() && CI->getParent() && CI->getFunction()) {
1962	DenormalMode DenormMode =
1963	CI->getFunction()->getDenormalMode(FPType: Src.getSemantics());
1964
1965	if (DenormMode == DenormalMode::getIEEE())
1966	return ConstantFP::get(Context&: CI->getContext(), V: Src);
1967
1968	if (DenormMode.Input == DenormalMode::Dynamic)
1969	return nullptr;
1970
1971	// If we know if either input or output is flushed, we can fold.
1972	if ((DenormMode.Input == DenormalMode::Dynamic &&
1973	DenormMode.Output == DenormalMode::IEEE) ||
1974	(DenormMode.Input == DenormalMode::IEEE &&
1975	DenormMode.Output == DenormalMode::Dynamic))
1976	return nullptr;
1977
1978	bool IsPositive =
1979	(!Src.isNegative() || DenormMode.Input == DenormalMode::PositiveZero ||
1980	(DenormMode.Output == DenormalMode::PositiveZero &&
1981	DenormMode.Input == DenormalMode::IEEE));
1982
1983	return ConstantFP::get(Context&: CI->getContext(),
1984	V: APFloat::getZero(Sem: Src.getSemantics(), Negative: !IsPositive));
1985	}
1986
1987	return nullptr;
1988	}
1989
1990	static Constant *ConstantFoldScalarCall1(StringRef Name,
1991	Intrinsic::ID IntrinsicID,
1992	Type *Ty,
1993	ArrayRef<Constant *> Operands,
1994	const TargetLibraryInfo *TLI,
1995	const CallBase *Call) {
1996	assert(Operands.size() == 1 && "Wrong number of operands.");
1997
1998	if (IntrinsicID == Intrinsic::is_constant) {
1999	// We know we have a "Constant" argument. But we want to only
2000	// return true for manifest constants, not those that depend on
2001	// constants with unknowable values, e.g. GlobalValue or BlockAddress.
2002	if (Operands[0]->isManifestConstant())
2003	return ConstantInt::getTrue(Context&: Ty->getContext());
2004	return nullptr;
2005	}
2006
2007	if (isa<PoisonValue>(Val: Operands[0])) {
2008	// TODO: All of these operations should probably propagate poison.
2009	if (IntrinsicID == Intrinsic::canonicalize)
2010	return PoisonValue::get(T: Ty);
2011	}
2012
2013	if (isa<UndefValue>(Val: Operands[0])) {
2014	// cosine(arg) is between -1 and 1. cosine(invalid arg) is NaN.
2015	// ctpop() is between 0 and bitwidth, pick 0 for undef.
2016	// fptoui.sat and fptosi.sat can always fold to zero (for a zero input).
2017	if (IntrinsicID == Intrinsic::cos ||
2018	IntrinsicID == Intrinsic::ctpop ||
2019	IntrinsicID == Intrinsic::fptoui_sat ||
2020	IntrinsicID == Intrinsic::fptosi_sat ||
2021	IntrinsicID == Intrinsic::canonicalize)
2022	return Constant::getNullValue(Ty);
2023	if (IntrinsicID == Intrinsic::bswap ||
2024	IntrinsicID == Intrinsic::bitreverse ||
2025	IntrinsicID == Intrinsic::launder_invariant_group ||
2026	IntrinsicID == Intrinsic::strip_invariant_group)
2027	return Operands[0];
2028	}
2029
2030	if (isa<ConstantPointerNull>(Val: Operands[0])) {
2031	// launder(null) == null == strip(null) iff in addrspace 0
2032	if (IntrinsicID == Intrinsic::launder_invariant_group ||
2033	IntrinsicID == Intrinsic::strip_invariant_group) {
2034	// If instruction is not yet put in a basic block (e.g. when cloning
2035	// a function during inlining), Call's caller may not be available.
2036	// So check Call's BB first before querying Call->getCaller.
2037	const Function *Caller =
2038	Call->getParent() ? Call->getCaller() : nullptr;
2039	if (Caller &&
2040	!NullPointerIsDefined(
2041	F: Caller, AS: Operands[0]->getType()->getPointerAddressSpace())) {
2042	return Operands[0];
2043	}
2044	return nullptr;
2045	}
2046	}
2047
2048	if (auto *Op = dyn_cast<ConstantFP>(Val: Operands[0])) {
2049	if (IntrinsicID == Intrinsic::convert_to_fp16) {
2050	APFloat Val(Op->getValueAPF());
2051
2052	bool lost = false;
2053	Val.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmNearestTiesToEven, losesInfo: &lost);
2054
2055	return ConstantInt::get(Context&: Ty->getContext(), V: Val.bitcastToAPInt());
2056	}
2057
2058	APFloat U = Op->getValueAPF();
2059
2060	if (IntrinsicID == Intrinsic::wasm_trunc_signed ||
2061	IntrinsicID == Intrinsic::wasm_trunc_unsigned) {
2062	bool Signed = IntrinsicID == Intrinsic::wasm_trunc_signed;
2063
2064	if (U.isNaN())
2065	return nullptr;
2066
2067	unsigned Width = Ty->getIntegerBitWidth();
2068	APSInt Int(Width, !Signed);
2069	bool IsExact = false;
2070	APFloat::opStatus Status =
2071	U.convertToInteger(Result&: Int, RM: APFloat::rmTowardZero, IsExact: &IsExact);
2072
2073	if (Status == APFloat::opOK || Status == APFloat::opInexact)
2074	return ConstantInt::get(Ty, V: Int);
2075
2076	return nullptr;
2077	}
2078
2079	if (IntrinsicID == Intrinsic::fptoui_sat ||
2080	IntrinsicID == Intrinsic::fptosi_sat) {
2081	// convertToInteger() already has the desired saturation semantics.
2082	APSInt Int(Ty->getIntegerBitWidth(),
2083	IntrinsicID == Intrinsic::fptoui_sat);
2084	bool IsExact;
2085	U.convertToInteger(Result&: Int, RM: APFloat::rmTowardZero, IsExact: &IsExact);
2086	return ConstantInt::get(Ty, V: Int);
2087	}
2088
2089	if (IntrinsicID == Intrinsic::canonicalize)
2090	return constantFoldCanonicalize(Ty, CI: Call, Src: U);
2091
2092	if (!Ty->isHalfTy() && !Ty->isFloatTy() && !Ty->isDoubleTy())
2093	return nullptr;
2094
2095	// Use internal versions of these intrinsics.
2096
2097	if (IntrinsicID == Intrinsic::nearbyint || IntrinsicID == Intrinsic::rint) {
2098	U.roundToIntegral(RM: APFloat::rmNearestTiesToEven);
2099	return ConstantFP::get(Context&: Ty->getContext(), V: U);
2100	}
2101
2102	if (IntrinsicID == Intrinsic::round) {
2103	U.roundToIntegral(RM: APFloat::rmNearestTiesToAway);
2104	return ConstantFP::get(Context&: Ty->getContext(), V: U);
2105	}
2106
2107	if (IntrinsicID == Intrinsic::roundeven) {
2108	U.roundToIntegral(RM: APFloat::rmNearestTiesToEven);
2109	return ConstantFP::get(Context&: Ty->getContext(), V: U);
2110	}
2111
2112	if (IntrinsicID == Intrinsic::ceil) {
2113	U.roundToIntegral(RM: APFloat::rmTowardPositive);
2114	return ConstantFP::get(Context&: Ty->getContext(), V: U);
2115	}
2116
2117	if (IntrinsicID == Intrinsic::floor) {
2118	U.roundToIntegral(RM: APFloat::rmTowardNegative);
2119	return ConstantFP::get(Context&: Ty->getContext(), V: U);
2120	}
2121
2122	if (IntrinsicID == Intrinsic::trunc) {
2123	U.roundToIntegral(RM: APFloat::rmTowardZero);
2124	return ConstantFP::get(Context&: Ty->getContext(), V: U);
2125	}
2126
2127	if (IntrinsicID == Intrinsic::fabs) {
2128	U.clearSign();
2129	return ConstantFP::get(Context&: Ty->getContext(), V: U);
2130	}
2131
2132	if (IntrinsicID == Intrinsic::amdgcn_fract) {
2133	// The v_fract instruction behaves like the OpenCL spec, which defines
2134	// fract(x) as fmin(x - floor(x), 0x1.fffffep-1f): "The min() operator is
2135	// there to prevent fract(-small) from returning 1.0. It returns the
2136	// largest positive floating-point number less than 1.0."
2137	APFloat FloorU(U);
2138	FloorU.roundToIntegral(RM: APFloat::rmTowardNegative);
2139	APFloat FractU(U - FloorU);
2140	APFloat AlmostOne(U.getSemantics(), 1);
2141	AlmostOne.next(/*nextDown*/ true);
2142	return ConstantFP::get(Context&: Ty->getContext(), V: minimum(A: FractU, B: AlmostOne));
2143	}
2144
2145	// Rounding operations (floor, trunc, ceil, round and nearbyint) do not
2146	// raise FP exceptions, unless the argument is signaling NaN.
2147
2148	std::optional<APFloat::roundingMode> RM;
2149	switch (IntrinsicID) {
2150	default:
2151	break;
2152	case Intrinsic::experimental_constrained_nearbyint:
2153	case Intrinsic::experimental_constrained_rint: {
2154	auto CI = cast<ConstrainedFPIntrinsic>(Val: Call);
2155	RM = CI->getRoundingMode();
2156	if (!RM || *RM == RoundingMode::Dynamic)
2157	return nullptr;
2158	break;
2159	}
2160	case Intrinsic::experimental_constrained_round:
2161	RM = APFloat::rmNearestTiesToAway;
2162	break;
2163	case Intrinsic::experimental_constrained_ceil:
2164	RM = APFloat::rmTowardPositive;
2165	break;
2166	case Intrinsic::experimental_constrained_floor:
2167	RM = APFloat::rmTowardNegative;
2168	break;
2169	case Intrinsic::experimental_constrained_trunc:
2170	RM = APFloat::rmTowardZero;
2171	break;
2172	}
2173	if (RM) {
2174	auto CI = cast<ConstrainedFPIntrinsic>(Val: Call);
2175	if (U.isFinite()) {
2176	APFloat::opStatus St = U.roundToIntegral(RM: *RM);
2177	if (IntrinsicID == Intrinsic::experimental_constrained_rint &&
2178	St == APFloat::opInexact) {
2179	std::optional<fp::ExceptionBehavior> EB = CI->getExceptionBehavior();
2180	if (EB && *EB == fp::ebStrict)
2181	return nullptr;
2182	}
2183	} else if (U.isSignaling()) {
2184	std::optional<fp::ExceptionBehavior> EB = CI->getExceptionBehavior();
2185	if (EB && *EB != fp::ebIgnore)
2186	return nullptr;
2187	U = APFloat::getQNaN(Sem: U.getSemantics());
2188	}
2189	return ConstantFP::get(Context&: Ty->getContext(), V: U);
2190	}
2191
2192	/// We only fold functions with finite arguments. Folding NaN and inf is
2193	/// likely to be aborted with an exception anyway, and some host libms
2194	/// have known errors raising exceptions.
2195	if (!U.isFinite())
2196	return nullptr;
2197
2198	/// Currently APFloat versions of these functions do not exist, so we use
2199	/// the host native double versions. Float versions are not called
2200	/// directly but for all these it is true (float)(f((double)arg)) ==
2201	/// f(arg). Long double not supported yet.
2202	const APFloat &APF = Op->getValueAPF();
2203
2204	switch (IntrinsicID) {
2205	default: break;
2206	case Intrinsic::log:
2207	return ConstantFoldFP(NativeFP: log, V: APF, Ty);
2208	case Intrinsic::log2:
2209	// TODO: What about hosts that lack a C99 library?
2210	return ConstantFoldFP(NativeFP: log2, V: APF, Ty);
2211	case Intrinsic::log10:
2212	// TODO: What about hosts that lack a C99 library?
2213	return ConstantFoldFP(NativeFP: log10, V: APF, Ty);
2214	case Intrinsic::exp:
2215	return ConstantFoldFP(NativeFP: exp, V: APF, Ty);
2216	case Intrinsic::exp2:
2217	// Fold exp2(x) as pow(2, x), in case the host lacks a C99 library.
2218	return ConstantFoldBinaryFP(NativeFP: pow, V: APFloat(2.0), W: APF, Ty);
2219	case Intrinsic::exp10:
2220	// Fold exp10(x) as pow(10, x), in case the host lacks a C99 library.
2221	return ConstantFoldBinaryFP(NativeFP: pow, V: APFloat(10.0), W: APF, Ty);
2222	case Intrinsic::sin:
2223	return ConstantFoldFP(NativeFP: sin, V: APF, Ty);
2224	case Intrinsic::cos:
2225	return ConstantFoldFP(NativeFP: cos, V: APF, Ty);
2226	case Intrinsic::sqrt:
2227	return ConstantFoldFP(NativeFP: sqrt, V: APF, Ty);
2228	case Intrinsic::amdgcn_cos:
2229	case Intrinsic::amdgcn_sin: {
2230	double V = getValueAsDouble(Op);
2231	if (V < -256.0 || V > 256.0)
2232	// The gfx8 and gfx9 architectures handle arguments outside the range
2233	// [-256, 256] differently. This should be a rare case so bail out
2234	// rather than trying to handle the difference.
2235	return nullptr;
2236	bool IsCos = IntrinsicID == Intrinsic::amdgcn_cos;
2237	double V4 = V * 4.0;
2238	if (V4 == floor(x: V4)) {
2239	// Force exact results for quarter-integer inputs.
2240	const double SinVals[4] = { 0.0, 1.0, 0.0, -1.0 };
2241	V = SinVals[((int)V4 + (IsCos ? 1 : 0)) & 3];
2242	} else {
2243	if (IsCos)
2244	V = cos(x: V * 2.0 * numbers::pi);
2245	else
2246	V = sin(x: V * 2.0 * numbers::pi);
2247	}
2248	return GetConstantFoldFPValue(V, Ty);
2249	}
2250	}
2251
2252	if (!TLI)
2253	return nullptr;
2254
2255	LibFunc Func = NotLibFunc;
2256	if (!TLI->getLibFunc(funcName: Name, F&: Func))
2257	return nullptr;
2258
2259	switch (Func) {
2260	default:
2261	break;
2262	case LibFunc_acos:
2263	case LibFunc_acosf:
2264	case LibFunc_acos_finite:
2265	case LibFunc_acosf_finite:
2266	if (TLI->has(F: Func))
2267	return ConstantFoldFP(NativeFP: acos, V: APF, Ty);
2268	break;
2269	case LibFunc_asin:
2270	case LibFunc_asinf:
2271	case LibFunc_asin_finite:
2272	case LibFunc_asinf_finite:
2273	if (TLI->has(F: Func))
2274	return ConstantFoldFP(NativeFP: asin, V: APF, Ty);
2275	break;
2276	case LibFunc_atan:
2277	case LibFunc_atanf:
2278	if (TLI->has(F: Func))
2279	return ConstantFoldFP(NativeFP: atan, V: APF, Ty);
2280	break;
2281	case LibFunc_ceil:
2282	case LibFunc_ceilf:
2283	if (TLI->has(F: Func)) {
2284	U.roundToIntegral(RM: APFloat::rmTowardPositive);
2285	return ConstantFP::get(Context&: Ty->getContext(), V: U);
2286	}
2287	break;
2288	case LibFunc_cos:
2289	case LibFunc_cosf:
2290	if (TLI->has(F: Func))
2291	return ConstantFoldFP(NativeFP: cos, V: APF, Ty);
2292	break;
2293	case LibFunc_cosh:
2294	case LibFunc_coshf:
2295	case LibFunc_cosh_finite:
2296	case LibFunc_coshf_finite:
2297	if (TLI->has(F: Func))
2298	return ConstantFoldFP(NativeFP: cosh, V: APF, Ty);
2299	break;
2300	case LibFunc_exp:
2301	case LibFunc_expf:
2302	case LibFunc_exp_finite:
2303	case LibFunc_expf_finite:
2304	if (TLI->has(F: Func))
2305	return ConstantFoldFP(NativeFP: exp, V: APF, Ty);
2306	break;
2307	case LibFunc_exp2:
2308	case LibFunc_exp2f:
2309	case LibFunc_exp2_finite:
2310	case LibFunc_exp2f_finite:
2311	if (TLI->has(F: Func))
2312	// Fold exp2(x) as pow(2, x), in case the host lacks a C99 library.
2313	return ConstantFoldBinaryFP(NativeFP: pow, V: APFloat(2.0), W: APF, Ty);
2314	break;
2315	case LibFunc_fabs:
2316	case LibFunc_fabsf:
2317	if (TLI->has(F: Func)) {
2318	U.clearSign();
2319	return ConstantFP::get(Context&: Ty->getContext(), V: U);
2320	}
2321	break;
2322	case LibFunc_floor:
2323	case LibFunc_floorf:
2324	if (TLI->has(F: Func)) {
2325	U.roundToIntegral(RM: APFloat::rmTowardNegative);
2326	return ConstantFP::get(Context&: Ty->getContext(), V: U);
2327	}
2328	break;
2329	case LibFunc_log:
2330	case LibFunc_logf:
2331	case LibFunc_log_finite:
2332	case LibFunc_logf_finite:
2333	if (!APF.isNegative() && !APF.isZero() && TLI->has(F: Func))
2334	return ConstantFoldFP(NativeFP: log, V: APF, Ty);
2335	break;
2336	case LibFunc_log2:
2337	case LibFunc_log2f:
2338	case LibFunc_log2_finite:
2339	case LibFunc_log2f_finite:
2340	if (!APF.isNegative() && !APF.isZero() && TLI->has(F: Func))
2341	// TODO: What about hosts that lack a C99 library?
2342	return ConstantFoldFP(NativeFP: log2, V: APF, Ty);
2343	break;
2344	case LibFunc_log10:
2345	case LibFunc_log10f:
2346	case LibFunc_log10_finite:
2347	case LibFunc_log10f_finite:
2348	if (!APF.isNegative() && !APF.isZero() && TLI->has(F: Func))
2349	// TODO: What about hosts that lack a C99 library?
2350	return ConstantFoldFP(NativeFP: log10, V: APF, Ty);
2351	break;
2352	case LibFunc_nearbyint:
2353	case LibFunc_nearbyintf:
2354	case LibFunc_rint:
2355	case LibFunc_rintf:
2356	if (TLI->has(F: Func)) {
2357	U.roundToIntegral(RM: APFloat::rmNearestTiesToEven);
2358	return ConstantFP::get(Context&: Ty->getContext(), V: U);
2359	}
2360	break;
2361	case LibFunc_round:
2362	case LibFunc_roundf:
2363	if (TLI->has(F: Func)) {
2364	U.roundToIntegral(RM: APFloat::rmNearestTiesToAway);
2365	return ConstantFP::get(Context&: Ty->getContext(), V: U);
2366	}
2367	break;
2368	case LibFunc_sin:
2369	case LibFunc_sinf:
2370	if (TLI->has(F: Func))
2371	return ConstantFoldFP(NativeFP: sin, V: APF, Ty);
2372	break;
2373	case LibFunc_sinh:
2374	case LibFunc_sinhf:
2375	case LibFunc_sinh_finite:
2376	case LibFunc_sinhf_finite:
2377	if (TLI->has(F: Func))
2378	return ConstantFoldFP(NativeFP: sinh, V: APF, Ty);
2379	break;
2380	case LibFunc_sqrt:
2381	case LibFunc_sqrtf:
2382	if (!APF.isNegative() && TLI->has(F: Func))
2383	return ConstantFoldFP(NativeFP: sqrt, V: APF, Ty);
2384	break;
2385	case LibFunc_tan:
2386	case LibFunc_tanf:
2387	if (TLI->has(F: Func))
2388	return ConstantFoldFP(NativeFP: tan, V: APF, Ty);
2389	break;
2390	case LibFunc_tanh:
2391	case LibFunc_tanhf:
2392	if (TLI->has(F: Func))
2393	return ConstantFoldFP(NativeFP: tanh, V: APF, Ty);
2394	break;
2395	case LibFunc_trunc:
2396	case LibFunc_truncf:
2397	if (TLI->has(F: Func)) {
2398	U.roundToIntegral(RM: APFloat::rmTowardZero);
2399	return ConstantFP::get(Context&: Ty->getContext(), V: U);
2400	}
2401	break;
2402	}
2403	return nullptr;
2404	}
2405
2406	if (auto *Op = dyn_cast<ConstantInt>(Val: Operands[0])) {
2407	switch (IntrinsicID) {
2408	case Intrinsic::bswap:
2409	return ConstantInt::get(Context&: Ty->getContext(), V: Op->getValue().byteSwap());
2410	case Intrinsic::ctpop:
2411	return ConstantInt::get(Ty, V: Op->getValue().popcount());
2412	case Intrinsic::bitreverse:
2413	return ConstantInt::get(Context&: Ty->getContext(), V: Op->getValue().reverseBits());
2414	case Intrinsic::convert_from_fp16: {
2415	APFloat Val(APFloat::IEEEhalf(), Op->getValue());
2416
2417	bool lost = false;
2418	APFloat::opStatus status = Val.convert(
2419	ToSemantics: Ty->getFltSemantics(), RM: APFloat::rmNearestTiesToEven, losesInfo: &lost);
2420
2421	// Conversion is always precise.
2422	(void)status;
2423	assert(status != APFloat::opInexact && !lost &&
2424	"Precision lost during fp16 constfolding");
2425
2426	return ConstantFP::get(Context&: Ty->getContext(), V: Val);
2427	}
2428
2429	case Intrinsic::amdgcn_s_wqm: {
2430	uint64_t Val = Op->getZExtValue();
2431	Val |= (Val & 0x5555555555555555ULL) << 1 |
2432	((Val >> 1) & 0x5555555555555555ULL);
2433	Val |= (Val & 0x3333333333333333ULL) << 2 |
2434	((Val >> 2) & 0x3333333333333333ULL);
2435	return ConstantInt::get(Ty, V: Val);
2436	}
2437
2438	case Intrinsic::amdgcn_s_quadmask: {
2439	uint64_t Val = Op->getZExtValue();
2440	uint64_t QuadMask = 0;
2441	for (unsigned I = 0; I < Op->getBitWidth() / 4; ++I, Val >>= 4) {
2442	if (!(Val & 0xF))
2443	continue;
2444
2445	QuadMask |= (1ULL << I);
2446	}
2447	return ConstantInt::get(Ty, V: QuadMask);
2448	}
2449
2450	case Intrinsic::amdgcn_s_bitreplicate: {
2451	uint64_t Val = Op->getZExtValue();
2452	Val = (Val & 0x000000000000FFFFULL) | (Val & 0x00000000FFFF0000ULL) << 16;
2453	Val = (Val & 0x000000FF000000FFULL) | (Val & 0x0000FF000000FF00ULL) << 8;
2454	Val = (Val & 0x000F000F000F000FULL) | (Val & 0x00F000F000F000F0ULL) << 4;
2455	Val = (Val & 0x0303030303030303ULL) | (Val & 0x0C0C0C0C0C0C0C0CULL) << 2;
2456	Val = (Val & 0x1111111111111111ULL) | (Val & 0x2222222222222222ULL) << 1;
2457	Val = Val | Val << 1;
2458	return ConstantInt::get(Ty, V: Val);
2459	}
2460
2461	default:
2462	return nullptr;
2463	}
2464	}
2465
2466	switch (IntrinsicID) {
2467	default: break;
2468	case Intrinsic::vector_reduce_add:
2469	case Intrinsic::vector_reduce_mul:
2470	case Intrinsic::vector_reduce_and:
2471	case Intrinsic::vector_reduce_or:
2472	case Intrinsic::vector_reduce_xor:
2473	case Intrinsic::vector_reduce_smin:
2474	case Intrinsic::vector_reduce_smax:
2475	case Intrinsic::vector_reduce_umin:
2476	case Intrinsic::vector_reduce_umax:
2477	if (Constant *C = constantFoldVectorReduce(IID: IntrinsicID, Op: Operands[0]))
2478	return C;
2479	break;
2480	}
2481
2482	// Support ConstantVector in case we have an Undef in the top.
2483	if (isa<ConstantVector>(Val: Operands[0]) ||
2484	isa<ConstantDataVector>(Val: Operands[0])) {
2485	auto *Op = cast<Constant>(Val: Operands[0]);
2486	switch (IntrinsicID) {
2487	default: break;
2488	case Intrinsic::x86_sse_cvtss2si:
2489	case Intrinsic::x86_sse_cvtss2si64:
2490	case Intrinsic::x86_sse2_cvtsd2si:
2491	case Intrinsic::x86_sse2_cvtsd2si64:
2492	if (ConstantFP *FPOp =
2493	dyn_cast_or_null<ConstantFP>(Val: Op->getAggregateElement(Elt: 0U)))
2494	return ConstantFoldSSEConvertToInt(Val: FPOp->getValueAPF(),
2495	/*roundTowardZero=*/false, Ty,
2496	/*IsSigned*/true);
2497	break;
2498	case Intrinsic::x86_sse_cvttss2si:
2499	case Intrinsic::x86_sse_cvttss2si64:
2500	case Intrinsic::x86_sse2_cvttsd2si:
2501	case Intrinsic::x86_sse2_cvttsd2si64:
2502	if (ConstantFP *FPOp =
2503	dyn_cast_or_null<ConstantFP>(Val: Op->getAggregateElement(Elt: 0U)))
2504	return ConstantFoldSSEConvertToInt(Val: FPOp->getValueAPF(),
2505	/*roundTowardZero=*/true, Ty,
2506	/*IsSigned*/true);
2507	break;
2508	}
2509	}
2510
2511	return nullptr;
2512	}
2513
2514	static Constant *evaluateCompare(const APFloat &Op1, const APFloat &Op2,
2515	const ConstrainedFPIntrinsic *Call) {
2516	APFloat::opStatus St = APFloat::opOK;
2517	auto *FCmp = cast<ConstrainedFPCmpIntrinsic>(Val: Call);
2518	FCmpInst::Predicate Cond = FCmp->getPredicate();
2519	if (FCmp->isSignaling()) {
2520	if (Op1.isNaN() || Op2.isNaN())
2521	St = APFloat::opInvalidOp;
2522	} else {
2523	if (Op1.isSignaling() || Op2.isSignaling())
2524	St = APFloat::opInvalidOp;
2525	}
2526	bool Result = FCmpInst::compare(LHS: Op1, RHS: Op2, Pred: Cond);
2527	if (mayFoldConstrained(CI: const_cast<ConstrainedFPCmpIntrinsic *>(FCmp), St))
2528	return ConstantInt::get(Ty: Call->getType()->getScalarType(), V: Result);
2529	return nullptr;
2530	}
2531
2532	static Constant *ConstantFoldScalarCall2(StringRef Name,
2533	Intrinsic::ID IntrinsicID,
2534	Type *Ty,
2535	ArrayRef<Constant *> Operands,
2536	const TargetLibraryInfo *TLI,
2537	const CallBase *Call) {
2538	assert(Operands.size() == 2 && "Wrong number of operands.");
2539
2540	if (Ty->isFloatingPointTy()) {
2541	// TODO: We should have undef handling for all of the FP intrinsics that
2542	// are attempted to be folded in this function.
2543	bool IsOp0Undef = isa<UndefValue>(Val: Operands[0]);
2544	bool IsOp1Undef = isa<UndefValue>(Val: Operands[1]);
2545	switch (IntrinsicID) {
2546	case Intrinsic::maxnum:
2547	case Intrinsic::minnum:
2548	case Intrinsic::maximum:
2549	case Intrinsic::minimum:
2550	// If one argument is undef, return the other argument.
2551	if (IsOp0Undef)
2552	return Operands[1];
2553	if (IsOp1Undef)
2554	return Operands[0];
2555	break;
2556	}
2557	}
2558
2559	if (const auto *Op1 = dyn_cast<ConstantFP>(Val: Operands[0])) {
2560	const APFloat &Op1V = Op1->getValueAPF();
2561
2562	if (const auto *Op2 = dyn_cast<ConstantFP>(Val: Operands[1])) {
2563	if (Op2->getType() != Op1->getType())
2564	return nullptr;
2565	const APFloat &Op2V = Op2->getValueAPF();
2566
2567	if (const auto *ConstrIntr = dyn_cast<ConstrainedFPIntrinsic>(Val: Call)) {
2568	RoundingMode RM = getEvaluationRoundingMode(CI: ConstrIntr);
2569	APFloat Res = Op1V;
2570	APFloat::opStatus St;
2571	switch (IntrinsicID) {
2572	default:
2573	return nullptr;
2574	case Intrinsic::experimental_constrained_fadd:
2575	St = Res.add(RHS: Op2V, RM);
2576	break;
2577	case Intrinsic::experimental_constrained_fsub:
2578	St = Res.subtract(RHS: Op2V, RM);
2579	break;
2580	case Intrinsic::experimental_constrained_fmul:
2581	St = Res.multiply(RHS: Op2V, RM);
2582	break;
2583	case Intrinsic::experimental_constrained_fdiv:
2584	St = Res.divide(RHS: Op2V, RM);
2585	break;
2586	case Intrinsic::experimental_constrained_frem:
2587	St = Res.mod(RHS: Op2V);
2588	break;
2589	case Intrinsic::experimental_constrained_fcmp:
2590	case Intrinsic::experimental_constrained_fcmps:
2591	return evaluateCompare(Op1: Op1V, Op2: Op2V, Call: ConstrIntr);
2592	}
2593	if (mayFoldConstrained(CI: const_cast<ConstrainedFPIntrinsic *>(ConstrIntr),
2594	St))
2595	return ConstantFP::get(Context&: Ty->getContext(), V: Res);
2596	return nullptr;
2597	}
2598
2599	switch (IntrinsicID) {
2600	default:
2601	break;
2602	case Intrinsic::copysign:
2603	return ConstantFP::get(Context&: Ty->getContext(), V: APFloat::copySign(Value: Op1V, Sign: Op2V));
2604	case Intrinsic::minnum:
2605	return ConstantFP::get(Context&: Ty->getContext(), V: minnum(A: Op1V, B: Op2V));
2606	case Intrinsic::maxnum:
2607	return ConstantFP::get(Context&: Ty->getContext(), V: maxnum(A: Op1V, B: Op2V));
2608	case Intrinsic::minimum:
2609	return ConstantFP::get(Context&: Ty->getContext(), V: minimum(A: Op1V, B: Op2V));
2610	case Intrinsic::maximum:
2611	return ConstantFP::get(Context&: Ty->getContext(), V: maximum(A: Op1V, B: Op2V));
2612	}
2613
2614	if (!Ty->isHalfTy() && !Ty->isFloatTy() && !Ty->isDoubleTy())
2615	return nullptr;
2616
2617	switch (IntrinsicID) {
2618	default:
2619	break;
2620	case Intrinsic::pow:
2621	return ConstantFoldBinaryFP(NativeFP: pow, V: Op1V, W: Op2V, Ty);
2622	case Intrinsic::amdgcn_fmul_legacy:
2623	// The legacy behaviour is that multiplying +/- 0.0 by anything, even
2624	// NaN or infinity, gives +0.0.
2625	if (Op1V.isZero() || Op2V.isZero())
2626	return ConstantFP::getZero(Ty);
2627	return ConstantFP::get(Context&: Ty->getContext(), V: Op1V * Op2V);
2628	}
2629
2630	if (!TLI)
2631	return nullptr;
2632
2633	LibFunc Func = NotLibFunc;
2634	if (!TLI->getLibFunc(funcName: Name, F&: Func))
2635	return nullptr;
2636
2637	switch (Func) {
2638	default:
2639	break;
2640	case LibFunc_pow:
2641	case LibFunc_powf:
2642	case LibFunc_pow_finite:
2643	case LibFunc_powf_finite:
2644	if (TLI->has(F: Func))
2645	return ConstantFoldBinaryFP(NativeFP: pow, V: Op1V, W: Op2V, Ty);
2646	break;
2647	case LibFunc_fmod:
2648	case LibFunc_fmodf:
2649	if (TLI->has(F: Func)) {
2650	APFloat V = Op1->getValueAPF();
2651	if (APFloat::opStatus::opOK == V.mod(RHS: Op2->getValueAPF()))
2652	return ConstantFP::get(Context&: Ty->getContext(), V);
2653	}
2654	break;
2655	case LibFunc_remainder:
2656	case LibFunc_remainderf:
2657	if (TLI->has(F: Func)) {
2658	APFloat V = Op1->getValueAPF();
2659	if (APFloat::opStatus::opOK == V.remainder(RHS: Op2->getValueAPF()))
2660	return ConstantFP::get(Context&: Ty->getContext(), V);
2661	}
2662	break;
2663	case LibFunc_atan2:
2664	case LibFunc_atan2f:
2665	// atan2(+/-0.0, +/-0.0) is known to raise an exception on some libm
2666	// (Solaris), so we do not assume a known result for that.
2667	if (Op1V.isZero() && Op2V.isZero())
2668	return nullptr;
2669	[[fallthrough]];
2670	case LibFunc_atan2_finite:
2671	case LibFunc_atan2f_finite:
2672	if (TLI->has(F: Func))
2673	return ConstantFoldBinaryFP(NativeFP: atan2, V: Op1V, W: Op2V, Ty);
2674	break;
2675	}
2676	} else if (auto *Op2C = dyn_cast<ConstantInt>(Val: Operands[1])) {
2677	switch (IntrinsicID) {
2678	case Intrinsic::ldexp: {
2679	return ConstantFP::get(
2680	Context&: Ty->getContext(),
2681	V: scalbn(X: Op1V, Exp: Op2C->getSExtValue(), RM: APFloat::rmNearestTiesToEven));
2682	}
2683	case Intrinsic::is_fpclass: {
2684	FPClassTest Mask = static_cast<FPClassTest>(Op2C->getZExtValue());
2685	bool Result =
2686	((Mask & fcSNan) && Op1V.isNaN() && Op1V.isSignaling()) ||
2687	((Mask & fcQNan) && Op1V.isNaN() && !Op1V.isSignaling()) ||
2688	((Mask & fcNegInf) && Op1V.isNegInfinity()) ||
2689	((Mask & fcNegNormal) && Op1V.isNormal() && Op1V.isNegative()) ||
2690	((Mask & fcNegSubnormal) && Op1V.isDenormal() && Op1V.isNegative()) ||
2691	((Mask & fcNegZero) && Op1V.isZero() && Op1V.isNegative()) ||
2692	((Mask & fcPosZero) && Op1V.isZero() && !Op1V.isNegative()) ||
2693	((Mask & fcPosSubnormal) && Op1V.isDenormal() && !Op1V.isNegative()) ||
2694	((Mask & fcPosNormal) && Op1V.isNormal() && !Op1V.isNegative()) ||
2695	((Mask & fcPosInf) && Op1V.isPosInfinity());
2696	return ConstantInt::get(Ty, V: Result);
2697	}
2698	default:
2699	break;
2700	}
2701
2702	if (!Ty->isHalfTy() && !Ty->isFloatTy() && !Ty->isDoubleTy())
2703	return nullptr;
2704	if (IntrinsicID == Intrinsic::powi && Ty->isHalfTy())
2705	return ConstantFP::get(
2706	Context&: Ty->getContext(),
2707	V: APFloat((float)std::pow(x: (float)Op1V.convertToDouble(),
2708	y: (int)Op2C->getZExtValue())));
2709	if (IntrinsicID == Intrinsic::powi && Ty->isFloatTy())
2710	return ConstantFP::get(
2711	Context&: Ty->getContext(),
2712	V: APFloat((float)std::pow(x: (float)Op1V.convertToDouble(),
2713	y: (int)Op2C->getZExtValue())));
2714	if (IntrinsicID == Intrinsic::powi && Ty->isDoubleTy())
2715	return ConstantFP::get(
2716	Context&: Ty->getContext(),
2717	V: APFloat((double)std::pow(x: Op1V.convertToDouble(),
2718	y: (int)Op2C->getZExtValue())));
2719	}
2720	return nullptr;
2721	}
2722
2723	if (Operands[0]->getType()->isIntegerTy() &&
2724	Operands[1]->getType()->isIntegerTy()) {
2725	const APInt *C0, *C1;
2726	if (!getConstIntOrUndef(Op: Operands[0], C&: C0) ||
2727	!getConstIntOrUndef(Op: Operands[1], C&: C1))
2728	return nullptr;
2729
2730	switch (IntrinsicID) {
2731	default: break;
2732	case Intrinsic::smax:
2733	case Intrinsic::smin:
2734	case Intrinsic::umax:
2735	case Intrinsic::umin:
2736	// This is the same as for binary ops - poison propagates.
2737	// TODO: Poison handling should be consolidated.
2738	if (isa<PoisonValue>(Val: Operands[0]) || isa<PoisonValue>(Val: Operands[1]))
2739	return PoisonValue::get(T: Ty);
2740
2741	if (!C0 && !C1)
2742	return UndefValue::get(T: Ty);
2743	if (!C0 || !C1)
2744	return MinMaxIntrinsic::getSaturationPoint(ID: IntrinsicID, Ty);
2745	return ConstantInt::get(
2746	Ty, V: ICmpInst::compare(LHS: *C0, RHS: *C1,
2747	Pred: MinMaxIntrinsic::getPredicate(ID: IntrinsicID))
2748	? *C0
2749	: *C1);
2750
2751	case Intrinsic::usub_with_overflow:
2752	case Intrinsic::ssub_with_overflow:
2753	// X - undef -> { 0, false }
2754	// undef - X -> { 0, false }
2755	if (!C0 || !C1)
2756	return Constant::getNullValue(Ty);
2757	[[fallthrough]];
2758	case Intrinsic::uadd_with_overflow:
2759	case Intrinsic::sadd_with_overflow:
2760	// X + undef -> { -1, false }
2761	// undef + x -> { -1, false }
2762	if (!C0 || !C1) {
2763	return ConstantStruct::get(
2764	T: cast<StructType>(Val: Ty),
2765	V: {Constant::getAllOnesValue(Ty: Ty->getStructElementType(N: 0)),
2766	Constant::getNullValue(Ty: Ty->getStructElementType(N: 1))});
2767	}
2768	[[fallthrough]];
2769	case Intrinsic::smul_with_overflow:
2770	case Intrinsic::umul_with_overflow: {
2771	// undef * X -> { 0, false }
2772	// X * undef -> { 0, false }
2773	if (!C0 || !C1)
2774	return Constant::getNullValue(Ty);
2775
2776	APInt Res;
2777	bool Overflow;
2778	switch (IntrinsicID) {
2779	default: llvm_unreachable("Invalid case");
2780	case Intrinsic::sadd_with_overflow:
2781	Res = C0->sadd_ov(RHS: *C1, Overflow);
2782	break;
2783	case Intrinsic::uadd_with_overflow:
2784	Res = C0->uadd_ov(RHS: *C1, Overflow);
2785	break;
2786	case Intrinsic::ssub_with_overflow:
2787	Res = C0->ssub_ov(RHS: *C1, Overflow);
2788	break;
2789	case Intrinsic::usub_with_overflow:
2790	Res = C0->usub_ov(RHS: *C1, Overflow);
2791	break;
2792	case Intrinsic::smul_with_overflow:
2793	Res = C0->smul_ov(RHS: *C1, Overflow);
2794	break;
2795	case Intrinsic::umul_with_overflow:
2796	Res = C0->umul_ov(RHS: *C1, Overflow);
2797	break;
2798	}
2799	Constant *Ops[] = {
2800	ConstantInt::get(Context&: Ty->getContext(), V: Res),
2801	ConstantInt::get(Ty: Type::getInt1Ty(C&: Ty->getContext()), V: Overflow)
2802	};
2803	return ConstantStruct::get(T: cast<StructType>(Val: Ty), V: Ops);
2804	}
2805	case Intrinsic::uadd_sat:
2806	case Intrinsic::sadd_sat:
2807	// This is the same as for binary ops - poison propagates.
2808	// TODO: Poison handling should be consolidated.
2809	if (isa<PoisonValue>(Val: Operands[0]) || isa<PoisonValue>(Val: Operands[1]))
2810	return PoisonValue::get(T: Ty);
2811
2812	if (!C0 && !C1)
2813	return UndefValue::get(T: Ty);
2814	if (!C0 || !C1)
2815	return Constant::getAllOnesValue(Ty);
2816	if (IntrinsicID == Intrinsic::uadd_sat)
2817	return ConstantInt::get(Ty, V: C0->uadd_sat(RHS: *C1));
2818	else
2819	return ConstantInt::get(Ty, V: C0->sadd_sat(RHS: *C1));
2820	case Intrinsic::usub_sat:
2821	case Intrinsic::ssub_sat:
2822	// This is the same as for binary ops - poison propagates.
2823	// TODO: Poison handling should be consolidated.
2824	if (isa<PoisonValue>(Val: Operands[0]) || isa<PoisonValue>(Val: Operands[1]))
2825	return PoisonValue::get(T: Ty);
2826
2827	if (!C0 && !C1)
2828	return UndefValue::get(T: Ty);
2829	if (!C0 || !C1)
2830	return Constant::getNullValue(Ty);
2831	if (IntrinsicID == Intrinsic::usub_sat)
2832	return ConstantInt::get(Ty, V: C0->usub_sat(RHS: *C1));
2833	else
2834	return ConstantInt::get(Ty, V: C0->ssub_sat(RHS: *C1));
2835	case Intrinsic::cttz:
2836	case Intrinsic::ctlz:
2837	assert(C1 && "Must be constant int");
2838
2839	// cttz(0, 1) and ctlz(0, 1) are poison.
2840	if (C1->isOne() && (!C0 || C0->isZero()))
2841	return PoisonValue::get(T: Ty);
2842	if (!C0)
2843	return Constant::getNullValue(Ty);
2844	if (IntrinsicID == Intrinsic::cttz)
2845	return ConstantInt::get(Ty, V: C0->countr_zero());
2846	else
2847	return ConstantInt::get(Ty, V: C0->countl_zero());
2848
2849	case Intrinsic::abs:
2850	assert(C1 && "Must be constant int");
2851	assert((C1->isOne() || C1->isZero()) && "Must be 0 or 1");
2852
2853	// Undef or minimum val operand with poison min --> undef
2854	if (C1->isOne() && (!C0 || C0->isMinSignedValue()))
2855	return UndefValue::get(T: Ty);
2856
2857	// Undef operand with no poison min --> 0 (sign bit must be clear)
2858	if (!C0)
2859	return Constant::getNullValue(Ty);
2860
2861	return ConstantInt::get(Ty, V: C0->abs());
2862	case Intrinsic::amdgcn_wave_reduce_umin:
2863	case Intrinsic::amdgcn_wave_reduce_umax:
2864	return dyn_cast<Constant>(Val: Operands[0]);
2865	}
2866
2867	return nullptr;
2868	}
2869
2870	// Support ConstantVector in case we have an Undef in the top.
2871	if ((isa<ConstantVector>(Val: Operands[0]) ||
2872	isa<ConstantDataVector>(Val: Operands[0])) &&
2873	// Check for default rounding mode.
2874	// FIXME: Support other rounding modes?
2875	isa<ConstantInt>(Val: Operands[1]) &&
2876	cast<ConstantInt>(Val: Operands[1])->getValue() == 4) {
2877	auto *Op = cast<Constant>(Val: Operands[0]);
2878	switch (IntrinsicID) {
2879	default: break;
2880	case Intrinsic::x86_avx512_vcvtss2si32:
2881	case Intrinsic::x86_avx512_vcvtss2si64:
2882	case Intrinsic::x86_avx512_vcvtsd2si32:
2883	case Intrinsic::x86_avx512_vcvtsd2si64:
2884	if (ConstantFP *FPOp =
2885	dyn_cast_or_null<ConstantFP>(Val: Op->getAggregateElement(Elt: 0U)))
2886	return ConstantFoldSSEConvertToInt(Val: FPOp->getValueAPF(),
2887	/*roundTowardZero=*/false, Ty,
2888	/*IsSigned*/true);
2889	break;
2890	case Intrinsic::x86_avx512_vcvtss2usi32:
2891	case Intrinsic::x86_avx512_vcvtss2usi64:
2892	case Intrinsic::x86_avx512_vcvtsd2usi32:
2893	case Intrinsic::x86_avx512_vcvtsd2usi64:
2894	if (ConstantFP *FPOp =
2895	dyn_cast_or_null<ConstantFP>(Val: Op->getAggregateElement(Elt: 0U)))
2896	return ConstantFoldSSEConvertToInt(Val: FPOp->getValueAPF(),
2897	/*roundTowardZero=*/false, Ty,
2898	/*IsSigned*/false);
2899	break;
2900	case Intrinsic::x86_avx512_cvttss2si:
2901	case Intrinsic::x86_avx512_cvttss2si64:
2902	case Intrinsic::x86_avx512_cvttsd2si:
2903	case Intrinsic::x86_avx512_cvttsd2si64:
2904	if (ConstantFP *FPOp =
2905	dyn_cast_or_null<ConstantFP>(Val: Op->getAggregateElement(Elt: 0U)))
2906	return ConstantFoldSSEConvertToInt(Val: FPOp->getValueAPF(),
2907	/*roundTowardZero=*/true, Ty,
2908	/*IsSigned*/true);
2909	break;
2910	case Intrinsic::x86_avx512_cvttss2usi:
2911	case Intrinsic::x86_avx512_cvttss2usi64:
2912	case Intrinsic::x86_avx512_cvttsd2usi:
2913	case Intrinsic::x86_avx512_cvttsd2usi64:
2914	if (ConstantFP *FPOp =
2915	dyn_cast_or_null<ConstantFP>(Val: Op->getAggregateElement(Elt: 0U)))
2916	return ConstantFoldSSEConvertToInt(Val: FPOp->getValueAPF(),
2917	/*roundTowardZero=*/true, Ty,
2918	/*IsSigned*/false);
2919	break;
2920	}
2921	}
2922	return nullptr;
2923	}
2924
2925	static APFloat ConstantFoldAMDGCNCubeIntrinsic(Intrinsic::ID IntrinsicID,
2926	const APFloat &S0,
2927	const APFloat &S1,
2928	const APFloat &S2) {
2929	unsigned ID;
2930	const fltSemantics &Sem = S0.getSemantics();
2931	APFloat MA(Sem), SC(Sem), TC(Sem);
2932	if (abs(X: S2) >= abs(X: S0) && abs(X: S2) >= abs(X: S1)) {
2933	if (S2.isNegative() && S2.isNonZero() && !S2.isNaN()) {
2934	// S2 < 0
2935	ID = 5;
2936	SC = -S0;
2937	} else {
2938	ID = 4;
2939	SC = S0;
2940	}
2941	MA = S2;
2942	TC = -S1;
2943	} else if (abs(X: S1) >= abs(X: S0)) {
2944	if (S1.isNegative() && S1.isNonZero() && !S1.isNaN()) {
2945	// S1 < 0
2946	ID = 3;
2947	TC = -S2;
2948	} else {
2949	ID = 2;
2950	TC = S2;
2951	}
2952	MA = S1;
2953	SC = S0;
2954	} else {
2955	if (S0.isNegative() && S0.isNonZero() && !S0.isNaN()) {
2956	// S0 < 0
2957	ID = 1;
2958	SC = S2;
2959	} else {
2960	ID = 0;
2961	SC = -S2;
2962	}
2963	MA = S0;
2964	TC = -S1;
2965	}
2966	switch (IntrinsicID) {
2967	default:
2968	llvm_unreachable("unhandled amdgcn cube intrinsic");
2969	case Intrinsic::amdgcn_cubeid:
2970	return APFloat(Sem, ID);
2971	case Intrinsic::amdgcn_cubema:
2972	return MA + MA;
2973	case Intrinsic::amdgcn_cubesc:
2974	return SC;
2975	case Intrinsic::amdgcn_cubetc:
2976	return TC;
2977	}
2978	}
2979
2980	static Constant *ConstantFoldAMDGCNPermIntrinsic(ArrayRef<Constant *> Operands,
2981	Type *Ty) {
2982	const APInt *C0, *C1, *C2;
2983	if (!getConstIntOrUndef(Op: Operands[0], C&: C0) ||
2984	!getConstIntOrUndef(Op: Operands[1], C&: C1) ||
2985	!getConstIntOrUndef(Op: Operands[2], C&: C2))
2986	return nullptr;
2987
2988	if (!C2)
2989	return UndefValue::get(T: Ty);
2990
2991	APInt Val(32, 0);
2992	unsigned NumUndefBytes = 0;
2993	for (unsigned I = 0; I < 32; I += 8) {
2994	unsigned Sel = C2->extractBitsAsZExtValue(numBits: 8, bitPosition: I);
2995	unsigned B = 0;
2996
2997	if (Sel >= 13)
2998	B = 0xff;
2999	else if (Sel == 12)
3000	B = 0x00;
3001	else {
3002	const APInt *Src = ((Sel & 10) == 10 || (Sel & 12) == 4) ? C0 : C1;
3003	if (!Src)
3004	++NumUndefBytes;
3005	else if (Sel < 8)
3006	B = Src->extractBitsAsZExtValue(numBits: 8, bitPosition: (Sel & 3) * 8);
3007	else
3008	B = Src->extractBitsAsZExtValue(numBits: 1, bitPosition: (Sel & 1) ? 31 : 15) * 0xff;
3009	}
3010
3011	Val.insertBits(SubBits: B, bitPosition: I, numBits: 8);
3012	}
3013
3014	if (NumUndefBytes == 4)
3015	return UndefValue::get(T: Ty);
3016
3017	return ConstantInt::get(Ty, V: Val);
3018	}
3019
3020	static Constant *ConstantFoldScalarCall3(StringRef Name,
3021	Intrinsic::ID IntrinsicID,
3022	Type *Ty,
3023	ArrayRef<Constant *> Operands,
3024	const TargetLibraryInfo *TLI,
3025	const CallBase *Call) {
3026	assert(Operands.size() == 3 && "Wrong number of operands.");
3027
3028	if (const auto *Op1 = dyn_cast<ConstantFP>(Val: Operands[0])) {
3029	if (const auto *Op2 = dyn_cast<ConstantFP>(Val: Operands[1])) {
3030	if (const auto *Op3 = dyn_cast<ConstantFP>(Val: Operands[2])) {
3031	const APFloat &C1 = Op1->getValueAPF();
3032	const APFloat &C2 = Op2->getValueAPF();
3033	const APFloat &C3 = Op3->getValueAPF();
3034
3035	if (const auto *ConstrIntr = dyn_cast<ConstrainedFPIntrinsic>(Val: Call)) {
3036	RoundingMode RM = getEvaluationRoundingMode(CI: ConstrIntr);
3037	APFloat Res = C1;
3038	APFloat::opStatus St;
3039	switch (IntrinsicID) {
3040	default:
3041	return nullptr;
3042	case Intrinsic::experimental_constrained_fma:
3043	case Intrinsic::experimental_constrained_fmuladd:
3044	St = Res.fusedMultiplyAdd(Multiplicand: C2, Addend: C3, RM);
3045	break;
3046	}
3047	if (mayFoldConstrained(
3048	CI: const_cast<ConstrainedFPIntrinsic *>(ConstrIntr), St))
3049	return ConstantFP::get(Context&: Ty->getContext(), V: Res);
3050	return nullptr;
3051	}
3052
3053	switch (IntrinsicID) {
3054	default: break;
3055	case Intrinsic::amdgcn_fma_legacy: {
3056	// The legacy behaviour is that multiplying +/- 0.0 by anything, even
3057	// NaN or infinity, gives +0.0.
3058	if (C1.isZero() || C2.isZero()) {
3059	// It's tempting to just return C3 here, but that would give the
3060	// wrong result if C3 was -0.0.
3061	return ConstantFP::get(Context&: Ty->getContext(), V: APFloat(0.0f) + C3);
3062	}
3063	[[fallthrough]];
3064	}
3065	case Intrinsic::fma:
3066	case Intrinsic::fmuladd: {
3067	APFloat V = C1;
3068	V.fusedMultiplyAdd(Multiplicand: C2, Addend: C3, RM: APFloat::rmNearestTiesToEven);
3069	return ConstantFP::get(Context&: Ty->getContext(), V);
3070	}
3071	case Intrinsic::amdgcn_cubeid:
3072	case Intrinsic::amdgcn_cubema:
3073	case Intrinsic::amdgcn_cubesc:
3074	case Intrinsic::amdgcn_cubetc: {
3075	APFloat V = ConstantFoldAMDGCNCubeIntrinsic(IntrinsicID, S0: C1, S1: C2, S2: C3);
3076	return ConstantFP::get(Context&: Ty->getContext(), V);
3077	}
3078	}
3079	}
3080	}
3081	}
3082
3083	if (IntrinsicID == Intrinsic::smul_fix ||
3084	IntrinsicID == Intrinsic::smul_fix_sat) {
3085	// poison * C -> poison
3086	// C * poison -> poison
3087	if (isa<PoisonValue>(Val: Operands[0]) || isa<PoisonValue>(Val: Operands[1]))
3088	return PoisonValue::get(T: Ty);
3089
3090	const APInt *C0, *C1;
3091	if (!getConstIntOrUndef(Op: Operands[0], C&: C0) ||
3092	!getConstIntOrUndef(Op: Operands[1], C&: C1))
3093	return nullptr;
3094
3095	// undef * C -> 0
3096	// C * undef -> 0
3097	if (!C0 || !C1)
3098	return Constant::getNullValue(Ty);
3099
3100	// This code performs rounding towards negative infinity in case the result
3101	// cannot be represented exactly for the given scale. Targets that do care
3102	// about rounding should use a target hook for specifying how rounding
3103	// should be done, and provide their own folding to be consistent with
3104	// rounding. This is the same approach as used by
3105	// DAGTypeLegalizer::ExpandIntRes_MULFIX.
3106	unsigned Scale = cast<ConstantInt>(Val: Operands[2])->getZExtValue();
3107	unsigned Width = C0->getBitWidth();
3108	assert(Scale < Width && "Illegal scale.");
3109	unsigned ExtendedWidth = Width * 2;
3110	APInt Product =
3111	(C0->sext(width: ExtendedWidth) * C1->sext(width: ExtendedWidth)).ashr(ShiftAmt: Scale);
3112	if (IntrinsicID == Intrinsic::smul_fix_sat) {
3113	APInt Max = APInt::getSignedMaxValue(numBits: Width).sext(width: ExtendedWidth);
3114	APInt Min = APInt::getSignedMinValue(numBits: Width).sext(width: ExtendedWidth);
3115	Product = APIntOps::smin(A: Product, B: Max);
3116	Product = APIntOps::smax(A: Product, B: Min);
3117	}
3118	return ConstantInt::get(Context&: Ty->getContext(), V: Product.sextOrTrunc(width: Width));
3119	}
3120
3121	if (IntrinsicID == Intrinsic::fshl || IntrinsicID == Intrinsic::fshr) {
3122	const APInt *C0, *C1, *C2;
3123	if (!getConstIntOrUndef(Op: Operands[0], C&: C0) ||
3124	!getConstIntOrUndef(Op: Operands[1], C&: C1) ||
3125	!getConstIntOrUndef(Op: Operands[2], C&: C2))
3126	return nullptr;
3127
3128	bool IsRight = IntrinsicID == Intrinsic::fshr;
3129	if (!C2)
3130	return Operands[IsRight ? 1 : 0];
3131	if (!C0 && !C1)
3132	return UndefValue::get(T: Ty);
3133
3134	// The shift amount is interpreted as modulo the bitwidth. If the shift
3135	// amount is effectively 0, avoid UB due to oversized inverse shift below.
3136	unsigned BitWidth = C2->getBitWidth();
3137	unsigned ShAmt = C2->urem(RHS: BitWidth);
3138	if (!ShAmt)
3139	return Operands[IsRight ? 1 : 0];
3140
3141	// (C0 << ShlAmt) | (C1 >> LshrAmt)
3142	unsigned LshrAmt = IsRight ? ShAmt : BitWidth - ShAmt;
3143	unsigned ShlAmt = !IsRight ? ShAmt : BitWidth - ShAmt;
3144	if (!C0)
3145	return ConstantInt::get(Ty, V: C1->lshr(shiftAmt: LshrAmt));
3146	if (!C1)
3147	return ConstantInt::get(Ty, V: C0->shl(shiftAmt: ShlAmt));
3148	return ConstantInt::get(Ty, V: C0->shl(shiftAmt: ShlAmt) | C1->lshr(shiftAmt: LshrAmt));
3149	}
3150
3151	if (IntrinsicID == Intrinsic::amdgcn_perm)
3152	return ConstantFoldAMDGCNPermIntrinsic(Operands, Ty);
3153
3154	return nullptr;
3155	}
3156
3157	static Constant *ConstantFoldScalarCall(StringRef Name,
3158	Intrinsic::ID IntrinsicID,
3159	Type *Ty,
3160	ArrayRef<Constant *> Operands,
3161	const TargetLibraryInfo *TLI,
3162	const CallBase *Call) {
3163	if (Operands.size() == 1)
3164	return ConstantFoldScalarCall1(Name, IntrinsicID, Ty, Operands, TLI, Call);
3165
3166	if (Operands.size() == 2)
3167	return ConstantFoldScalarCall2(Name, IntrinsicID, Ty, Operands, TLI, Call);
3168
3169	if (Operands.size() == 3)
3170	return ConstantFoldScalarCall3(Name, IntrinsicID, Ty, Operands, TLI, Call);
3171
3172	return nullptr;
3173	}
3174
3175	static Constant *ConstantFoldFixedVectorCall(
3176	StringRef Name, Intrinsic::ID IntrinsicID, FixedVectorType *FVTy,
3177	ArrayRef<Constant *> Operands, const DataLayout &DL,
3178	const TargetLibraryInfo *TLI, const CallBase *Call) {
3179	SmallVector<Constant *, 4> Result(FVTy->getNumElements());
3180	SmallVector<Constant *, 4> Lane(Operands.size());
3181	Type *Ty = FVTy->getElementType();
3182
3183	switch (IntrinsicID) {
3184	case Intrinsic::masked_load: {
3185	auto *SrcPtr = Operands[0];
3186	auto *Mask = Operands[2];
3187	auto *Passthru = Operands[3];
3188
3189	Constant *VecData = ConstantFoldLoadFromConstPtr(C: SrcPtr, Ty: FVTy, DL);
3190
3191	SmallVector<Constant *, 32> NewElements;
3192	for (unsigned I = 0, E = FVTy->getNumElements(); I != E; ++I) {
3193	auto *MaskElt = Mask->getAggregateElement(Elt: I);
3194	if (!MaskElt)
3195	break;
3196	auto *PassthruElt = Passthru->getAggregateElement(Elt: I);
3197	auto *VecElt = VecData ? VecData->getAggregateElement(Elt: I) : nullptr;
3198	if (isa<UndefValue>(Val: MaskElt)) {
3199	if (PassthruElt)
3200	NewElements.push_back(Elt: PassthruElt);
3201	else if (VecElt)
3202	NewElements.push_back(Elt: VecElt);
3203	else
3204	return nullptr;
3205	}
3206	if (MaskElt->isNullValue()) {
3207	if (!PassthruElt)
3208	return nullptr;
3209	NewElements.push_back(Elt: PassthruElt);
3210	} else if (MaskElt->isOneValue()) {
3211	if (!VecElt)
3212	return nullptr;
3213	NewElements.push_back(Elt: VecElt);
3214	} else {
3215	return nullptr;
3216	}
3217	}
3218	if (NewElements.size() != FVTy->getNumElements())
3219	return nullptr;
3220	return ConstantVector::get(V: NewElements);
3221	}
3222	case Intrinsic::arm_mve_vctp8:
3223	case Intrinsic::arm_mve_vctp16:
3224	case Intrinsic::arm_mve_vctp32:
3225	case Intrinsic::arm_mve_vctp64: {
3226	if (auto *Op = dyn_cast<ConstantInt>(Val: Operands[0])) {
3227	unsigned Lanes = FVTy->getNumElements();
3228	uint64_t Limit = Op->getZExtValue();
3229
3230	SmallVector<Constant *, 16> NCs;
3231	for (unsigned i = 0; i < Lanes; i++) {
3232	if (i < Limit)
3233	NCs.push_back(Elt: ConstantInt::getTrue(Ty));
3234	else
3235	NCs.push_back(Elt: ConstantInt::getFalse(Ty));
3236	}
3237	return ConstantVector::get(V: NCs);
3238	}
3239	return nullptr;
3240	}
3241	case Intrinsic::get_active_lane_mask: {
3242	auto *Op0 = dyn_cast<ConstantInt>(Val: Operands[0]);
3243	auto *Op1 = dyn_cast<ConstantInt>(Val: Operands[1]);
3244	if (Op0 && Op1) {
3245	unsigned Lanes = FVTy->getNumElements();
3246	uint64_t Base = Op0->getZExtValue();
3247	uint64_t Limit = Op1->getZExtValue();
3248
3249	SmallVector<Constant *, 16> NCs;
3250	for (unsigned i = 0; i < Lanes; i++) {
3251	if (Base + i < Limit)
3252	NCs.push_back(Elt: ConstantInt::getTrue(Ty));
3253	else
3254	NCs.push_back(Elt: ConstantInt::getFalse(Ty));
3255	}
3256	return ConstantVector::get(V: NCs);
3257	}
3258	return nullptr;
3259	}
3260	default:
3261	break;
3262	}
3263
3264	for (unsigned I = 0, E = FVTy->getNumElements(); I != E; ++I) {
3265	// Gather a column of constants.
3266	for (unsigned J = 0, JE = Operands.size(); J != JE; ++J) {
3267	// Some intrinsics use a scalar type for certain arguments.
3268	if (isVectorIntrinsicWithScalarOpAtArg(ID: IntrinsicID, ScalarOpdIdx: J)) {
3269	Lane[J] = Operands[J];
3270	continue;
3271	}
3272
3273	Constant *Agg = Operands[J]->getAggregateElement(Elt: I);
3274	if (!Agg)
3275	return nullptr;
3276
3277	Lane[J] = Agg;
3278	}
3279
3280	// Use the regular scalar folding to simplify this column.
3281	Constant *Folded =
3282	ConstantFoldScalarCall(Name, IntrinsicID, Ty, Operands: Lane, TLI, Call);
3283	if (!Folded)
3284	return nullptr;
3285	Result[I] = Folded;
3286	}
3287
3288	return ConstantVector::get(V: Result);
3289	}
3290
3291	static Constant *ConstantFoldScalableVectorCall(
3292	StringRef Name, Intrinsic::ID IntrinsicID, ScalableVectorType *SVTy,
3293	ArrayRef<Constant *> Operands, const DataLayout &DL,
3294	const TargetLibraryInfo *TLI, const CallBase *Call) {
3295	switch (IntrinsicID) {
3296	case Intrinsic::aarch64_sve_convert_from_svbool: {
3297	auto *Src = dyn_cast<Constant>(Val: Operands[0]);
3298	if (!Src || !Src->isNullValue())
3299	break;
3300
3301	return ConstantInt::getFalse(Ty: SVTy);
3302	}
3303	default:
3304	break;
3305	}
3306	return nullptr;
3307	}
3308
3309	static std::pair<Constant *, Constant *>
3310	ConstantFoldScalarFrexpCall(Constant *Op, Type *IntTy) {
3311	if (isa<PoisonValue>(Val: Op))
3312	return {Op, PoisonValue::get(T: IntTy)};
3313
3314	auto *ConstFP = dyn_cast<ConstantFP>(Val: Op);
3315	if (!ConstFP)
3316	return {};
3317
3318	const APFloat &U = ConstFP->getValueAPF();
3319	int FrexpExp;
3320	APFloat FrexpMant = frexp(X: U, Exp&: FrexpExp, RM: APFloat::rmNearestTiesToEven);
3321	Constant *Result0 = ConstantFP::get(Ty: ConstFP->getType(), V: FrexpMant);
3322
3323	// The exponent is an "unspecified value" for inf/nan. We use zero to avoid
3324	// using undef.
3325	Constant *Result1 = FrexpMant.isFinite() ? ConstantInt::get(Ty: IntTy, V: FrexpExp)
3326	: ConstantInt::getNullValue(Ty: IntTy);
3327	return {Result0, Result1};
3328	}
3329
3330	/// Handle intrinsics that return tuples, which may be tuples of vectors.
3331	static Constant *
3332	ConstantFoldStructCall(StringRef Name, Intrinsic::ID IntrinsicID,
3333	StructType *StTy, ArrayRef<Constant *> Operands,
3334	const DataLayout &DL, const TargetLibraryInfo *TLI,
3335	const CallBase *Call) {
3336
3337	switch (IntrinsicID) {
3338	case Intrinsic::frexp: {
3339	Type *Ty0 = StTy->getContainedType(i: 0);
3340	Type *Ty1 = StTy->getContainedType(i: 1)->getScalarType();
3341
3342	if (auto *FVTy0 = dyn_cast<FixedVectorType>(Val: Ty0)) {
3343	SmallVector<Constant *, 4> Results0(FVTy0->getNumElements());
3344	SmallVector<Constant *, 4> Results1(FVTy0->getNumElements());
3345
3346	for (unsigned I = 0, E = FVTy0->getNumElements(); I != E; ++I) {
3347	Constant *Lane = Operands[0]->getAggregateElement(Elt: I);
3348	std::tie(args&: Results0[I], args&: Results1[I]) =
3349	ConstantFoldScalarFrexpCall(Op: Lane, IntTy: Ty1);
3350	if (!Results0[I])
3351	return nullptr;
3352	}
3353
3354	return ConstantStruct::get(T: StTy, Vs: ConstantVector::get(V: Results0),
3355	Vs: ConstantVector::get(V: Results1));
3356	}
3357
3358	auto [Result0, Result1] = ConstantFoldScalarFrexpCall(Op: Operands[0], IntTy: Ty1);
3359	if (!Result0)
3360	return nullptr;
3361	return ConstantStruct::get(T: StTy, Vs: Result0, Vs: Result1);
3362	}
3363	default:
3364	// TODO: Constant folding of vector intrinsics that fall through here does
3365	// not work (e.g. overflow intrinsics)
3366	return ConstantFoldScalarCall(Name, IntrinsicID, Ty: StTy, Operands, TLI, Call);
3367	}
3368
3369	return nullptr;
3370	}
3371
3372	} // end anonymous namespace
3373
3374	Constant *llvm::ConstantFoldCall(const CallBase *Call, Function *F,
3375	ArrayRef<Constant *> Operands,
3376	const TargetLibraryInfo *TLI) {
3377	if (Call->isNoBuiltin())
3378	return nullptr;
3379	if (!F->hasName())
3380	return nullptr;
3381
3382	// If this is not an intrinsic and not recognized as a library call, bail out.
3383	Intrinsic::ID IID = F->getIntrinsicID();
3384	if (IID == Intrinsic::not_intrinsic) {
3385	if (!TLI)
3386	return nullptr;
3387	LibFunc LibF;
3388	if (!TLI->getLibFunc(FDecl: *F, F&: LibF))
3389	return nullptr;
3390	}
3391
3392	StringRef Name = F->getName();
3393	Type *Ty = F->getReturnType();
3394	if (auto *FVTy = dyn_cast<FixedVectorType>(Val: Ty))
3395	return ConstantFoldFixedVectorCall(
3396	Name, IntrinsicID: IID, FVTy, Operands, DL: F->getParent()->getDataLayout(), TLI, Call);
3397
3398	if (auto *SVTy = dyn_cast<ScalableVectorType>(Val: Ty))
3399	return ConstantFoldScalableVectorCall(
3400	Name, IntrinsicID: IID, SVTy, Operands, DL: F->getParent()->getDataLayout(), TLI, Call);
3401
3402	if (auto *StTy = dyn_cast<StructType>(Val: Ty))
3403	return ConstantFoldStructCall(Name, IntrinsicID: IID, StTy, Operands,
3404	DL: F->getParent()->getDataLayout(), TLI, Call);
3405
3406	// TODO: If this is a library function, we already discovered that above,
3407	// so we should pass the LibFunc, not the name (and it might be better
3408	// still to separate intrinsic handling from libcalls).
3409	return ConstantFoldScalarCall(Name, IntrinsicID: IID, Ty, Operands, TLI, Call);
3410	}
3411
3412	bool llvm::isMathLibCallNoop(const CallBase *Call,
3413	const TargetLibraryInfo *TLI) {
3414	// FIXME: Refactor this code; this duplicates logic in LibCallsShrinkWrap
3415	// (and to some extent ConstantFoldScalarCall).
3416	if (Call->isNoBuiltin() || Call->isStrictFP())
3417	return false;
3418	Function *F = Call->getCalledFunction();
3419	if (!F)
3420	return false;
3421
3422	LibFunc Func;
3423	if (!TLI || !TLI->getLibFunc(FDecl: *F, F&: Func))
3424	return false;
3425
3426	if (Call->arg_size() == 1) {
3427	if (ConstantFP *OpC = dyn_cast<ConstantFP>(Val: Call->getArgOperand(i: 0))) {
3428	const APFloat &Op = OpC->getValueAPF();
3429	switch (Func) {
3430	case LibFunc_logl:
3431	case LibFunc_log:
3432	case LibFunc_logf:
3433	case LibFunc_log2l:
3434	case LibFunc_log2:
3435	case LibFunc_log2f:
3436	case LibFunc_log10l:
3437	case LibFunc_log10:
3438	case LibFunc_log10f:
3439	return Op.isNaN() || (!Op.isZero() && !Op.isNegative());
3440
3441	case LibFunc_expl:
3442	case LibFunc_exp:
3443	case LibFunc_expf:
3444	// FIXME: These boundaries are slightly conservative.
3445	if (OpC->getType()->isDoubleTy())
3446	return !(Op < APFloat(-745.0) || Op > APFloat(709.0));
3447	if (OpC->getType()->isFloatTy())
3448	return !(Op < APFloat(-103.0f) || Op > APFloat(88.0f));
3449	break;
3450
3451	case LibFunc_exp2l:
3452	case LibFunc_exp2:
3453	case LibFunc_exp2f:
3454	// FIXME: These boundaries are slightly conservative.
3455	if (OpC->getType()->isDoubleTy())
3456	return !(Op < APFloat(-1074.0) || Op > APFloat(1023.0));
3457	if (OpC->getType()->isFloatTy())
3458	return !(Op < APFloat(-149.0f) || Op > APFloat(127.0f));
3459	break;
3460
3461	case LibFunc_sinl:
3462	case LibFunc_sin:
3463	case LibFunc_sinf:
3464	case LibFunc_cosl:
3465	case LibFunc_cos:
3466	case LibFunc_cosf:
3467	return !Op.isInfinity();
3468
3469	case LibFunc_tanl:
3470	case LibFunc_tan:
3471	case LibFunc_tanf: {
3472	// FIXME: Stop using the host math library.
3473	// FIXME: The computation isn't done in the right precision.
3474	Type *Ty = OpC->getType();
3475	if (Ty->isDoubleTy() || Ty->isFloatTy() || Ty->isHalfTy())
3476	return ConstantFoldFP(NativeFP: tan, V: OpC->getValueAPF(), Ty) != nullptr;
3477	break;
3478	}
3479
3480	case LibFunc_atan:
3481	case LibFunc_atanf:
3482	case LibFunc_atanl:
3483	// Per POSIX, this MAY fail if Op is denormal. We choose not failing.
3484	return true;
3485
3486
3487	case LibFunc_asinl:
3488	case LibFunc_asin:
3489	case LibFunc_asinf:
3490	case LibFunc_acosl:
3491	case LibFunc_acos:
3492	case LibFunc_acosf:
3493	return !(Op < APFloat(Op.getSemantics(), "-1") ||
3494	Op > APFloat(Op.getSemantics(), "1"));
3495
3496	case LibFunc_sinh:
3497	case LibFunc_cosh:
3498	case LibFunc_sinhf:
3499	case LibFunc_coshf:
3500	case LibFunc_sinhl:
3501	case LibFunc_coshl:
3502	// FIXME: These boundaries are slightly conservative.
3503	if (OpC->getType()->isDoubleTy())
3504	return !(Op < APFloat(-710.0) || Op > APFloat(710.0));
3505	if (OpC->getType()->isFloatTy())
3506	return !(Op < APFloat(-89.0f) || Op > APFloat(89.0f));
3507	break;
3508
3509	case LibFunc_sqrtl:
3510	case LibFunc_sqrt:
3511	case LibFunc_sqrtf:
3512	return Op.isNaN() || Op.isZero() || !Op.isNegative();
3513
3514	// FIXME: Add more functions: sqrt_finite, atanh, expm1, log1p,
3515	// maybe others?
3516	default:
3517	break;
3518	}
3519	}
3520	}
3521
3522	if (Call->arg_size() == 2) {
3523	ConstantFP *Op0C = dyn_cast<ConstantFP>(Val: Call->getArgOperand(i: 0));
3524	ConstantFP *Op1C = dyn_cast<ConstantFP>(Val: Call->getArgOperand(i: 1));
3525	if (Op0C && Op1C) {
3526	const APFloat &Op0 = Op0C->getValueAPF();
3527	const APFloat &Op1 = Op1C->getValueAPF();
3528
3529	switch (Func) {
3530	case LibFunc_powl:
3531	case LibFunc_pow:
3532	case LibFunc_powf: {
3533	// FIXME: Stop using the host math library.
3534	// FIXME: The computation isn't done in the right precision.
3535	Type *Ty = Op0C->getType();
3536	if (Ty->isDoubleTy() || Ty->isFloatTy() || Ty->isHalfTy()) {
3537	if (Ty == Op1C->getType())
3538	return ConstantFoldBinaryFP(NativeFP: pow, V: Op0, W: Op1, Ty) != nullptr;
3539	}
3540	break;
3541	}
3542
3543	case LibFunc_fmodl:
3544	case LibFunc_fmod:
3545	case LibFunc_fmodf:
3546	case LibFunc_remainderl:
3547	case LibFunc_remainder:
3548	case LibFunc_remainderf:
3549	return Op0.isNaN() || Op1.isNaN() ||
3550	(!Op0.isInfinity() && !Op1.isZero());
3551
3552	case LibFunc_atan2:
3553	case LibFunc_atan2f:
3554	case LibFunc_atan2l:
3555	// Although IEEE-754 says atan2(+/-0.0, +/-0.0) are well-defined, and
3556	// GLIBC and MSVC do not appear to raise an error on those, we
3557	// cannot rely on that behavior. POSIX and C11 say that a domain error
3558	// may occur, so allow for that possibility.
3559	return !Op0.isZero() || !Op1.isZero();
3560
3561	default:
3562	break;
3563	}
3564	}
3565	}
3566
3567	return false;
3568	}
3569
3570	void TargetFolder::anchor() {}
3571