1	//===-- CodeGenFunction.h - Per-Function state for LLVM CodeGen -*- C++ -*-===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This is the internal per-function state used for llvm translation.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#ifndef LLVM_CLANG_LIB_CODEGEN_CODEGENFUNCTION_H
14	#define LLVM_CLANG_LIB_CODEGEN_CODEGENFUNCTION_H
15
16	#include "CGBuilder.h"
17	#include "CGDebugInfo.h"
18	#include "CGLoopInfo.h"
19	#include "CGValue.h"
20	#include "CodeGenModule.h"
21	#include "CodeGenPGO.h"
22	#include "EHScopeStack.h"
23	#include "VarBypassDetector.h"
24	#include "clang/AST/CharUnits.h"
25	#include "clang/AST/CurrentSourceLocExprScope.h"
26	#include "clang/AST/ExprCXX.h"
27	#include "clang/AST/ExprObjC.h"
28	#include "clang/AST/ExprOpenMP.h"
29	#include "clang/AST/StmtOpenACC.h"
30	#include "clang/AST/StmtOpenMP.h"
31	#include "clang/AST/Type.h"
32	#include "clang/Basic/ABI.h"
33	#include "clang/Basic/CapturedStmt.h"
34	#include "clang/Basic/CodeGenOptions.h"
35	#include "clang/Basic/OpenMPKinds.h"
36	#include "clang/Basic/TargetInfo.h"
37	#include "llvm/ADT/ArrayRef.h"
38	#include "llvm/ADT/DenseMap.h"
39	#include "llvm/ADT/MapVector.h"
40	#include "llvm/ADT/SmallVector.h"
41	#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
42	#include "llvm/IR/ValueHandle.h"
43	#include "llvm/Support/Debug.h"
44	#include "llvm/Transforms/Utils/SanitizerStats.h"
45	#include <optional>
46
47	namespace llvm {
48	class BasicBlock;
49	class LLVMContext;
50	class MDNode;
51	class SwitchInst;
52	class Twine;
53	class Value;
54	class CanonicalLoopInfo;
55	}
56
57	namespace clang {
58	class ASTContext;
59	class CXXDestructorDecl;
60	class CXXForRangeStmt;
61	class CXXTryStmt;
62	class Decl;
63	class LabelDecl;
64	class FunctionDecl;
65	class FunctionProtoType;
66	class LabelStmt;
67	class ObjCContainerDecl;
68	class ObjCInterfaceDecl;
69	class ObjCIvarDecl;
70	class ObjCMethodDecl;
71	class ObjCImplementationDecl;
72	class ObjCPropertyImplDecl;
73	class TargetInfo;
74	class VarDecl;
75	class ObjCForCollectionStmt;
76	class ObjCAtTryStmt;
77	class ObjCAtThrowStmt;
78	class ObjCAtSynchronizedStmt;
79	class ObjCAutoreleasePoolStmt;
80	class OMPUseDevicePtrClause;
81	class OMPUseDeviceAddrClause;
82	class SVETypeFlags;
83	class OMPExecutableDirective;
84
85	namespace analyze_os_log {
86	class OSLogBufferLayout;
87	}
88
89	namespace CodeGen {
90	class CodeGenTypes;
91	class CGCallee;
92	class CGFunctionInfo;
93	class CGBlockInfo;
94	class CGCXXABI;
95	class BlockByrefHelpers;
96	class BlockByrefInfo;
97	class BlockFieldFlags;
98	class RegionCodeGenTy;
99	class TargetCodeGenInfo;
100	struct OMPTaskDataTy;
101	struct CGCoroData;
102
103	/// The kind of evaluation to perform on values of a particular
104	/// type. Basically, is the code in CGExprScalar, CGExprComplex, or
105	/// CGExprAgg?
106	///
107	/// TODO: should vectors maybe be split out into their own thing?
108	enum TypeEvaluationKind {
109	TEK_Scalar,
110	TEK_Complex,
111	TEK_Aggregate
112	};
113
114	#define LIST_SANITIZER_CHECKS \
115	SANITIZER_CHECK(AddOverflow, add_overflow, 0) \
116	SANITIZER_CHECK(BuiltinUnreachable, builtin_unreachable, 0) \
117	SANITIZER_CHECK(CFICheckFail, cfi_check_fail, 0) \
118	SANITIZER_CHECK(DivremOverflow, divrem_overflow, 0) \
119	SANITIZER_CHECK(DynamicTypeCacheMiss, dynamic_type_cache_miss, 0) \
120	SANITIZER_CHECK(FloatCastOverflow, float_cast_overflow, 0) \
121	SANITIZER_CHECK(FunctionTypeMismatch, function_type_mismatch, 0) \
122	SANITIZER_CHECK(ImplicitConversion, implicit_conversion, 0) \
123	SANITIZER_CHECK(InvalidBuiltin, invalid_builtin, 0) \
124	SANITIZER_CHECK(InvalidObjCCast, invalid_objc_cast, 0) \
125	SANITIZER_CHECK(LoadInvalidValue, load_invalid_value, 0) \
126	SANITIZER_CHECK(MissingReturn, missing_return, 0) \
127	SANITIZER_CHECK(MulOverflow, mul_overflow, 0) \
128	SANITIZER_CHECK(NegateOverflow, negate_overflow, 0) \
129	SANITIZER_CHECK(NullabilityArg, nullability_arg, 0) \
130	SANITIZER_CHECK(NullabilityReturn, nullability_return, 1) \
131	SANITIZER_CHECK(NonnullArg, nonnull_arg, 0) \
132	SANITIZER_CHECK(NonnullReturn, nonnull_return, 1) \
133	SANITIZER_CHECK(OutOfBounds, out_of_bounds, 0) \
134	SANITIZER_CHECK(PointerOverflow, pointer_overflow, 0) \
135	SANITIZER_CHECK(ShiftOutOfBounds, shift_out_of_bounds, 0) \
136	SANITIZER_CHECK(SubOverflow, sub_overflow, 0) \
137	SANITIZER_CHECK(TypeMismatch, type_mismatch, 1) \
138	SANITIZER_CHECK(AlignmentAssumption, alignment_assumption, 0) \
139	SANITIZER_CHECK(VLABoundNotPositive, vla_bound_not_positive, 0)
140
141	enum SanitizerHandler {
142	#define SANITIZER_CHECK(Enum, Name, Version) Enum,
143	LIST_SANITIZER_CHECKS
144	#undef SANITIZER_CHECK
145	};
146
147	/// Helper class with most of the code for saving a value for a
148	/// conditional expression cleanup.
149	struct DominatingLLVMValue {
150	typedef llvm::PointerIntPair<llvm::Value*, 1, bool> saved_type;
151
152	/// Answer whether the given value needs extra work to be saved.
153	static bool needsSaving(llvm::Value *value) {
154	// If it's not an instruction, we don't need to save.
155	if (!isa<llvm::Instruction>(Val: value)) return false;
156
157	// If it's an instruction in the entry block, we don't need to save.
158	llvm::BasicBlock *block = cast<llvm::Instruction>(Val: value)->getParent();
159	return (block != &block->getParent()->getEntryBlock());
160	}
161
162	static saved_type save(CodeGenFunction &CGF, llvm::Value *value);
163	static llvm::Value *restore(CodeGenFunction &CGF, saved_type value);
164	};
165
166	/// A partial specialization of DominatingValue for llvm::Values that
167	/// might be llvm::Instructions.
168	template <class T> struct DominatingPointer<T,true> : DominatingLLVMValue {
169	typedef T *type;
170	static type restore(CodeGenFunction &CGF, saved_type value) {
171	return static_cast<T*>(DominatingLLVMValue::restore(CGF, value));
172	}
173	};
174
175	/// A specialization of DominatingValue for Address.
176	template <> struct DominatingValue<Address> {
177	typedef Address type;
178
179	struct saved_type {
180	DominatingLLVMValue::saved_type SavedValue;
181	llvm::Type *ElementType;
182	CharUnits Alignment;
183	};
184
185	static bool needsSaving(type value) {
186	return DominatingLLVMValue::needsSaving(value: value.getPointer());
187	}
188	static saved_type save(CodeGenFunction &CGF, type value) {
189	return { .SavedValue: DominatingLLVMValue::save(CGF, value: value.getPointer()),
190	.ElementType: value.getElementType(), .Alignment: value.getAlignment() };
191	}
192	static type restore(CodeGenFunction &CGF, saved_type value) {
193	return Address(DominatingLLVMValue::restore(CGF, value: value.SavedValue),
194	value.ElementType, value.Alignment);
195	}
196	};
197
198	/// A specialization of DominatingValue for RValue.
199	template <> struct DominatingValue<RValue> {
200	typedef RValue type;
201	class saved_type {
202	enum Kind { ScalarLiteral, ScalarAddress, AggregateLiteral,
203	AggregateAddress, ComplexAddress };
204
205	llvm::Value *Value;
206	llvm::Type *ElementType;
207	LLVM_PREFERRED_TYPE(Kind)
208	unsigned K : 3;
209	unsigned Align : 29;
210	saved_type(llvm::Value *v, llvm::Type *e, Kind k, unsigned a = 0)
211	: Value(v), ElementType(e), K(k), Align(a) {}
212
213	public:
214	static bool needsSaving(RValue value);
215	static saved_type save(CodeGenFunction &CGF, RValue value);
216	RValue restore(CodeGenFunction &CGF);
217
218	// implementations in CGCleanup.cpp
219	};
220
221	static bool needsSaving(type value) {
222	return saved_type::needsSaving(value);
223	}
224	static saved_type save(CodeGenFunction &CGF, type value) {
225	return saved_type::save(CGF, value);
226	}
227	static type restore(CodeGenFunction &CGF, saved_type value) {
228	return value.restore(CGF);
229	}
230	};
231
232	/// CodeGenFunction - This class organizes the per-function state that is used
233	/// while generating LLVM code.
234	class CodeGenFunction : public CodeGenTypeCache {
235	CodeGenFunction(const CodeGenFunction &) = delete;
236	void operator=(const CodeGenFunction &) = delete;
237
238	friend class CGCXXABI;
239	public:
240	/// A jump destination is an abstract label, branching to which may
241	/// require a jump out through normal cleanups.
242	struct JumpDest {
243	JumpDest() : Block(nullptr), Index(0) {}
244	JumpDest(llvm::BasicBlock *Block, EHScopeStack::stable_iterator Depth,
245	unsigned Index)
246	: Block(Block), ScopeDepth(Depth), Index(Index) {}
247
248	bool isValid() const { return Block != nullptr; }
249	llvm::BasicBlock *getBlock() const { return Block; }
250	EHScopeStack::stable_iterator getScopeDepth() const { return ScopeDepth; }
251	unsigned getDestIndex() const { return Index; }
252
253	// This should be used cautiously.
254	void setScopeDepth(EHScopeStack::stable_iterator depth) {
255	ScopeDepth = depth;
256	}
257
258	private:
259	llvm::BasicBlock *Block;
260	EHScopeStack::stable_iterator ScopeDepth;
261	unsigned Index;
262	};
263
264	CodeGenModule &CGM; // Per-module state.
265	const TargetInfo &Target;
266
267	// For EH/SEH outlined funclets, this field points to parent's CGF
268	CodeGenFunction *ParentCGF = nullptr;
269
270	typedef std::pair<llvm::Value *, llvm::Value *> ComplexPairTy;
271	LoopInfoStack LoopStack;
272	CGBuilderTy Builder;
273
274	// Stores variables for which we can't generate correct lifetime markers
275	// because of jumps.
276	VarBypassDetector Bypasses;
277
278	/// List of recently emitted OMPCanonicalLoops.
279	///
280	/// Since OMPCanonicalLoops are nested inside other statements (in particular
281	/// CapturedStmt generated by OMPExecutableDirective and non-perfectly nested
282	/// loops), we cannot directly call OMPEmitOMPCanonicalLoop and receive its
283	/// llvm::CanonicalLoopInfo. Instead, we call EmitStmt and any
284	/// OMPEmitOMPCanonicalLoop called by it will add its CanonicalLoopInfo to
285	/// this stack when done. Entering a new loop requires clearing this list; it
286	/// either means we start parsing a new loop nest (in which case the previous
287	/// loop nest goes out of scope) or a second loop in the same level in which
288	/// case it would be ambiguous into which of the two (or more) loops the loop
289	/// nest would extend.
290	SmallVector<llvm::CanonicalLoopInfo *, 4> OMPLoopNestStack;
291
292	/// Stack to track the Logical Operator recursion nest for MC/DC.
293	SmallVector<const BinaryOperator *, 16> MCDCLogOpStack;
294
295	/// Number of nested loop to be consumed by the last surrounding
296	/// loop-associated directive.
297	int ExpectedOMPLoopDepth = 0;
298
299	// CodeGen lambda for loops and support for ordered clause
300	typedef llvm::function_ref<void(CodeGenFunction &, const OMPLoopDirective &,
301	JumpDest)>
302	CodeGenLoopTy;
303	typedef llvm::function_ref<void(CodeGenFunction &, SourceLocation,
304	const unsigned, const bool)>
305	CodeGenOrderedTy;
306
307	// Codegen lambda for loop bounds in worksharing loop constructs
308	typedef llvm::function_ref<std::pair<LValue, LValue>(
309	CodeGenFunction &, const OMPExecutableDirective &S)>
310	CodeGenLoopBoundsTy;
311
312	// Codegen lambda for loop bounds in dispatch-based loop implementation
313	typedef llvm::function_ref<std::pair<llvm::Value *, llvm::Value *>(
314	CodeGenFunction &, const OMPExecutableDirective &S, Address LB,
315	Address UB)>
316	CodeGenDispatchBoundsTy;
317
318	/// CGBuilder insert helper. This function is called after an
319	/// instruction is created using Builder.
320	void InsertHelper(llvm::Instruction *I, const llvm::Twine &Name,
321	llvm::BasicBlock *BB,
322	llvm::BasicBlock::iterator InsertPt) const;
323
324	/// CurFuncDecl - Holds the Decl for the current outermost
325	/// non-closure context.
326	const Decl *CurFuncDecl = nullptr;
327	/// CurCodeDecl - This is the inner-most code context, which includes blocks.
328	const Decl *CurCodeDecl = nullptr;
329	const CGFunctionInfo *CurFnInfo = nullptr;
330	QualType FnRetTy;
331	llvm::Function *CurFn = nullptr;
332
333	/// Save Parameter Decl for coroutine.
334	llvm::SmallVector<const ParmVarDecl *, 4> FnArgs;
335
336	// Holds coroutine data if the current function is a coroutine. We use a
337	// wrapper to manage its lifetime, so that we don't have to define CGCoroData
338	// in this header.
339	struct CGCoroInfo {
340	std::unique_ptr<CGCoroData> Data;
341	bool InSuspendBlock = false;
342	CGCoroInfo();
343	~CGCoroInfo();
344	};
345	CGCoroInfo CurCoro;
346
347	bool isCoroutine() const {
348	return CurCoro.Data != nullptr;
349	}
350
351	bool inSuspendBlock() const {
352	return isCoroutine() && CurCoro.InSuspendBlock;
353	}
354
355	/// CurGD - The GlobalDecl for the current function being compiled.
356	GlobalDecl CurGD;
357
358	/// PrologueCleanupDepth - The cleanup depth enclosing all the
359	/// cleanups associated with the parameters.
360	EHScopeStack::stable_iterator PrologueCleanupDepth;
361
362	/// ReturnBlock - Unified return block.
363	JumpDest ReturnBlock;
364
365	/// ReturnValue - The temporary alloca to hold the return
366	/// value. This is invalid iff the function has no return value.
367	Address ReturnValue = Address::invalid();
368
369	/// ReturnValuePointer - The temporary alloca to hold a pointer to sret.
370	/// This is invalid if sret is not in use.
371	Address ReturnValuePointer = Address::invalid();
372
373	/// If a return statement is being visited, this holds the return statment's
374	/// result expression.
375	const Expr *RetExpr = nullptr;
376
377	/// Return true if a label was seen in the current scope.
378	bool hasLabelBeenSeenInCurrentScope() const {
379	if (CurLexicalScope)
380	return CurLexicalScope->hasLabels();
381	return !LabelMap.empty();
382	}
383
384	/// AllocaInsertPoint - This is an instruction in the entry block before which
385	/// we prefer to insert allocas.
386	llvm::AssertingVH<llvm::Instruction> AllocaInsertPt;
387
388	private:
389	/// PostAllocaInsertPt - This is a place in the prologue where code can be
390	/// inserted that will be dominated by all the static allocas. This helps
391	/// achieve two things:
392	/// 1. Contiguity of all static allocas (within the prologue) is maintained.
393	/// 2. All other prologue code (which are dominated by static allocas) do
394	/// appear in the source order immediately after all static allocas.
395	///
396	/// PostAllocaInsertPt will be lazily created when it is *really* required.
397	llvm::AssertingVH<llvm::Instruction> PostAllocaInsertPt = nullptr;
398
399	public:
400	/// Return PostAllocaInsertPt. If it is not yet created, then insert it
401	/// immediately after AllocaInsertPt.
402	llvm::Instruction *getPostAllocaInsertPoint() {
403	if (!PostAllocaInsertPt) {
404	assert(AllocaInsertPt &&
405	"Expected static alloca insertion point at function prologue");
406	assert(AllocaInsertPt->getParent()->isEntryBlock() &&
407	"EBB should be entry block of the current code gen function");
408	PostAllocaInsertPt = AllocaInsertPt->clone();
409	PostAllocaInsertPt->setName("postallocapt");
410	PostAllocaInsertPt->insertAfter(InsertPos: AllocaInsertPt);
411	}
412
413	return PostAllocaInsertPt;
414	}
415
416	/// API for captured statement code generation.
417	class CGCapturedStmtInfo {
418	public:
419	explicit CGCapturedStmtInfo(CapturedRegionKind K = CR_Default)
420	: Kind(K), ThisValue(nullptr), CXXThisFieldDecl(nullptr) {}
421	explicit CGCapturedStmtInfo(const CapturedStmt &S,
422	CapturedRegionKind K = CR_Default)
423	: Kind(K), ThisValue(nullptr), CXXThisFieldDecl(nullptr) {
424
425	RecordDecl::field_iterator Field =
426	S.getCapturedRecordDecl()->field_begin();
427	for (CapturedStmt::const_capture_iterator I = S.capture_begin(),
428	E = S.capture_end();
429	I != E; ++I, ++Field) {
430	if (I->capturesThis())
431	CXXThisFieldDecl = *Field;
432	else if (I->capturesVariable())
433	CaptureFields[I->getCapturedVar()->getCanonicalDecl()] = *Field;
434	else if (I->capturesVariableByCopy())
435	CaptureFields[I->getCapturedVar()->getCanonicalDecl()] = *Field;
436	}
437	}
438
439	virtual ~CGCapturedStmtInfo();
440
441	CapturedRegionKind getKind() const { return Kind; }
442
443	virtual void setContextValue(llvm::Value *V) { ThisValue = V; }
444	// Retrieve the value of the context parameter.
445	virtual llvm::Value *getContextValue() const { return ThisValue; }
446
447	/// Lookup the captured field decl for a variable.
448	virtual const FieldDecl *lookup(const VarDecl *VD) const {
449	return CaptureFields.lookup(Val: VD->getCanonicalDecl());
450	}
451
452	bool isCXXThisExprCaptured() const { return getThisFieldDecl() != nullptr; }
453	virtual FieldDecl *getThisFieldDecl() const { return CXXThisFieldDecl; }
454
455	static bool classof(const CGCapturedStmtInfo *) {
456	return true;
457	}
458
459	/// Emit the captured statement body.
460	virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S) {
461	CGF.incrementProfileCounter(S);
462	CGF.EmitStmt(S);
463	}
464
465	/// Get the name of the capture helper.
466	virtual StringRef getHelperName() const { return "__captured_stmt"; }
467
468	/// Get the CaptureFields
469	llvm::SmallDenseMap<const VarDecl *, FieldDecl *> getCaptureFields() {
470	return CaptureFields;
471	}
472
473	private:
474	/// The kind of captured statement being generated.
475	CapturedRegionKind Kind;
476
477	/// Keep the map between VarDecl and FieldDecl.
478	llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields;
479
480	/// The base address of the captured record, passed in as the first
481	/// argument of the parallel region function.
482	llvm::Value *ThisValue;
483
484	/// Captured 'this' type.
485	FieldDecl *CXXThisFieldDecl;
486	};
487	CGCapturedStmtInfo *CapturedStmtInfo = nullptr;
488
489	/// RAII for correct setting/restoring of CapturedStmtInfo.
490	class CGCapturedStmtRAII {
491	private:
492	CodeGenFunction &CGF;
493	CGCapturedStmtInfo *PrevCapturedStmtInfo;
494	public:
495	CGCapturedStmtRAII(CodeGenFunction &CGF,
496	CGCapturedStmtInfo *NewCapturedStmtInfo)
497	: CGF(CGF), PrevCapturedStmtInfo(CGF.CapturedStmtInfo) {
498	CGF.CapturedStmtInfo = NewCapturedStmtInfo;
499	}
500	~CGCapturedStmtRAII() { CGF.CapturedStmtInfo = PrevCapturedStmtInfo; }
501	};
502
503	/// An abstract representation of regular/ObjC call/message targets.
504	class AbstractCallee {
505	/// The function declaration of the callee.
506	const Decl *CalleeDecl;
507
508	public:
509	AbstractCallee() : CalleeDecl(nullptr) {}
510	AbstractCallee(const FunctionDecl *FD) : CalleeDecl(FD) {}
511	AbstractCallee(const ObjCMethodDecl *OMD) : CalleeDecl(OMD) {}
512	bool hasFunctionDecl() const {
513	return isa_and_nonnull<FunctionDecl>(Val: CalleeDecl);
514	}
515	const Decl *getDecl() const { return CalleeDecl; }
516	unsigned getNumParams() const {
517	if (const auto *FD = dyn_cast<FunctionDecl>(Val: CalleeDecl))
518	return FD->getNumParams();
519	return cast<ObjCMethodDecl>(Val: CalleeDecl)->param_size();
520	}
521	const ParmVarDecl *getParamDecl(unsigned I) const {
522	if (const auto *FD = dyn_cast<FunctionDecl>(Val: CalleeDecl))
523	return FD->getParamDecl(i: I);
524	return *(cast<ObjCMethodDecl>(Val: CalleeDecl)->param_begin() + I);
525	}
526	};
527
528	/// Sanitizers enabled for this function.
529	SanitizerSet SanOpts;
530
531	/// True if CodeGen currently emits code implementing sanitizer checks.
532	bool IsSanitizerScope = false;
533
534	/// RAII object to set/unset CodeGenFunction::IsSanitizerScope.
535	class SanitizerScope {
536	CodeGenFunction *CGF;
537	public:
538	SanitizerScope(CodeGenFunction *CGF);
539	~SanitizerScope();
540	};
541
542	/// In C++, whether we are code generating a thunk. This controls whether we
543	/// should emit cleanups.
544	bool CurFuncIsThunk = false;
545
546	/// In ARC, whether we should autorelease the return value.
547	bool AutoreleaseResult = false;
548
549	/// Whether we processed a Microsoft-style asm block during CodeGen. These can
550	/// potentially set the return value.
551	bool SawAsmBlock = false;
552
553	GlobalDecl CurSEHParent;
554
555	/// True if the current function is an outlined SEH helper. This can be a
556	/// finally block or filter expression.
557	bool IsOutlinedSEHHelper = false;
558
559	/// True if CodeGen currently emits code inside presereved access index
560	/// region.
561	bool IsInPreservedAIRegion = false;
562
563	/// True if the current statement has nomerge attribute.
564	bool InNoMergeAttributedStmt = false;
565
566	/// True if the current statement has noinline attribute.
567	bool InNoInlineAttributedStmt = false;
568
569	/// True if the current statement has always_inline attribute.
570	bool InAlwaysInlineAttributedStmt = false;
571
572	// The CallExpr within the current statement that the musttail attribute
573	// applies to. nullptr if there is no 'musttail' on the current statement.
574	const CallExpr *MustTailCall = nullptr;
575
576	/// Returns true if a function must make progress, which means the
577	/// mustprogress attribute can be added.
578	bool checkIfFunctionMustProgress() {
579	if (CGM.getCodeGenOpts().getFiniteLoops() ==
580	CodeGenOptions::FiniteLoopsKind::Never)
581	return false;
582
583	// C++11 and later guarantees that a thread eventually will do one of the
584	// following (C++11 [intro.multithread]p24 and C++17 [intro.progress]p1):
585	// - terminate,
586	// - make a call to a library I/O function,
587	// - perform an access through a volatile glvalue, or
588	// - perform a synchronization operation or an atomic operation.
589	//
590	// Hence each function is 'mustprogress' in C++11 or later.
591	return getLangOpts().CPlusPlus11;
592	}
593
594	/// Returns true if a loop must make progress, which means the mustprogress
595	/// attribute can be added. \p HasConstantCond indicates whether the branch
596	/// condition is a known constant.
597	bool checkIfLoopMustProgress(bool HasConstantCond) {
598	if (CGM.getCodeGenOpts().getFiniteLoops() ==
599	CodeGenOptions::FiniteLoopsKind::Always)
600	return true;
601	if (CGM.getCodeGenOpts().getFiniteLoops() ==
602	CodeGenOptions::FiniteLoopsKind::Never)
603	return false;
604
605	// If the containing function must make progress, loops also must make
606	// progress (as in C++11 and later).
607	if (checkIfFunctionMustProgress())
608	return true;
609
610	// Now apply rules for plain C (see 6.8.5.6 in C11).
611	// Loops with constant conditions do not have to make progress in any C
612	// version.
613	if (HasConstantCond)
614	return false;
615
616	// Loops with non-constant conditions must make progress in C11 and later.
617	return getLangOpts().C11;
618	}
619
620	const CodeGen::CGBlockInfo *BlockInfo = nullptr;
621	llvm::Value *BlockPointer = nullptr;
622
623	llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;
624	FieldDecl *LambdaThisCaptureField = nullptr;
625
626	/// A mapping from NRVO variables to the flags used to indicate
627	/// when the NRVO has been applied to this variable.
628	llvm::DenseMap<const VarDecl *, llvm::Value *> NRVOFlags;
629
630	EHScopeStack EHStack;
631	llvm::SmallVector<char, 256> LifetimeExtendedCleanupStack;
632	llvm::SmallVector<const JumpDest *, 2> SEHTryEpilogueStack;
633
634	llvm::Instruction *CurrentFuncletPad = nullptr;
635
636	class CallLifetimeEnd final : public EHScopeStack::Cleanup {
637	bool isRedundantBeforeReturn() override { return true; }
638
639	llvm::Value *Addr;
640	llvm::Value *Size;
641
642	public:
643	CallLifetimeEnd(Address addr, llvm::Value *size)
644	: Addr(addr.getPointer()), Size(size) {}
645
646	void Emit(CodeGenFunction &CGF, Flags flags) override {
647	CGF.EmitLifetimeEnd(Size, Addr);
648	}
649	};
650
651	/// Header for data within LifetimeExtendedCleanupStack.
652	struct LifetimeExtendedCleanupHeader {
653	/// The size of the following cleanup object.
654	unsigned Size;
655	/// The kind of cleanup to push.
656	LLVM_PREFERRED_TYPE(CleanupKind)
657	unsigned Kind : 31;
658	/// Whether this is a conditional cleanup.
659	LLVM_PREFERRED_TYPE(bool)
660	unsigned IsConditional : 1;
661
662	size_t getSize() const { return Size; }
663	CleanupKind getKind() const { return (CleanupKind)Kind; }
664	bool isConditional() const { return IsConditional; }
665	};
666
667	/// i32s containing the indexes of the cleanup destinations.
668	Address NormalCleanupDest = Address::invalid();
669
670	unsigned NextCleanupDestIndex = 1;
671
672	/// EHResumeBlock - Unified block containing a call to llvm.eh.resume.
673	llvm::BasicBlock *EHResumeBlock = nullptr;
674
675	/// The exception slot. All landing pads write the current exception pointer
676	/// into this alloca.
677	llvm::Value *ExceptionSlot = nullptr;
678
679	/// The selector slot. Under the MandatoryCleanup model, all landing pads
680	/// write the current selector value into this alloca.
681	llvm::AllocaInst *EHSelectorSlot = nullptr;
682
683	/// A stack of exception code slots. Entering an __except block pushes a slot
684	/// on the stack and leaving pops one. The __exception_code() intrinsic loads
685	/// a value from the top of the stack.
686	SmallVector<Address, 1> SEHCodeSlotStack;
687
688	/// Value returned by __exception_info intrinsic.
689	llvm::Value *SEHInfo = nullptr;
690
691	/// Emits a landing pad for the current EH stack.
692	llvm::BasicBlock *EmitLandingPad();
693
694	llvm::BasicBlock *getInvokeDestImpl();
695
696	/// Parent loop-based directive for scan directive.
697	const OMPExecutableDirective *OMPParentLoopDirectiveForScan = nullptr;
698	llvm::BasicBlock *OMPBeforeScanBlock = nullptr;
699	llvm::BasicBlock *OMPAfterScanBlock = nullptr;
700	llvm::BasicBlock *OMPScanExitBlock = nullptr;
701	llvm::BasicBlock *OMPScanDispatch = nullptr;
702	bool OMPFirstScanLoop = false;
703
704	/// Manages parent directive for scan directives.
705	class ParentLoopDirectiveForScanRegion {
706	CodeGenFunction &CGF;
707	const OMPExecutableDirective *ParentLoopDirectiveForScan;
708
709	public:
710	ParentLoopDirectiveForScanRegion(
711	CodeGenFunction &CGF,
712	const OMPExecutableDirective &ParentLoopDirectiveForScan)
713	: CGF(CGF),
714	ParentLoopDirectiveForScan(CGF.OMPParentLoopDirectiveForScan) {
715	CGF.OMPParentLoopDirectiveForScan = &ParentLoopDirectiveForScan;
716	}
717	~ParentLoopDirectiveForScanRegion() {
718	CGF.OMPParentLoopDirectiveForScan = ParentLoopDirectiveForScan;
719	}
720	};
721
722	template <class T>
723	typename DominatingValue<T>::saved_type saveValueInCond(T value) {
724	return DominatingValue<T>::save(*this, value);
725	}
726
727	class CGFPOptionsRAII {
728	public:
729	CGFPOptionsRAII(CodeGenFunction &CGF, FPOptions FPFeatures);
730	CGFPOptionsRAII(CodeGenFunction &CGF, const Expr *E);
731	~CGFPOptionsRAII();
732
733	private:
734	void ConstructorHelper(FPOptions FPFeatures);
735	CodeGenFunction &CGF;
736	FPOptions OldFPFeatures;
737	llvm::fp::ExceptionBehavior OldExcept;
738	llvm::RoundingMode OldRounding;
739	std::optional<CGBuilderTy::FastMathFlagGuard> FMFGuard;
740	};
741	FPOptions CurFPFeatures;
742
743	public:
744	/// ObjCEHValueStack - Stack of Objective-C exception values, used for
745	/// rethrows.
746	SmallVector<llvm::Value*, 8> ObjCEHValueStack;
747
748	/// A class controlling the emission of a finally block.
749	class FinallyInfo {
750	/// Where the catchall's edge through the cleanup should go.
751	JumpDest RethrowDest;
752
753	/// A function to call to enter the catch.
754	llvm::FunctionCallee BeginCatchFn;
755
756	/// An i1 variable indicating whether or not the @finally is
757	/// running for an exception.
758	llvm::AllocaInst *ForEHVar = nullptr;
759
760	/// An i8* variable into which the exception pointer to rethrow
761	/// has been saved.
762	llvm::AllocaInst *SavedExnVar = nullptr;
763
764	public:
765	void enter(CodeGenFunction &CGF, const Stmt *Finally,
766	llvm::FunctionCallee beginCatchFn,
767	llvm::FunctionCallee endCatchFn, llvm::FunctionCallee rethrowFn);
768	void exit(CodeGenFunction &CGF);
769	};
770
771	/// Returns true inside SEH __try blocks.
772	bool isSEHTryScope() const { return !SEHTryEpilogueStack.empty(); }
773
774	/// Returns true while emitting a cleanuppad.
775	bool isCleanupPadScope() const {
776	return CurrentFuncletPad && isa<llvm::CleanupPadInst>(Val: CurrentFuncletPad);
777	}
778
779	/// pushFullExprCleanup - Push a cleanup to be run at the end of the
780	/// current full-expression. Safe against the possibility that
781	/// we're currently inside a conditionally-evaluated expression.
782	template <class T, class... As>
783	void pushFullExprCleanup(CleanupKind kind, As... A) {
784	// If we're not in a conditional branch, or if none of the
785	// arguments requires saving, then use the unconditional cleanup.
786	if (!isInConditionalBranch())
787	return EHStack.pushCleanup<T>(kind, A...);
788
789	// Stash values in a tuple so we can guarantee the order of saves.
790	typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple;
791	SavedTuple Saved{saveValueInCond(A)...};
792
793	typedef EHScopeStack::ConditionalCleanup<T, As...> CleanupType;
794	EHStack.pushCleanupTuple<CleanupType>(kind, Saved);
795	initFullExprCleanup();
796	}
797
798	/// Queue a cleanup to be pushed after finishing the current full-expression,
799	/// potentially with an active flag.
800	template <class T, class... As>
801	void pushCleanupAfterFullExpr(CleanupKind Kind, As... A) {
802	if (!isInConditionalBranch())
803	return pushCleanupAfterFullExprWithActiveFlag<T>(Kind, Address::invalid(),
804	A...);
805
806	Address ActiveFlag = createCleanupActiveFlag();
807	assert(!DominatingValue<Address>::needsSaving(ActiveFlag) &&
808	"cleanup active flag should never need saving");
809
810	typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple;
811	SavedTuple Saved{saveValueInCond(A)...};
812
813	typedef EHScopeStack::ConditionalCleanup<T, As...> CleanupType;
814	pushCleanupAfterFullExprWithActiveFlag<CleanupType>(Kind, ActiveFlag, Saved);
815	}
816
817	template <class T, class... As>
818	void pushCleanupAfterFullExprWithActiveFlag(CleanupKind Kind,
819	Address ActiveFlag, As... A) {
820	LifetimeExtendedCleanupHeader Header = {.Size: sizeof(T), .Kind: Kind,
821	.IsConditional: ActiveFlag.isValid()};
822
823	size_t OldSize = LifetimeExtendedCleanupStack.size();
824	LifetimeExtendedCleanupStack.resize(
825	N: LifetimeExtendedCleanupStack.size() + sizeof(Header) + Header.Size +
826	(Header.IsConditional ? sizeof(ActiveFlag) : 0));
827
828	static_assert(sizeof(Header) % alignof(T) == 0,
829	"Cleanup will be allocated on misaligned address");
830	char *Buffer = &LifetimeExtendedCleanupStack[OldSize];
831	new (Buffer) LifetimeExtendedCleanupHeader(Header);
832	new (Buffer + sizeof(Header)) T(A...);
833	if (Header.IsConditional)
834	new (Buffer + sizeof(Header) + sizeof(T)) Address(ActiveFlag);
835	}
836
837	/// Set up the last cleanup that was pushed as a conditional
838	/// full-expression cleanup.
839	void initFullExprCleanup() {
840	initFullExprCleanupWithFlag(ActiveFlag: createCleanupActiveFlag());
841	}
842
843	void initFullExprCleanupWithFlag(Address ActiveFlag);
844	Address createCleanupActiveFlag();
845
846	/// PushDestructorCleanup - Push a cleanup to call the
847	/// complete-object destructor of an object of the given type at the
848	/// given address. Does nothing if T is not a C++ class type with a
849	/// non-trivial destructor.
850	void PushDestructorCleanup(QualType T, Address Addr);
851
852	/// PushDestructorCleanup - Push a cleanup to call the
853	/// complete-object variant of the given destructor on the object at
854	/// the given address.
855	void PushDestructorCleanup(const CXXDestructorDecl *Dtor, QualType T,
856	Address Addr);
857
858	/// PopCleanupBlock - Will pop the cleanup entry on the stack and
859	/// process all branch fixups.
860	void PopCleanupBlock(bool FallThroughIsBranchThrough = false);
861
862	/// DeactivateCleanupBlock - Deactivates the given cleanup block.
863	/// The block cannot be reactivated. Pops it if it's the top of the
864	/// stack.
865	///
866	/// \param DominatingIP - An instruction which is known to
867	/// dominate the current IP (if set) and which lies along
868	/// all paths of execution between the current IP and the
869	/// the point at which the cleanup comes into scope.
870	void DeactivateCleanupBlock(EHScopeStack::stable_iterator Cleanup,
871	llvm::Instruction *DominatingIP);
872
873	/// ActivateCleanupBlock - Activates an initially-inactive cleanup.
874	/// Cannot be used to resurrect a deactivated cleanup.
875	///
876	/// \param DominatingIP - An instruction which is known to
877	/// dominate the current IP (if set) and which lies along
878	/// all paths of execution between the current IP and the
879	/// the point at which the cleanup comes into scope.
880	void ActivateCleanupBlock(EHScopeStack::stable_iterator Cleanup,
881	llvm::Instruction *DominatingIP);
882
883	/// Enters a new scope for capturing cleanups, all of which
884	/// will be executed once the scope is exited.
885	class RunCleanupsScope {
886	EHScopeStack::stable_iterator CleanupStackDepth, OldCleanupScopeDepth;
887	size_t LifetimeExtendedCleanupStackSize;
888	bool OldDidCallStackSave;
889	protected:
890	bool PerformCleanup;
891	private:
892
893	RunCleanupsScope(const RunCleanupsScope &) = delete;
894	void operator=(const RunCleanupsScope &) = delete;
895
896	protected:
897	CodeGenFunction& CGF;
898
899	public:
900	/// Enter a new cleanup scope.
901	explicit RunCleanupsScope(CodeGenFunction &CGF)
902	: PerformCleanup(true), CGF(CGF)
903	{
904	CleanupStackDepth = CGF.EHStack.stable_begin();
905	LifetimeExtendedCleanupStackSize =
906	CGF.LifetimeExtendedCleanupStack.size();
907	OldDidCallStackSave = CGF.DidCallStackSave;
908	CGF.DidCallStackSave = false;
909	OldCleanupScopeDepth = CGF.CurrentCleanupScopeDepth;
910	CGF.CurrentCleanupScopeDepth = CleanupStackDepth;
911	}
912
913	/// Exit this cleanup scope, emitting any accumulated cleanups.
914	~RunCleanupsScope() {
915	if (PerformCleanup)
916	ForceCleanup();
917	}
918
919	/// Determine whether this scope requires any cleanups.
920	bool requiresCleanups() const {
921	return CGF.EHStack.stable_begin() != CleanupStackDepth;
922	}
923
924	/// Force the emission of cleanups now, instead of waiting
925	/// until this object is destroyed.
926	/// \param ValuesToReload - A list of values that need to be available at
927	/// the insertion point after cleanup emission. If cleanup emission created
928	/// a shared cleanup block, these value pointers will be rewritten.
929	/// Otherwise, they not will be modified.
930	void ForceCleanup(std::initializer_list<llvm::Value**> ValuesToReload = {}) {
931	assert(PerformCleanup && "Already forced cleanup");
932	CGF.DidCallStackSave = OldDidCallStackSave;
933	CGF.PopCleanupBlocks(OldCleanupStackSize: CleanupStackDepth, OldLifetimeExtendedStackSize: LifetimeExtendedCleanupStackSize,
934	ValuesToReload);
935	PerformCleanup = false;
936	CGF.CurrentCleanupScopeDepth = OldCleanupScopeDepth;
937	}
938	};
939
940	// Cleanup stack depth of the RunCleanupsScope that was pushed most recently.
941	EHScopeStack::stable_iterator CurrentCleanupScopeDepth =
942	EHScopeStack::stable_end();
943
944	class LexicalScope : public RunCleanupsScope {
945	SourceRange Range;
946	SmallVector<const LabelDecl*, 4> Labels;
947	LexicalScope *ParentScope;
948
949	LexicalScope(const LexicalScope &) = delete;
950	void operator=(const LexicalScope &) = delete;
951
952	public:
953	/// Enter a new cleanup scope.
954	explicit LexicalScope(CodeGenFunction &CGF, SourceRange Range)
955	: RunCleanupsScope(CGF), Range(Range), ParentScope(CGF.CurLexicalScope) {
956	CGF.CurLexicalScope = this;
957	if (CGDebugInfo *DI = CGF.getDebugInfo())
958	DI->EmitLexicalBlockStart(Builder&: CGF.Builder, Loc: Range.getBegin());
959	}
960
961	void addLabel(const LabelDecl *label) {
962	assert(PerformCleanup && "adding label to dead scope?");
963	Labels.push_back(Elt: label);
964	}
965
966	/// Exit this cleanup scope, emitting any accumulated
967	/// cleanups.
968	~LexicalScope() {
969	if (CGDebugInfo *DI = CGF.getDebugInfo())
970	DI->EmitLexicalBlockEnd(Builder&: CGF.Builder, Loc: Range.getEnd());
971
972	// If we should perform a cleanup, force them now. Note that
973	// this ends the cleanup scope before rescoping any labels.
974	if (PerformCleanup) {
975	ApplyDebugLocation DL(CGF, Range.getEnd());
976	ForceCleanup();
977	}
978	}
979
980	/// Force the emission of cleanups now, instead of waiting
981	/// until this object is destroyed.
982	void ForceCleanup() {
983	CGF.CurLexicalScope = ParentScope;
984	RunCleanupsScope::ForceCleanup();
985
986	if (!Labels.empty())
987	rescopeLabels();
988	}
989
990	bool hasLabels() const {
991	return !Labels.empty();
992	}
993
994	void rescopeLabels();
995	};
996
997	typedef llvm::DenseMap<const Decl *, Address> DeclMapTy;
998
999	/// The class used to assign some variables some temporarily addresses.
1000	class OMPMapVars {
1001	DeclMapTy SavedLocals;
1002	DeclMapTy SavedTempAddresses;
1003	OMPMapVars(const OMPMapVars &) = delete;
1004	void operator=(const OMPMapVars &) = delete;
1005
1006	public:
1007	explicit OMPMapVars() = default;
1008	~OMPMapVars() {
1009	assert(SavedLocals.empty() && "Did not restored original addresses.");
1010	};
1011
1012	/// Sets the address of the variable \p LocalVD to be \p TempAddr in
1013	/// function \p CGF.
1014	/// \return true if at least one variable was set already, false otherwise.
1015	bool setVarAddr(CodeGenFunction &CGF, const VarDecl *LocalVD,
1016	Address TempAddr) {
1017	LocalVD = LocalVD->getCanonicalDecl();
1018	// Only save it once.
1019	if (SavedLocals.count(LocalVD)) return false;
1020
1021	// Copy the existing local entry to SavedLocals.
1022	auto it = CGF.LocalDeclMap.find(LocalVD);
1023	if (it != CGF.LocalDeclMap.end())
1024	SavedLocals.try_emplace(LocalVD, it->second);
1025	else
1026	SavedLocals.try_emplace(LocalVD, Address::invalid());
1027
1028	// Generate the private entry.
1029	QualType VarTy = LocalVD->getType();
1030	if (VarTy->isReferenceType()) {
1031	Address Temp = CGF.CreateMemTemp(T: VarTy);
1032	CGF.Builder.CreateStore(Val: TempAddr.getPointer(), Addr: Temp);
1033	TempAddr = Temp;
1034	}
1035	SavedTempAddresses.try_emplace(LocalVD, TempAddr);
1036
1037	return true;
1038	}
1039
1040	/// Applies new addresses to the list of the variables.
1041	/// \return true if at least one variable is using new address, false
1042	/// otherwise.
1043	bool apply(CodeGenFunction &CGF) {
1044	copyInto(Src: SavedTempAddresses, Dest&: CGF.LocalDeclMap);
1045	SavedTempAddresses.clear();
1046	return !SavedLocals.empty();
1047	}
1048
1049	/// Restores original addresses of the variables.
1050	void restore(CodeGenFunction &CGF) {
1051	if (!SavedLocals.empty()) {
1052	copyInto(Src: SavedLocals, Dest&: CGF.LocalDeclMap);
1053	SavedLocals.clear();
1054	}
1055	}
1056
1057	private:
1058	/// Copy all the entries in the source map over the corresponding
1059	/// entries in the destination, which must exist.
1060	static void copyInto(const DeclMapTy &Src, DeclMapTy &Dest) {
1061	for (auto &Pair : Src) {
1062	if (!Pair.second.isValid()) {
1063	Dest.erase(Val: Pair.first);
1064	continue;
1065	}
1066
1067	auto I = Dest.find(Val: Pair.first);
1068	if (I != Dest.end())
1069	I->second = Pair.second;
1070	else
1071	Dest.insert(KV: Pair);
1072	}
1073	}
1074	};
1075
1076	/// The scope used to remap some variables as private in the OpenMP loop body
1077	/// (or other captured region emitted without outlining), and to restore old
1078	/// vars back on exit.
1079	class OMPPrivateScope : public RunCleanupsScope {
1080	OMPMapVars MappedVars;
1081	OMPPrivateScope(const OMPPrivateScope &) = delete;
1082	void operator=(const OMPPrivateScope &) = delete;
1083
1084	public:
1085	/// Enter a new OpenMP private scope.
1086	explicit OMPPrivateScope(CodeGenFunction &CGF) : RunCleanupsScope(CGF) {}
1087
1088	/// Registers \p LocalVD variable as a private with \p Addr as the address
1089	/// of the corresponding private variable. \p
1090	/// PrivateGen is the address of the generated private variable.
1091	/// \return true if the variable is registered as private, false if it has
1092	/// been privatized already.
1093	bool addPrivate(const VarDecl *LocalVD, Address Addr) {
1094	assert(PerformCleanup && "adding private to dead scope");
1095	return MappedVars.setVarAddr(CGF, LocalVD, TempAddr: Addr);
1096	}
1097
1098	/// Privatizes local variables previously registered as private.
1099	/// Registration is separate from the actual privatization to allow
1100	/// initializers use values of the original variables, not the private one.
1101	/// This is important, for example, if the private variable is a class
1102	/// variable initialized by a constructor that references other private
1103	/// variables. But at initialization original variables must be used, not
1104	/// private copies.
1105	/// \return true if at least one variable was privatized, false otherwise.
1106	bool Privatize() { return MappedVars.apply(CGF); }
1107
1108	void ForceCleanup() {
1109	RunCleanupsScope::ForceCleanup();
1110	restoreMap();
1111	}
1112
1113	/// Exit scope - all the mapped variables are restored.
1114	~OMPPrivateScope() {
1115	if (PerformCleanup)
1116	ForceCleanup();
1117	}
1118
1119	/// Checks if the global variable is captured in current function.
1120	bool isGlobalVarCaptured(const VarDecl *VD) const {
1121	VD = VD->getCanonicalDecl();
1122	return !VD->isLocalVarDeclOrParm() && CGF.LocalDeclMap.count(VD) > 0;
1123	}
1124
1125	/// Restore all mapped variables w/o clean up. This is usefully when we want
1126	/// to reference the original variables but don't want the clean up because
1127	/// that could emit lifetime end too early, causing backend issue #56913.
1128	void restoreMap() { MappedVars.restore(CGF); }
1129	};
1130
1131	/// Save/restore original map of previously emitted local vars in case when we
1132	/// need to duplicate emission of the same code several times in the same
1133	/// function for OpenMP code.
1134	class OMPLocalDeclMapRAII {
1135	CodeGenFunction &CGF;
1136	DeclMapTy SavedMap;
1137
1138	public:
1139	OMPLocalDeclMapRAII(CodeGenFunction &CGF)
1140	: CGF(CGF), SavedMap(CGF.LocalDeclMap) {}
1141	~OMPLocalDeclMapRAII() { SavedMap.swap(RHS&: CGF.LocalDeclMap); }
1142	};
1143
1144	/// Takes the old cleanup stack size and emits the cleanup blocks
1145	/// that have been added.
1146	void
1147	PopCleanupBlocks(EHScopeStack::stable_iterator OldCleanupStackSize,
1148	std::initializer_list<llvm::Value **> ValuesToReload = {});
1149
1150	/// Takes the old cleanup stack size and emits the cleanup blocks
1151	/// that have been added, then adds all lifetime-extended cleanups from
1152	/// the given position to the stack.
1153	void
1154	PopCleanupBlocks(EHScopeStack::stable_iterator OldCleanupStackSize,
1155	size_t OldLifetimeExtendedStackSize,
1156	std::initializer_list<llvm::Value **> ValuesToReload = {});
1157
1158	void ResolveBranchFixups(llvm::BasicBlock *Target);
1159
1160	/// The given basic block lies in the current EH scope, but may be a
1161	/// target of a potentially scope-crossing jump; get a stable handle
1162	/// to which we can perform this jump later.
1163	JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target) {
1164	return JumpDest(Target,
1165	EHStack.getInnermostNormalCleanup(),
1166	NextCleanupDestIndex++);
1167	}
1168
1169	/// The given basic block lies in the current EH scope, but may be a
1170	/// target of a potentially scope-crossing jump; get a stable handle
1171	/// to which we can perform this jump later.
1172	JumpDest getJumpDestInCurrentScope(StringRef Name = StringRef()) {
1173	return getJumpDestInCurrentScope(Target: createBasicBlock(name: Name));
1174	}
1175
1176	/// EmitBranchThroughCleanup - Emit a branch from the current insert
1177	/// block through the normal cleanup handling code (if any) and then
1178	/// on to \arg Dest.
1179	void EmitBranchThroughCleanup(JumpDest Dest);
1180
1181	/// isObviouslyBranchWithoutCleanups - Return true if a branch to the
1182	/// specified destination obviously has no cleanups to run. 'false' is always
1183	/// a conservatively correct answer for this method.
1184	bool isObviouslyBranchWithoutCleanups(JumpDest Dest) const;
1185
1186	/// popCatchScope - Pops the catch scope at the top of the EHScope
1187	/// stack, emitting any required code (other than the catch handlers
1188	/// themselves).
1189	void popCatchScope();
1190
1191	llvm::BasicBlock *getEHResumeBlock(bool isCleanup);
1192	llvm::BasicBlock *getEHDispatchBlock(EHScopeStack::stable_iterator scope);
1193	llvm::BasicBlock *
1194	getFuncletEHDispatchBlock(EHScopeStack::stable_iterator scope);
1195
1196	/// An object to manage conditionally-evaluated expressions.
1197	class ConditionalEvaluation {
1198	llvm::BasicBlock *StartBB;
1199
1200	public:
1201	ConditionalEvaluation(CodeGenFunction &CGF)
1202	: StartBB(CGF.Builder.GetInsertBlock()) {}
1203
1204	void begin(CodeGenFunction &CGF) {
1205	assert(CGF.OutermostConditional != this);
1206	if (!CGF.OutermostConditional)
1207	CGF.OutermostConditional = this;
1208	}
1209
1210	void end(CodeGenFunction &CGF) {
1211	assert(CGF.OutermostConditional != nullptr);
1212	if (CGF.OutermostConditional == this)
1213	CGF.OutermostConditional = nullptr;
1214	}
1215
1216	/// Returns a block which will be executed prior to each
1217	/// evaluation of the conditional code.
1218	llvm::BasicBlock *getStartingBlock() const {
1219	return StartBB;
1220	}
1221	};
1222
1223	/// isInConditionalBranch - Return true if we're currently emitting
1224	/// one branch or the other of a conditional expression.
1225	bool isInConditionalBranch() const { return OutermostConditional != nullptr; }
1226
1227	void setBeforeOutermostConditional(llvm::Value *value, Address addr) {
1228	assert(isInConditionalBranch());
1229	llvm::BasicBlock *block = OutermostConditional->getStartingBlock();
1230	auto store = new llvm::StoreInst(value, addr.getPointer(), &block->back());
1231	store->setAlignment(addr.getAlignment().getAsAlign());
1232	}
1233
1234	/// An RAII object to record that we're evaluating a statement
1235	/// expression.
1236	class StmtExprEvaluation {
1237	CodeGenFunction &CGF;
1238
1239	/// We have to save the outermost conditional: cleanups in a
1240	/// statement expression aren't conditional just because the
1241	/// StmtExpr is.
1242	ConditionalEvaluation *SavedOutermostConditional;
1243
1244	public:
1245	StmtExprEvaluation(CodeGenFunction &CGF)
1246	: CGF(CGF), SavedOutermostConditional(CGF.OutermostConditional) {
1247	CGF.OutermostConditional = nullptr;
1248	}
1249
1250	~StmtExprEvaluation() {
1251	CGF.OutermostConditional = SavedOutermostConditional;
1252	CGF.EnsureInsertPoint();
1253	}
1254	};
1255
1256	/// An object which temporarily prevents a value from being
1257	/// destroyed by aggressive peephole optimizations that assume that
1258	/// all uses of a value have been realized in the IR.
1259	class PeepholeProtection {
1260	llvm::Instruction *Inst = nullptr;
1261	friend class CodeGenFunction;
1262
1263	public:
1264	PeepholeProtection() = default;
1265	};
1266
1267	/// A non-RAII class containing all the information about a bound
1268	/// opaque value. OpaqueValueMapping, below, is a RAII wrapper for
1269	/// this which makes individual mappings very simple; using this
1270	/// class directly is useful when you have a variable number of
1271	/// opaque values or don't want the RAII functionality for some
1272	/// reason.
1273	class OpaqueValueMappingData {
1274	const OpaqueValueExpr *OpaqueValue;
1275	bool BoundLValue;
1276	CodeGenFunction::PeepholeProtection Protection;
1277
1278	OpaqueValueMappingData(const OpaqueValueExpr *ov,
1279	bool boundLValue)
1280	: OpaqueValue(ov), BoundLValue(boundLValue) {}
1281	public:
1282	OpaqueValueMappingData() : OpaqueValue(nullptr) {}
1283
1284	static bool shouldBindAsLValue(const Expr *expr) {
1285	// gl-values should be bound as l-values for obvious reasons.
1286	// Records should be bound as l-values because IR generation
1287	// always keeps them in memory. Expressions of function type
1288	// act exactly like l-values but are formally required to be
1289	// r-values in C.
1290	return expr->isGLValue() ||
1291	expr->getType()->isFunctionType() ||
1292	hasAggregateEvaluationKind(T: expr->getType());
1293	}
1294
1295	static OpaqueValueMappingData bind(CodeGenFunction &CGF,
1296	const OpaqueValueExpr *ov,
1297	const Expr *e) {
1298	if (shouldBindAsLValue(ov))
1299	return bind(CGF, ov, lv: CGF.EmitLValue(E: e));
1300	return bind(CGF, ov, rv: CGF.EmitAnyExpr(E: e));
1301	}
1302
1303	static OpaqueValueMappingData bind(CodeGenFunction &CGF,
1304	const OpaqueValueExpr *ov,
1305	const LValue &lv) {
1306	assert(shouldBindAsLValue(ov));
1307	CGF.OpaqueLValues.insert(KV: std::make_pair(x&: ov, y: lv));
1308	return OpaqueValueMappingData(ov, true);
1309	}
1310
1311	static OpaqueValueMappingData bind(CodeGenFunction &CGF,
1312	const OpaqueValueExpr *ov,
1313	const RValue &rv) {
1314	assert(!shouldBindAsLValue(ov));
1315	CGF.OpaqueRValues.insert(KV: std::make_pair(x&: ov, y: rv));
1316
1317	OpaqueValueMappingData data(ov, false);
1318
1319	// Work around an extremely aggressive peephole optimization in
1320	// EmitScalarConversion which assumes that all other uses of a
1321	// value are extant.
1322	data.Protection = CGF.protectFromPeepholes(rvalue: rv);
1323
1324	return data;
1325	}
1326
1327	bool isValid() const { return OpaqueValue != nullptr; }
1328	void clear() { OpaqueValue = nullptr; }
1329
1330	void unbind(CodeGenFunction &CGF) {
1331	assert(OpaqueValue && "no data to unbind!");
1332
1333	if (BoundLValue) {
1334	CGF.OpaqueLValues.erase(Val: OpaqueValue);
1335	} else {
1336	CGF.OpaqueRValues.erase(Val: OpaqueValue);
1337	CGF.unprotectFromPeepholes(protection: Protection);
1338	}
1339	}
1340	};
1341
1342	/// An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
1343	class OpaqueValueMapping {
1344	CodeGenFunction &CGF;
1345	OpaqueValueMappingData Data;
1346
1347	public:
1348	static bool shouldBindAsLValue(const Expr *expr) {
1349	return OpaqueValueMappingData::shouldBindAsLValue(expr);
1350	}
1351
1352	/// Build the opaque value mapping for the given conditional
1353	/// operator if it's the GNU ?: extension. This is a common
1354	/// enough pattern that the convenience operator is really
1355	/// helpful.
1356	///
1357	OpaqueValueMapping(CodeGenFunction &CGF,
1358	const AbstractConditionalOperator *op) : CGF(CGF) {
1359	if (isa<ConditionalOperator>(Val: op))
1360	// Leave Data empty.
1361	return;
1362
1363	const BinaryConditionalOperator *e = cast<BinaryConditionalOperator>(Val: op);
1364	Data = OpaqueValueMappingData::bind(CGF, ov: e->getOpaqueValue(),
1365	e: e->getCommon());
1366	}
1367
1368	/// Build the opaque value mapping for an OpaqueValueExpr whose source
1369	/// expression is set to the expression the OVE represents.
1370	OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *OV)
1371	: CGF(CGF) {
1372	if (OV) {
1373	assert(OV->getSourceExpr() && "wrong form of OpaqueValueMapping used "
1374	"for OVE with no source expression");
1375	Data = OpaqueValueMappingData::bind(CGF, ov: OV, e: OV->getSourceExpr());
1376	}
1377	}
1378
1379	OpaqueValueMapping(CodeGenFunction &CGF,
1380	const OpaqueValueExpr *opaqueValue,
1381	LValue lvalue)
1382	: CGF(CGF), Data(OpaqueValueMappingData::bind(CGF, ov: opaqueValue, lv: lvalue)) {
1383	}
1384
1385	OpaqueValueMapping(CodeGenFunction &CGF,
1386	const OpaqueValueExpr *opaqueValue,
1387	RValue rvalue)
1388	: CGF(CGF), Data(OpaqueValueMappingData::bind(CGF, ov: opaqueValue, rv: rvalue)) {
1389	}
1390
1391	void pop() {
1392	Data.unbind(CGF);
1393	Data.clear();
1394	}
1395
1396	~OpaqueValueMapping() {
1397	if (Data.isValid()) Data.unbind(CGF);
1398	}
1399	};
1400
1401	private:
1402	CGDebugInfo *DebugInfo;
1403	/// Used to create unique names for artificial VLA size debug info variables.
1404	unsigned VLAExprCounter = 0;
1405	bool DisableDebugInfo = false;
1406
1407	/// DidCallStackSave - Whether llvm.stacksave has been called. Used to avoid
1408	/// calling llvm.stacksave for multiple VLAs in the same scope.
1409	bool DidCallStackSave = false;
1410
1411	/// IndirectBranch - The first time an indirect goto is seen we create a block
1412	/// with an indirect branch. Every time we see the address of a label taken,
1413	/// we add the label to the indirect goto. Every subsequent indirect goto is
1414	/// codegen'd as a jump to the IndirectBranch's basic block.
1415	llvm::IndirectBrInst *IndirectBranch = nullptr;
1416
1417	/// LocalDeclMap - This keeps track of the LLVM allocas or globals for local C
1418	/// decls.
1419	DeclMapTy LocalDeclMap;
1420
1421	// Keep track of the cleanups for callee-destructed parameters pushed to the
1422	// cleanup stack so that they can be deactivated later.
1423	llvm::DenseMap<const ParmVarDecl *, EHScopeStack::stable_iterator>
1424	CalleeDestructedParamCleanups;
1425
1426	/// SizeArguments - If a ParmVarDecl had the pass_object_size attribute, this
1427	/// will contain a mapping from said ParmVarDecl to its implicit "object_size"
1428	/// parameter.
1429	llvm::SmallDenseMap<const ParmVarDecl *, const ImplicitParamDecl *, 2>
1430	SizeArguments;
1431
1432	/// Track escaped local variables with auto storage. Used during SEH
1433	/// outlining to produce a call to llvm.localescape.
1434	llvm::DenseMap<llvm::AllocaInst *, int> EscapedLocals;
1435
1436	/// LabelMap - This keeps track of the LLVM basic block for each C label.
1437	llvm::DenseMap<const LabelDecl*, JumpDest> LabelMap;
1438
1439	// BreakContinueStack - This keeps track of where break and continue
1440	// statements should jump to.
1441	struct BreakContinue {
1442	BreakContinue(JumpDest Break, JumpDest Continue)
1443	: BreakBlock(Break), ContinueBlock(Continue) {}
1444
1445	JumpDest BreakBlock;
1446	JumpDest ContinueBlock;
1447	};
1448	SmallVector<BreakContinue, 8> BreakContinueStack;
1449
1450	/// Handles cancellation exit points in OpenMP-related constructs.
1451	class OpenMPCancelExitStack {
1452	/// Tracks cancellation exit point and join point for cancel-related exit
1453	/// and normal exit.
1454	struct CancelExit {
1455	CancelExit() = default;
1456	CancelExit(OpenMPDirectiveKind Kind, JumpDest ExitBlock,
1457	JumpDest ContBlock)
1458	: Kind(Kind), ExitBlock(ExitBlock), ContBlock(ContBlock) {}
1459	OpenMPDirectiveKind Kind = llvm::omp::OMPD_unknown;
1460	/// true if the exit block has been emitted already by the special
1461	/// emitExit() call, false if the default codegen is used.
1462	bool HasBeenEmitted = false;
1463	JumpDest ExitBlock;
1464	JumpDest ContBlock;
1465	};
1466
1467	SmallVector<CancelExit, 8> Stack;
1468
1469	public:
1470	OpenMPCancelExitStack() : Stack(1) {}
1471	~OpenMPCancelExitStack() = default;
1472	/// Fetches the exit block for the current OpenMP construct.
1473	JumpDest getExitBlock() const { return Stack.back().ExitBlock; }
1474	/// Emits exit block with special codegen procedure specific for the related
1475	/// OpenMP construct + emits code for normal construct cleanup.
1476	void emitExit(CodeGenFunction &CGF, OpenMPDirectiveKind Kind,
1477	const llvm::function_ref<void(CodeGenFunction &)> CodeGen) {
1478	if (Stack.back().Kind == Kind && getExitBlock().isValid()) {
1479	assert(CGF.getOMPCancelDestination(Kind).isValid());
1480	assert(CGF.HaveInsertPoint());
1481	assert(!Stack.back().HasBeenEmitted);
1482	auto IP = CGF.Builder.saveAndClearIP();
1483	CGF.EmitBlock(BB: Stack.back().ExitBlock.getBlock());
1484	CodeGen(CGF);
1485	CGF.EmitBranch(Block: Stack.back().ContBlock.getBlock());
1486	CGF.Builder.restoreIP(IP);
1487	Stack.back().HasBeenEmitted = true;
1488	}
1489	CodeGen(CGF);
1490	}
1491	/// Enter the cancel supporting \a Kind construct.
1492	/// \param Kind OpenMP directive that supports cancel constructs.
1493	/// \param HasCancel true, if the construct has inner cancel directive,
1494	/// false otherwise.
1495	void enter(CodeGenFunction &CGF, OpenMPDirectiveKind Kind, bool HasCancel) {
1496	Stack.push_back(Elt: {Kind,
1497	HasCancel ? CGF.getJumpDestInCurrentScope(Name: "cancel.exit")
1498	: JumpDest(),
1499	HasCancel ? CGF.getJumpDestInCurrentScope(Name: "cancel.cont")
1500	: JumpDest()});
1501	}
1502	/// Emits default exit point for the cancel construct (if the special one
1503	/// has not be used) + join point for cancel/normal exits.
1504	void exit(CodeGenFunction &CGF) {
1505	if (getExitBlock().isValid()) {
1506	assert(CGF.getOMPCancelDestination(Stack.back().Kind).isValid());
1507	bool HaveIP = CGF.HaveInsertPoint();
1508	if (!Stack.back().HasBeenEmitted) {
1509	if (HaveIP)
1510	CGF.EmitBranchThroughCleanup(Dest: Stack.back().ContBlock);
1511	CGF.EmitBlock(BB: Stack.back().ExitBlock.getBlock());
1512	CGF.EmitBranchThroughCleanup(Dest: Stack.back().ContBlock);
1513	}
1514	CGF.EmitBlock(BB: Stack.back().ContBlock.getBlock());
1515	if (!HaveIP) {
1516	CGF.Builder.CreateUnreachable();
1517	CGF.Builder.ClearInsertionPoint();
1518	}
1519	}
1520	Stack.pop_back();
1521	}
1522	};
1523	OpenMPCancelExitStack OMPCancelStack;
1524
1525	/// Lower the Likelihood knowledge about the \p Cond via llvm.expect intrin.
1526	llvm::Value *emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
1527	Stmt::Likelihood LH);
1528
1529	CodeGenPGO PGO;
1530
1531	/// Bitmap used by MC/DC to track condition outcomes of a boolean expression.
1532	Address MCDCCondBitmapAddr = Address::invalid();
1533
1534	/// Calculate branch weights appropriate for PGO data
1535	llvm::MDNode *createProfileWeights(uint64_t TrueCount,
1536	uint64_t FalseCount) const;
1537	llvm::MDNode *createProfileWeights(ArrayRef<uint64_t> Weights) const;
1538	llvm::MDNode *createProfileWeightsForLoop(const Stmt *Cond,
1539	uint64_t LoopCount) const;
1540
1541	public:
1542	/// Increment the profiler's counter for the given statement by \p StepV.
1543	/// If \p StepV is null, the default increment is 1.
1544	void incrementProfileCounter(const Stmt *S, llvm::Value *StepV = nullptr) {
1545	if (CGM.getCodeGenOpts().hasProfileClangInstr() &&
1546	!CurFn->hasFnAttribute(llvm::Attribute::NoProfile) &&
1547	!CurFn->hasFnAttribute(llvm::Attribute::SkipProfile))
1548	PGO.emitCounterIncrement(Builder, S, StepV);
1549	PGO.setCurrentStmt(S);
1550	}
1551
1552	bool isMCDCCoverageEnabled() const {
1553	return (CGM.getCodeGenOpts().hasProfileClangInstr() &&
1554	CGM.getCodeGenOpts().MCDCCoverage &&
1555	!CurFn->hasFnAttribute(llvm::Attribute::NoProfile));
1556	}
1557
1558	/// Allocate a temp value on the stack that MCDC can use to track condition
1559	/// results.
1560	void maybeCreateMCDCCondBitmap() {
1561	if (isMCDCCoverageEnabled()) {
1562	PGO.emitMCDCParameters(Builder);
1563	MCDCCondBitmapAddr =
1564	CreateIRTemp(T: getContext().UnsignedIntTy, Name: "mcdc.addr");
1565	}
1566	}
1567
1568	bool isBinaryLogicalOp(const Expr *E) const {
1569	const BinaryOperator *BOp = dyn_cast<BinaryOperator>(Val: E->IgnoreParens());
1570	return (BOp && BOp->isLogicalOp());
1571	}
1572
1573	/// Zero-init the MCDC temp value.
1574	void maybeResetMCDCCondBitmap(const Expr *E) {
1575	if (isMCDCCoverageEnabled() && isBinaryLogicalOp(E)) {
1576	PGO.emitMCDCCondBitmapReset(Builder, S: E, MCDCCondBitmapAddr);
1577	PGO.setCurrentStmt(E);
1578	}
1579	}
1580
1581	/// Increment the profiler's counter for the given expression by \p StepV.
1582	/// If \p StepV is null, the default increment is 1.
1583	void maybeUpdateMCDCTestVectorBitmap(const Expr *E) {
1584	if (isMCDCCoverageEnabled() && isBinaryLogicalOp(E)) {
1585	PGO.emitMCDCTestVectorBitmapUpdate(Builder, S: E, MCDCCondBitmapAddr);
1586	PGO.setCurrentStmt(E);
1587	}
1588	}
1589
1590	/// Update the MCDC temp value with the condition's evaluated result.
1591	void maybeUpdateMCDCCondBitmap(const Expr *E, llvm::Value *Val) {
1592	if (isMCDCCoverageEnabled()) {
1593	PGO.emitMCDCCondBitmapUpdate(Builder, S: E, MCDCCondBitmapAddr, Val);
1594	PGO.setCurrentStmt(E);
1595	}
1596	}
1597
1598	/// Get the profiler's count for the given statement.
1599	uint64_t getProfileCount(const Stmt *S) {
1600	return PGO.getStmtCount(S).value_or(u: 0);
1601	}
1602
1603	/// Set the profiler's current count.
1604	void setCurrentProfileCount(uint64_t Count) {
1605	PGO.setCurrentRegionCount(Count);
1606	}
1607
1608	/// Get the profiler's current count. This is generally the count for the most
1609	/// recently incremented counter.
1610	uint64_t getCurrentProfileCount() {
1611	return PGO.getCurrentRegionCount();
1612	}
1613
1614	private:
1615
1616	/// SwitchInsn - This is nearest current switch instruction. It is null if
1617	/// current context is not in a switch.
1618	llvm::SwitchInst *SwitchInsn = nullptr;
1619	/// The branch weights of SwitchInsn when doing instrumentation based PGO.
1620	SmallVector<uint64_t, 16> *SwitchWeights = nullptr;
1621
1622	/// The likelihood attributes of the SwitchCase.
1623	SmallVector<Stmt::Likelihood, 16> *SwitchLikelihood = nullptr;
1624
1625	/// CaseRangeBlock - This block holds if condition check for last case
1626	/// statement range in current switch instruction.
1627	llvm::BasicBlock *CaseRangeBlock = nullptr;
1628
1629	/// OpaqueLValues - Keeps track of the current set of opaque value
1630	/// expressions.
1631	llvm::DenseMap<const OpaqueValueExpr *, LValue> OpaqueLValues;
1632	llvm::DenseMap<const OpaqueValueExpr *, RValue> OpaqueRValues;
1633
1634	// VLASizeMap - This keeps track of the associated size for each VLA type.
1635	// We track this by the size expression rather than the type itself because
1636	// in certain situations, like a const qualifier applied to an VLA typedef,
1637	// multiple VLA types can share the same size expression.
1638	// FIXME: Maybe this could be a stack of maps that is pushed/popped as we
1639	// enter/leave scopes.
1640	llvm::DenseMap<const Expr*, llvm::Value*> VLASizeMap;
1641
1642	/// A block containing a single 'unreachable' instruction. Created
1643	/// lazily by getUnreachableBlock().
1644	llvm::BasicBlock *UnreachableBlock = nullptr;
1645
1646	/// Counts of the number return expressions in the function.
1647	unsigned NumReturnExprs = 0;
1648
1649	/// Count the number of simple (constant) return expressions in the function.
1650	unsigned NumSimpleReturnExprs = 0;
1651
1652	/// The last regular (non-return) debug location (breakpoint) in the function.
1653	SourceLocation LastStopPoint;
1654
1655	public:
1656	/// Source location information about the default argument or member
1657	/// initializer expression we're evaluating, if any.
1658	CurrentSourceLocExprScope CurSourceLocExprScope;
1659	using SourceLocExprScopeGuard =
1660	CurrentSourceLocExprScope::SourceLocExprScopeGuard;
1661
1662	/// A scope within which we are constructing the fields of an object which
1663	/// might use a CXXDefaultInitExpr. This stashes away a 'this' value to use
1664	/// if we need to evaluate a CXXDefaultInitExpr within the evaluation.
1665	class FieldConstructionScope {
1666	public:
1667	FieldConstructionScope(CodeGenFunction &CGF, Address This)
1668	: CGF(CGF), OldCXXDefaultInitExprThis(CGF.CXXDefaultInitExprThis) {
1669	CGF.CXXDefaultInitExprThis = This;
1670	}
1671	~FieldConstructionScope() {
1672	CGF.CXXDefaultInitExprThis = OldCXXDefaultInitExprThis;
1673	}
1674
1675	private:
1676	CodeGenFunction &CGF;
1677	Address OldCXXDefaultInitExprThis;
1678	};
1679
1680	/// The scope of a CXXDefaultInitExpr. Within this scope, the value of 'this'
1681	/// is overridden to be the object under construction.
1682	class CXXDefaultInitExprScope {
1683	public:
1684	CXXDefaultInitExprScope(CodeGenFunction &CGF, const CXXDefaultInitExpr *E)
1685	: CGF(CGF), OldCXXThisValue(CGF.CXXThisValue),
1686	OldCXXThisAlignment(CGF.CXXThisAlignment),
1687	SourceLocScope(E, CGF.CurSourceLocExprScope) {
1688	CGF.CXXThisValue = CGF.CXXDefaultInitExprThis.getPointer();
1689	CGF.CXXThisAlignment = CGF.CXXDefaultInitExprThis.getAlignment();
1690	}
1691	~CXXDefaultInitExprScope() {
1692	CGF.CXXThisValue = OldCXXThisValue;
1693	CGF.CXXThisAlignment = OldCXXThisAlignment;
1694	}
1695
1696	public:
1697	CodeGenFunction &CGF;
1698	llvm::Value *OldCXXThisValue;
1699	CharUnits OldCXXThisAlignment;
1700	SourceLocExprScopeGuard SourceLocScope;
1701	};
1702
1703	struct CXXDefaultArgExprScope : SourceLocExprScopeGuard {
1704	CXXDefaultArgExprScope(CodeGenFunction &CGF, const CXXDefaultArgExpr *E)
1705	: SourceLocExprScopeGuard(E, CGF.CurSourceLocExprScope) {}
1706	};
1707
1708	/// The scope of an ArrayInitLoopExpr. Within this scope, the value of the
1709	/// current loop index is overridden.
1710	class ArrayInitLoopExprScope {
1711	public:
1712	ArrayInitLoopExprScope(CodeGenFunction &CGF, llvm::Value *Index)
1713	: CGF(CGF), OldArrayInitIndex(CGF.ArrayInitIndex) {
1714	CGF.ArrayInitIndex = Index;
1715	}
1716	~ArrayInitLoopExprScope() {
1717	CGF.ArrayInitIndex = OldArrayInitIndex;
1718	}
1719
1720	private:
1721	CodeGenFunction &CGF;
1722	llvm::Value *OldArrayInitIndex;
1723	};
1724
1725	class InlinedInheritingConstructorScope {
1726	public:
1727	InlinedInheritingConstructorScope(CodeGenFunction &CGF, GlobalDecl GD)
1728	: CGF(CGF), OldCurGD(CGF.CurGD), OldCurFuncDecl(CGF.CurFuncDecl),
1729	OldCurCodeDecl(CGF.CurCodeDecl),
1730	OldCXXABIThisDecl(CGF.CXXABIThisDecl),
1731	OldCXXABIThisValue(CGF.CXXABIThisValue),
1732	OldCXXThisValue(CGF.CXXThisValue),
1733	OldCXXABIThisAlignment(CGF.CXXABIThisAlignment),
1734	OldCXXThisAlignment(CGF.CXXThisAlignment),
1735	OldReturnValue(CGF.ReturnValue), OldFnRetTy(CGF.FnRetTy),
1736	OldCXXInheritedCtorInitExprArgs(
1737	std::move(CGF.CXXInheritedCtorInitExprArgs)) {
1738	CGF.CurGD = GD;
1739	CGF.CurFuncDecl = CGF.CurCodeDecl =
1740	cast<CXXConstructorDecl>(Val: GD.getDecl());
1741	CGF.CXXABIThisDecl = nullptr;
1742	CGF.CXXABIThisValue = nullptr;
1743	CGF.CXXThisValue = nullptr;
1744	CGF.CXXABIThisAlignment = CharUnits();
1745	CGF.CXXThisAlignment = CharUnits();
1746	CGF.ReturnValue = Address::invalid();
1747	CGF.FnRetTy = QualType();
1748	CGF.CXXInheritedCtorInitExprArgs.clear();
1749	}
1750	~InlinedInheritingConstructorScope() {
1751	CGF.CurGD = OldCurGD;
1752	CGF.CurFuncDecl = OldCurFuncDecl;
1753	CGF.CurCodeDecl = OldCurCodeDecl;
1754	CGF.CXXABIThisDecl = OldCXXABIThisDecl;
1755	CGF.CXXABIThisValue = OldCXXABIThisValue;
1756	CGF.CXXThisValue = OldCXXThisValue;
1757	CGF.CXXABIThisAlignment = OldCXXABIThisAlignment;
1758	CGF.CXXThisAlignment = OldCXXThisAlignment;
1759	CGF.ReturnValue = OldReturnValue;
1760	CGF.FnRetTy = OldFnRetTy;
1761	CGF.CXXInheritedCtorInitExprArgs =
1762	std::move(OldCXXInheritedCtorInitExprArgs);
1763	}
1764
1765	private:
1766	CodeGenFunction &CGF;
1767	GlobalDecl OldCurGD;
1768	const Decl *OldCurFuncDecl;
1769	const Decl *OldCurCodeDecl;
1770	ImplicitParamDecl *OldCXXABIThisDecl;
1771	llvm::Value *OldCXXABIThisValue;
1772	llvm::Value *OldCXXThisValue;
1773	CharUnits OldCXXABIThisAlignment;
1774	CharUnits OldCXXThisAlignment;
1775	Address OldReturnValue;
1776	QualType OldFnRetTy;
1777	CallArgList OldCXXInheritedCtorInitExprArgs;
1778	};
1779
1780	// Helper class for the OpenMP IR Builder. Allows reusability of code used for
1781	// region body, and finalization codegen callbacks. This will class will also
1782	// contain privatization functions used by the privatization call backs
1783	//
1784	// TODO: this is temporary class for things that are being moved out of
1785	// CGOpenMPRuntime, new versions of current CodeGenFunction methods, or
1786	// utility function for use with the OMPBuilder. Once that move to use the
1787	// OMPBuilder is done, everything here will either become part of CodeGenFunc.
1788	// directly, or a new helper class that will contain functions used by both
1789	// this and the OMPBuilder
1790
1791	struct OMPBuilderCBHelpers {
1792
1793	OMPBuilderCBHelpers() = delete;
1794	OMPBuilderCBHelpers(const OMPBuilderCBHelpers &) = delete;
1795	OMPBuilderCBHelpers &operator=(const OMPBuilderCBHelpers &) = delete;
1796
1797	using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1798
1799	/// Cleanup action for allocate support.
1800	class OMPAllocateCleanupTy final : public EHScopeStack::Cleanup {
1801
1802	private:
1803	llvm::CallInst *RTLFnCI;
1804
1805	public:
1806	OMPAllocateCleanupTy(llvm::CallInst *RLFnCI) : RTLFnCI(RLFnCI) {
1807	RLFnCI->removeFromParent();
1808	}
1809
1810	void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
1811	if (!CGF.HaveInsertPoint())
1812	return;
1813	CGF.Builder.Insert(I: RTLFnCI);
1814	}
1815	};
1816
1817	/// Returns address of the threadprivate variable for the current
1818	/// thread. This Also create any necessary OMP runtime calls.
1819	///
1820	/// \param VD VarDecl for Threadprivate variable.
1821	/// \param VDAddr Address of the Vardecl
1822	/// \param Loc The location where the barrier directive was encountered
1823	static Address getAddrOfThreadPrivate(CodeGenFunction &CGF,
1824	const VarDecl *VD, Address VDAddr,
1825	SourceLocation Loc);
1826
1827	/// Gets the OpenMP-specific address of the local variable /p VD.
1828	static Address getAddressOfLocalVariable(CodeGenFunction &CGF,
1829	const VarDecl *VD);
1830	/// Get the platform-specific name separator.
1831	/// \param Parts different parts of the final name that needs separation
1832	/// \param FirstSeparator First separator used between the initial two
1833	/// parts of the name.
1834	/// \param Separator separator used between all of the rest consecutinve
1835	/// parts of the name
1836	static std::string getNameWithSeparators(ArrayRef<StringRef> Parts,
1837	StringRef FirstSeparator = ".",
1838	StringRef Separator = ".");
1839	/// Emit the Finalization for an OMP region
1840	/// \param CGF The Codegen function this belongs to
1841	/// \param IP Insertion point for generating the finalization code.
1842	static void FinalizeOMPRegion(CodeGenFunction &CGF, InsertPointTy IP) {
1843	CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1844	assert(IP.getBlock()->end() != IP.getPoint() &&
1845	"OpenMP IR Builder should cause terminated block!");
1846
1847	llvm::BasicBlock *IPBB = IP.getBlock();
1848	llvm::BasicBlock *DestBB = IPBB->getUniqueSuccessor();
1849	assert(DestBB && "Finalization block should have one successor!");
1850
1851	// erase and replace with cleanup branch.
1852	IPBB->getTerminator()->eraseFromParent();
1853	CGF.Builder.SetInsertPoint(IPBB);
1854	CodeGenFunction::JumpDest Dest = CGF.getJumpDestInCurrentScope(Target: DestBB);
1855	CGF.EmitBranchThroughCleanup(Dest);
1856	}
1857
1858	/// Emit the body of an OMP region
1859	/// \param CGF The Codegen function this belongs to
1860	/// \param RegionBodyStmt The body statement for the OpenMP region being
1861	/// generated
1862	/// \param AllocaIP Where to insert alloca instructions
1863	/// \param CodeGenIP Where to insert the region code
1864	/// \param RegionName Name to be used for new blocks
1865	static void EmitOMPInlinedRegionBody(CodeGenFunction &CGF,
1866	const Stmt *RegionBodyStmt,
1867	InsertPointTy AllocaIP,
1868	InsertPointTy CodeGenIP,
1869	Twine RegionName);
1870
1871	static void EmitCaptureStmt(CodeGenFunction &CGF, InsertPointTy CodeGenIP,
1872	llvm::BasicBlock &FiniBB, llvm::Function *Fn,
1873	ArrayRef<llvm::Value *> Args) {
1874	llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
1875	if (llvm::Instruction *CodeGenIPBBTI = CodeGenIPBB->getTerminator())
1876	CodeGenIPBBTI->eraseFromParent();
1877
1878	CGF.Builder.SetInsertPoint(CodeGenIPBB);
1879
1880	if (Fn->doesNotThrow())
1881	CGF.EmitNounwindRuntimeCall(callee: Fn, args: Args);
1882	else
1883	CGF.EmitRuntimeCall(callee: Fn, args: Args);
1884
1885	if (CGF.Builder.saveIP().isSet())
1886	CGF.Builder.CreateBr(Dest: &FiniBB);
1887	}
1888
1889	/// Emit the body of an OMP region that will be outlined in
1890	/// OpenMPIRBuilder::finalize().
1891	/// \param CGF The Codegen function this belongs to
1892	/// \param RegionBodyStmt The body statement for the OpenMP region being
1893	/// generated
1894	/// \param AllocaIP Where to insert alloca instructions
1895	/// \param CodeGenIP Where to insert the region code
1896	/// \param RegionName Name to be used for new blocks
1897	static void EmitOMPOutlinedRegionBody(CodeGenFunction &CGF,
1898	const Stmt *RegionBodyStmt,
1899	InsertPointTy AllocaIP,
1900	InsertPointTy CodeGenIP,
1901	Twine RegionName);
1902
1903	/// RAII for preserving necessary info during Outlined region body codegen.
1904	class OutlinedRegionBodyRAII {
1905
1906	llvm::AssertingVH<llvm::Instruction> OldAllocaIP;
1907	CodeGenFunction::JumpDest OldReturnBlock;
1908	CodeGenFunction &CGF;
1909
1910	public:
1911	OutlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP,
1912	llvm::BasicBlock &RetBB)
1913	: CGF(cgf) {
1914	assert(AllocaIP.isSet() &&
1915	"Must specify Insertion point for allocas of outlined function");
1916	OldAllocaIP = CGF.AllocaInsertPt;
1917	CGF.AllocaInsertPt = &*AllocaIP.getPoint();
1918
1919	OldReturnBlock = CGF.ReturnBlock;
1920	CGF.ReturnBlock = CGF.getJumpDestInCurrentScope(Target: &RetBB);
1921	}
1922
1923	~OutlinedRegionBodyRAII() {
1924	CGF.AllocaInsertPt = OldAllocaIP;
1925	CGF.ReturnBlock = OldReturnBlock;
1926	}
1927	};
1928
1929	/// RAII for preserving necessary info during inlined region body codegen.
1930	class InlinedRegionBodyRAII {
1931
1932	llvm::AssertingVH<llvm::Instruction> OldAllocaIP;
1933	CodeGenFunction &CGF;
1934
1935	public:
1936	InlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP,
1937	llvm::BasicBlock &FiniBB)
1938	: CGF(cgf) {
1939	// Alloca insertion block should be in the entry block of the containing
1940	// function so it expects an empty AllocaIP in which case will reuse the
1941	// old alloca insertion point, or a new AllocaIP in the same block as
1942	// the old one
1943	assert((!AllocaIP.isSet() ||
1944	CGF.AllocaInsertPt->getParent() == AllocaIP.getBlock()) &&
1945	"Insertion point should be in the entry block of containing "
1946	"function!");
1947	OldAllocaIP = CGF.AllocaInsertPt;
1948	if (AllocaIP.isSet())
1949	CGF.AllocaInsertPt = &*AllocaIP.getPoint();
1950
1951	// TODO: Remove the call, after making sure the counter is not used by
1952	// the EHStack.
1953	// Since this is an inlined region, it should not modify the
1954	// ReturnBlock, and should reuse the one for the enclosing outlined
1955	// region. So, the JumpDest being return by the function is discarded
1956	(void)CGF.getJumpDestInCurrentScope(Target: &FiniBB);
1957	}
1958
1959	~InlinedRegionBodyRAII() { CGF.AllocaInsertPt = OldAllocaIP; }
1960	};
1961	};
1962
1963	private:
1964	/// CXXThisDecl - When generating code for a C++ member function,
1965	/// this will hold the implicit 'this' declaration.
1966	ImplicitParamDecl *CXXABIThisDecl = nullptr;
1967	llvm::Value *CXXABIThisValue = nullptr;
1968	llvm::Value *CXXThisValue = nullptr;
1969	CharUnits CXXABIThisAlignment;
1970	CharUnits CXXThisAlignment;
1971
1972	/// The value of 'this' to use when evaluating CXXDefaultInitExprs within
1973	/// this expression.
1974	Address CXXDefaultInitExprThis = Address::invalid();
1975
1976	/// The current array initialization index when evaluating an
1977	/// ArrayInitIndexExpr within an ArrayInitLoopExpr.
1978	llvm::Value *ArrayInitIndex = nullptr;
1979
1980	/// The values of function arguments to use when evaluating
1981	/// CXXInheritedCtorInitExprs within this context.
1982	CallArgList CXXInheritedCtorInitExprArgs;
1983
1984	/// CXXStructorImplicitParamDecl - When generating code for a constructor or
1985	/// destructor, this will hold the implicit argument (e.g. VTT).
1986	ImplicitParamDecl *CXXStructorImplicitParamDecl = nullptr;
1987	llvm::Value *CXXStructorImplicitParamValue = nullptr;
1988
1989	/// OutermostConditional - Points to the outermost active
1990	/// conditional control. This is used so that we know if a
1991	/// temporary should be destroyed conditionally.
1992	ConditionalEvaluation *OutermostConditional = nullptr;
1993
1994	/// The current lexical scope.
1995	LexicalScope *CurLexicalScope = nullptr;
1996
1997	/// The current source location that should be used for exception
1998	/// handling code.
1999	SourceLocation CurEHLocation;
2000
2001	/// BlockByrefInfos - For each __block variable, contains
2002	/// information about the layout of the variable.
2003	llvm::DenseMap<const ValueDecl *, BlockByrefInfo> BlockByrefInfos;
2004
2005	/// Used by -fsanitize=nullability-return to determine whether the return
2006	/// value can be checked.
2007	llvm::Value *RetValNullabilityPrecondition = nullptr;
2008
2009	/// Check if -fsanitize=nullability-return instrumentation is required for
2010	/// this function.
2011	bool requiresReturnValueNullabilityCheck() const {
2012	return RetValNullabilityPrecondition;
2013	}
2014
2015	/// Used to store precise source locations for return statements by the
2016	/// runtime return value checks.
2017	Address ReturnLocation = Address::invalid();
2018
2019	/// Check if the return value of this function requires sanitization.
2020	bool requiresReturnValueCheck() const;
2021
2022	bool isInAllocaArgument(CGCXXABI &ABI, QualType Ty);
2023	bool hasInAllocaArg(const CXXMethodDecl *MD);
2024
2025	llvm::BasicBlock *TerminateLandingPad = nullptr;
2026	llvm::BasicBlock *TerminateHandler = nullptr;
2027	llvm::SmallVector<llvm::BasicBlock *, 2> TrapBBs;
2028
2029	/// Terminate funclets keyed by parent funclet pad.
2030	llvm::MapVector<llvm::Value *, llvm::BasicBlock *> TerminateFunclets;
2031
2032	/// Largest vector width used in ths function. Will be used to create a
2033	/// function attribute.
2034	unsigned LargestVectorWidth = 0;
2035
2036	/// True if we need emit the life-time markers. This is initially set in
2037	/// the constructor, but could be overwritten to true if this is a coroutine.
2038	bool ShouldEmitLifetimeMarkers;
2039
2040	/// Add OpenCL kernel arg metadata and the kernel attribute metadata to
2041	/// the function metadata.
2042	void EmitKernelMetadata(const FunctionDecl *FD, llvm::Function *Fn);
2043
2044	public:
2045	CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext=false);
2046	~CodeGenFunction();
2047
2048	CodeGenTypes &getTypes() const { return CGM.getTypes(); }
2049	ASTContext &getContext() const { return CGM.getContext(); }
2050	CGDebugInfo *getDebugInfo() {
2051	if (DisableDebugInfo)
2052	return nullptr;
2053	return DebugInfo;
2054	}
2055	void disableDebugInfo() { DisableDebugInfo = true; }
2056	void enableDebugInfo() { DisableDebugInfo = false; }
2057
2058	bool shouldUseFusedARCCalls() {
2059	return CGM.getCodeGenOpts().OptimizationLevel == 0;
2060	}
2061
2062	const LangOptions &getLangOpts() const { return CGM.getLangOpts(); }
2063
2064	/// Returns a pointer to the function's exception object and selector slot,
2065	/// which is assigned in every landing pad.
2066	Address getExceptionSlot();
2067	Address getEHSelectorSlot();
2068
2069	/// Returns the contents of the function's exception object and selector
2070	/// slots.
2071	llvm::Value *getExceptionFromSlot();
2072	llvm::Value *getSelectorFromSlot();
2073
2074	Address getNormalCleanupDestSlot();
2075
2076	llvm::BasicBlock *getUnreachableBlock() {
2077	if (!UnreachableBlock) {
2078	UnreachableBlock = createBasicBlock(name: "unreachable");
2079	new llvm::UnreachableInst(getLLVMContext(), UnreachableBlock);
2080	}
2081	return UnreachableBlock;
2082	}
2083
2084	llvm::BasicBlock *getInvokeDest() {
2085	if (!EHStack.requiresLandingPad()) return nullptr;
2086	return getInvokeDestImpl();
2087	}
2088
2089	bool currentFunctionUsesSEHTry() const { return !!CurSEHParent; }
2090
2091	const TargetInfo &getTarget() const { return Target; }
2092	llvm::LLVMContext &getLLVMContext() { return CGM.getLLVMContext(); }
2093	const TargetCodeGenInfo &getTargetHooks() const {
2094	return CGM.getTargetCodeGenInfo();
2095	}
2096
2097	//===--------------------------------------------------------------------===//
2098	// Cleanups
2099	//===--------------------------------------------------------------------===//
2100
2101	typedef void Destroyer(CodeGenFunction &CGF, Address addr, QualType ty);
2102
2103	void pushIrregularPartialArrayCleanup(llvm::Value *arrayBegin,
2104	Address arrayEndPointer,
2105	QualType elementType,
2106	CharUnits elementAlignment,
2107	Destroyer *destroyer);
2108	void pushRegularPartialArrayCleanup(llvm::Value *arrayBegin,
2109	llvm::Value *arrayEnd,
2110	QualType elementType,
2111	CharUnits elementAlignment,
2112	Destroyer *destroyer);
2113
2114	void pushDestroy(QualType::DestructionKind dtorKind,
2115	Address addr, QualType type);
2116	void pushEHDestroy(QualType::DestructionKind dtorKind,
2117	Address addr, QualType type);
2118	void pushDestroy(CleanupKind kind, Address addr, QualType type,
2119	Destroyer *destroyer, bool useEHCleanupForArray);
2120	void pushLifetimeExtendedDestroy(CleanupKind kind, Address addr,
2121	QualType type, Destroyer *destroyer,
2122	bool useEHCleanupForArray);
2123	void pushCallObjectDeleteCleanup(const FunctionDecl *OperatorDelete,
2124	llvm::Value *CompletePtr,
2125	QualType ElementType);
2126	void pushStackRestore(CleanupKind kind, Address SPMem);
2127	void pushKmpcAllocFree(CleanupKind Kind,
2128	std::pair<llvm::Value *, llvm::Value *> AddrSizePair);
2129	void emitDestroy(Address addr, QualType type, Destroyer *destroyer,
2130	bool useEHCleanupForArray);
2131	llvm::Function *generateDestroyHelper(Address addr, QualType type,
2132	Destroyer *destroyer,
2133	bool useEHCleanupForArray,
2134	const VarDecl *VD);
2135	void emitArrayDestroy(llvm::Value *begin, llvm::Value *end,
2136	QualType elementType, CharUnits elementAlign,
2137	Destroyer *destroyer,
2138	bool checkZeroLength, bool useEHCleanup);
2139
2140	Destroyer *getDestroyer(QualType::DestructionKind destructionKind);
2141
2142	/// Determines whether an EH cleanup is required to destroy a type
2143	/// with the given destruction kind.
2144	bool needsEHCleanup(QualType::DestructionKind kind) {
2145	switch (kind) {
2146	case QualType::DK_none:
2147	return false;
2148	case QualType::DK_cxx_destructor:
2149	case QualType::DK_objc_weak_lifetime:
2150	case QualType::DK_nontrivial_c_struct:
2151	return getLangOpts().Exceptions;
2152	case QualType::DK_objc_strong_lifetime:
2153	return getLangOpts().Exceptions &&
2154	CGM.getCodeGenOpts().ObjCAutoRefCountExceptions;
2155	}
2156	llvm_unreachable("bad destruction kind");
2157	}
2158
2159	CleanupKind getCleanupKind(QualType::DestructionKind kind) {
2160	return (needsEHCleanup(kind) ? NormalAndEHCleanup : NormalCleanup);
2161	}
2162
2163	//===--------------------------------------------------------------------===//
2164	// Objective-C
2165	//===--------------------------------------------------------------------===//
2166
2167	void GenerateObjCMethod(const ObjCMethodDecl *OMD);
2168
2169	void StartObjCMethod(const ObjCMethodDecl *MD, const ObjCContainerDecl *CD);
2170
2171	/// GenerateObjCGetter - Synthesize an Objective-C property getter function.
2172	void GenerateObjCGetter(ObjCImplementationDecl *IMP,
2173	const ObjCPropertyImplDecl *PID);
2174	void generateObjCGetterBody(const ObjCImplementationDecl *classImpl,
2175	const ObjCPropertyImplDecl *propImpl,
2176	const ObjCMethodDecl *GetterMothodDecl,
2177	llvm::Constant *AtomicHelperFn);
2178
2179	void GenerateObjCCtorDtorMethod(ObjCImplementationDecl *IMP,
2180	ObjCMethodDecl *MD, bool ctor);
2181
2182	/// GenerateObjCSetter - Synthesize an Objective-C property setter function
2183	/// for the given property.
2184	void GenerateObjCSetter(ObjCImplementationDecl *IMP,
2185	const ObjCPropertyImplDecl *PID);
2186	void generateObjCSetterBody(const ObjCImplementationDecl *classImpl,
2187	const ObjCPropertyImplDecl *propImpl,
2188	llvm::Constant *AtomicHelperFn);
2189
2190	//===--------------------------------------------------------------------===//
2191	// Block Bits
2192	//===--------------------------------------------------------------------===//
2193
2194	/// Emit block literal.
2195	/// \return an LLVM value which is a pointer to a struct which contains
2196	/// information about the block, including the block invoke function, the
2197	/// captured variables, etc.
2198	llvm::Value *EmitBlockLiteral(const BlockExpr *);
2199
2200	llvm::Function *GenerateBlockFunction(GlobalDecl GD,
2201	const CGBlockInfo &Info,
2202	const DeclMapTy &ldm,
2203	bool IsLambdaConversionToBlock,
2204	bool BuildGlobalBlock);
2205
2206	/// Check if \p T is a C++ class that has a destructor that can throw.
2207	static bool cxxDestructorCanThrow(QualType T);
2208
2209	llvm::Constant *GenerateCopyHelperFunction(const CGBlockInfo &blockInfo);
2210	llvm::Constant *GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo);
2211	llvm::Constant *GenerateObjCAtomicSetterCopyHelperFunction(
2212	const ObjCPropertyImplDecl *PID);
2213	llvm::Constant *GenerateObjCAtomicGetterCopyHelperFunction(
2214	const ObjCPropertyImplDecl *PID);
2215	llvm::Value *EmitBlockCopyAndAutorelease(llvm::Value *Block, QualType Ty);
2216
2217	void BuildBlockRelease(llvm::Value *DeclPtr, BlockFieldFlags flags,
2218	bool CanThrow);
2219
2220	class AutoVarEmission;
2221
2222	void emitByrefStructureInit(const AutoVarEmission &emission);
2223
2224	/// Enter a cleanup to destroy a __block variable. Note that this
2225	/// cleanup should be a no-op if the variable hasn't left the stack
2226	/// yet; if a cleanup is required for the variable itself, that needs
2227	/// to be done externally.
2228	///
2229	/// \param Kind Cleanup kind.
2230	///
2231	/// \param Addr When \p LoadBlockVarAddr is false, the address of the __block
2232	/// structure that will be passed to _Block_object_dispose. When
2233	/// \p LoadBlockVarAddr is true, the address of the field of the block
2234	/// structure that holds the address of the __block structure.
2235	///
2236	/// \param Flags The flag that will be passed to _Block_object_dispose.
2237	///
2238	/// \param LoadBlockVarAddr Indicates whether we need to emit a load from
2239	/// \p Addr to get the address of the __block structure.
2240	void enterByrefCleanup(CleanupKind Kind, Address Addr, BlockFieldFlags Flags,
2241	bool LoadBlockVarAddr, bool CanThrow);
2242
2243	void setBlockContextParameter(const ImplicitParamDecl *D, unsigned argNum,
2244	llvm::Value *ptr);
2245
2246	Address LoadBlockStruct();
2247	Address GetAddrOfBlockDecl(const VarDecl *var);
2248
2249	/// BuildBlockByrefAddress - Computes the location of the
2250	/// data in a variable which is declared as __block.
2251	Address emitBlockByrefAddress(Address baseAddr, const VarDecl *V,
2252	bool followForward = true);
2253	Address emitBlockByrefAddress(Address baseAddr,
2254	const BlockByrefInfo &info,
2255	bool followForward,
2256	const llvm::Twine &name);
2257
2258	const BlockByrefInfo &getBlockByrefInfo(const VarDecl *var);
2259
2260	QualType BuildFunctionArgList(GlobalDecl GD, FunctionArgList &Args);
2261
2262	void GenerateCode(GlobalDecl GD, llvm::Function *Fn,
2263	const CGFunctionInfo &FnInfo);
2264
2265	/// Annotate the function with an attribute that disables TSan checking at
2266	/// runtime.
2267	void markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn);
2268
2269	/// Emit code for the start of a function.
2270	/// \param Loc The location to be associated with the function.
2271	/// \param StartLoc The location of the function body.
2272	void StartFunction(GlobalDecl GD,
2273	QualType RetTy,
2274	llvm::Function *Fn,
2275	const CGFunctionInfo &FnInfo,
2276	const FunctionArgList &Args,
2277	SourceLocation Loc = SourceLocation(),
2278	SourceLocation StartLoc = SourceLocation());
2279
2280	static bool IsConstructorDelegationValid(const CXXConstructorDecl *Ctor);
2281
2282	void EmitConstructorBody(FunctionArgList &Args);
2283	void EmitDestructorBody(FunctionArgList &Args);
2284	void emitImplicitAssignmentOperatorBody(FunctionArgList &Args);
2285	void EmitFunctionBody(const Stmt *Body);
2286	void EmitBlockWithFallThrough(llvm::BasicBlock *BB, const Stmt *S);
2287
2288	void EmitForwardingCallToLambda(const CXXMethodDecl *LambdaCallOperator,
2289	CallArgList &CallArgs,
2290	const CGFunctionInfo *CallOpFnInfo = nullptr,
2291	llvm::Constant *CallOpFn = nullptr);
2292	void EmitLambdaBlockInvokeBody();
2293	void EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD);
2294	void EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD,
2295	CallArgList &CallArgs);
2296	void EmitLambdaInAllocaImplFn(const CXXMethodDecl *CallOp,
2297	const CGFunctionInfo **ImplFnInfo,
2298	llvm::Function **ImplFn);
2299	void EmitLambdaInAllocaCallOpBody(const CXXMethodDecl *MD);
2300	void EmitLambdaVLACapture(const VariableArrayType *VAT, LValue LV) {
2301	EmitStoreThroughLValue(Src: RValue::get(V: VLASizeMap[VAT->getSizeExpr()]), Dst: LV);
2302	}
2303	void EmitAsanPrologueOrEpilogue(bool Prologue);
2304
2305	/// Emit the unified return block, trying to avoid its emission when
2306	/// possible.
2307	/// \return The debug location of the user written return statement if the
2308	/// return block is avoided.
2309	llvm::DebugLoc EmitReturnBlock();
2310
2311	/// FinishFunction - Complete IR generation of the current function. It is
2312	/// legal to call this function even if there is no current insertion point.
2313	void FinishFunction(SourceLocation EndLoc=SourceLocation());
2314
2315	void StartThunk(llvm::Function *Fn, GlobalDecl GD,
2316	const CGFunctionInfo &FnInfo, bool IsUnprototyped);
2317
2318	void EmitCallAndReturnForThunk(llvm::FunctionCallee Callee,
2319	const ThunkInfo *Thunk, bool IsUnprototyped);
2320
2321	void FinishThunk();
2322
2323	/// Emit a musttail call for a thunk with a potentially adjusted this pointer.
2324	void EmitMustTailThunk(GlobalDecl GD, llvm::Value *AdjustedThisPtr,
2325	llvm::FunctionCallee Callee);
2326
2327	/// Generate a thunk for the given method.
2328	void generateThunk(llvm::Function *Fn, const CGFunctionInfo &FnInfo,
2329	GlobalDecl GD, const ThunkInfo &Thunk,
2330	bool IsUnprototyped);
2331
2332	llvm::Function *GenerateVarArgsThunk(llvm::Function *Fn,
2333	const CGFunctionInfo &FnInfo,
2334	GlobalDecl GD, const ThunkInfo &Thunk);
2335
2336	void EmitCtorPrologue(const CXXConstructorDecl *CD, CXXCtorType Type,
2337	FunctionArgList &Args);
2338
2339	void EmitInitializerForField(FieldDecl *Field, LValue LHS, Expr *Init);
2340
2341	/// Struct with all information about dynamic [sub]class needed to set vptr.
2342	struct VPtr {
2343	BaseSubobject Base;
2344	const CXXRecordDecl *NearestVBase;
2345	CharUnits OffsetFromNearestVBase;
2346	const CXXRecordDecl *VTableClass;
2347	};
2348
2349	/// Initialize the vtable pointer of the given subobject.
2350	void InitializeVTablePointer(const VPtr &vptr);
2351
2352	typedef llvm::SmallVector<VPtr, 4> VPtrsVector;
2353
2354	typedef llvm::SmallPtrSet<const CXXRecordDecl *, 4> VisitedVirtualBasesSetTy;
2355	VPtrsVector getVTablePointers(const CXXRecordDecl *VTableClass);
2356
2357	void getVTablePointers(BaseSubobject Base, const CXXRecordDecl *NearestVBase,
2358	CharUnits OffsetFromNearestVBase,
2359	bool BaseIsNonVirtualPrimaryBase,
2360	const CXXRecordDecl *VTableClass,
2361	VisitedVirtualBasesSetTy &VBases, VPtrsVector &vptrs);
2362
2363	void InitializeVTablePointers(const CXXRecordDecl *ClassDecl);
2364
2365	/// GetVTablePtr - Return the Value of the vtable pointer member pointed
2366	/// to by This.
2367	llvm::Value *GetVTablePtr(Address This, llvm::Type *VTableTy,
2368	const CXXRecordDecl *VTableClass);
2369
2370	enum CFITypeCheckKind {
2371	CFITCK_VCall,
2372	CFITCK_NVCall,
2373	CFITCK_DerivedCast,
2374	CFITCK_UnrelatedCast,
2375	CFITCK_ICall,
2376	CFITCK_NVMFCall,
2377	CFITCK_VMFCall,
2378	};
2379
2380	/// Derived is the presumed address of an object of type T after a
2381	/// cast. If T is a polymorphic class type, emit a check that the virtual
2382	/// table for Derived belongs to a class derived from T.
2383	void EmitVTablePtrCheckForCast(QualType T, Address Derived, bool MayBeNull,
2384	CFITypeCheckKind TCK, SourceLocation Loc);
2385
2386	/// EmitVTablePtrCheckForCall - Virtual method MD is being called via VTable.
2387	/// If vptr CFI is enabled, emit a check that VTable is valid.
2388	void EmitVTablePtrCheckForCall(const CXXRecordDecl *RD, llvm::Value *VTable,
2389	CFITypeCheckKind TCK, SourceLocation Loc);
2390
2391	/// EmitVTablePtrCheck - Emit a check that VTable is a valid virtual table for
2392	/// RD using llvm.type.test.
2393	void EmitVTablePtrCheck(const CXXRecordDecl *RD, llvm::Value *VTable,
2394	CFITypeCheckKind TCK, SourceLocation Loc);
2395
2396	/// If whole-program virtual table optimization is enabled, emit an assumption
2397	/// that VTable is a member of RD's type identifier. Or, if vptr CFI is
2398	/// enabled, emit a check that VTable is a member of RD's type identifier.
2399	void EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD,
2400	llvm::Value *VTable, SourceLocation Loc);
2401
2402	/// Returns whether we should perform a type checked load when loading a
2403	/// virtual function for virtual calls to members of RD. This is generally
2404	/// true when both vcall CFI and whole-program-vtables are enabled.
2405	bool ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD);
2406
2407	/// Emit a type checked load from the given vtable.
2408	llvm::Value *EmitVTableTypeCheckedLoad(const CXXRecordDecl *RD,
2409	llvm::Value *VTable,
2410	llvm::Type *VTableTy,
2411	uint64_t VTableByteOffset);
2412
2413	/// EnterDtorCleanups - Enter the cleanups necessary to complete the
2414	/// given phase of destruction for a destructor. The end result
2415	/// should call destructors on members and base classes in reverse
2416	/// order of their construction.
2417	void EnterDtorCleanups(const CXXDestructorDecl *Dtor, CXXDtorType Type);
2418
2419	/// ShouldInstrumentFunction - Return true if the current function should be
2420	/// instrumented with __cyg_profile_func_* calls
2421	bool ShouldInstrumentFunction();
2422
2423	/// ShouldSkipSanitizerInstrumentation - Return true if the current function
2424	/// should not be instrumented with sanitizers.
2425	bool ShouldSkipSanitizerInstrumentation();
2426
2427	/// ShouldXRayInstrument - Return true if the current function should be
2428	/// instrumented with XRay nop sleds.
2429	bool ShouldXRayInstrumentFunction() const;
2430
2431	/// AlwaysEmitXRayCustomEvents - Return true if we must unconditionally emit
2432	/// XRay custom event handling calls.
2433	bool AlwaysEmitXRayCustomEvents() const;
2434
2435	/// AlwaysEmitXRayTypedEvents - Return true if clang must unconditionally emit
2436	/// XRay typed event handling calls.
2437	bool AlwaysEmitXRayTypedEvents() const;
2438
2439	/// Return a type hash constant for a function instrumented by
2440	/// -fsanitize=function.
2441	llvm::ConstantInt *getUBSanFunctionTypeHash(QualType T) const;
2442
2443	/// EmitFunctionProlog - Emit the target specific LLVM code to load the
2444	/// arguments for the given function. This is also responsible for naming the
2445	/// LLVM function arguments.
2446	void EmitFunctionProlog(const CGFunctionInfo &FI,
2447	llvm::Function *Fn,
2448	const FunctionArgList &Args);
2449
2450	/// EmitFunctionEpilog - Emit the target specific LLVM code to return the
2451	/// given temporary.
2452	void EmitFunctionEpilog(const CGFunctionInfo &FI, bool EmitRetDbgLoc,
2453	SourceLocation EndLoc);
2454
2455	/// Emit a test that checks if the return value \p RV is nonnull.
2456	void EmitReturnValueCheck(llvm::Value *RV);
2457
2458	/// EmitStartEHSpec - Emit the start of the exception spec.
2459	void EmitStartEHSpec(const Decl *D);
2460
2461	/// EmitEndEHSpec - Emit the end of the exception spec.
2462	void EmitEndEHSpec(const Decl *D);
2463
2464	/// getTerminateLandingPad - Return a landing pad that just calls terminate.
2465	llvm::BasicBlock *getTerminateLandingPad();
2466
2467	/// getTerminateLandingPad - Return a cleanup funclet that just calls
2468	/// terminate.
2469	llvm::BasicBlock *getTerminateFunclet();
2470
2471	/// getTerminateHandler - Return a handler (not a landing pad, just
2472	/// a catch handler) that just calls terminate. This is used when
2473	/// a terminate scope encloses a try.
2474	llvm::BasicBlock *getTerminateHandler();
2475
2476	llvm::Type *ConvertTypeForMem(QualType T);
2477	llvm::Type *ConvertType(QualType T);
2478	llvm::Type *ConvertType(const TypeDecl *T) {
2479	return ConvertType(T: getContext().getTypeDeclType(Decl: T));
2480	}
2481
2482	/// LoadObjCSelf - Load the value of self. This function is only valid while
2483	/// generating code for an Objective-C method.
2484	llvm::Value *LoadObjCSelf();
2485
2486	/// TypeOfSelfObject - Return type of object that this self represents.
2487	QualType TypeOfSelfObject();
2488
2489	/// getEvaluationKind - Return the TypeEvaluationKind of QualType \c T.
2490	static TypeEvaluationKind getEvaluationKind(QualType T);
2491
2492	static bool hasScalarEvaluationKind(QualType T) {
2493	return getEvaluationKind(T) == TEK_Scalar;
2494	}
2495
2496	static bool hasAggregateEvaluationKind(QualType T) {
2497	return getEvaluationKind(T) == TEK_Aggregate;
2498	}
2499
2500	/// createBasicBlock - Create an LLVM basic block.
2501	llvm::BasicBlock *createBasicBlock(const Twine &name = "",
2502	llvm::Function *parent = nullptr,
2503	llvm::BasicBlock *before = nullptr) {
2504	return llvm::BasicBlock::Create(Context&: getLLVMContext(), Name: name, Parent: parent, InsertBefore: before);
2505	}
2506
2507	/// getBasicBlockForLabel - Return the LLVM basicblock that the specified
2508	/// label maps to.
2509	JumpDest getJumpDestForLabel(const LabelDecl *S);
2510
2511	/// SimplifyForwardingBlocks - If the given basic block is only a branch to
2512	/// another basic block, simplify it. This assumes that no other code could
2513	/// potentially reference the basic block.
2514	void SimplifyForwardingBlocks(llvm::BasicBlock *BB);
2515
2516	/// EmitBlock - Emit the given block \arg BB and set it as the insert point,
2517	/// adding a fall-through branch from the current insert block if
2518	/// necessary. It is legal to call this function even if there is no current
2519	/// insertion point.
2520	///
2521	/// IsFinished - If true, indicates that the caller has finished emitting
2522	/// branches to the given block and does not expect to emit code into it. This
2523	/// means the block can be ignored if it is unreachable.
2524	void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false);
2525
2526	/// EmitBlockAfterUses - Emit the given block somewhere hopefully
2527	/// near its uses, and leave the insertion point in it.
2528	void EmitBlockAfterUses(llvm::BasicBlock *BB);
2529
2530	/// EmitBranch - Emit a branch to the specified basic block from the current
2531	/// insert block, taking care to avoid creation of branches from dummy
2532	/// blocks. It is legal to call this function even if there is no current
2533	/// insertion point.
2534	///
2535	/// This function clears the current insertion point. The caller should follow
2536	/// calls to this function with calls to Emit*Block prior to generation new
2537	/// code.
2538	void EmitBranch(llvm::BasicBlock *Block);
2539
2540	/// HaveInsertPoint - True if an insertion point is defined. If not, this
2541	/// indicates that the current code being emitted is unreachable.
2542	bool HaveInsertPoint() const {
2543	return Builder.GetInsertBlock() != nullptr;
2544	}
2545
2546	/// EnsureInsertPoint - Ensure that an insertion point is defined so that
2547	/// emitted IR has a place to go. Note that by definition, if this function
2548	/// creates a block then that block is unreachable; callers may do better to
2549	/// detect when no insertion point is defined and simply skip IR generation.
2550	void EnsureInsertPoint() {
2551	if (!HaveInsertPoint())
2552	EmitBlock(BB: createBasicBlock());
2553	}
2554
2555	/// ErrorUnsupported - Print out an error that codegen doesn't support the
2556	/// specified stmt yet.
2557	void ErrorUnsupported(const Stmt *S, const char *Type);
2558
2559	//===--------------------------------------------------------------------===//
2560	// Helpers
2561	//===--------------------------------------------------------------------===//
2562
2563	LValue MakeAddrLValue(Address Addr, QualType T,
2564	AlignmentSource Source = AlignmentSource::Type) {
2565	return LValue::MakeAddr(address: Addr, type: T, Context&: getContext(), BaseInfo: LValueBaseInfo(Source),
2566	TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2567	}
2568
2569	LValue MakeAddrLValue(Address Addr, QualType T, LValueBaseInfo BaseInfo,
2570	TBAAAccessInfo TBAAInfo) {
2571	return LValue::MakeAddr(address: Addr, type: T, Context&: getContext(), BaseInfo, TBAAInfo);
2572	}
2573
2574	LValue MakeAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,
2575	AlignmentSource Source = AlignmentSource::Type) {
2576	Address Addr(V, ConvertTypeForMem(T), Alignment);
2577	return LValue::MakeAddr(address: Addr, type: T, Context&: getContext(), BaseInfo: LValueBaseInfo(Source),
2578	TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2579	}
2580
2581	LValue
2582	MakeAddrLValueWithoutTBAA(Address Addr, QualType T,
2583	AlignmentSource Source = AlignmentSource::Type) {
2584	return LValue::MakeAddr(address: Addr, type: T, Context&: getContext(), BaseInfo: LValueBaseInfo(Source),
2585	TBAAInfo: TBAAAccessInfo());
2586	}
2587
2588	LValue MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T);
2589	LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T);
2590
2591	Address EmitLoadOfReference(LValue RefLVal,
2592	LValueBaseInfo *PointeeBaseInfo = nullptr,
2593	TBAAAccessInfo *PointeeTBAAInfo = nullptr);
2594	LValue EmitLoadOfReferenceLValue(LValue RefLVal);
2595	LValue EmitLoadOfReferenceLValue(Address RefAddr, QualType RefTy,
2596	AlignmentSource Source =
2597	AlignmentSource::Type) {
2598	LValue RefLVal = MakeAddrLValue(Addr: RefAddr, T: RefTy, BaseInfo: LValueBaseInfo(Source),
2599	TBAAInfo: CGM.getTBAAAccessInfo(AccessType: RefTy));
2600	return EmitLoadOfReferenceLValue(RefLVal);
2601	}
2602
2603	/// Load a pointer with type \p PtrTy stored at address \p Ptr.
2604	/// Note that \p PtrTy is the type of the loaded pointer, not the addresses
2605	/// it is loaded from.
2606	Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy,
2607	LValueBaseInfo *BaseInfo = nullptr,
2608	TBAAAccessInfo *TBAAInfo = nullptr);
2609	LValue EmitLoadOfPointerLValue(Address Ptr, const PointerType *PtrTy);
2610
2611	/// CreateTempAlloca - This creates an alloca and inserts it into the entry
2612	/// block if \p ArraySize is nullptr, otherwise inserts it at the current
2613	/// insertion point of the builder. The caller is responsible for setting an
2614	/// appropriate alignment on
2615	/// the alloca.
2616	///
2617	/// \p ArraySize is the number of array elements to be allocated if it
2618	/// is not nullptr.
2619	///
2620	/// LangAS::Default is the address space of pointers to local variables and
2621	/// temporaries, as exposed in the source language. In certain
2622	/// configurations, this is not the same as the alloca address space, and a
2623	/// cast is needed to lift the pointer from the alloca AS into
2624	/// LangAS::Default. This can happen when the target uses a restricted
2625	/// address space for the stack but the source language requires
2626	/// LangAS::Default to be a generic address space. The latter condition is
2627	/// common for most programming languages; OpenCL is an exception in that
2628	/// LangAS::Default is the private address space, which naturally maps
2629	/// to the stack.
2630	///
2631	/// Because the address of a temporary is often exposed to the program in
2632	/// various ways, this function will perform the cast. The original alloca
2633	/// instruction is returned through \p Alloca if it is not nullptr.
2634	///
2635	/// The cast is not performaed in CreateTempAllocaWithoutCast. This is
2636	/// more efficient if the caller knows that the address will not be exposed.
2637	llvm::AllocaInst *CreateTempAlloca(llvm::Type *Ty, const Twine &Name = "tmp",
2638	llvm::Value *ArraySize = nullptr);
2639	Address CreateTempAlloca(llvm::Type *Ty, CharUnits align,
2640	const Twine &Name = "tmp",
2641	llvm::Value *ArraySize = nullptr,
2642	Address *Alloca = nullptr);
2643	Address CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits align,
2644	const Twine &Name = "tmp",
2645	llvm::Value *ArraySize = nullptr);
2646
2647	/// CreateDefaultAlignedTempAlloca - This creates an alloca with the
2648	/// default ABI alignment of the given LLVM type.
2649	///
2650	/// IMPORTANT NOTE: This is *not* generally the right alignment for
2651	/// any given AST type that happens to have been lowered to the
2652	/// given IR type. This should only ever be used for function-local,
2653	/// IR-driven manipulations like saving and restoring a value. Do
2654	/// not hand this address off to arbitrary IRGen routines, and especially
2655	/// do not pass it as an argument to a function that might expect a
2656	/// properly ABI-aligned value.
2657	Address CreateDefaultAlignTempAlloca(llvm::Type *Ty,
2658	const Twine &Name = "tmp");
2659
2660	/// CreateIRTemp - Create a temporary IR object of the given type, with
2661	/// appropriate alignment. This routine should only be used when an temporary
2662	/// value needs to be stored into an alloca (for example, to avoid explicit
2663	/// PHI construction), but the type is the IR type, not the type appropriate
2664	/// for storing in memory.
2665	///
2666	/// That is, this is exactly equivalent to CreateMemTemp, but calling
2667	/// ConvertType instead of ConvertTypeForMem.
2668	Address CreateIRTemp(QualType T, const Twine &Name = "tmp");
2669
2670	/// CreateMemTemp - Create a temporary memory object of the given type, with
2671	/// appropriate alignmen and cast it to the default address space. Returns
2672	/// the original alloca instruction by \p Alloca if it is not nullptr.
2673	Address CreateMemTemp(QualType T, const Twine &Name = "tmp",
2674	Address *Alloca = nullptr);
2675	Address CreateMemTemp(QualType T, CharUnits Align, const Twine &Name = "tmp",
2676	Address *Alloca = nullptr);
2677
2678	/// CreateMemTemp - Create a temporary memory object of the given type, with
2679	/// appropriate alignmen without casting it to the default address space.
2680	Address CreateMemTempWithoutCast(QualType T, const Twine &Name = "tmp");
2681	Address CreateMemTempWithoutCast(QualType T, CharUnits Align,
2682	const Twine &Name = "tmp");
2683
2684	/// CreateAggTemp - Create a temporary memory object for the given
2685	/// aggregate type.
2686	AggValueSlot CreateAggTemp(QualType T, const Twine &Name = "tmp",
2687	Address *Alloca = nullptr) {
2688	return AggValueSlot::forAddr(addr: CreateMemTemp(T, Name, Alloca),
2689	quals: T.getQualifiers(),
2690	isDestructed: AggValueSlot::IsNotDestructed,
2691	needsGC: AggValueSlot::DoesNotNeedGCBarriers,
2692	isAliased: AggValueSlot::IsNotAliased,
2693	mayOverlap: AggValueSlot::DoesNotOverlap);
2694	}
2695
2696	/// EvaluateExprAsBool - Perform the usual unary conversions on the specified
2697	/// expression and compare the result against zero, returning an Int1Ty value.
2698	llvm::Value *EvaluateExprAsBool(const Expr *E);
2699
2700	/// EmitIgnoredExpr - Emit an expression in a context which ignores the result.
2701	void EmitIgnoredExpr(const Expr *E);
2702
2703	/// EmitAnyExpr - Emit code to compute the specified expression which can have
2704	/// any type. The result is returned as an RValue struct. If this is an
2705	/// aggregate expression, the aggloc/agglocvolatile arguments indicate where
2706	/// the result should be returned.
2707	///
2708	/// \param ignoreResult True if the resulting value isn't used.
2709	RValue EmitAnyExpr(const Expr *E,
2710	AggValueSlot aggSlot = AggValueSlot::ignored(),
2711	bool ignoreResult = false);
2712
2713	// EmitVAListRef - Emit a "reference" to a va_list; this is either the address
2714	// or the value of the expression, depending on how va_list is defined.
2715	Address EmitVAListRef(const Expr *E);
2716
2717	/// Emit a "reference" to a __builtin_ms_va_list; this is
2718	/// always the value of the expression, because a __builtin_ms_va_list is a
2719	/// pointer to a char.
2720	Address EmitMSVAListRef(const Expr *E);
2721
2722	/// EmitAnyExprToTemp - Similarly to EmitAnyExpr(), however, the result will
2723	/// always be accessible even if no aggregate location is provided.
2724	RValue EmitAnyExprToTemp(const Expr *E);
2725
2726	/// EmitAnyExprToMem - Emits the code necessary to evaluate an
2727	/// arbitrary expression into the given memory location.
2728	void EmitAnyExprToMem(const Expr *E, Address Location,
2729	Qualifiers Quals, bool IsInitializer);
2730
2731	void EmitAnyExprToExn(const Expr *E, Address Addr);
2732
2733	/// EmitExprAsInit - Emits the code necessary to initialize a
2734	/// location in memory with the given initializer.
2735	void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue,
2736	bool capturedByInit);
2737
2738	/// hasVolatileMember - returns true if aggregate type has a volatile
2739	/// member.
2740	bool hasVolatileMember(QualType T) {
2741	if (const RecordType *RT = T->getAs<RecordType>()) {
2742	const RecordDecl *RD = cast<RecordDecl>(Val: RT->getDecl());
2743	return RD->hasVolatileMember();
2744	}
2745	return false;
2746	}
2747
2748	/// Determine whether a return value slot may overlap some other object.
2749	AggValueSlot::Overlap_t getOverlapForReturnValue() {
2750	// FIXME: Assuming no overlap here breaks guaranteed copy elision for base
2751	// class subobjects. These cases may need to be revisited depending on the
2752	// resolution of the relevant core issue.
2753	return AggValueSlot::DoesNotOverlap;
2754	}
2755
2756	/// Determine whether a field initialization may overlap some other object.
2757	AggValueSlot::Overlap_t getOverlapForFieldInit(const FieldDecl *FD);
2758
2759	/// Determine whether a base class initialization may overlap some other
2760	/// object.
2761	AggValueSlot::Overlap_t getOverlapForBaseInit(const CXXRecordDecl *RD,
2762	const CXXRecordDecl *BaseRD,
2763	bool IsVirtual);
2764
2765	/// Emit an aggregate assignment.
2766	void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy) {
2767	bool IsVolatile = hasVolatileMember(T: EltTy);
2768	EmitAggregateCopy(Dest, Src, EltTy, MayOverlap: AggValueSlot::MayOverlap, isVolatile: IsVolatile);
2769	}
2770
2771	void EmitAggregateCopyCtor(LValue Dest, LValue Src,
2772	AggValueSlot::Overlap_t MayOverlap) {
2773	EmitAggregateCopy(Dest, Src, EltTy: Src.getType(), MayOverlap);
2774	}
2775
2776	/// EmitAggregateCopy - Emit an aggregate copy.
2777	///
2778	/// \param isVolatile \c true iff either the source or the destination is
2779	/// volatile.
2780	/// \param MayOverlap Whether the tail padding of the destination might be
2781	/// occupied by some other object. More efficient code can often be
2782	/// generated if not.
2783	void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy,
2784	AggValueSlot::Overlap_t MayOverlap,
2785	bool isVolatile = false);
2786
2787	/// GetAddrOfLocalVar - Return the address of a local variable.
2788	Address GetAddrOfLocalVar(const VarDecl *VD) {
2789	auto it = LocalDeclMap.find(VD);
2790	assert(it != LocalDeclMap.end() &&
2791	"Invalid argument to GetAddrOfLocalVar(), no decl!");
2792	return it->second;
2793	}
2794
2795	/// Given an opaque value expression, return its LValue mapping if it exists,
2796	/// otherwise create one.
2797	LValue getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e);
2798
2799	/// Given an opaque value expression, return its RValue mapping if it exists,
2800	/// otherwise create one.
2801	RValue getOrCreateOpaqueRValueMapping(const OpaqueValueExpr *e);
2802
2803	/// Get the index of the current ArrayInitLoopExpr, if any.
2804	llvm::Value *getArrayInitIndex() { return ArrayInitIndex; }
2805
2806	/// getAccessedFieldNo - Given an encoded value and a result number, return
2807	/// the input field number being accessed.
2808	static unsigned getAccessedFieldNo(unsigned Idx, const llvm::Constant *Elts);
2809
2810	llvm::BlockAddress *GetAddrOfLabel(const LabelDecl *L);
2811	llvm::BasicBlock *GetIndirectGotoBlock();
2812
2813	/// Check if \p E is a C++ "this" pointer wrapped in value-preserving casts.
2814	static bool IsWrappedCXXThis(const Expr *E);
2815
2816	/// EmitNullInitialization - Generate code to set a value of the given type to
2817	/// null, If the type contains data member pointers, they will be initialized
2818	/// to -1 in accordance with the Itanium C++ ABI.
2819	void EmitNullInitialization(Address DestPtr, QualType Ty);
2820
2821	/// Emits a call to an LLVM variable-argument intrinsic, either
2822	/// \c llvm.va_start or \c llvm.va_end.
2823	/// \param ArgValue A reference to the \c va_list as emitted by either
2824	/// \c EmitVAListRef or \c EmitMSVAListRef.
2825	/// \param IsStart If \c true, emits a call to \c llvm.va_start; otherwise,
2826	/// calls \c llvm.va_end.
2827	llvm::Value *EmitVAStartEnd(llvm::Value *ArgValue, bool IsStart);
2828
2829	/// Generate code to get an argument from the passed in pointer
2830	/// and update it accordingly.
2831	/// \param VE The \c VAArgExpr for which to generate code.
2832	/// \param VAListAddr Receives a reference to the \c va_list as emitted by
2833	/// either \c EmitVAListRef or \c EmitMSVAListRef.
2834	/// \returns A pointer to the argument.
2835	// FIXME: We should be able to get rid of this method and use the va_arg
2836	// instruction in LLVM instead once it works well enough.
2837	Address EmitVAArg(VAArgExpr *VE, Address &VAListAddr);
2838
2839	/// emitArrayLength - Compute the length of an array, even if it's a
2840	/// VLA, and drill down to the base element type.
2841	llvm::Value *emitArrayLength(const ArrayType *arrayType,
2842	QualType &baseType,
2843	Address &addr);
2844
2845	/// EmitVLASize - Capture all the sizes for the VLA expressions in
2846	/// the given variably-modified type and store them in the VLASizeMap.
2847	///
2848	/// This function can be called with a null (unreachable) insert point.
2849	void EmitVariablyModifiedType(QualType Ty);
2850
2851	struct VlaSizePair {
2852	llvm::Value *NumElts;
2853	QualType Type;
2854
2855	VlaSizePair(llvm::Value *NE, QualType T) : NumElts(NE), Type(T) {}
2856	};
2857
2858	/// Return the number of elements for a single dimension
2859	/// for the given array type.
2860	VlaSizePair getVLAElements1D(const VariableArrayType *vla);
2861	VlaSizePair getVLAElements1D(QualType vla);
2862
2863	/// Returns an LLVM value that corresponds to the size,
2864	/// in non-variably-sized elements, of a variable length array type,
2865	/// plus that largest non-variably-sized element type. Assumes that
2866	/// the type has already been emitted with EmitVariablyModifiedType.
2867	VlaSizePair getVLASize(const VariableArrayType *vla);
2868	VlaSizePair getVLASize(QualType vla);
2869
2870	/// LoadCXXThis - Load the value of 'this'. This function is only valid while
2871	/// generating code for an C++ member function.
2872	llvm::Value *LoadCXXThis() {
2873	assert(CXXThisValue && "no 'this' value for this function");
2874	return CXXThisValue;
2875	}
2876	Address LoadCXXThisAddress();
2877
2878	/// LoadCXXVTT - Load the VTT parameter to base constructors/destructors have
2879	/// virtual bases.
2880	// FIXME: Every place that calls LoadCXXVTT is something
2881	// that needs to be abstracted properly.
2882	llvm::Value *LoadCXXVTT() {
2883	assert(CXXStructorImplicitParamValue && "no VTT value for this function");
2884	return CXXStructorImplicitParamValue;
2885	}
2886
2887	/// GetAddressOfBaseOfCompleteClass - Convert the given pointer to a
2888	/// complete class to the given direct base.
2889	Address
2890	GetAddressOfDirectBaseInCompleteClass(Address Value,
2891	const CXXRecordDecl *Derived,
2892	const CXXRecordDecl *Base,
2893	bool BaseIsVirtual);
2894
2895	static bool ShouldNullCheckClassCastValue(const CastExpr *Cast);
2896
2897	/// GetAddressOfBaseClass - This function will add the necessary delta to the
2898	/// load of 'this' and returns address of the base class.
2899	Address GetAddressOfBaseClass(Address Value,
2900	const CXXRecordDecl *Derived,
2901	CastExpr::path_const_iterator PathBegin,
2902	CastExpr::path_const_iterator PathEnd,
2903	bool NullCheckValue, SourceLocation Loc);
2904
2905	Address GetAddressOfDerivedClass(Address Value,
2906	const CXXRecordDecl *Derived,
2907	CastExpr::path_const_iterator PathBegin,
2908	CastExpr::path_const_iterator PathEnd,
2909	bool NullCheckValue);
2910
2911	/// GetVTTParameter - Return the VTT parameter that should be passed to a
2912	/// base constructor/destructor with virtual bases.
2913	/// FIXME: VTTs are Itanium ABI-specific, so the definition should move
2914	/// to ItaniumCXXABI.cpp together with all the references to VTT.
2915	llvm::Value *GetVTTParameter(GlobalDecl GD, bool ForVirtualBase,
2916	bool Delegating);
2917
2918	void EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor,
2919	CXXCtorType CtorType,
2920	const FunctionArgList &Args,
2921	SourceLocation Loc);
2922	// It's important not to confuse this and the previous function. Delegating
2923	// constructors are the C++0x feature. The constructor delegate optimization
2924	// is used to reduce duplication in the base and complete consturctors where
2925	// they are substantially the same.
2926	void EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor,
2927	const FunctionArgList &Args);
2928
2929	/// Emit a call to an inheriting constructor (that is, one that invokes a
2930	/// constructor inherited from a base class) by inlining its definition. This
2931	/// is necessary if the ABI does not support forwarding the arguments to the
2932	/// base class constructor (because they're variadic or similar).
2933	void EmitInlinedInheritingCXXConstructorCall(const CXXConstructorDecl *Ctor,
2934	CXXCtorType CtorType,
2935	bool ForVirtualBase,
2936	bool Delegating,
2937	CallArgList &Args);
2938
2939	/// Emit a call to a constructor inherited from a base class, passing the
2940	/// current constructor's arguments along unmodified (without even making
2941	/// a copy).
2942	void EmitInheritedCXXConstructorCall(const CXXConstructorDecl *D,
2943	bool ForVirtualBase, Address This,
2944	bool InheritedFromVBase,
2945	const CXXInheritedCtorInitExpr *E);
2946
2947	void EmitCXXConstructorCall(const CXXConstructorDecl *D, CXXCtorType Type,
2948	bool ForVirtualBase, bool Delegating,
2949	AggValueSlot ThisAVS, const CXXConstructExpr *E);
2950
2951	void EmitCXXConstructorCall(const CXXConstructorDecl *D, CXXCtorType Type,
2952	bool ForVirtualBase, bool Delegating,
2953	Address This, CallArgList &Args,
2954	AggValueSlot::Overlap_t Overlap,
2955	SourceLocation Loc, bool NewPointerIsChecked);
2956
2957	/// Emit assumption load for all bases. Requires to be called only on
2958	/// most-derived class and not under construction of the object.
2959	void EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl, Address This);
2960
2961	/// Emit assumption that vptr load == global vtable.
2962	void EmitVTableAssumptionLoad(const VPtr &vptr, Address This);
2963
2964	void EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D,
2965	Address This, Address Src,
2966	const CXXConstructExpr *E);
2967
2968	void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
2969	const ArrayType *ArrayTy,
2970	Address ArrayPtr,
2971	const CXXConstructExpr *E,
2972	bool NewPointerIsChecked,
2973	bool ZeroInitialization = false);
2974
2975	void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
2976	llvm::Value *NumElements,
2977	Address ArrayPtr,
2978	const CXXConstructExpr *E,
2979	bool NewPointerIsChecked,
2980	bool ZeroInitialization = false);
2981
2982	static Destroyer destroyCXXObject;
2983
2984	void EmitCXXDestructorCall(const CXXDestructorDecl *D, CXXDtorType Type,
2985	bool ForVirtualBase, bool Delegating, Address This,
2986	QualType ThisTy);
2987
2988	void EmitNewArrayInitializer(const CXXNewExpr *E, QualType elementType,
2989	llvm::Type *ElementTy, Address NewPtr,
2990	llvm::Value *NumElements,
2991	llvm::Value *AllocSizeWithoutCookie);
2992
2993	void EmitCXXTemporary(const CXXTemporary *Temporary, QualType TempType,
2994	Address Ptr);
2995
2996	void EmitSehCppScopeBegin();
2997	void EmitSehCppScopeEnd();
2998	void EmitSehTryScopeBegin();
2999	void EmitSehTryScopeEnd();
3000
3001	llvm::Value *EmitLifetimeStart(llvm::TypeSize Size, llvm::Value *Addr);
3002	void EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr);
3003
3004	llvm::Value *EmitCXXNewExpr(const CXXNewExpr *E);
3005	void EmitCXXDeleteExpr(const CXXDeleteExpr *E);
3006
3007	void EmitDeleteCall(const FunctionDecl *DeleteFD, llvm::Value *Ptr,
3008	QualType DeleteTy, llvm::Value *NumElements = nullptr,
3009	CharUnits CookieSize = CharUnits());
3010
3011	RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type,
3012	const CallExpr *TheCallExpr, bool IsDelete);
3013
3014	llvm::Value *EmitCXXTypeidExpr(const CXXTypeidExpr *E);
3015	llvm::Value *EmitDynamicCast(Address V, const CXXDynamicCastExpr *DCE);
3016	Address EmitCXXUuidofExpr(const CXXUuidofExpr *E);
3017
3018	/// Situations in which we might emit a check for the suitability of a
3019	/// pointer or glvalue. Needs to be kept in sync with ubsan_handlers.cpp in
3020	/// compiler-rt.
3021	enum TypeCheckKind {
3022	/// Checking the operand of a load. Must be suitably sized and aligned.
3023	TCK_Load,
3024	/// Checking the destination of a store. Must be suitably sized and aligned.
3025	TCK_Store,
3026	/// Checking the bound value in a reference binding. Must be suitably sized
3027	/// and aligned, but is not required to refer to an object (until the
3028	/// reference is used), per core issue 453.
3029	TCK_ReferenceBinding,
3030	/// Checking the object expression in a non-static data member access. Must
3031	/// be an object within its lifetime.
3032	TCK_MemberAccess,
3033	/// Checking the 'this' pointer for a call to a non-static member function.
3034	/// Must be an object within its lifetime.
3035	TCK_MemberCall,
3036	/// Checking the 'this' pointer for a constructor call.
3037	TCK_ConstructorCall,
3038	/// Checking the operand of a static_cast to a derived pointer type. Must be
3039	/// null or an object within its lifetime.
3040	TCK_DowncastPointer,
3041	/// Checking the operand of a static_cast to a derived reference type. Must
3042	/// be an object within its lifetime.
3043	TCK_DowncastReference,
3044	/// Checking the operand of a cast to a base object. Must be suitably sized
3045	/// and aligned.
3046	TCK_Upcast,
3047	/// Checking the operand of a cast to a virtual base object. Must be an
3048	/// object within its lifetime.
3049	TCK_UpcastToVirtualBase,
3050	/// Checking the value assigned to a _Nonnull pointer. Must not be null.
3051	TCK_NonnullAssign,
3052	/// Checking the operand of a dynamic_cast or a typeid expression. Must be
3053	/// null or an object within its lifetime.
3054	TCK_DynamicOperation
3055	};
3056
3057	/// Determine whether the pointer type check \p TCK permits null pointers.
3058	static bool isNullPointerAllowed(TypeCheckKind TCK);
3059
3060	/// Determine whether the pointer type check \p TCK requires a vptr check.
3061	static bool isVptrCheckRequired(TypeCheckKind TCK, QualType Ty);
3062
3063	/// Whether any type-checking sanitizers are enabled. If \c false,
3064	/// calls to EmitTypeCheck can be skipped.
3065	bool sanitizePerformTypeCheck() const;
3066
3067	/// Emit a check that \p V is the address of storage of the
3068	/// appropriate size and alignment for an object of type \p Type
3069	/// (or if ArraySize is provided, for an array of that bound).
3070	void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, llvm::Value *V,
3071	QualType Type, CharUnits Alignment = CharUnits::Zero(),
3072	SanitizerSet SkippedChecks = SanitizerSet(),
3073	llvm::Value *ArraySize = nullptr);
3074
3075	/// Emit a check that \p Base points into an array object, which
3076	/// we can access at index \p Index. \p Accessed should be \c false if we
3077	/// this expression is used as an lvalue, for instance in "&Arr[Idx]".
3078	void EmitBoundsCheck(const Expr *E, const Expr *Base, llvm::Value *Index,
3079	QualType IndexType, bool Accessed);
3080	void EmitBoundsCheckImpl(const Expr *E, llvm::Value *Bound,
3081	llvm::Value *Index, QualType IndexType,
3082	QualType IndexedType, bool Accessed);
3083
3084	// Find a struct's flexible array member. It may be embedded inside multiple
3085	// sub-structs, but must still be the last field.
3086	const FieldDecl *FindFlexibleArrayMemberField(ASTContext &Ctx,
3087	const RecordDecl *RD,
3088	StringRef Name,
3089	uint64_t &Offset);
3090
3091	/// Find the FieldDecl specified in a FAM's "counted_by" attribute. Returns
3092	/// \p nullptr if either the attribute or the field doesn't exist.
3093	const FieldDecl *FindCountedByField(const FieldDecl *FD);
3094
3095	/// Build an expression accessing the "counted_by" field.
3096	llvm::Value *EmitCountedByFieldExpr(const Expr *Base,
3097	const FieldDecl *FAMDecl,
3098	const FieldDecl *CountDecl);
3099
3100	llvm::Value *EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
3101	bool isInc, bool isPre);
3102	ComplexPairTy EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,
3103	bool isInc, bool isPre);
3104
3105	/// Converts Location to a DebugLoc, if debug information is enabled.
3106	llvm::DebugLoc SourceLocToDebugLoc(SourceLocation Location);
3107
3108	/// Get the record field index as represented in debug info.
3109	unsigned getDebugInfoFIndex(const RecordDecl *Rec, unsigned FieldIndex);
3110
3111
3112	//===--------------------------------------------------------------------===//
3113	// Declaration Emission
3114	//===--------------------------------------------------------------------===//
3115
3116	/// EmitDecl - Emit a declaration.
3117	///
3118	/// This function can be called with a null (unreachable) insert point.
3119	void EmitDecl(const Decl &D);
3120
3121	/// EmitVarDecl - Emit a local variable declaration.
3122	///
3123	/// This function can be called with a null (unreachable) insert point.
3124	void EmitVarDecl(const VarDecl &D);
3125
3126	void EmitScalarInit(const Expr *init, const ValueDecl *D, LValue lvalue,
3127	bool capturedByInit);
3128
3129	typedef void SpecialInitFn(CodeGenFunction &Init, const VarDecl &D,
3130	llvm::Value *Address);
3131
3132	/// Determine whether the given initializer is trivial in the sense
3133	/// that it requires no code to be generated.
3134	bool isTrivialInitializer(const Expr *Init);
3135
3136	/// EmitAutoVarDecl - Emit an auto variable declaration.
3137	///
3138	/// This function can be called with a null (unreachable) insert point.
3139	void EmitAutoVarDecl(const VarDecl &D);
3140
3141	class AutoVarEmission {
3142	friend class CodeGenFunction;
3143
3144	const VarDecl *Variable;
3145
3146	/// The address of the alloca for languages with explicit address space
3147	/// (e.g. OpenCL) or alloca casted to generic pointer for address space
3148	/// agnostic languages (e.g. C++). Invalid if the variable was emitted
3149	/// as a global constant.
3150	Address Addr;
3151
3152	llvm::Value *NRVOFlag;
3153
3154	/// True if the variable is a __block variable that is captured by an
3155	/// escaping block.
3156	bool IsEscapingByRef;
3157
3158	/// True if the variable is of aggregate type and has a constant
3159	/// initializer.
3160	bool IsConstantAggregate;
3161
3162	/// Non-null if we should use lifetime annotations.
3163	llvm::Value *SizeForLifetimeMarkers;
3164
3165	/// Address with original alloca instruction. Invalid if the variable was
3166	/// emitted as a global constant.
3167	Address AllocaAddr;
3168
3169	struct Invalid {};
3170	AutoVarEmission(Invalid)
3171	: Variable(nullptr), Addr(Address::invalid()),
3172	AllocaAddr(Address::invalid()) {}
3173
3174	AutoVarEmission(const VarDecl &variable)
3175	: Variable(&variable), Addr(Address::invalid()), NRVOFlag(nullptr),
3176	IsEscapingByRef(false), IsConstantAggregate(false),
3177	SizeForLifetimeMarkers(nullptr), AllocaAddr(Address::invalid()) {}
3178
3179	bool wasEmittedAsGlobal() const { return !Addr.isValid(); }
3180
3181	public:
3182	static AutoVarEmission invalid() { return AutoVarEmission(Invalid()); }
3183
3184	bool useLifetimeMarkers() const {
3185	return SizeForLifetimeMarkers != nullptr;
3186	}
3187	llvm::Value *getSizeForLifetimeMarkers() const {
3188	assert(useLifetimeMarkers());
3189	return SizeForLifetimeMarkers;
3190	}
3191
3192	/// Returns the raw, allocated address, which is not necessarily
3193	/// the address of the object itself. It is casted to default
3194	/// address space for address space agnostic languages.
3195	Address getAllocatedAddress() const {
3196	return Addr;
3197	}
3198
3199	/// Returns the address for the original alloca instruction.
3200	Address getOriginalAllocatedAddress() const { return AllocaAddr; }
3201
3202	/// Returns the address of the object within this declaration.
3203	/// Note that this does not chase the forwarding pointer for
3204	/// __block decls.
3205	Address getObjectAddress(CodeGenFunction &CGF) const {
3206	if (!IsEscapingByRef) return Addr;
3207
3208	return CGF.emitBlockByrefAddress(baseAddr: Addr, V: Variable, /*forward*/ followForward: false);
3209	}
3210	};
3211	AutoVarEmission EmitAutoVarAlloca(const VarDecl &var);
3212	void EmitAutoVarInit(const AutoVarEmission &emission);
3213	void EmitAutoVarCleanups(const AutoVarEmission &emission);
3214	void emitAutoVarTypeCleanup(const AutoVarEmission &emission,
3215	QualType::DestructionKind dtorKind);
3216
3217	/// Emits the alloca and debug information for the size expressions for each
3218	/// dimension of an array. It registers the association of its (1-dimensional)
3219	/// QualTypes and size expression's debug node, so that CGDebugInfo can
3220	/// reference this node when creating the DISubrange object to describe the
3221	/// array types.
3222	void EmitAndRegisterVariableArrayDimensions(CGDebugInfo *DI,
3223	const VarDecl &D,
3224	bool EmitDebugInfo);
3225
3226	void EmitStaticVarDecl(const VarDecl &D,
3227	llvm::GlobalValue::LinkageTypes Linkage);
3228
3229	class ParamValue {
3230	llvm::Value *Value;
3231	llvm::Type *ElementType;
3232	unsigned Alignment;
3233	ParamValue(llvm::Value *V, llvm::Type *T, unsigned A)
3234	: Value(V), ElementType(T), Alignment(A) {}
3235	public:
3236	static ParamValue forDirect(llvm::Value *value) {
3237	return ParamValue(value, nullptr, 0);
3238	}
3239	static ParamValue forIndirect(Address addr) {
3240	assert(!addr.getAlignment().isZero());
3241	return ParamValue(addr.getPointer(), addr.getElementType(),
3242	addr.getAlignment().getQuantity());
3243	}
3244
3245	bool isIndirect() const { return Alignment != 0; }
3246	llvm::Value *getAnyValue() const { return Value; }
3247
3248	llvm::Value *getDirectValue() const {
3249	assert(!isIndirect());
3250	return Value;
3251	}
3252
3253	Address getIndirectAddress() const {
3254	assert(isIndirect());
3255	return Address(Value, ElementType, CharUnits::fromQuantity(Quantity: Alignment),
3256	KnownNonNull);
3257	}
3258	};
3259
3260	/// EmitParmDecl - Emit a ParmVarDecl or an ImplicitParamDecl.
3261	void EmitParmDecl(const VarDecl &D, ParamValue Arg, unsigned ArgNo);
3262
3263	/// protectFromPeepholes - Protect a value that we're intending to
3264	/// store to the side, but which will probably be used later, from
3265	/// aggressive peepholing optimizations that might delete it.
3266	///
3267	/// Pass the result to unprotectFromPeepholes to declare that
3268	/// protection is no longer required.
3269	///
3270	/// There's no particular reason why this shouldn't apply to
3271	/// l-values, it's just that no existing peepholes work on pointers.
3272	PeepholeProtection protectFromPeepholes(RValue rvalue);
3273	void unprotectFromPeepholes(PeepholeProtection protection);
3274
3275	void emitAlignmentAssumptionCheck(llvm::Value *Ptr, QualType Ty,
3276	SourceLocation Loc,
3277	SourceLocation AssumptionLoc,
3278	llvm::Value *Alignment,
3279	llvm::Value *OffsetValue,
3280	llvm::Value *TheCheck,
3281	llvm::Instruction *Assumption);
3282
3283	void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty,
3284	SourceLocation Loc, SourceLocation AssumptionLoc,
3285	llvm::Value *Alignment,
3286	llvm::Value *OffsetValue = nullptr);
3287
3288	void emitAlignmentAssumption(llvm::Value *PtrValue, const Expr *E,
3289	SourceLocation AssumptionLoc,
3290	llvm::Value *Alignment,
3291	llvm::Value *OffsetValue = nullptr);
3292
3293	//===--------------------------------------------------------------------===//
3294	// Statement Emission
3295	//===--------------------------------------------------------------------===//
3296
3297	/// EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
3298	void EmitStopPoint(const Stmt *S);
3299
3300	/// EmitStmt - Emit the code for the statement \arg S. It is legal to call
3301	/// this function even if there is no current insertion point.
3302	///
3303	/// This function may clear the current insertion point; callers should use
3304	/// EnsureInsertPoint if they wish to subsequently generate code without first
3305	/// calling EmitBlock, EmitBranch, or EmitStmt.
3306	void EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs = std::nullopt);
3307
3308	/// EmitSimpleStmt - Try to emit a "simple" statement which does not
3309	/// necessarily require an insertion point or debug information; typically
3310	/// because the statement amounts to a jump or a container of other
3311	/// statements.
3312	///
3313	/// \return True if the statement was handled.
3314	bool EmitSimpleStmt(const Stmt *S, ArrayRef<const Attr *> Attrs);
3315
3316	Address EmitCompoundStmt(const CompoundStmt &S, bool GetLast = false,
3317	AggValueSlot AVS = AggValueSlot::ignored());
3318	Address EmitCompoundStmtWithoutScope(const CompoundStmt &S,
3319	bool GetLast = false,
3320	AggValueSlot AVS =
3321	AggValueSlot::ignored());
3322
3323	/// EmitLabel - Emit the block for the given label. It is legal to call this
3324	/// function even if there is no current insertion point.
3325	void EmitLabel(const LabelDecl *D); // helper for EmitLabelStmt.
3326
3327	void EmitLabelStmt(const LabelStmt &S);
3328	void EmitAttributedStmt(const AttributedStmt &S);
3329	void EmitGotoStmt(const GotoStmt &S);
3330	void EmitIndirectGotoStmt(const IndirectGotoStmt &S);
3331	void EmitIfStmt(const IfStmt &S);
3332
3333	void EmitWhileStmt(const WhileStmt &S,
3334	ArrayRef<const Attr *> Attrs = std::nullopt);
3335	void EmitDoStmt(const DoStmt &S, ArrayRef<const Attr *> Attrs = std::nullopt);
3336	void EmitForStmt(const ForStmt &S,
3337	ArrayRef<const Attr *> Attrs = std::nullopt);
3338	void EmitReturnStmt(const ReturnStmt &S);
3339	void EmitDeclStmt(const DeclStmt &S);
3340	void EmitBreakStmt(const BreakStmt &S);
3341	void EmitContinueStmt(const ContinueStmt &S);
3342	void EmitSwitchStmt(const SwitchStmt &S);
3343	void EmitDefaultStmt(const DefaultStmt &S, ArrayRef<const Attr *> Attrs);
3344	void EmitCaseStmt(const CaseStmt &S, ArrayRef<const Attr *> Attrs);
3345	void EmitCaseStmtRange(const CaseStmt &S, ArrayRef<const Attr *> Attrs);
3346	void EmitAsmStmt(const AsmStmt &S);
3347
3348	void EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S);
3349	void EmitObjCAtTryStmt(const ObjCAtTryStmt &S);
3350	void EmitObjCAtThrowStmt(const ObjCAtThrowStmt &S);
3351	void EmitObjCAtSynchronizedStmt(const ObjCAtSynchronizedStmt &S);
3352	void EmitObjCAutoreleasePoolStmt(const ObjCAutoreleasePoolStmt &S);
3353
3354	void EmitCoroutineBody(const CoroutineBodyStmt &S);
3355	void EmitCoreturnStmt(const CoreturnStmt &S);
3356	RValue EmitCoawaitExpr(const CoawaitExpr &E,
3357	AggValueSlot aggSlot = AggValueSlot::ignored(),
3358	bool ignoreResult = false);
3359	LValue EmitCoawaitLValue(const CoawaitExpr *E);
3360	RValue EmitCoyieldExpr(const CoyieldExpr &E,
3361	AggValueSlot aggSlot = AggValueSlot::ignored(),
3362	bool ignoreResult = false);
3363	LValue EmitCoyieldLValue(const CoyieldExpr *E);
3364	RValue EmitCoroutineIntrinsic(const CallExpr *E, unsigned int IID);
3365
3366	void EnterCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false);
3367	void ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false);
3368
3369	void EmitCXXTryStmt(const CXXTryStmt &S);
3370	void EmitSEHTryStmt(const SEHTryStmt &S);
3371	void EmitSEHLeaveStmt(const SEHLeaveStmt &S);
3372	void EnterSEHTryStmt(const SEHTryStmt &S);
3373	void ExitSEHTryStmt(const SEHTryStmt &S);
3374	void VolatilizeTryBlocks(llvm::BasicBlock *BB,
3375	llvm::SmallPtrSet<llvm::BasicBlock *, 10> &V);
3376
3377	void pushSEHCleanup(CleanupKind kind,
3378	llvm::Function *FinallyFunc);
3379	void startOutlinedSEHHelper(CodeGenFunction &ParentCGF, bool IsFilter,
3380	const Stmt *OutlinedStmt);
3381
3382	llvm::Function *GenerateSEHFilterFunction(CodeGenFunction &ParentCGF,
3383	const SEHExceptStmt &Except);
3384
3385	llvm::Function *GenerateSEHFinallyFunction(CodeGenFunction &ParentCGF,
3386	const SEHFinallyStmt &Finally);
3387
3388	void EmitSEHExceptionCodeSave(CodeGenFunction &ParentCGF,
3389	llvm::Value *ParentFP,
3390	llvm::Value *EntryEBP);
3391	llvm::Value *EmitSEHExceptionCode();
3392	llvm::Value *EmitSEHExceptionInfo();
3393	llvm::Value *EmitSEHAbnormalTermination();
3394
3395	/// Emit simple code for OpenMP directives in Simd-only mode.
3396	void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D);
3397
3398	/// Scan the outlined statement for captures from the parent function. For
3399	/// each capture, mark the capture as escaped and emit a call to
3400	/// llvm.localrecover. Insert the localrecover result into the LocalDeclMap.
3401	void EmitCapturedLocals(CodeGenFunction &ParentCGF, const Stmt *OutlinedStmt,
3402	bool IsFilter);
3403
3404	/// Recovers the address of a local in a parent function. ParentVar is the
3405	/// address of the variable used in the immediate parent function. It can
3406	/// either be an alloca or a call to llvm.localrecover if there are nested
3407	/// outlined functions. ParentFP is the frame pointer of the outermost parent
3408	/// frame.
3409	Address recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF,
3410	Address ParentVar,
3411	llvm::Value *ParentFP);
3412
3413	void EmitCXXForRangeStmt(const CXXForRangeStmt &S,
3414	ArrayRef<const Attr *> Attrs = std::nullopt);
3415
3416	/// Controls insertion of cancellation exit blocks in worksharing constructs.
3417	class OMPCancelStackRAII {
3418	CodeGenFunction &CGF;
3419
3420	public:
3421	OMPCancelStackRAII(CodeGenFunction &CGF, OpenMPDirectiveKind Kind,
3422	bool HasCancel)
3423	: CGF(CGF) {
3424	CGF.OMPCancelStack.enter(CGF, Kind, HasCancel);
3425	}
3426	~OMPCancelStackRAII() { CGF.OMPCancelStack.exit(CGF); }
3427	};
3428
3429	/// Returns calculated size of the specified type.
3430	llvm::Value *getTypeSize(QualType Ty);
3431	LValue InitCapturedStruct(const CapturedStmt &S);
3432	llvm::Function *EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K);
3433	llvm::Function *GenerateCapturedStmtFunction(const CapturedStmt &S);
3434	Address GenerateCapturedStmtArgument(const CapturedStmt &S);
3435	llvm::Function *GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S,
3436	SourceLocation Loc);
3437	void GenerateOpenMPCapturedVars(const CapturedStmt &S,
3438	SmallVectorImpl<llvm::Value *> &CapturedVars);
3439	void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy,
3440	SourceLocation Loc);
3441	/// Perform element by element copying of arrays with type \a
3442	/// OriginalType from \a SrcAddr to \a DestAddr using copying procedure
3443	/// generated by \a CopyGen.
3444	///
3445	/// \param DestAddr Address of the destination array.
3446	/// \param SrcAddr Address of the source array.
3447	/// \param OriginalType Type of destination and source arrays.
3448	/// \param CopyGen Copying procedure that copies value of single array element
3449	/// to another single array element.
3450	void EmitOMPAggregateAssign(
3451	Address DestAddr, Address SrcAddr, QualType OriginalType,
3452	const llvm::function_ref<void(Address, Address)> CopyGen);
3453	/// Emit proper copying of data from one variable to another.
3454	///
3455	/// \param OriginalType Original type of the copied variables.
3456	/// \param DestAddr Destination address.
3457	/// \param SrcAddr Source address.
3458	/// \param DestVD Destination variable used in \a CopyExpr (for arrays, has
3459	/// type of the base array element).
3460	/// \param SrcVD Source variable used in \a CopyExpr (for arrays, has type of
3461	/// the base array element).
3462	/// \param Copy Actual copygin expression for copying data from \a SrcVD to \a
3463	/// DestVD.
3464	void EmitOMPCopy(QualType OriginalType,
3465	Address DestAddr, Address SrcAddr,
3466	const VarDecl *DestVD, const VarDecl *SrcVD,
3467	const Expr *Copy);
3468	/// Emit atomic update code for constructs: \a X = \a X \a BO \a E or
3469	/// \a X = \a E \a BO \a E.
3470	///
3471	/// \param X Value to be updated.
3472	/// \param E Update value.
3473	/// \param BO Binary operation for update operation.
3474	/// \param IsXLHSInRHSPart true if \a X is LHS in RHS part of the update
3475	/// expression, false otherwise.
3476	/// \param AO Atomic ordering of the generated atomic instructions.
3477	/// \param CommonGen Code generator for complex expressions that cannot be
3478	/// expressed through atomicrmw instruction.
3479	/// \returns <true, OldAtomicValue> if simple 'atomicrmw' instruction was
3480	/// generated, <false, RValue::get(nullptr)> otherwise.
3481	std::pair<bool, RValue> EmitOMPAtomicSimpleUpdateExpr(
3482	LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3483	llvm::AtomicOrdering AO, SourceLocation Loc,
3484	const llvm::function_ref<RValue(RValue)> CommonGen);
3485	bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D,
3486	OMPPrivateScope &PrivateScope);
3487	void EmitOMPPrivateClause(const OMPExecutableDirective &D,
3488	OMPPrivateScope &PrivateScope);
3489	void EmitOMPUseDevicePtrClause(
3490	const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope,
3491	const llvm::DenseMap<const ValueDecl *, llvm::Value *>
3492	CaptureDeviceAddrMap);
3493	void EmitOMPUseDeviceAddrClause(
3494	const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope,
3495	const llvm::DenseMap<const ValueDecl *, llvm::Value *>
3496	CaptureDeviceAddrMap);
3497	/// Emit code for copyin clause in \a D directive. The next code is
3498	/// generated at the start of outlined functions for directives:
3499	/// \code
3500	/// threadprivate_var1 = master_threadprivate_var1;
3501	/// operator=(threadprivate_var2, master_threadprivate_var2);
3502	/// ...
3503	/// __kmpc_barrier(&loc, global_tid);
3504	/// \endcode
3505	///
3506	/// \param D OpenMP directive possibly with 'copyin' clause(s).
3507	/// \returns true if at least one copyin variable is found, false otherwise.
3508	bool EmitOMPCopyinClause(const OMPExecutableDirective &D);
3509	/// Emit initial code for lastprivate variables. If some variable is
3510	/// not also firstprivate, then the default initialization is used. Otherwise
3511	/// initialization of this variable is performed by EmitOMPFirstprivateClause
3512	/// method.
3513	///
3514	/// \param D Directive that may have 'lastprivate' directives.
3515	/// \param PrivateScope Private scope for capturing lastprivate variables for
3516	/// proper codegen in internal captured statement.
3517	///
3518	/// \returns true if there is at least one lastprivate variable, false
3519	/// otherwise.
3520	bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D,
3521	OMPPrivateScope &PrivateScope);
3522	/// Emit final copying of lastprivate values to original variables at
3523	/// the end of the worksharing or simd directive.
3524	///
3525	/// \param D Directive that has at least one 'lastprivate' directives.
3526	/// \param IsLastIterCond Boolean condition that must be set to 'i1 true' if
3527	/// it is the last iteration of the loop code in associated directive, or to
3528	/// 'i1 false' otherwise. If this item is nullptr, no final check is required.
3529	void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D,
3530	bool NoFinals,
3531	llvm::Value *IsLastIterCond = nullptr);
3532	/// Emit initial code for linear clauses.
3533	void EmitOMPLinearClause(const OMPLoopDirective &D,
3534	CodeGenFunction::OMPPrivateScope &PrivateScope);
3535	/// Emit final code for linear clauses.
3536	/// \param CondGen Optional conditional code for final part of codegen for
3537	/// linear clause.
3538	void EmitOMPLinearClauseFinal(
3539	const OMPLoopDirective &D,
3540	const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen);
3541	/// Emit initial code for reduction variables. Creates reduction copies
3542	/// and initializes them with the values according to OpenMP standard.
3543	///
3544	/// \param D Directive (possibly) with the 'reduction' clause.
3545	/// \param PrivateScope Private scope for capturing reduction variables for
3546	/// proper codegen in internal captured statement.
3547	///
3548	void EmitOMPReductionClauseInit(const OMPExecutableDirective &D,
3549	OMPPrivateScope &PrivateScope,
3550	bool ForInscan = false);
3551	/// Emit final update of reduction values to original variables at
3552	/// the end of the directive.
3553	///
3554	/// \param D Directive that has at least one 'reduction' directives.
3555	/// \param ReductionKind The kind of reduction to perform.
3556	void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D,
3557	const OpenMPDirectiveKind ReductionKind);
3558	/// Emit initial code for linear variables. Creates private copies
3559	/// and initializes them with the values according to OpenMP standard.
3560	///
3561	/// \param D Directive (possibly) with the 'linear' clause.
3562	/// \return true if at least one linear variable is found that should be
3563	/// initialized with the value of the original variable, false otherwise.
3564	bool EmitOMPLinearClauseInit(const OMPLoopDirective &D);
3565
3566	typedef const llvm::function_ref<void(CodeGenFunction & /*CGF*/,
3567	llvm::Function * /*OutlinedFn*/,
3568	const OMPTaskDataTy & /*Data*/)>
3569	TaskGenTy;
3570	void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S,
3571	const OpenMPDirectiveKind CapturedRegion,
3572	const RegionCodeGenTy &BodyGen,
3573	const TaskGenTy &TaskGen, OMPTaskDataTy &Data);
3574	struct OMPTargetDataInfo {
3575	Address BasePointersArray = Address::invalid();
3576	Address PointersArray = Address::invalid();
3577	Address SizesArray = Address::invalid();
3578	Address MappersArray = Address::invalid();
3579	unsigned NumberOfTargetItems = 0;
3580	explicit OMPTargetDataInfo() = default;
3581	OMPTargetDataInfo(Address BasePointersArray, Address PointersArray,
3582	Address SizesArray, Address MappersArray,
3583	unsigned NumberOfTargetItems)
3584	: BasePointersArray(BasePointersArray), PointersArray(PointersArray),
3585	SizesArray(SizesArray), MappersArray(MappersArray),
3586	NumberOfTargetItems(NumberOfTargetItems) {}
3587	};
3588	void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S,
3589	const RegionCodeGenTy &BodyGen,
3590	OMPTargetDataInfo &InputInfo);
3591	void processInReduction(const OMPExecutableDirective &S,
3592	OMPTaskDataTy &Data,
3593	CodeGenFunction &CGF,
3594	const CapturedStmt *CS,
3595	OMPPrivateScope &Scope);
3596	void EmitOMPMetaDirective(const OMPMetaDirective &S);
3597	void EmitOMPParallelDirective(const OMPParallelDirective &S);
3598	void EmitOMPSimdDirective(const OMPSimdDirective &S);
3599	void EmitOMPTileDirective(const OMPTileDirective &S);
3600	void EmitOMPUnrollDirective(const OMPUnrollDirective &S);
3601	void EmitOMPForDirective(const OMPForDirective &S);
3602	void EmitOMPForSimdDirective(const OMPForSimdDirective &S);
3603	void EmitOMPSectionsDirective(const OMPSectionsDirective &S);
3604	void EmitOMPSectionDirective(const OMPSectionDirective &S);
3605	void EmitOMPSingleDirective(const OMPSingleDirective &S);
3606	void EmitOMPMasterDirective(const OMPMasterDirective &S);
3607	void EmitOMPMaskedDirective(const OMPMaskedDirective &S);
3608	void EmitOMPCriticalDirective(const OMPCriticalDirective &S);
3609	void EmitOMPParallelForDirective(const OMPParallelForDirective &S);
3610	void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S);
3611	void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S);
3612	void EmitOMPParallelMasterDirective(const OMPParallelMasterDirective &S);
3613	void EmitOMPTaskDirective(const OMPTaskDirective &S);
3614	void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S);
3615	void EmitOMPErrorDirective(const OMPErrorDirective &S);
3616	void EmitOMPBarrierDirective(const OMPBarrierDirective &S);
3617	void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S);
3618	void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S);
3619	void EmitOMPFlushDirective(const OMPFlushDirective &S);
3620	void EmitOMPDepobjDirective(const OMPDepobjDirective &S);
3621	void EmitOMPScanDirective(const OMPScanDirective &S);
3622	void EmitOMPOrderedDirective(const OMPOrderedDirective &S);
3623	void EmitOMPAtomicDirective(const OMPAtomicDirective &S);
3624	void EmitOMPTargetDirective(const OMPTargetDirective &S);
3625	void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S);
3626	void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S);
3627	void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S);
3628	void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S);
3629	void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S);
3630	void
3631	EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S);
3632	void EmitOMPTeamsDirective(const OMPTeamsDirective &S);
3633	void
3634	EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S);
3635	void EmitOMPCancelDirective(const OMPCancelDirective &S);
3636	void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S);
3637	void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S);
3638	void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S);
3639	void EmitOMPMasterTaskLoopDirective(const OMPMasterTaskLoopDirective &S);
3640	void
3641	EmitOMPMasterTaskLoopSimdDirective(const OMPMasterTaskLoopSimdDirective &S);
3642	void EmitOMPParallelMasterTaskLoopDirective(
3643	const OMPParallelMasterTaskLoopDirective &S);
3644	void EmitOMPParallelMasterTaskLoopSimdDirective(
3645	const OMPParallelMasterTaskLoopSimdDirective &S);
3646	void EmitOMPDistributeDirective(const OMPDistributeDirective &S);
3647	void EmitOMPDistributeParallelForDirective(
3648	const OMPDistributeParallelForDirective &S);
3649	void EmitOMPDistributeParallelForSimdDirective(
3650	const OMPDistributeParallelForSimdDirective &S);
3651	void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S);
3652	void EmitOMPTargetParallelForSimdDirective(
3653	const OMPTargetParallelForSimdDirective &S);
3654	void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S);
3655	void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S);
3656	void
3657	EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S);
3658	void EmitOMPTeamsDistributeParallelForSimdDirective(
3659	const OMPTeamsDistributeParallelForSimdDirective &S);
3660	void EmitOMPTeamsDistributeParallelForDirective(
3661	const OMPTeamsDistributeParallelForDirective &S);
3662	void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S);
3663	void EmitOMPTargetTeamsDistributeDirective(
3664	const OMPTargetTeamsDistributeDirective &S);
3665	void EmitOMPTargetTeamsDistributeParallelForDirective(
3666	const OMPTargetTeamsDistributeParallelForDirective &S);
3667	void EmitOMPTargetTeamsDistributeParallelForSimdDirective(
3668	const OMPTargetTeamsDistributeParallelForSimdDirective &S);
3669	void EmitOMPTargetTeamsDistributeSimdDirective(
3670	const OMPTargetTeamsDistributeSimdDirective &S);
3671	void EmitOMPGenericLoopDirective(const OMPGenericLoopDirective &S);
3672	void EmitOMPParallelGenericLoopDirective(const OMPLoopDirective &S);
3673	void EmitOMPTargetParallelGenericLoopDirective(
3674	const OMPTargetParallelGenericLoopDirective &S);
3675	void EmitOMPTargetTeamsGenericLoopDirective(
3676	const OMPTargetTeamsGenericLoopDirective &S);
3677	void EmitOMPTeamsGenericLoopDirective(const OMPTeamsGenericLoopDirective &S);
3678	void EmitOMPInteropDirective(const OMPInteropDirective &S);
3679	void EmitOMPParallelMaskedDirective(const OMPParallelMaskedDirective &S);
3680
3681	/// Emit device code for the target directive.
3682	static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM,
3683	StringRef ParentName,
3684	const OMPTargetDirective &S);
3685	static void
3686	EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName,
3687	const OMPTargetParallelDirective &S);
3688	/// Emit device code for the target parallel for directive.
3689	static void EmitOMPTargetParallelForDeviceFunction(
3690	CodeGenModule &CGM, StringRef ParentName,
3691	const OMPTargetParallelForDirective &S);
3692	/// Emit device code for the target parallel for simd directive.
3693	static void EmitOMPTargetParallelForSimdDeviceFunction(
3694	CodeGenModule &CGM, StringRef ParentName,
3695	const OMPTargetParallelForSimdDirective &S);
3696	/// Emit device code for the target teams directive.
3697	static void
3698	EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName,
3699	const OMPTargetTeamsDirective &S);
3700	/// Emit device code for the target teams distribute directive.
3701	static void EmitOMPTargetTeamsDistributeDeviceFunction(
3702	CodeGenModule &CGM, StringRef ParentName,
3703	const OMPTargetTeamsDistributeDirective &S);
3704	/// Emit device code for the target teams distribute simd directive.
3705	static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(
3706	CodeGenModule &CGM, StringRef ParentName,
3707	const OMPTargetTeamsDistributeSimdDirective &S);
3708	/// Emit device code for the target simd directive.
3709	static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM,
3710	StringRef ParentName,
3711	const OMPTargetSimdDirective &S);
3712	/// Emit device code for the target teams distribute parallel for simd
3713	/// directive.
3714	static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
3715	CodeGenModule &CGM, StringRef ParentName,
3716	const OMPTargetTeamsDistributeParallelForSimdDirective &S);
3717
3718	/// Emit device code for the target teams loop directive.
3719	static void EmitOMPTargetTeamsGenericLoopDeviceFunction(
3720	CodeGenModule &CGM, StringRef ParentName,
3721	const OMPTargetTeamsGenericLoopDirective &S);
3722
3723	/// Emit device code for the target parallel loop directive.
3724	static void EmitOMPTargetParallelGenericLoopDeviceFunction(
3725	CodeGenModule &CGM, StringRef ParentName,
3726	const OMPTargetParallelGenericLoopDirective &S);
3727
3728	static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
3729	CodeGenModule &CGM, StringRef ParentName,
3730	const OMPTargetTeamsDistributeParallelForDirective &S);
3731
3732	/// Emit the Stmt \p S and return its topmost canonical loop, if any.
3733	/// TODO: The \p Depth paramter is not yet implemented and must be 1. In the
3734	/// future it is meant to be the number of loops expected in the loop nests
3735	/// (usually specified by the "collapse" clause) that are collapsed to a
3736	/// single loop by this function.
3737	llvm::CanonicalLoopInfo *EmitOMPCollapsedCanonicalLoopNest(const Stmt *S,
3738	int Depth);
3739
3740	/// Emit an OMPCanonicalLoop using the OpenMPIRBuilder.
3741	void EmitOMPCanonicalLoop(const OMPCanonicalLoop *S);
3742
3743	/// Emit inner loop of the worksharing/simd construct.
3744	///
3745	/// \param S Directive, for which the inner loop must be emitted.
3746	/// \param RequiresCleanup true, if directive has some associated private
3747	/// variables.
3748	/// \param LoopCond Bollean condition for loop continuation.
3749	/// \param IncExpr Increment expression for loop control variable.
3750	/// \param BodyGen Generator for the inner body of the inner loop.
3751	/// \param PostIncGen Genrator for post-increment code (required for ordered
3752	/// loop directvies).
3753	void EmitOMPInnerLoop(
3754	const OMPExecutableDirective &S, bool RequiresCleanup,
3755	const Expr *LoopCond, const Expr *IncExpr,
3756	const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
3757	const llvm::function_ref<void(CodeGenFunction &)> PostIncGen);
3758
3759	JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind);
3760	/// Emit initial code for loop counters of loop-based directives.
3761	void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S,
3762	OMPPrivateScope &LoopScope);
3763
3764	/// Helper for the OpenMP loop directives.
3765	void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit);
3766
3767	/// Emit code for the worksharing loop-based directive.
3768	/// \return true, if this construct has any lastprivate clause, false -
3769	/// otherwise.
3770	bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB,
3771	const CodeGenLoopBoundsTy &CodeGenLoopBounds,
3772	const CodeGenDispatchBoundsTy &CGDispatchBounds);
3773
3774	/// Emit code for the distribute loop-based directive.
3775	void EmitOMPDistributeLoop(const OMPLoopDirective &S,
3776	const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr);
3777
3778	/// Helpers for the OpenMP loop directives.
3779	void EmitOMPSimdInit(const OMPLoopDirective &D);
3780	void EmitOMPSimdFinal(
3781	const OMPLoopDirective &D,
3782	const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen);
3783
3784	/// Emits the lvalue for the expression with possibly captured variable.
3785	LValue EmitOMPSharedLValue(const Expr *E);
3786
3787	private:
3788	/// Helpers for blocks.
3789	llvm::Value *EmitBlockLiteral(const CGBlockInfo &Info);
3790
3791	/// struct with the values to be passed to the OpenMP loop-related functions
3792	struct OMPLoopArguments {
3793	/// loop lower bound
3794	Address LB = Address::invalid();
3795	/// loop upper bound
3796	Address UB = Address::invalid();
3797	/// loop stride
3798	Address ST = Address::invalid();
3799	/// isLastIteration argument for runtime functions
3800	Address IL = Address::invalid();
3801	/// Chunk value generated by sema
3802	llvm::Value *Chunk = nullptr;
3803	/// EnsureUpperBound
3804	Expr *EUB = nullptr;
3805	/// IncrementExpression
3806	Expr *IncExpr = nullptr;
3807	/// Loop initialization
3808	Expr *Init = nullptr;
3809	/// Loop exit condition
3810	Expr *Cond = nullptr;
3811	/// Update of LB after a whole chunk has been executed
3812	Expr *NextLB = nullptr;
3813	/// Update of UB after a whole chunk has been executed
3814	Expr *NextUB = nullptr;
3815	OMPLoopArguments() = default;
3816	OMPLoopArguments(Address LB, Address UB, Address ST, Address IL,
3817	llvm::Value *Chunk = nullptr, Expr *EUB = nullptr,
3818	Expr *IncExpr = nullptr, Expr *Init = nullptr,
3819	Expr *Cond = nullptr, Expr *NextLB = nullptr,
3820	Expr *NextUB = nullptr)
3821	: LB(LB), UB(UB), ST(ST), IL(IL), Chunk(Chunk), EUB(EUB),
3822	IncExpr(IncExpr), Init(Init), Cond(Cond), NextLB(NextLB),
3823	NextUB(NextUB) {}
3824	};
3825	void EmitOMPOuterLoop(bool DynamicOrOrdered, bool IsMonotonic,
3826	const OMPLoopDirective &S, OMPPrivateScope &LoopScope,
3827	const OMPLoopArguments &LoopArgs,
3828	const CodeGenLoopTy &CodeGenLoop,
3829	const CodeGenOrderedTy &CodeGenOrdered);
3830	void EmitOMPForOuterLoop(const OpenMPScheduleTy &ScheduleKind,
3831	bool IsMonotonic, const OMPLoopDirective &S,
3832	OMPPrivateScope &LoopScope, bool Ordered,
3833	const OMPLoopArguments &LoopArgs,
3834	const CodeGenDispatchBoundsTy &CGDispatchBounds);
3835	void EmitOMPDistributeOuterLoop(OpenMPDistScheduleClauseKind ScheduleKind,
3836	const OMPLoopDirective &S,
3837	OMPPrivateScope &LoopScope,
3838	const OMPLoopArguments &LoopArgs,
3839	const CodeGenLoopTy &CodeGenLoopContent);
3840	/// Emit code for sections directive.
3841	void EmitSections(const OMPExecutableDirective &S);
3842
3843	public:
3844	//===--------------------------------------------------------------------===//
3845	// OpenACC Emission
3846	//===--------------------------------------------------------------------===//
3847	void EmitOpenACCComputeConstruct(const OpenACCComputeConstruct &S) {
3848	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
3849	// simply emitting its structured block, but in the future we will implement
3850	// some sort of IR.
3851	EmitStmt(S: S.getStructuredBlock());
3852	}
3853
3854	//===--------------------------------------------------------------------===//
3855	// LValue Expression Emission
3856	//===--------------------------------------------------------------------===//
3857
3858	/// Create a check that a scalar RValue is non-null.
3859	llvm::Value *EmitNonNullRValueCheck(RValue RV, QualType T);
3860
3861	/// GetUndefRValue - Get an appropriate 'undef' rvalue for the given type.
3862	RValue GetUndefRValue(QualType Ty);
3863
3864	/// EmitUnsupportedRValue - Emit a dummy r-value using the type of E
3865	/// and issue an ErrorUnsupported style diagnostic (using the
3866	/// provided Name).
3867	RValue EmitUnsupportedRValue(const Expr *E,
3868	const char *Name);
3869
3870	/// EmitUnsupportedLValue - Emit a dummy l-value using the type of E and issue
3871	/// an ErrorUnsupported style diagnostic (using the provided Name).
3872	LValue EmitUnsupportedLValue(const Expr *E,
3873	const char *Name);
3874
3875	/// EmitLValue - Emit code to compute a designator that specifies the location
3876	/// of the expression.
3877	///
3878	/// This can return one of two things: a simple address or a bitfield
3879	/// reference. In either case, the LLVM Value* in the LValue structure is
3880	/// guaranteed to be an LLVM pointer type.
3881	///
3882	/// If this returns a bitfield reference, nothing about the pointee type of
3883	/// the LLVM value is known: For example, it may not be a pointer to an
3884	/// integer.
3885	///
3886	/// If this returns a normal address, and if the lvalue's C type is fixed
3887	/// size, this method guarantees that the returned pointer type will point to
3888	/// an LLVM type of the same size of the lvalue's type. If the lvalue has a
3889	/// variable length type, this is not possible.
3890	///
3891	LValue EmitLValue(const Expr *E,
3892	KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
3893
3894	private:
3895	LValue EmitLValueHelper(const Expr *E, KnownNonNull_t IsKnownNonNull);
3896
3897	public:
3898	/// Same as EmitLValue but additionally we generate checking code to
3899	/// guard against undefined behavior. This is only suitable when we know
3900	/// that the address will be used to access the object.
3901	LValue EmitCheckedLValue(const Expr *E, TypeCheckKind TCK);
3902
3903	RValue convertTempToRValue(Address addr, QualType type,
3904	SourceLocation Loc);
3905
3906	void EmitAtomicInit(Expr *E, LValue lvalue);
3907
3908	bool LValueIsSuitableForInlineAtomic(LValue Src);
3909
3910	RValue EmitAtomicLoad(LValue LV, SourceLocation SL,
3911	AggValueSlot Slot = AggValueSlot::ignored());
3912
3913	RValue EmitAtomicLoad(LValue lvalue, SourceLocation loc,
3914	llvm::AtomicOrdering AO, bool IsVolatile = false,
3915	AggValueSlot slot = AggValueSlot::ignored());
3916
3917	void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit);
3918
3919	void EmitAtomicStore(RValue rvalue, LValue lvalue, llvm::AtomicOrdering AO,
3920	bool IsVolatile, bool isInit);
3921
3922	std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange(
3923	LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
3924	llvm::AtomicOrdering Success =
3925	llvm::AtomicOrdering::SequentiallyConsistent,
3926	llvm::AtomicOrdering Failure =
3927	llvm::AtomicOrdering::SequentiallyConsistent,
3928	bool IsWeak = false, AggValueSlot Slot = AggValueSlot::ignored());
3929
3930	void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO,
3931	const llvm::function_ref<RValue(RValue)> &UpdateOp,
3932	bool IsVolatile);
3933
3934	/// EmitToMemory - Change a scalar value from its value
3935	/// representation to its in-memory representation.
3936	llvm::Value *EmitToMemory(llvm::Value *Value, QualType Ty);
3937
3938	/// EmitFromMemory - Change a scalar value from its memory
3939	/// representation to its value representation.
3940	llvm::Value *EmitFromMemory(llvm::Value *Value, QualType Ty);
3941
3942	/// Check if the scalar \p Value is within the valid range for the given
3943	/// type \p Ty.
3944	///
3945	/// Returns true if a check is needed (even if the range is unknown).
3946	bool EmitScalarRangeCheck(llvm::Value *Value, QualType Ty,
3947	SourceLocation Loc);
3948
3949	/// EmitLoadOfScalar - Load a scalar value from an address, taking
3950	/// care to appropriately convert from the memory representation to
3951	/// the LLVM value representation.
3952	llvm::Value *EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty,
3953	SourceLocation Loc,
3954	AlignmentSource Source = AlignmentSource::Type,
3955	bool isNontemporal = false) {
3956	return EmitLoadOfScalar(Addr, Volatile, Ty, Loc, BaseInfo: LValueBaseInfo(Source),
3957	TBAAInfo: CGM.getTBAAAccessInfo(AccessType: Ty), isNontemporal);
3958	}
3959
3960	llvm::Value *EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty,
3961	SourceLocation Loc, LValueBaseInfo BaseInfo,
3962	TBAAAccessInfo TBAAInfo,
3963	bool isNontemporal = false);
3964
3965	/// EmitLoadOfScalar - Load a scalar value from an address, taking
3966	/// care to appropriately convert from the memory representation to
3967	/// the LLVM value representation. The l-value must be a simple
3968	/// l-value.
3969	llvm::Value *EmitLoadOfScalar(LValue lvalue, SourceLocation Loc);
3970
3971	/// EmitStoreOfScalar - Store a scalar value to an address, taking
3972	/// care to appropriately convert from the memory representation to
3973	/// the LLVM value representation.
3974	void EmitStoreOfScalar(llvm::Value *Value, Address Addr,
3975	bool Volatile, QualType Ty,
3976	AlignmentSource Source = AlignmentSource::Type,
3977	bool isInit = false, bool isNontemporal = false) {
3978	EmitStoreOfScalar(Value, Addr, Volatile, Ty, BaseInfo: LValueBaseInfo(Source),
3979	TBAAInfo: CGM.getTBAAAccessInfo(AccessType: Ty), isInit, isNontemporal);
3980	}
3981
3982	void EmitStoreOfScalar(llvm::Value *Value, Address Addr,
3983	bool Volatile, QualType Ty,
3984	LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo,
3985	bool isInit = false, bool isNontemporal = false);
3986
3987	/// EmitStoreOfScalar - Store a scalar value to an address, taking
3988	/// care to appropriately convert from the memory representation to
3989	/// the LLVM value representation. The l-value must be a simple
3990	/// l-value. The isInit flag indicates whether this is an initialization.
3991	/// If so, atomic qualifiers are ignored and the store is always non-atomic.
3992	void EmitStoreOfScalar(llvm::Value *value, LValue lvalue, bool isInit=false);
3993
3994	/// EmitLoadOfLValue - Given an expression that represents a value lvalue,
3995	/// this method emits the address of the lvalue, then loads the result as an
3996	/// rvalue, returning the rvalue.
3997	RValue EmitLoadOfLValue(LValue V, SourceLocation Loc);
3998	RValue EmitLoadOfExtVectorElementLValue(LValue V);
3999	RValue EmitLoadOfBitfieldLValue(LValue LV, SourceLocation Loc);
4000	RValue EmitLoadOfGlobalRegLValue(LValue LV);
4001
4002	/// EmitStoreThroughLValue - Store the specified rvalue into the specified
4003	/// lvalue, where both are guaranteed to the have the same type, and that type
4004	/// is 'Ty'.
4005	void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit = false);
4006	void EmitStoreThroughExtVectorComponentLValue(RValue Src, LValue Dst);
4007	void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst);
4008
4009	/// EmitStoreThroughBitfieldLValue - Store Src into Dst with same constraints
4010	/// as EmitStoreThroughLValue.
4011	///
4012	/// \param Result [out] - If non-null, this will be set to a Value* for the
4013	/// bit-field contents after the store, appropriate for use as the result of
4014	/// an assignment to the bit-field.
4015	void EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
4016	llvm::Value **Result=nullptr);
4017
4018	/// Emit an l-value for an assignment (simple or compound) of complex type.
4019	LValue EmitComplexAssignmentLValue(const BinaryOperator *E);
4020	LValue EmitComplexCompoundAssignmentLValue(const CompoundAssignOperator *E);
4021	LValue EmitScalarCompoundAssignWithComplex(const CompoundAssignOperator *E,
4022	llvm::Value *&Result);
4023
4024	// Note: only available for agg return types
4025	LValue EmitBinaryOperatorLValue(const BinaryOperator *E);
4026	LValue EmitCompoundAssignmentLValue(const CompoundAssignOperator *E);
4027	// Note: only available for agg return types
4028	LValue EmitCallExprLValue(const CallExpr *E);
4029	// Note: only available for agg return types
4030	LValue EmitVAArgExprLValue(const VAArgExpr *E);
4031	LValue EmitDeclRefLValue(const DeclRefExpr *E);
4032	LValue EmitStringLiteralLValue(const StringLiteral *E);
4033	LValue EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E);
4034	LValue EmitPredefinedLValue(const PredefinedExpr *E);
4035	LValue EmitUnaryOpLValue(const UnaryOperator *E);
4036	LValue EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
4037	bool Accessed = false);
4038	LValue EmitMatrixSubscriptExpr(const MatrixSubscriptExpr *E);
4039	LValue EmitOMPArraySectionExpr(const OMPArraySectionExpr *E,
4040	bool IsLowerBound = true);
4041	LValue EmitExtVectorElementExpr(const ExtVectorElementExpr *E);
4042	LValue EmitMemberExpr(const MemberExpr *E);
4043	LValue EmitObjCIsaExpr(const ObjCIsaExpr *E);
4044	LValue EmitCompoundLiteralLValue(const CompoundLiteralExpr *E);
4045	LValue EmitInitListLValue(const InitListExpr *E);
4046	void EmitIgnoredConditionalOperator(const AbstractConditionalOperator *E);
4047	LValue EmitConditionalOperatorLValue(const AbstractConditionalOperator *E);
4048	LValue EmitCastLValue(const CastExpr *E);
4049	LValue EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *E);
4050	LValue EmitOpaqueValueLValue(const OpaqueValueExpr *e);
4051
4052	Address EmitExtVectorElementLValue(LValue V);
4053
4054	RValue EmitRValueForField(LValue LV, const FieldDecl *FD, SourceLocation Loc);
4055
4056	Address EmitArrayToPointerDecay(const Expr *Array,
4057	LValueBaseInfo *BaseInfo = nullptr,
4058	TBAAAccessInfo *TBAAInfo = nullptr);
4059
4060	class ConstantEmission {
4061	llvm::PointerIntPair<llvm::Constant*, 1, bool> ValueAndIsReference;
4062	ConstantEmission(llvm::Constant *C, bool isReference)
4063	: ValueAndIsReference(C, isReference) {}
4064	public:
4065	ConstantEmission() {}
4066	static ConstantEmission forReference(llvm::Constant *C) {
4067	return ConstantEmission(C, true);
4068	}
4069	static ConstantEmission forValue(llvm::Constant *C) {
4070	return ConstantEmission(C, false);
4071	}
4072
4073	explicit operator bool() const {
4074	return ValueAndIsReference.getOpaqueValue() != nullptr;
4075	}
4076
4077	bool isReference() const { return ValueAndIsReference.getInt(); }
4078	LValue getReferenceLValue(CodeGenFunction &CGF, Expr *refExpr) const {
4079	assert(isReference());
4080	return CGF.MakeNaturalAlignAddrLValue(V: ValueAndIsReference.getPointer(),
4081	T: refExpr->getType());
4082	}
4083
4084	llvm::Constant *getValue() const {
4085	assert(!isReference());
4086	return ValueAndIsReference.getPointer();
4087	}
4088	};
4089
4090	ConstantEmission tryEmitAsConstant(DeclRefExpr *refExpr);
4091	ConstantEmission tryEmitAsConstant(const MemberExpr *ME);
4092	llvm::Value *emitScalarConstant(const ConstantEmission &Constant, Expr *E);
4093
4094	RValue EmitPseudoObjectRValue(const PseudoObjectExpr *e,
4095	AggValueSlot slot = AggValueSlot::ignored());
4096	LValue EmitPseudoObjectLValue(const PseudoObjectExpr *e);
4097
4098	llvm::Value *EmitIvarOffset(const ObjCInterfaceDecl *Interface,
4099	const ObjCIvarDecl *Ivar);
4100	llvm::Value *EmitIvarOffsetAsPointerDiff(const ObjCInterfaceDecl *Interface,
4101	const ObjCIvarDecl *Ivar);
4102	LValue EmitLValueForField(LValue Base, const FieldDecl* Field);
4103	LValue EmitLValueForLambdaField(const FieldDecl *Field);
4104	LValue EmitLValueForLambdaField(const FieldDecl *Field,
4105	llvm::Value *ThisValue);
4106
4107	/// EmitLValueForFieldInitialization - Like EmitLValueForField, except that
4108	/// if the Field is a reference, this will return the address of the reference
4109	/// and not the address of the value stored in the reference.
4110	LValue EmitLValueForFieldInitialization(LValue Base,
4111	const FieldDecl* Field);
4112
4113	LValue EmitLValueForIvar(QualType ObjectTy,
4114	llvm::Value* Base, const ObjCIvarDecl *Ivar,
4115	unsigned CVRQualifiers);
4116
4117	LValue EmitCXXConstructLValue(const CXXConstructExpr *E);
4118	LValue EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E);
4119	LValue EmitCXXTypeidLValue(const CXXTypeidExpr *E);
4120	LValue EmitCXXUuidofLValue(const CXXUuidofExpr *E);
4121
4122	LValue EmitObjCMessageExprLValue(const ObjCMessageExpr *E);
4123	LValue EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E);
4124	LValue EmitStmtExprLValue(const StmtExpr *E);
4125	LValue EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E);
4126	LValue EmitObjCSelectorLValue(const ObjCSelectorExpr *E);
4127	void EmitDeclRefExprDbgValue(const DeclRefExpr *E, const APValue &Init);
4128
4129	//===--------------------------------------------------------------------===//
4130	// Scalar Expression Emission
4131	//===--------------------------------------------------------------------===//
4132
4133	/// EmitCall - Generate a call of the given function, expecting the given
4134	/// result type, and using the given argument list which specifies both the
4135	/// LLVM arguments and the types they were derived from.
4136	RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee,
4137	ReturnValueSlot ReturnValue, const CallArgList &Args,
4138	llvm::CallBase **callOrInvoke, bool IsMustTail,
4139	SourceLocation Loc);
4140	RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee,
4141	ReturnValueSlot ReturnValue, const CallArgList &Args,
4142	llvm::CallBase **callOrInvoke = nullptr,
4143	bool IsMustTail = false) {
4144	return EmitCall(CallInfo, Callee, ReturnValue, Args, callOrInvoke,
4145	IsMustTail, Loc: SourceLocation());
4146	}
4147	RValue EmitCall(QualType FnType, const CGCallee &Callee, const CallExpr *E,
4148	ReturnValueSlot ReturnValue, llvm::Value *Chain = nullptr);
4149	RValue EmitCallExpr(const CallExpr *E,
4150	ReturnValueSlot ReturnValue = ReturnValueSlot());
4151	RValue EmitSimpleCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue);
4152	CGCallee EmitCallee(const Expr *E);
4153
4154	void checkTargetFeatures(const CallExpr *E, const FunctionDecl *TargetDecl);
4155	void checkTargetFeatures(SourceLocation Loc, const FunctionDecl *TargetDecl);
4156
4157	llvm::CallInst *EmitRuntimeCall(llvm::FunctionCallee callee,
4158	const Twine &name = "");
4159	llvm::CallInst *EmitRuntimeCall(llvm::FunctionCallee callee,
4160	ArrayRef<llvm::Value *> args,
4161	const Twine &name = "");
4162	llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4163	const Twine &name = "");
4164	llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4165	ArrayRef<llvm::Value *> args,
4166	const Twine &name = "");
4167
4168	SmallVector<llvm::OperandBundleDef, 1>
4169	getBundlesForFunclet(llvm::Value *Callee);
4170
4171	llvm::CallBase *EmitCallOrInvoke(llvm::FunctionCallee Callee,
4172	ArrayRef<llvm::Value *> Args,
4173	const Twine &Name = "");
4174	llvm::CallBase *EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4175	ArrayRef<llvm::Value *> args,
4176	const Twine &name = "");
4177	llvm::CallBase *EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4178	const Twine &name = "");
4179	void EmitNoreturnRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4180	ArrayRef<llvm::Value *> args);
4181
4182	CGCallee BuildAppleKextVirtualCall(const CXXMethodDecl *MD,
4183	NestedNameSpecifier *Qual,
4184	llvm::Type *Ty);
4185
4186	CGCallee BuildAppleKextVirtualDestructorCall(const CXXDestructorDecl *DD,
4187	CXXDtorType Type,
4188	const CXXRecordDecl *RD);
4189
4190	// Return the copy constructor name with the prefix "__copy_constructor_"
4191	// removed.
4192	static std::string getNonTrivialCopyConstructorStr(QualType QT,
4193	CharUnits Alignment,
4194	bool IsVolatile,
4195	ASTContext &Ctx);
4196
4197	// Return the destructor name with the prefix "__destructor_" removed.
4198	static std::string getNonTrivialDestructorStr(QualType QT,
4199	CharUnits Alignment,
4200	bool IsVolatile,
4201	ASTContext &Ctx);
4202
4203	// These functions emit calls to the special functions of non-trivial C
4204	// structs.
4205	void defaultInitNonTrivialCStructVar(LValue Dst);
4206	void callCStructDefaultConstructor(LValue Dst);
4207	void callCStructDestructor(LValue Dst);
4208	void callCStructCopyConstructor(LValue Dst, LValue Src);
4209	void callCStructMoveConstructor(LValue Dst, LValue Src);
4210	void callCStructCopyAssignmentOperator(LValue Dst, LValue Src);
4211	void callCStructMoveAssignmentOperator(LValue Dst, LValue Src);
4212
4213	RValue
4214	EmitCXXMemberOrOperatorCall(const CXXMethodDecl *Method,
4215	const CGCallee &Callee,
4216	ReturnValueSlot ReturnValue, llvm::Value *This,
4217	llvm::Value *ImplicitParam,
4218	QualType ImplicitParamTy, const CallExpr *E,
4219	CallArgList *RtlArgs);
4220	RValue EmitCXXDestructorCall(GlobalDecl Dtor, const CGCallee &Callee,
4221	llvm::Value *This, QualType ThisTy,
4222	llvm::Value *ImplicitParam,
4223	QualType ImplicitParamTy, const CallExpr *E);
4224	RValue EmitCXXMemberCallExpr(const CXXMemberCallExpr *E,
4225	ReturnValueSlot ReturnValue);
4226	RValue EmitCXXMemberOrOperatorMemberCallExpr(const CallExpr *CE,
4227	const CXXMethodDecl *MD,
4228	ReturnValueSlot ReturnValue,
4229	bool HasQualifier,
4230	NestedNameSpecifier *Qualifier,
4231	bool IsArrow, const Expr *Base);
4232	// Compute the object pointer.
4233	Address EmitCXXMemberDataPointerAddress(const Expr *E, Address base,
4234	llvm::Value *memberPtr,
4235	const MemberPointerType *memberPtrType,
4236	LValueBaseInfo *BaseInfo = nullptr,
4237	TBAAAccessInfo *TBAAInfo = nullptr);
4238	RValue EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E,
4239	ReturnValueSlot ReturnValue);
4240
4241	RValue EmitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *E,
4242	const CXXMethodDecl *MD,
4243	ReturnValueSlot ReturnValue);
4244	RValue EmitCXXPseudoDestructorExpr(const CXXPseudoDestructorExpr *E);
4245
4246	RValue EmitCUDAKernelCallExpr(const CUDAKernelCallExpr *E,
4247	ReturnValueSlot ReturnValue);
4248
4249	RValue EmitNVPTXDevicePrintfCallExpr(const CallExpr *E);
4250	RValue EmitAMDGPUDevicePrintfCallExpr(const CallExpr *E);
4251	RValue EmitOpenMPDevicePrintfCallExpr(const CallExpr *E);
4252
4253	RValue EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
4254	const CallExpr *E, ReturnValueSlot ReturnValue);
4255
4256	RValue emitRotate(const CallExpr *E, bool IsRotateRight);
4257
4258	/// Emit IR for __builtin_os_log_format.
4259	RValue emitBuiltinOSLogFormat(const CallExpr &E);
4260
4261	/// Emit IR for __builtin_is_aligned.
4262	RValue EmitBuiltinIsAligned(const CallExpr *E);
4263	/// Emit IR for __builtin_align_up/__builtin_align_down.
4264	RValue EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp);
4265
4266	llvm::Function *generateBuiltinOSLogHelperFunction(
4267	const analyze_os_log::OSLogBufferLayout &Layout,
4268	CharUnits BufferAlignment);
4269
4270	RValue EmitBlockCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue);
4271
4272	/// EmitTargetBuiltinExpr - Emit the given builtin call. Returns 0 if the call
4273	/// is unhandled by the current target.
4274	llvm::Value *EmitTargetBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4275	ReturnValueSlot ReturnValue);
4276
4277	llvm::Value *EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty,
4278	const llvm::CmpInst::Predicate Fp,
4279	const llvm::CmpInst::Predicate Ip,
4280	const llvm::Twine &Name = "");
4281	llvm::Value *EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4282	ReturnValueSlot ReturnValue,
4283	llvm::Triple::ArchType Arch);
4284	llvm::Value *EmitARMMVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4285	ReturnValueSlot ReturnValue,
4286	llvm::Triple::ArchType Arch);
4287	llvm::Value *EmitARMCDEBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4288	ReturnValueSlot ReturnValue,
4289	llvm::Triple::ArchType Arch);
4290	llvm::Value *EmitCMSEClearRecord(llvm::Value *V, llvm::IntegerType *ITy,
4291	QualType RTy);
4292	llvm::Value *EmitCMSEClearRecord(llvm::Value *V, llvm::ArrayType *ATy,
4293	QualType RTy);
4294
4295	llvm::Value *EmitCommonNeonBuiltinExpr(unsigned BuiltinID,
4296	unsigned LLVMIntrinsic,
4297	unsigned AltLLVMIntrinsic,
4298	const char *NameHint,
4299	unsigned Modifier,
4300	const CallExpr *E,
4301	SmallVectorImpl<llvm::Value *> &Ops,
4302	Address PtrOp0, Address PtrOp1,
4303	llvm::Triple::ArchType Arch);
4304
4305	llvm::Function *LookupNeonLLVMIntrinsic(unsigned IntrinsicID,
4306	unsigned Modifier, llvm::Type *ArgTy,
4307	const CallExpr *E);
4308	llvm::Value *EmitNeonCall(llvm::Function *F,
4309	SmallVectorImpl<llvm::Value*> &O,
4310	const char *name,
4311	unsigned shift = 0, bool rightshift = false);
4312	llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx,
4313	const llvm::ElementCount &Count);
4314	llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx);
4315	llvm::Value *EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty,
4316	bool negateForRightShift);
4317	llvm::Value *EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt,
4318	llvm::Type *Ty, bool usgn, const char *name);
4319	llvm::Value *vectorWrapScalar16(llvm::Value *Op);
4320	/// SVEBuiltinMemEltTy - Returns the memory element type for this memory
4321	/// access builtin. Only required if it can't be inferred from the base
4322	/// pointer operand.
4323	llvm::Type *SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags);
4324
4325	SmallVector<llvm::Type *, 2>
4326	getSVEOverloadTypes(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType,
4327	ArrayRef<llvm::Value *> Ops);
4328	llvm::Type *getEltType(const SVETypeFlags &TypeFlags);
4329	llvm::ScalableVectorType *getSVEType(const SVETypeFlags &TypeFlags);
4330	llvm::ScalableVectorType *getSVEPredType(const SVETypeFlags &TypeFlags);
4331	llvm::Value *EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags,
4332	llvm::Type *ReturnType,
4333	ArrayRef<llvm::Value *> Ops);
4334	llvm::Value *EmitSVETupleCreate(const SVETypeFlags &TypeFlags,
4335	llvm::Type *ReturnType,
4336	ArrayRef<llvm::Value *> Ops);
4337	llvm::Value *EmitSVEAllTruePred(const SVETypeFlags &TypeFlags);
4338	llvm::Value *EmitSVEDupX(llvm::Value *Scalar);
4339	llvm::Value *EmitSVEDupX(llvm::Value *Scalar, llvm::Type *Ty);
4340	llvm::Value *EmitSVEReinterpret(llvm::Value *Val, llvm::Type *Ty);
4341	llvm::Value *EmitSVEPMull(const SVETypeFlags &TypeFlags,
4342	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4343	unsigned BuiltinID);
4344	llvm::Value *EmitSVEMovl(const SVETypeFlags &TypeFlags,
4345	llvm::ArrayRef<llvm::Value *> Ops,
4346	unsigned BuiltinID);
4347	llvm::Value *EmitSVEPredicateCast(llvm::Value *Pred,
4348	llvm::ScalableVectorType *VTy);
4349	llvm::Value *EmitSVEGatherLoad(const SVETypeFlags &TypeFlags,
4350	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4351	unsigned IntID);
4352	llvm::Value *EmitSVEScatterStore(const SVETypeFlags &TypeFlags,
4353	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4354	unsigned IntID);
4355	llvm::Value *EmitSVEMaskedLoad(const CallExpr *, llvm::Type *ReturnTy,
4356	SmallVectorImpl<llvm::Value *> &Ops,
4357	unsigned BuiltinID, bool IsZExtReturn);
4358	llvm::Value *EmitSVEMaskedStore(const CallExpr *,
4359	SmallVectorImpl<llvm::Value *> &Ops,
4360	unsigned BuiltinID);
4361	llvm::Value *EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags,
4362	SmallVectorImpl<llvm::Value *> &Ops,
4363	unsigned BuiltinID);
4364	llvm::Value *EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags,
4365	SmallVectorImpl<llvm::Value *> &Ops,
4366	unsigned IntID);
4367	llvm::Value *EmitSVEStructLoad(const SVETypeFlags &TypeFlags,
4368	SmallVectorImpl<llvm::Value *> &Ops,
4369	unsigned IntID);
4370	llvm::Value *EmitSVEStructStore(const SVETypeFlags &TypeFlags,
4371	SmallVectorImpl<llvm::Value *> &Ops,
4372	unsigned IntID);
4373	/// FormSVEBuiltinResult - Returns the struct of scalable vectors as a wider
4374	/// vector. It extracts the scalable vector from the struct and inserts into
4375	/// the wider vector. This avoids the error when allocating space in llvm
4376	/// for struct of scalable vectors if a function returns struct.
4377	llvm::Value *FormSVEBuiltinResult(llvm::Value *Call);
4378
4379	llvm::Value *EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4380
4381	llvm::Value *EmitSMELd1St1(const SVETypeFlags &TypeFlags,
4382	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4383	unsigned IntID);
4384	llvm::Value *EmitSMEReadWrite(const SVETypeFlags &TypeFlags,
4385	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4386	unsigned IntID);
4387	llvm::Value *EmitSMEZero(const SVETypeFlags &TypeFlags,
4388	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4389	unsigned IntID);
4390	llvm::Value *EmitSMELdrStr(const SVETypeFlags &TypeFlags,
4391	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4392	unsigned IntID);
4393
4394	void GetAArch64SVEProcessedOperands(unsigned BuiltinID, const CallExpr *E,
4395	SmallVectorImpl<llvm::Value *> &Ops,
4396	SVETypeFlags TypeFlags);
4397
4398	llvm::Value *EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4399
4400	llvm::Value *EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4401	llvm::Triple::ArchType Arch);
4402	llvm::Value *EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4403
4404	llvm::Value *BuildVector(ArrayRef<llvm::Value*> Ops);
4405	llvm::Value *EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4406	llvm::Value *EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4407	llvm::Value *EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4408	llvm::Value *EmitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned Idx,
4409	const CallExpr *E);
4410	llvm::Value *EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4411	llvm::Value *EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4412	llvm::Value *EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
4413	const CallExpr *E);
4414	llvm::Value *EmitHexagonBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4415	llvm::Value *EmitRISCVBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4416	ReturnValueSlot ReturnValue);
4417	void ProcessOrderScopeAMDGCN(llvm::Value *Order, llvm::Value *Scope,
4418	llvm::AtomicOrdering &AO,
4419	llvm::SyncScope::ID &SSID);
4420
4421	enum class MSVCIntrin;
4422	llvm::Value *EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E);
4423
4424	llvm::Value *EmitBuiltinAvailable(const VersionTuple &Version);
4425
4426	llvm::Value *EmitObjCProtocolExpr(const ObjCProtocolExpr *E);
4427	llvm::Value *EmitObjCStringLiteral(const ObjCStringLiteral *E);
4428	llvm::Value *EmitObjCBoxedExpr(const ObjCBoxedExpr *E);
4429	llvm::Value *EmitObjCArrayLiteral(const ObjCArrayLiteral *E);
4430	llvm::Value *EmitObjCDictionaryLiteral(const ObjCDictionaryLiteral *E);
4431	llvm::Value *EmitObjCCollectionLiteral(const Expr *E,
4432	const ObjCMethodDecl *MethodWithObjects);
4433	llvm::Value *EmitObjCSelectorExpr(const ObjCSelectorExpr *E);
4434	RValue EmitObjCMessageExpr(const ObjCMessageExpr *E,
4435	ReturnValueSlot Return = ReturnValueSlot());
4436
4437	/// Retrieves the default cleanup kind for an ARC cleanup.
4438	/// Except under -fobjc-arc-eh, ARC cleanups are normal-only.
4439	CleanupKind getARCCleanupKind() {
4440	return CGM.getCodeGenOpts().ObjCAutoRefCountExceptions
4441	? NormalAndEHCleanup : NormalCleanup;
4442	}
4443
4444	// ARC primitives.
4445	void EmitARCInitWeak(Address addr, llvm::Value *value);
4446	void EmitARCDestroyWeak(Address addr);
4447	llvm::Value *EmitARCLoadWeak(Address addr);
4448	llvm::Value *EmitARCLoadWeakRetained(Address addr);
4449	llvm::Value *EmitARCStoreWeak(Address addr, llvm::Value *value, bool ignored);
4450	void emitARCCopyAssignWeak(QualType Ty, Address DstAddr, Address SrcAddr);
4451	void emitARCMoveAssignWeak(QualType Ty, Address DstAddr, Address SrcAddr);
4452	void EmitARCCopyWeak(Address dst, Address src);
4453	void EmitARCMoveWeak(Address dst, Address src);
4454	llvm::Value *EmitARCRetainAutorelease(QualType type, llvm::Value *value);
4455	llvm::Value *EmitARCRetainAutoreleaseNonBlock(llvm::Value *value);
4456	llvm::Value *EmitARCStoreStrong(LValue lvalue, llvm::Value *value,
4457	bool resultIgnored);
4458	llvm::Value *EmitARCStoreStrongCall(Address addr, llvm::Value *value,
4459	bool resultIgnored);
4460	llvm::Value *EmitARCRetain(QualType type, llvm::Value *value);
4461	llvm::Value *EmitARCRetainNonBlock(llvm::Value *value);
4462	llvm::Value *EmitARCRetainBlock(llvm::Value *value, bool mandatory);
4463	void EmitARCDestroyStrong(Address addr, ARCPreciseLifetime_t precise);
4464	void EmitARCRelease(llvm::Value *value, ARCPreciseLifetime_t precise);
4465	llvm::Value *EmitARCAutorelease(llvm::Value *value);
4466	llvm::Value *EmitARCAutoreleaseReturnValue(llvm::Value *value);
4467	llvm::Value *EmitARCRetainAutoreleaseReturnValue(llvm::Value *value);
4468	llvm::Value *EmitARCRetainAutoreleasedReturnValue(llvm::Value *value);
4469	llvm::Value *EmitARCUnsafeClaimAutoreleasedReturnValue(llvm::Value *value);
4470
4471	llvm::Value *EmitObjCAutorelease(llvm::Value *value, llvm::Type *returnType);
4472	llvm::Value *EmitObjCRetainNonBlock(llvm::Value *value,
4473	llvm::Type *returnType);
4474	void EmitObjCRelease(llvm::Value *value, ARCPreciseLifetime_t precise);
4475
4476	std::pair<LValue,llvm::Value*>
4477	EmitARCStoreAutoreleasing(const BinaryOperator *e);
4478	std::pair<LValue,llvm::Value*>
4479	EmitARCStoreStrong(const BinaryOperator *e, bool ignored);
4480	std::pair<LValue,llvm::Value*>
4481	EmitARCStoreUnsafeUnretained(const BinaryOperator *e, bool ignored);
4482
4483	llvm::Value *EmitObjCAlloc(llvm::Value *value,
4484	llvm::Type *returnType);
4485	llvm::Value *EmitObjCAllocWithZone(llvm::Value *value,
4486	llvm::Type *returnType);
4487	llvm::Value *EmitObjCAllocInit(llvm::Value *value, llvm::Type *resultType);
4488
4489	llvm::Value *EmitObjCThrowOperand(const Expr *expr);
4490	llvm::Value *EmitObjCConsumeObject(QualType T, llvm::Value *Ptr);
4491	llvm::Value *EmitObjCExtendObjectLifetime(QualType T, llvm::Value *Ptr);
4492
4493	llvm::Value *EmitARCExtendBlockObject(const Expr *expr);
4494	llvm::Value *EmitARCReclaimReturnedObject(const Expr *e,
4495	bool allowUnsafeClaim);
4496	llvm::Value *EmitARCRetainScalarExpr(const Expr *expr);
4497	llvm::Value *EmitARCRetainAutoreleaseScalarExpr(const Expr *expr);
4498	llvm::Value *EmitARCUnsafeUnretainedScalarExpr(const Expr *expr);
4499
4500	void EmitARCIntrinsicUse(ArrayRef<llvm::Value*> values);
4501
4502	void EmitARCNoopIntrinsicUse(ArrayRef<llvm::Value *> values);
4503
4504	static Destroyer destroyARCStrongImprecise;
4505	static Destroyer destroyARCStrongPrecise;
4506	static Destroyer destroyARCWeak;
4507	static Destroyer emitARCIntrinsicUse;
4508	static Destroyer destroyNonTrivialCStruct;
4509
4510	void EmitObjCAutoreleasePoolPop(llvm::Value *Ptr);
4511	llvm::Value *EmitObjCAutoreleasePoolPush();
4512	llvm::Value *EmitObjCMRRAutoreleasePoolPush();
4513	void EmitObjCAutoreleasePoolCleanup(llvm::Value *Ptr);
4514	void EmitObjCMRRAutoreleasePoolPop(llvm::Value *Ptr);
4515
4516	/// Emits a reference binding to the passed in expression.
4517	RValue EmitReferenceBindingToExpr(const Expr *E);
4518
4519	//===--------------------------------------------------------------------===//
4520	// Expression Emission
4521	//===--------------------------------------------------------------------===//
4522
4523	// Expressions are broken into three classes: scalar, complex, aggregate.
4524
4525	/// EmitScalarExpr - Emit the computation of the specified expression of LLVM
4526	/// scalar type, returning the result.
4527	llvm::Value *EmitScalarExpr(const Expr *E , bool IgnoreResultAssign = false);
4528
4529	/// Emit a conversion from the specified type to the specified destination
4530	/// type, both of which are LLVM scalar types.
4531	llvm::Value *EmitScalarConversion(llvm::Value *Src, QualType SrcTy,
4532	QualType DstTy, SourceLocation Loc);
4533
4534	/// Emit a conversion from the specified complex type to the specified
4535	/// destination type, where the destination type is an LLVM scalar type.
4536	llvm::Value *EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy,
4537	QualType DstTy,
4538	SourceLocation Loc);
4539
4540	/// EmitAggExpr - Emit the computation of the specified expression
4541	/// of aggregate type. The result is computed into the given slot,
4542	/// which may be null to indicate that the value is not needed.
4543	void EmitAggExpr(const Expr *E, AggValueSlot AS);
4544
4545	/// EmitAggExprToLValue - Emit the computation of the specified expression of
4546	/// aggregate type into a temporary LValue.
4547	LValue EmitAggExprToLValue(const Expr *E);
4548
4549	/// Build all the stores needed to initialize an aggregate at Dest with the
4550	/// value Val.
4551	void EmitAggregateStore(llvm::Value *Val, Address Dest, bool DestIsVolatile);
4552
4553	/// EmitExtendGCLifetime - Given a pointer to an Objective-C object,
4554	/// make sure it survives garbage collection until this point.
4555	void EmitExtendGCLifetime(llvm::Value *object);
4556
4557	/// EmitComplexExpr - Emit the computation of the specified expression of
4558	/// complex type, returning the result.
4559	ComplexPairTy EmitComplexExpr(const Expr *E,
4560	bool IgnoreReal = false,
4561	bool IgnoreImag = false);
4562
4563	/// EmitComplexExprIntoLValue - Emit the given expression of complex
4564	/// type and place its result into the specified l-value.
4565	void EmitComplexExprIntoLValue(const Expr *E, LValue dest, bool isInit);
4566
4567	/// EmitStoreOfComplex - Store a complex number into the specified l-value.
4568	void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit);
4569
4570	/// EmitLoadOfComplex - Load a complex number from the specified l-value.
4571	ComplexPairTy EmitLoadOfComplex(LValue src, SourceLocation loc);
4572
4573	ComplexPairTy EmitPromotedComplexExpr(const Expr *E, QualType PromotionType);
4574	llvm::Value *EmitPromotedScalarExpr(const Expr *E, QualType PromotionType);
4575	ComplexPairTy EmitPromotedValue(ComplexPairTy result, QualType PromotionType);
4576	ComplexPairTy EmitUnPromotedValue(ComplexPairTy result, QualType PromotionType);
4577
4578	Address emitAddrOfRealComponent(Address complex, QualType complexType);
4579	Address emitAddrOfImagComponent(Address complex, QualType complexType);
4580
4581	/// AddInitializerToStaticVarDecl - Add the initializer for 'D' to the
4582	/// global variable that has already been created for it. If the initializer
4583	/// has a different type than GV does, this may free GV and return a different
4584	/// one. Otherwise it just returns GV.
4585	llvm::GlobalVariable *
4586	AddInitializerToStaticVarDecl(const VarDecl &D,
4587	llvm::GlobalVariable *GV);
4588
4589	// Emit an @llvm.invariant.start call for the given memory region.
4590	void EmitInvariantStart(llvm::Constant *Addr, CharUnits Size);
4591
4592	/// EmitCXXGlobalVarDeclInit - Create the initializer for a C++
4593	/// variable with global storage.
4594	void EmitCXXGlobalVarDeclInit(const VarDecl &D, llvm::GlobalVariable *GV,
4595	bool PerformInit);
4596
4597	llvm::Function *createAtExitStub(const VarDecl &VD, llvm::FunctionCallee Dtor,
4598	llvm::Constant *Addr);
4599
4600	llvm::Function *createTLSAtExitStub(const VarDecl &VD,
4601	llvm::FunctionCallee Dtor,
4602	llvm::Constant *Addr,
4603	llvm::FunctionCallee &AtExit);
4604
4605	/// Call atexit() with a function that passes the given argument to
4606	/// the given function.
4607	void registerGlobalDtorWithAtExit(const VarDecl &D, llvm::FunctionCallee fn,
4608	llvm::Constant *addr);
4609
4610	/// Registers the dtor using 'llvm.global_dtors' for platforms that do not
4611	/// support an 'atexit()' function.
4612	void registerGlobalDtorWithLLVM(const VarDecl &D, llvm::FunctionCallee fn,
4613	llvm::Constant *addr);
4614
4615	/// Call atexit() with function dtorStub.
4616	void registerGlobalDtorWithAtExit(llvm::Constant *dtorStub);
4617
4618	/// Call unatexit() with function dtorStub.
4619	llvm::Value *unregisterGlobalDtorWithUnAtExit(llvm::Constant *dtorStub);
4620
4621	/// Emit code in this function to perform a guarded variable
4622	/// initialization. Guarded initializations are used when it's not
4623	/// possible to prove that an initialization will be done exactly
4624	/// once, e.g. with a static local variable or a static data member
4625	/// of a class template.
4626	void EmitCXXGuardedInit(const VarDecl &D, llvm::GlobalVariable *DeclPtr,
4627	bool PerformInit);
4628
4629	enum class GuardKind { VariableGuard, TlsGuard };
4630
4631	/// Emit a branch to select whether or not to perform guarded initialization.
4632	void EmitCXXGuardedInitBranch(llvm::Value *NeedsInit,
4633	llvm::BasicBlock *InitBlock,
4634	llvm::BasicBlock *NoInitBlock,
4635	GuardKind Kind, const VarDecl *D);
4636
4637	/// GenerateCXXGlobalInitFunc - Generates code for initializing global
4638	/// variables.
4639	void
4640	GenerateCXXGlobalInitFunc(llvm::Function *Fn,
4641	ArrayRef<llvm::Function *> CXXThreadLocals,
4642	ConstantAddress Guard = ConstantAddress::invalid());
4643
4644	/// GenerateCXXGlobalCleanUpFunc - Generates code for cleaning up global
4645	/// variables.
4646	void GenerateCXXGlobalCleanUpFunc(
4647	llvm::Function *Fn,
4648	ArrayRef<std::tuple<llvm::FunctionType *, llvm::WeakTrackingVH,
4649	llvm::Constant *>>
4650	DtorsOrStermFinalizers);
4651
4652	void GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn,
4653	const VarDecl *D,
4654	llvm::GlobalVariable *Addr,
4655	bool PerformInit);
4656
4657	void EmitCXXConstructExpr(const CXXConstructExpr *E, AggValueSlot Dest);
4658
4659	void EmitSynthesizedCXXCopyCtor(Address Dest, Address Src, const Expr *Exp);
4660
4661	void EmitCXXThrowExpr(const CXXThrowExpr *E, bool KeepInsertionPoint = true);
4662
4663	RValue EmitAtomicExpr(AtomicExpr *E);
4664
4665	//===--------------------------------------------------------------------===//
4666	// Annotations Emission
4667	//===--------------------------------------------------------------------===//
4668
4669	/// Emit an annotation call (intrinsic).
4670	llvm::Value *EmitAnnotationCall(llvm::Function *AnnotationFn,
4671	llvm::Value *AnnotatedVal,
4672	StringRef AnnotationStr,
4673	SourceLocation Location,
4674	const AnnotateAttr *Attr);
4675
4676	/// Emit local annotations for the local variable V, declared by D.
4677	void EmitVarAnnotations(const VarDecl *D, llvm::Value *V);
4678
4679	/// Emit field annotations for the given field & value. Returns the
4680	/// annotation result.
4681	Address EmitFieldAnnotations(const FieldDecl *D, Address V);
4682
4683	//===--------------------------------------------------------------------===//
4684	// Internal Helpers
4685	//===--------------------------------------------------------------------===//
4686
4687	/// ContainsLabel - Return true if the statement contains a label in it. If
4688	/// this statement is not executed normally, it not containing a label means
4689	/// that we can just remove the code.
4690	static bool ContainsLabel(const Stmt *S, bool IgnoreCaseStmts = false);
4691
4692	/// containsBreak - Return true if the statement contains a break out of it.
4693	/// If the statement (recursively) contains a switch or loop with a break
4694	/// inside of it, this is fine.
4695	static bool containsBreak(const Stmt *S);
4696
4697	/// Determine if the given statement might introduce a declaration into the
4698	/// current scope, by being a (possibly-labelled) DeclStmt.
4699	static bool mightAddDeclToScope(const Stmt *S);
4700
4701	/// ConstantFoldsToSimpleInteger - If the specified expression does not fold
4702	/// to a constant, or if it does but contains a label, return false. If it
4703	/// constant folds return true and set the boolean result in Result.
4704	bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result,
4705	bool AllowLabels = false);
4706
4707	/// ConstantFoldsToSimpleInteger - If the specified expression does not fold
4708	/// to a constant, or if it does but contains a label, return false. If it
4709	/// constant folds return true and set the folded value.
4710	bool ConstantFoldsToSimpleInteger(const Expr *Cond, llvm::APSInt &Result,
4711	bool AllowLabels = false);
4712
4713	/// Ignore parentheses and logical-NOT to track conditions consistently.
4714	static const Expr *stripCond(const Expr *C);
4715
4716	/// isInstrumentedCondition - Determine whether the given condition is an
4717	/// instrumentable condition (i.e. no "&&" or "||").
4718	static bool isInstrumentedCondition(const Expr *C);
4719
4720	/// EmitBranchToCounterBlock - Emit a conditional branch to a new block that
4721	/// increments a profile counter based on the semantics of the given logical
4722	/// operator opcode. This is used to instrument branch condition coverage
4723	/// for logical operators.
4724	void EmitBranchToCounterBlock(const Expr *Cond, BinaryOperator::Opcode LOp,
4725	llvm::BasicBlock *TrueBlock,
4726	llvm::BasicBlock *FalseBlock,
4727	uint64_t TrueCount = 0,
4728	Stmt::Likelihood LH = Stmt::LH_None,
4729	const Expr *CntrIdx = nullptr);
4730
4731	/// EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g. for an
4732	/// if statement) to the specified blocks. Based on the condition, this might
4733	/// try to simplify the codegen of the conditional based on the branch.
4734	/// TrueCount should be the number of times we expect the condition to
4735	/// evaluate to true based on PGO data.
4736	void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock,
4737	llvm::BasicBlock *FalseBlock, uint64_t TrueCount,
4738	Stmt::Likelihood LH = Stmt::LH_None,
4739	const Expr *ConditionalOp = nullptr);
4740
4741	/// Given an assignment `*LHS = RHS`, emit a test that checks if \p RHS is
4742	/// nonnull, if \p LHS is marked _Nonnull.
4743	void EmitNullabilityCheck(LValue LHS, llvm::Value *RHS, SourceLocation Loc);
4744
4745	/// An enumeration which makes it easier to specify whether or not an
4746	/// operation is a subtraction.
4747	enum { NotSubtraction = false, IsSubtraction = true };
4748
4749	/// Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to
4750	/// detect undefined behavior when the pointer overflow sanitizer is enabled.
4751	/// \p SignedIndices indicates whether any of the GEP indices are signed.
4752	/// \p IsSubtraction indicates whether the expression used to form the GEP
4753	/// is a subtraction.
4754	llvm::Value *EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr,
4755	ArrayRef<llvm::Value *> IdxList,
4756	bool SignedIndices,
4757	bool IsSubtraction,
4758	SourceLocation Loc,
4759	const Twine &Name = "");
4760
4761	/// Specifies which type of sanitizer check to apply when handling a
4762	/// particular builtin.
4763	enum BuiltinCheckKind {
4764	BCK_CTZPassedZero,
4765	BCK_CLZPassedZero,
4766	};
4767
4768	/// Emits an argument for a call to a builtin. If the builtin sanitizer is
4769	/// enabled, a runtime check specified by \p Kind is also emitted.
4770	llvm::Value *EmitCheckedArgForBuiltin(const Expr *E, BuiltinCheckKind Kind);
4771
4772	/// Emit a description of a type in a format suitable for passing to
4773	/// a runtime sanitizer handler.
4774	llvm::Constant *EmitCheckTypeDescriptor(QualType T);
4775
4776	/// Convert a value into a format suitable for passing to a runtime
4777	/// sanitizer handler.
4778	llvm::Value *EmitCheckValue(llvm::Value *V);
4779
4780	/// Emit a description of a source location in a format suitable for
4781	/// passing to a runtime sanitizer handler.
4782	llvm::Constant *EmitCheckSourceLocation(SourceLocation Loc);
4783
4784	void EmitKCFIOperandBundle(const CGCallee &Callee,
4785	SmallVectorImpl<llvm::OperandBundleDef> &Bundles);
4786
4787	/// Create a basic block that will either trap or call a handler function in
4788	/// the UBSan runtime with the provided arguments, and create a conditional
4789	/// branch to it.
4790	void EmitCheck(ArrayRef<std::pair<llvm::Value *, SanitizerMask>> Checked,
4791	SanitizerHandler Check, ArrayRef<llvm::Constant *> StaticArgs,
4792	ArrayRef<llvm::Value *> DynamicArgs);
4793
4794	/// Emit a slow path cross-DSO CFI check which calls __cfi_slowpath
4795	/// if Cond if false.
4796	void EmitCfiSlowPathCheck(SanitizerMask Kind, llvm::Value *Cond,
4797	llvm::ConstantInt *TypeId, llvm::Value *Ptr,
4798	ArrayRef<llvm::Constant *> StaticArgs);
4799
4800	/// Emit a reached-unreachable diagnostic if \p Loc is valid and runtime
4801	/// checking is enabled. Otherwise, just emit an unreachable instruction.
4802	void EmitUnreachable(SourceLocation Loc);
4803
4804	/// Create a basic block that will call the trap intrinsic, and emit a
4805	/// conditional branch to it, for the -ftrapv checks.
4806	void EmitTrapCheck(llvm::Value *Checked, SanitizerHandler CheckHandlerID);
4807
4808	/// Emit a call to trap or debugtrap and attach function attribute
4809	/// "trap-func-name" if specified.
4810	llvm::CallInst *EmitTrapCall(llvm::Intrinsic::ID IntrID);
4811
4812	/// Emit a stub for the cross-DSO CFI check function.
4813	void EmitCfiCheckStub();
4814
4815	/// Emit a cross-DSO CFI failure handling function.
4816	void EmitCfiCheckFail();
4817
4818	/// Create a check for a function parameter that may potentially be
4819	/// declared as non-null.
4820	void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc,
4821	AbstractCallee AC, unsigned ParmNum);
4822
4823	/// EmitCallArg - Emit a single call argument.
4824	void EmitCallArg(CallArgList &args, const Expr *E, QualType ArgType);
4825
4826	/// EmitDelegateCallArg - We are performing a delegate call; that
4827	/// is, the current function is delegating to another one. Produce
4828	/// a r-value suitable for passing the given parameter.
4829	void EmitDelegateCallArg(CallArgList &args, const VarDecl *param,
4830	SourceLocation loc);
4831
4832	/// SetFPAccuracy - Set the minimum required accuracy of the given floating
4833	/// point operation, expressed as the maximum relative error in ulp.
4834	void SetFPAccuracy(llvm::Value *Val, float Accuracy);
4835
4836	/// Set the minimum required accuracy of the given sqrt operation
4837	/// based on CodeGenOpts.
4838	void SetSqrtFPAccuracy(llvm::Value *Val);
4839
4840	/// Set the minimum required accuracy of the given sqrt operation based on
4841	/// CodeGenOpts.
4842	void SetDivFPAccuracy(llvm::Value *Val);
4843
4844	/// Set the codegen fast-math flags.
4845	void SetFastMathFlags(FPOptions FPFeatures);
4846
4847	// Truncate or extend a boolean vector to the requested number of elements.
4848	llvm::Value *emitBoolVecConversion(llvm::Value *SrcVec,
4849	unsigned NumElementsDst,
4850	const llvm::Twine &Name = "");
4851
4852	private:
4853	llvm::MDNode *getRangeForLoadFromType(QualType Ty);
4854	void EmitReturnOfRValue(RValue RV, QualType Ty);
4855
4856	void deferPlaceholderReplacement(llvm::Instruction *Old, llvm::Value *New);
4857
4858	llvm::SmallVector<std::pair<llvm::WeakTrackingVH, llvm::Value *>, 4>
4859	DeferredReplacements;
4860
4861	/// Set the address of a local variable.
4862	void setAddrOfLocalVar(const VarDecl *VD, Address Addr) {
4863	assert(!LocalDeclMap.count(VD) && "Decl already exists in LocalDeclMap!");
4864	LocalDeclMap.insert({VD, Addr});
4865	}
4866
4867	/// ExpandTypeFromArgs - Reconstruct a structure of type \arg Ty
4868	/// from function arguments into \arg Dst. See ABIArgInfo::Expand.
4869	///
4870	/// \param AI - The first function argument of the expansion.
4871	void ExpandTypeFromArgs(QualType Ty, LValue Dst,
4872	llvm::Function::arg_iterator &AI);
4873
4874	/// ExpandTypeToArgs - Expand an CallArg \arg Arg, with the LLVM type for \arg
4875	/// Ty, into individual arguments on the provided vector \arg IRCallArgs,
4876	/// starting at index \arg IRCallArgPos. See ABIArgInfo::Expand.
4877	void ExpandTypeToArgs(QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
4878	SmallVectorImpl<llvm::Value *> &IRCallArgs,
4879	unsigned &IRCallArgPos);
4880
4881	std::pair<llvm::Value *, llvm::Type *>
4882	EmitAsmInput(const TargetInfo::ConstraintInfo &Info, const Expr *InputExpr,
4883	std::string &ConstraintStr);
4884
4885	std::pair<llvm::Value *, llvm::Type *>
4886	EmitAsmInputLValue(const TargetInfo::ConstraintInfo &Info, LValue InputValue,
4887	QualType InputType, std::string &ConstraintStr,
4888	SourceLocation Loc);
4889
4890	/// Attempts to statically evaluate the object size of E. If that
4891	/// fails, emits code to figure the size of E out for us. This is
4892	/// pass_object_size aware.
4893	///
4894	/// If EmittedExpr is non-null, this will use that instead of re-emitting E.
4895	llvm::Value *evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
4896	llvm::IntegerType *ResType,
4897	llvm::Value *EmittedE,
4898	bool IsDynamic);
4899
4900	/// Emits the size of E, as required by __builtin_object_size. This
4901	/// function is aware of pass_object_size parameters, and will act accordingly
4902	/// if E is a parameter with the pass_object_size attribute.
4903	llvm::Value *emitBuiltinObjectSize(const Expr *E, unsigned Type,
4904	llvm::IntegerType *ResType,
4905	llvm::Value *EmittedE,
4906	bool IsDynamic);
4907
4908	llvm::Value *emitFlexibleArrayMemberSize(const Expr *E, unsigned Type,
4909	llvm::IntegerType *ResType);
4910
4911	void emitZeroOrPatternForAutoVarInit(QualType type, const VarDecl &D,
4912	Address Loc);
4913
4914	public:
4915	enum class EvaluationOrder {
4916	///! No language constraints on evaluation order.
4917	Default,
4918	///! Language semantics require left-to-right evaluation.
4919	ForceLeftToRight,
4920	///! Language semantics require right-to-left evaluation.
4921	ForceRightToLeft
4922	};
4923
4924	// Wrapper for function prototype sources. Wraps either a FunctionProtoType or
4925	// an ObjCMethodDecl.
4926	struct PrototypeWrapper {
4927	llvm::PointerUnion<const FunctionProtoType *, const ObjCMethodDecl *> P;
4928
4929	PrototypeWrapper(const FunctionProtoType *FT) : P(FT) {}
4930	PrototypeWrapper(const ObjCMethodDecl *MD) : P(MD) {}
4931	};
4932
4933	void EmitCallArgs(CallArgList &Args, PrototypeWrapper Prototype,
4934	llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
4935	AbstractCallee AC = AbstractCallee(),
4936	unsigned ParamsToSkip = 0,
4937	EvaluationOrder Order = EvaluationOrder::Default);
4938
4939	/// EmitPointerWithAlignment - Given an expression with a pointer type,
4940	/// emit the value and compute our best estimate of the alignment of the
4941	/// pointee.
4942	///
4943	/// \param BaseInfo - If non-null, this will be initialized with
4944	/// information about the source of the alignment and the may-alias
4945	/// attribute. Note that this function will conservatively fall back on
4946	/// the type when it doesn't recognize the expression and may-alias will
4947	/// be set to false.
4948	///
4949	/// One reasonable way to use this information is when there's a language
4950	/// guarantee that the pointer must be aligned to some stricter value, and
4951	/// we're simply trying to ensure that sufficiently obvious uses of under-
4952	/// aligned objects don't get miscompiled; for example, a placement new
4953	/// into the address of a local variable. In such a case, it's quite
4954	/// reasonable to just ignore the returned alignment when it isn't from an
4955	/// explicit source.
4956	Address
4957	EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo = nullptr,
4958	TBAAAccessInfo *TBAAInfo = nullptr,
4959	KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
4960
4961	/// If \p E references a parameter with pass_object_size info or a constant
4962	/// array size modifier, emit the object size divided by the size of \p EltTy.
4963	/// Otherwise return null.
4964	llvm::Value *LoadPassedObjectSize(const Expr *E, QualType EltTy);
4965
4966	void EmitSanitizerStatReport(llvm::SanitizerStatKind SSK);
4967
4968	struct MultiVersionResolverOption {
4969	llvm::Function *Function;
4970	struct Conds {
4971	StringRef Architecture;
4972	llvm::SmallVector<StringRef, 8> Features;
4973
4974	Conds(StringRef Arch, ArrayRef<StringRef> Feats)
4975	: Architecture(Arch), Features(Feats.begin(), Feats.end()) {}
4976	} Conditions;
4977
4978	MultiVersionResolverOption(llvm::Function *F, StringRef Arch,
4979	ArrayRef<StringRef> Feats)
4980	: Function(F), Conditions(Arch, Feats) {}
4981	};
4982
4983	// Emits the body of a multiversion function's resolver. Assumes that the
4984	// options are already sorted in the proper order, with the 'default' option
4985	// last (if it exists).
4986	void EmitMultiVersionResolver(llvm::Function *Resolver,
4987	ArrayRef<MultiVersionResolverOption> Options);
4988	void
4989	EmitX86MultiVersionResolver(llvm::Function *Resolver,
4990	ArrayRef<MultiVersionResolverOption> Options);
4991	void
4992	EmitAArch64MultiVersionResolver(llvm::Function *Resolver,
4993	ArrayRef<MultiVersionResolverOption> Options);
4994
4995	private:
4996	QualType getVarArgType(const Expr *Arg);
4997
4998	void EmitDeclMetadata();
4999
5000	BlockByrefHelpers *buildByrefHelpers(llvm::StructType &byrefType,
5001	const AutoVarEmission &emission);
5002
5003	void AddObjCARCExceptionMetadata(llvm::Instruction *Inst);
5004
5005	llvm::Value *GetValueForARMHint(unsigned BuiltinID);
5006	llvm::Value *EmitX86CpuIs(const CallExpr *E);
5007	llvm::Value *EmitX86CpuIs(StringRef CPUStr);
5008	llvm::Value *EmitX86CpuSupports(const CallExpr *E);
5009	llvm::Value *EmitX86CpuSupports(ArrayRef<StringRef> FeatureStrs);
5010	llvm::Value *EmitX86CpuSupports(std::array<uint32_t, 4> FeatureMask);
5011	llvm::Value *EmitX86CpuInit();
5012	llvm::Value *FormX86ResolverCondition(const MultiVersionResolverOption &RO);
5013	llvm::Value *EmitAArch64CpuInit();
5014	llvm::Value *
5015	FormAArch64ResolverCondition(const MultiVersionResolverOption &RO);
5016	llvm::Value *EmitAArch64CpuSupports(ArrayRef<StringRef> FeatureStrs);
5017	};
5018
5019
5020	inline DominatingLLVMValue::saved_type
5021	DominatingLLVMValue::save(CodeGenFunction &CGF, llvm::Value *value) {
5022	if (!needsSaving(value)) return saved_type(value, false);
5023
5024	// Otherwise, we need an alloca.
5025	auto align = CharUnits::fromQuantity(
5026	Quantity: CGF.CGM.getDataLayout().getPrefTypeAlign(Ty: value->getType()));
5027	Address alloca =
5028	CGF.CreateTempAlloca(Ty: value->getType(), align, Name: "cond-cleanup.save");
5029	CGF.Builder.CreateStore(Val: value, Addr: alloca);
5030
5031	return saved_type(alloca.getPointer(), true);
5032	}
5033
5034	inline llvm::Value *DominatingLLVMValue::restore(CodeGenFunction &CGF,
5035	saved_type value) {
5036	// If the value says it wasn't saved, trust that it's still dominating.
5037	if (!value.getInt()) return value.getPointer();
5038
5039	// Otherwise, it should be an alloca instruction, as set up in save().
5040	auto alloca = cast<llvm::AllocaInst>(Val: value.getPointer());
5041	return CGF.Builder.CreateAlignedLoad(Ty: alloca->getAllocatedType(), Ptr: alloca,
5042	Align: alloca->getAlign());
5043	}
5044
5045	} // end namespace CodeGen
5046
5047	// Map the LangOption for floating point exception behavior into
5048	// the corresponding enum in the IR.
5049	llvm::fp::ExceptionBehavior
5050	ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind);
5051	} // end namespace clang
5052
5053	#endif
5054