1	//===--- CGExpr.cpp - Emit LLVM Code from Expressions ---------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This contains code to emit Expr nodes as LLVM code.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "CGCUDARuntime.h"
14	#include "CGCXXABI.h"
15	#include "CGCall.h"
16	#include "CGCleanup.h"
17	#include "CGDebugInfo.h"
18	#include "CGObjCRuntime.h"
19	#include "CGOpenMPRuntime.h"
20	#include "CGRecordLayout.h"
21	#include "CodeGenFunction.h"
22	#include "CodeGenModule.h"
23	#include "ConstantEmitter.h"
24	#include "TargetInfo.h"
25	#include "clang/AST/ASTContext.h"
26	#include "clang/AST/Attr.h"
27	#include "clang/AST/DeclObjC.h"
28	#include "clang/AST/NSAPI.h"
29	#include "clang/AST/StmtVisitor.h"
30	#include "clang/Basic/Builtins.h"
31	#include "clang/Basic/CodeGenOptions.h"
32	#include "clang/Basic/SourceManager.h"
33	#include "llvm/ADT/Hashing.h"
34	#include "llvm/ADT/STLExtras.h"
35	#include "llvm/ADT/StringExtras.h"
36	#include "llvm/IR/DataLayout.h"
37	#include "llvm/IR/Intrinsics.h"
38	#include "llvm/IR/IntrinsicsWebAssembly.h"
39	#include "llvm/IR/LLVMContext.h"
40	#include "llvm/IR/MDBuilder.h"
41	#include "llvm/IR/MatrixBuilder.h"
42	#include "llvm/Passes/OptimizationLevel.h"
43	#include "llvm/Support/ConvertUTF.h"
44	#include "llvm/Support/MathExtras.h"
45	#include "llvm/Support/Path.h"
46	#include "llvm/Support/SaveAndRestore.h"
47	#include "llvm/Support/xxhash.h"
48	#include "llvm/Transforms/Utils/SanitizerStats.h"
49
50	#include <optional>
51	#include <string>
52
53	using namespace clang;
54	using namespace CodeGen;
55
56	// Experiment to make sanitizers easier to debug
57	static llvm::cl::opt<bool> ClSanitizeDebugDeoptimization(
58	"ubsan-unique-traps", llvm::cl::Optional,
59	llvm::cl::desc("Deoptimize traps for UBSAN so there is 1 trap per check"),
60	llvm::cl::init(Val: false));
61
62	//===--------------------------------------------------------------------===//
63	// Miscellaneous Helper Methods
64	//===--------------------------------------------------------------------===//
65
66	/// CreateTempAlloca - This creates a alloca and inserts it into the entry
67	/// block.
68	Address CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty,
69	CharUnits Align,
70	const Twine &Name,
71	llvm::Value *ArraySize) {
72	auto Alloca = CreateTempAlloca(Ty, Name, ArraySize);
73	Alloca->setAlignment(Align.getAsAlign());
74	return Address(Alloca, Ty, Align, KnownNonNull);
75	}
76
77	/// CreateTempAlloca - This creates a alloca and inserts it into the entry
78	/// block. The alloca is casted to default address space if necessary.
79	Address CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align,
80	const Twine &Name,
81	llvm::Value *ArraySize,
82	Address *AllocaAddr) {
83	auto Alloca = CreateTempAllocaWithoutCast(Ty, Align, Name, ArraySize);
84	if (AllocaAddr)
85	*AllocaAddr = Alloca;
86	llvm::Value *V = Alloca.getPointer();
87	// Alloca always returns a pointer in alloca address space, which may
88	// be different from the type defined by the language. For example,
89	// in C++ the auto variables are in the default address space. Therefore
90	// cast alloca to the default address space when necessary.
91	if (getASTAllocaAddressSpace() != LangAS::Default) {
92	auto DestAddrSpace = getContext().getTargetAddressSpace(AS: LangAS::Default);
93	llvm::IRBuilderBase::InsertPointGuard IPG(Builder);
94	// When ArraySize is nullptr, alloca is inserted at AllocaInsertPt,
95	// otherwise alloca is inserted at the current insertion point of the
96	// builder.
97	if (!ArraySize)
98	Builder.SetInsertPoint(getPostAllocaInsertPoint());
99	V = getTargetHooks().performAddrSpaceCast(
100	*this, V, getASTAllocaAddressSpace(), LangAS::Default,
101	Ty->getPointerTo(AddrSpace: DestAddrSpace), /*non-null*/ true);
102	}
103
104	return Address(V, Ty, Align, KnownNonNull);
105	}
106
107	/// CreateTempAlloca - This creates an alloca and inserts it into the entry
108	/// block if \p ArraySize is nullptr, otherwise inserts it at the current
109	/// insertion point of the builder.
110	llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty,
111	const Twine &Name,
112	llvm::Value *ArraySize) {
113	if (ArraySize)
114	return Builder.CreateAlloca(Ty, ArraySize, Name);
115	return new llvm::AllocaInst(Ty, CGM.getDataLayout().getAllocaAddrSpace(),
116	ArraySize, Name, AllocaInsertPt);
117	}
118
119	/// CreateDefaultAlignTempAlloca - This creates an alloca with the
120	/// default alignment of the corresponding LLVM type, which is *not*
121	/// guaranteed to be related in any way to the expected alignment of
122	/// an AST type that might have been lowered to Ty.
123	Address CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty,
124	const Twine &Name) {
125	CharUnits Align =
126	CharUnits::fromQuantity(Quantity: CGM.getDataLayout().getPrefTypeAlign(Ty));
127	return CreateTempAlloca(Ty, Align, Name);
128	}
129
130	Address CodeGenFunction::CreateIRTemp(QualType Ty, const Twine &Name) {
131	CharUnits Align = getContext().getTypeAlignInChars(T: Ty);
132	return CreateTempAlloca(Ty: ConvertType(T: Ty), Align, Name);
133	}
134
135	Address CodeGenFunction::CreateMemTemp(QualType Ty, const Twine &Name,
136	Address *Alloca) {
137	// FIXME: Should we prefer the preferred type alignment here?
138	return CreateMemTemp(T: Ty, Align: getContext().getTypeAlignInChars(T: Ty), Name, Alloca);
139	}
140
141	Address CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align,
142	const Twine &Name, Address *Alloca) {
143	Address Result = CreateTempAlloca(Ty: ConvertTypeForMem(T: Ty), Align, Name,
144	/*ArraySize=*/nullptr, AllocaAddr: Alloca);
145
146	if (Ty->isConstantMatrixType()) {
147	auto *ArrayTy = cast<llvm::ArrayType>(Val: Result.getElementType());
148	auto *VectorTy = llvm::FixedVectorType::get(ElementType: ArrayTy->getElementType(),
149	NumElts: ArrayTy->getNumElements());
150
151	Result = Address(Result.getPointer(), VectorTy, Result.getAlignment(),
152	KnownNonNull);
153	}
154	return Result;
155	}
156
157	Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, CharUnits Align,
158	const Twine &Name) {
159	return CreateTempAllocaWithoutCast(Ty: ConvertTypeForMem(T: Ty), Align, Name);
160	}
161
162	Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty,
163	const Twine &Name) {
164	return CreateMemTempWithoutCast(Ty, Align: getContext().getTypeAlignInChars(T: Ty),
165	Name);
166	}
167
168	/// EvaluateExprAsBool - Perform the usual unary conversions on the specified
169	/// expression and compare the result against zero, returning an Int1Ty value.
170	llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) {
171	PGO.setCurrentStmt(E);
172	if (const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>()) {
173	llvm::Value *MemPtr = EmitScalarExpr(E);
174	return CGM.getCXXABI().EmitMemberPointerIsNotNull(CGF&: *this, MemPtr, MPT);
175	}
176
177	QualType BoolTy = getContext().BoolTy;
178	SourceLocation Loc = E->getExprLoc();
179	CGFPOptionsRAII FPOptsRAII(*this, E);
180	if (!E->getType()->isAnyComplexType())
181	return EmitScalarConversion(Src: EmitScalarExpr(E), SrcTy: E->getType(), DstTy: BoolTy, Loc);
182
183	return EmitComplexToScalarConversion(Src: EmitComplexExpr(E), SrcTy: E->getType(), DstTy: BoolTy,
184	Loc);
185	}
186
187	/// EmitIgnoredExpr - Emit code to compute the specified expression,
188	/// ignoring the result.
189	void CodeGenFunction::EmitIgnoredExpr(const Expr *E) {
190	if (E->isPRValue())
191	return (void)EmitAnyExpr(E, aggSlot: AggValueSlot::ignored(), ignoreResult: true);
192
193	// if this is a bitfield-resulting conditional operator, we can special case
194	// emit this. The normal 'EmitLValue' version of this is particularly
195	// difficult to codegen for, since creating a single "LValue" for two
196	// different sized arguments here is not particularly doable.
197	if (const auto *CondOp = dyn_cast<AbstractConditionalOperator>(
198	Val: E->IgnoreParenNoopCasts(Ctx: getContext()))) {
199	if (CondOp->getObjectKind() == OK_BitField)
200	return EmitIgnoredConditionalOperator(E: CondOp);
201	}
202
203	// Just emit it as an l-value and drop the result.
204	EmitLValue(E);
205	}
206
207	/// EmitAnyExpr - Emit code to compute the specified expression which
208	/// can have any type. The result is returned as an RValue struct.
209	/// If this is an aggregate expression, AggSlot indicates where the
210	/// result should be returned.
211	RValue CodeGenFunction::EmitAnyExpr(const Expr *E,
212	AggValueSlot aggSlot,
213	bool ignoreResult) {
214	switch (getEvaluationKind(T: E->getType())) {
215	case TEK_Scalar:
216	return RValue::get(V: EmitScalarExpr(E, IgnoreResultAssign: ignoreResult));
217	case TEK_Complex:
218	return RValue::getComplex(C: EmitComplexExpr(E, IgnoreReal: ignoreResult, IgnoreImag: ignoreResult));
219	case TEK_Aggregate:
220	if (!ignoreResult && aggSlot.isIgnored())
221	aggSlot = CreateAggTemp(T: E->getType(), Name: "agg-temp");
222	EmitAggExpr(E, AS: aggSlot);
223	return aggSlot.asRValue();
224	}
225	llvm_unreachable("bad evaluation kind");
226	}
227
228	/// EmitAnyExprToTemp - Similar to EmitAnyExpr(), however, the result will
229	/// always be accessible even if no aggregate location is provided.
230	RValue CodeGenFunction::EmitAnyExprToTemp(const Expr *E) {
231	AggValueSlot AggSlot = AggValueSlot::ignored();
232
233	if (hasAggregateEvaluationKind(T: E->getType()))
234	AggSlot = CreateAggTemp(T: E->getType(), Name: "agg.tmp");
235	return EmitAnyExpr(E, aggSlot: AggSlot);
236	}
237
238	/// EmitAnyExprToMem - Evaluate an expression into a given memory
239	/// location.
240	void CodeGenFunction::EmitAnyExprToMem(const Expr *E,
241	Address Location,
242	Qualifiers Quals,
243	bool IsInit) {
244	// FIXME: This function should take an LValue as an argument.
245	switch (getEvaluationKind(T: E->getType())) {
246	case TEK_Complex:
247	EmitComplexExprIntoLValue(E, dest: MakeAddrLValue(Addr: Location, T: E->getType()),
248	/*isInit*/ false);
249	return;
250
251	case TEK_Aggregate: {
252	EmitAggExpr(E, AS: AggValueSlot::forAddr(addr: Location, quals: Quals,
253	isDestructed: AggValueSlot::IsDestructed_t(IsInit),
254	needsGC: AggValueSlot::DoesNotNeedGCBarriers,
255	isAliased: AggValueSlot::IsAliased_t(!IsInit),
256	mayOverlap: AggValueSlot::MayOverlap));
257	return;
258	}
259
260	case TEK_Scalar: {
261	RValue RV = RValue::get(V: EmitScalarExpr(E, /*Ignore*/ IgnoreResultAssign: false));
262	LValue LV = MakeAddrLValue(Addr: Location, T: E->getType());
263	EmitStoreThroughLValue(Src: RV, Dst: LV);
264	return;
265	}
266	}
267	llvm_unreachable("bad evaluation kind");
268	}
269
270	static void
271	pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M,
272	const Expr *E, Address ReferenceTemporary) {
273	// Objective-C++ ARC:
274	// If we are binding a reference to a temporary that has ownership, we
275	// need to perform retain/release operations on the temporary.
276	//
277	// FIXME: This should be looking at E, not M.
278	if (auto Lifetime = M->getType().getObjCLifetime()) {
279	switch (Lifetime) {
280	case Qualifiers::OCL_None:
281	case Qualifiers::OCL_ExplicitNone:
282	// Carry on to normal cleanup handling.
283	break;
284
285	case Qualifiers::OCL_Autoreleasing:
286	// Nothing to do; cleaned up by an autorelease pool.
287	return;
288
289	case Qualifiers::OCL_Strong:
290	case Qualifiers::OCL_Weak:
291	switch (StorageDuration Duration = M->getStorageDuration()) {
292	case SD_Static:
293	// Note: we intentionally do not register a cleanup to release
294	// the object on program termination.
295	return;
296
297	case SD_Thread:
298	// FIXME: We should probably register a cleanup in this case.
299	return;
300
301	case SD_Automatic:
302	case SD_FullExpression:
303	CodeGenFunction::Destroyer *Destroy;
304	CleanupKind CleanupKind;
305	if (Lifetime == Qualifiers::OCL_Strong) {
306	const ValueDecl *VD = M->getExtendingDecl();
307	bool Precise =
308	VD && isa<VarDecl>(VD) && VD->hasAttr<ObjCPreciseLifetimeAttr>();
309	CleanupKind = CGF.getARCCleanupKind();
310	Destroy = Precise ? &CodeGenFunction::destroyARCStrongPrecise
311	: &CodeGenFunction::destroyARCStrongImprecise;
312	} else {
313	// __weak objects always get EH cleanups; otherwise, exceptions
314	// could cause really nasty crashes instead of mere leaks.
315	CleanupKind = NormalAndEHCleanup;
316	Destroy = &CodeGenFunction::destroyARCWeak;
317	}
318	if (Duration == SD_FullExpression)
319	CGF.pushDestroy(CleanupKind, ReferenceTemporary,
320	M->getType(), *Destroy,
321	CleanupKind & EHCleanup);
322	else
323	CGF.pushLifetimeExtendedDestroy(kind: CleanupKind, addr: ReferenceTemporary,
324	type: M->getType(),
325	destroyer: *Destroy, useEHCleanupForArray: CleanupKind & EHCleanup);
326	return;
327
328	case SD_Dynamic:
329	llvm_unreachable("temporary cannot have dynamic storage duration");
330	}
331	llvm_unreachable("unknown storage duration");
332	}
333	}
334
335	CXXDestructorDecl *ReferenceTemporaryDtor = nullptr;
336	if (const RecordType *RT =
337	E->getType()->getBaseElementTypeUnsafe()->getAs<RecordType>()) {
338	// Get the destructor for the reference temporary.
339	auto *ClassDecl = cast<CXXRecordDecl>(Val: RT->getDecl());
340	if (!ClassDecl->hasTrivialDestructor())
341	ReferenceTemporaryDtor = ClassDecl->getDestructor();
342	}
343
344	if (!ReferenceTemporaryDtor)
345	return;
346
347	// Call the destructor for the temporary.
348	switch (M->getStorageDuration()) {
349	case SD_Static:
350	case SD_Thread: {
351	llvm::FunctionCallee CleanupFn;
352	llvm::Constant *CleanupArg;
353	if (E->getType()->isArrayType()) {
354	CleanupFn = CodeGenFunction(CGF.CGM).generateDestroyHelper(
355	addr: ReferenceTemporary, type: E->getType(),
356	destroyer: CodeGenFunction::destroyCXXObject, useEHCleanupForArray: CGF.getLangOpts().Exceptions,
357	VD: dyn_cast_or_null<VarDecl>(Val: M->getExtendingDecl()));
358	CleanupArg = llvm::Constant::getNullValue(Ty: CGF.Int8PtrTy);
359	} else {
360	CleanupFn = CGF.CGM.getAddrAndTypeOfCXXStructor(
361	GD: GlobalDecl(ReferenceTemporaryDtor, Dtor_Complete));
362	CleanupArg = cast<llvm::Constant>(Val: ReferenceTemporary.getPointer());
363	}
364	CGF.CGM.getCXXABI().registerGlobalDtor(
365	CGF, D: *cast<VarDecl>(Val: M->getExtendingDecl()), Dtor: CleanupFn, Addr: CleanupArg);
366	break;
367	}
368
369	case SD_FullExpression:
370	CGF.pushDestroy(kind: NormalAndEHCleanup, addr: ReferenceTemporary, type: E->getType(),
371	destroyer: CodeGenFunction::destroyCXXObject,
372	useEHCleanupForArray: CGF.getLangOpts().Exceptions);
373	break;
374
375	case SD_Automatic:
376	CGF.pushLifetimeExtendedDestroy(kind: NormalAndEHCleanup,
377	addr: ReferenceTemporary, type: E->getType(),
378	destroyer: CodeGenFunction::destroyCXXObject,
379	useEHCleanupForArray: CGF.getLangOpts().Exceptions);
380	break;
381
382	case SD_Dynamic:
383	llvm_unreachable("temporary cannot have dynamic storage duration");
384	}
385	}
386
387	static Address createReferenceTemporary(CodeGenFunction &CGF,
388	const MaterializeTemporaryExpr *M,
389	const Expr *Inner,
390	Address *Alloca = nullptr) {
391	auto &TCG = CGF.getTargetHooks();
392	switch (M->getStorageDuration()) {
393	case SD_FullExpression:
394	case SD_Automatic: {
395	// If we have a constant temporary array or record try to promote it into a
396	// constant global under the same rules a normal constant would've been
397	// promoted. This is easier on the optimizer and generally emits fewer
398	// instructions.
399	QualType Ty = Inner->getType();
400	if (CGF.CGM.getCodeGenOpts().MergeAllConstants &&
401	(Ty->isArrayType() || Ty->isRecordType()) &&
402	Ty.isConstantStorage(Ctx: CGF.getContext(), ExcludeCtor: true, ExcludeDtor: false))
403	if (auto Init = ConstantEmitter(CGF).tryEmitAbstract(E: Inner, T: Ty)) {
404	auto AS = CGF.CGM.GetGlobalConstantAddressSpace();
405	auto *GV = new llvm::GlobalVariable(
406	CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
407	llvm::GlobalValue::PrivateLinkage, Init, ".ref.tmp", nullptr,
408	llvm::GlobalValue::NotThreadLocal,
409	CGF.getContext().getTargetAddressSpace(AS));
410	CharUnits alignment = CGF.getContext().getTypeAlignInChars(T: Ty);
411	GV->setAlignment(alignment.getAsAlign());
412	llvm::Constant *C = GV;
413	if (AS != LangAS::Default)
414	C = TCG.performAddrSpaceCast(
415	CGM&: CGF.CGM, V: GV, SrcAddr: AS, DestAddr: LangAS::Default,
416	DestTy: GV->getValueType()->getPointerTo(
417	AddrSpace: CGF.getContext().getTargetAddressSpace(AS: LangAS::Default)));
418	// FIXME: Should we put the new global into a COMDAT?
419	return Address(C, GV->getValueType(), alignment);
420	}
421	return CGF.CreateMemTemp(Ty, Name: "ref.tmp", Alloca);
422	}
423	case SD_Thread:
424	case SD_Static:
425	return CGF.CGM.GetAddrOfGlobalTemporary(E: M, Inner);
426
427	case SD_Dynamic:
428	llvm_unreachable("temporary can't have dynamic storage duration");
429	}
430	llvm_unreachable("unknown storage duration");
431	}
432
433	/// Helper method to check if the underlying ABI is AAPCS
434	static bool isAAPCS(const TargetInfo &TargetInfo) {
435	return TargetInfo.getABI().starts_with(Prefix: "aapcs");
436	}
437
438	LValue CodeGenFunction::
439	EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) {
440	const Expr *E = M->getSubExpr();
441
442	assert((!M->getExtendingDecl() || !isa<VarDecl>(M->getExtendingDecl()) ||
443	!cast<VarDecl>(M->getExtendingDecl())->isARCPseudoStrong()) &&
444	"Reference should never be pseudo-strong!");
445
446	// FIXME: ideally this would use EmitAnyExprToMem, however, we cannot do so
447	// as that will cause the lifetime adjustment to be lost for ARC
448	auto ownership = M->getType().getObjCLifetime();
449	if (ownership != Qualifiers::OCL_None &&
450	ownership != Qualifiers::OCL_ExplicitNone) {
451	Address Object = createReferenceTemporary(CGF&: *this, M, Inner: E);
452	if (auto *Var = dyn_cast<llvm::GlobalVariable>(Val: Object.getPointer())) {
453	llvm::Type *Ty = ConvertTypeForMem(T: E->getType());
454	Object = Object.withElementType(ElemTy: Ty);
455
456	// createReferenceTemporary will promote the temporary to a global with a
457	// constant initializer if it can. It can only do this to a value of
458	// ARC-manageable type if the value is global and therefore "immune" to
459	// ref-counting operations. Therefore we have no need to emit either a
460	// dynamic initialization or a cleanup and we can just return the address
461	// of the temporary.
462	if (Var->hasInitializer())
463	return MakeAddrLValue(Object, M->getType(), AlignmentSource::Decl);
464
465	Var->setInitializer(CGM.EmitNullConstant(T: E->getType()));
466	}
467	LValue RefTempDst = MakeAddrLValue(Object, M->getType(),
468	AlignmentSource::Decl);
469
470	switch (getEvaluationKind(T: E->getType())) {
471	default: llvm_unreachable("expected scalar or aggregate expression");
472	case TEK_Scalar:
473	EmitScalarInit(init: E, D: M->getExtendingDecl(), lvalue: RefTempDst, capturedByInit: false);
474	break;
475	case TEK_Aggregate: {
476	EmitAggExpr(E, AS: AggValueSlot::forAddr(addr: Object,
477	quals: E->getType().getQualifiers(),
478	isDestructed: AggValueSlot::IsDestructed,
479	needsGC: AggValueSlot::DoesNotNeedGCBarriers,
480	isAliased: AggValueSlot::IsNotAliased,
481	mayOverlap: AggValueSlot::DoesNotOverlap));
482	break;
483	}
484	}
485
486	pushTemporaryCleanup(CGF&: *this, M, E, ReferenceTemporary: Object);
487	return RefTempDst;
488	}
489
490	SmallVector<const Expr *, 2> CommaLHSs;
491	SmallVector<SubobjectAdjustment, 2> Adjustments;
492	E = E->skipRValueSubobjectAdjustments(CommaLHS&: CommaLHSs, Adjustments);
493
494	for (const auto &Ignored : CommaLHSs)
495	EmitIgnoredExpr(E: Ignored);
496
497	if (const auto *opaque = dyn_cast<OpaqueValueExpr>(Val: E)) {
498	if (opaque->getType()->isRecordType()) {
499	assert(Adjustments.empty());
500	return EmitOpaqueValueLValue(e: opaque);
501	}
502	}
503
504	// Create and initialize the reference temporary.
505	Address Alloca = Address::invalid();
506	Address Object = createReferenceTemporary(CGF&: *this, M, Inner: E, Alloca: &Alloca);
507	if (auto *Var = dyn_cast<llvm::GlobalVariable>(
508	Val: Object.getPointer()->stripPointerCasts())) {
509	llvm::Type *TemporaryType = ConvertTypeForMem(T: E->getType());
510	Object = Object.withElementType(ElemTy: TemporaryType);
511	// If the temporary is a global and has a constant initializer or is a
512	// constant temporary that we promoted to a global, we may have already
513	// initialized it.
514	if (!Var->hasInitializer()) {
515	Var->setInitializer(CGM.EmitNullConstant(T: E->getType()));
516	EmitAnyExprToMem(E, Location: Object, Quals: Qualifiers(), /*IsInit*/true);
517	}
518	} else {
519	switch (M->getStorageDuration()) {
520	case SD_Automatic:
521	if (auto *Size = EmitLifetimeStart(
522	Size: CGM.getDataLayout().getTypeAllocSize(Ty: Alloca.getElementType()),
523	Addr: Alloca.getPointer())) {
524	pushCleanupAfterFullExpr<CallLifetimeEnd>(Kind: NormalEHLifetimeMarker,
525	A: Alloca, A: Size);
526	}
527	break;
528
529	case SD_FullExpression: {
530	if (!ShouldEmitLifetimeMarkers)
531	break;
532
533	// Avoid creating a conditional cleanup just to hold an llvm.lifetime.end
534	// marker. Instead, start the lifetime of a conditional temporary earlier
535	// so that it's unconditional. Don't do this with sanitizers which need
536	// more precise lifetime marks. However when inside an "await.suspend"
537	// block, we should always avoid conditional cleanup because it creates
538	// boolean marker that lives across await_suspend, which can destroy coro
539	// frame.
540	ConditionalEvaluation *OldConditional = nullptr;
541	CGBuilderTy::InsertPoint OldIP;
542	if (isInConditionalBranch() && !E->getType().isDestructedType() &&
543	((!SanOpts.has(K: SanitizerKind::HWAddress) &&
544	!SanOpts.has(K: SanitizerKind::Memory) &&
545	!CGM.getCodeGenOpts().SanitizeAddressUseAfterScope) ||
546	inSuspendBlock())) {
547	OldConditional = OutermostConditional;
548	OutermostConditional = nullptr;
549
550	OldIP = Builder.saveIP();
551	llvm::BasicBlock *Block = OldConditional->getStartingBlock();
552	Builder.restoreIP(IP: CGBuilderTy::InsertPoint(
553	Block, llvm::BasicBlock::iterator(Block->back())));
554	}
555
556	if (auto *Size = EmitLifetimeStart(
557	Size: CGM.getDataLayout().getTypeAllocSize(Ty: Alloca.getElementType()),
558	Addr: Alloca.getPointer())) {
559	pushFullExprCleanup<CallLifetimeEnd>(kind: NormalEHLifetimeMarker, A: Alloca,
560	A: Size);
561	}
562
563	if (OldConditional) {
564	OutermostConditional = OldConditional;
565	Builder.restoreIP(IP: OldIP);
566	}
567	break;
568	}
569
570	default:
571	break;
572	}
573	EmitAnyExprToMem(E, Location: Object, Quals: Qualifiers(), /*IsInit*/true);
574	}
575	pushTemporaryCleanup(CGF&: *this, M, E, ReferenceTemporary: Object);
576
577	// Perform derived-to-base casts and/or field accesses, to get from the
578	// temporary object we created (and, potentially, for which we extended
579	// the lifetime) to the subobject we're binding the reference to.
580	for (SubobjectAdjustment &Adjustment : llvm::reverse(C&: Adjustments)) {
581	switch (Adjustment.Kind) {
582	case SubobjectAdjustment::DerivedToBaseAdjustment:
583	Object =
584	GetAddressOfBaseClass(Value: Object, Derived: Adjustment.DerivedToBase.DerivedClass,
585	PathBegin: Adjustment.DerivedToBase.BasePath->path_begin(),
586	PathEnd: Adjustment.DerivedToBase.BasePath->path_end(),
587	/*NullCheckValue=*/ false, Loc: E->getExprLoc());
588	break;
589
590	case SubobjectAdjustment::FieldAdjustment: {
591	LValue LV = MakeAddrLValue(Addr: Object, T: E->getType(), Source: AlignmentSource::Decl);
592	LV = EmitLValueForField(Base: LV, Field: Adjustment.Field);
593	assert(LV.isSimple() &&
594	"materialized temporary field is not a simple lvalue");
595	Object = LV.getAddress(CGF&: *this);
596	break;
597	}
598
599	case SubobjectAdjustment::MemberPointerAdjustment: {
600	llvm::Value *Ptr = EmitScalarExpr(E: Adjustment.Ptr.RHS);
601	Object = EmitCXXMemberDataPointerAddress(E, base: Object, memberPtr: Ptr,
602	memberPtrType: Adjustment.Ptr.MPT);
603	break;
604	}
605	}
606	}
607
608	return MakeAddrLValue(Object, M->getType(), AlignmentSource::Decl);
609	}
610
611	RValue
612	CodeGenFunction::EmitReferenceBindingToExpr(const Expr *E) {
613	// Emit the expression as an lvalue.
614	LValue LV = EmitLValue(E);
615	assert(LV.isSimple());
616	llvm::Value *Value = LV.getPointer(CGF&: *this);
617
618	if (sanitizePerformTypeCheck() && !E->getType()->isFunctionType()) {
619	// C++11 [dcl.ref]p5 (as amended by core issue 453):
620	// If a glvalue to which a reference is directly bound designates neither
621	// an existing object or function of an appropriate type nor a region of
622	// storage of suitable size and alignment to contain an object of the
623	// reference's type, the behavior is undefined.
624	QualType Ty = E->getType();
625	EmitTypeCheck(TCK: TCK_ReferenceBinding, Loc: E->getExprLoc(), V: Value, Type: Ty);
626	}
627
628	return RValue::get(V: Value);
629	}
630
631
632	/// getAccessedFieldNo - Given an encoded value and a result number, return the
633	/// input field number being accessed.
634	unsigned CodeGenFunction::getAccessedFieldNo(unsigned Idx,
635	const llvm::Constant *Elts) {
636	return cast<llvm::ConstantInt>(Val: Elts->getAggregateElement(Elt: Idx))
637	->getZExtValue();
638	}
639
640	/// Emit the hash_16_bytes function from include/llvm/ADT/Hashing.h.
641	static llvm::Value *emitHash16Bytes(CGBuilderTy &Builder, llvm::Value *Low,
642	llvm::Value *High) {
643	llvm::Value *KMul = Builder.getInt64(C: 0x9ddfea08eb382d69ULL);
644	llvm::Value *K47 = Builder.getInt64(C: 47);
645	llvm::Value *A0 = Builder.CreateMul(LHS: Builder.CreateXor(LHS: Low, RHS: High), RHS: KMul);
646	llvm::Value *A1 = Builder.CreateXor(LHS: Builder.CreateLShr(LHS: A0, RHS: K47), RHS: A0);
647	llvm::Value *B0 = Builder.CreateMul(LHS: Builder.CreateXor(LHS: High, RHS: A1), RHS: KMul);
648	llvm::Value *B1 = Builder.CreateXor(LHS: Builder.CreateLShr(LHS: B0, RHS: K47), RHS: B0);
649	return Builder.CreateMul(LHS: B1, RHS: KMul);
650	}
651
652	bool CodeGenFunction::isNullPointerAllowed(TypeCheckKind TCK) {
653	return TCK == TCK_DowncastPointer || TCK == TCK_Upcast ||
654	TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation;
655	}
656
657	bool CodeGenFunction::isVptrCheckRequired(TypeCheckKind TCK, QualType Ty) {
658	CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
659	return (RD && RD->hasDefinition() && RD->isDynamicClass()) &&
660	(TCK == TCK_MemberAccess || TCK == TCK_MemberCall ||
661	TCK == TCK_DowncastPointer || TCK == TCK_DowncastReference ||
662	TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation);
663	}
664
665	bool CodeGenFunction::sanitizePerformTypeCheck() const {
666	return SanOpts.has(K: SanitizerKind::Null) ||
667	SanOpts.has(K: SanitizerKind::Alignment) ||
668	SanOpts.has(K: SanitizerKind::ObjectSize) ||
669	SanOpts.has(K: SanitizerKind::Vptr);
670	}
671
672	void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc,
673	llvm::Value *Ptr, QualType Ty,
674	CharUnits Alignment,
675	SanitizerSet SkippedChecks,
676	llvm::Value *ArraySize) {
677	if (!sanitizePerformTypeCheck())
678	return;
679
680	// Don't check pointers outside the default address space. The null check
681	// isn't correct, the object-size check isn't supported by LLVM, and we can't
682	// communicate the addresses to the runtime handler for the vptr check.
683	if (Ptr->getType()->getPointerAddressSpace())
684	return;
685
686	// Don't check pointers to volatile data. The behavior here is implementation-
687	// defined.
688	if (Ty.isVolatileQualified())
689	return;
690
691	SanitizerScope SanScope(this);
692
693	SmallVector<std::pair<llvm::Value *, SanitizerMask>, 3> Checks;
694	llvm::BasicBlock *Done = nullptr;
695
696	// Quickly determine whether we have a pointer to an alloca. It's possible
697	// to skip null checks, and some alignment checks, for these pointers. This
698	// can reduce compile-time significantly.
699	auto PtrToAlloca = dyn_cast<llvm::AllocaInst>(Val: Ptr->stripPointerCasts());
700
701	llvm::Value *True = llvm::ConstantInt::getTrue(Context&: getLLVMContext());
702	llvm::Value *IsNonNull = nullptr;
703	bool IsGuaranteedNonNull =
704	SkippedChecks.has(K: SanitizerKind::Null) || PtrToAlloca;
705	bool AllowNullPointers = isNullPointerAllowed(TCK);
706	if ((SanOpts.has(K: SanitizerKind::Null) || AllowNullPointers) &&
707	!IsGuaranteedNonNull) {
708	// The glvalue must not be an empty glvalue.
709	IsNonNull = Builder.CreateIsNotNull(Arg: Ptr);
710
711	// The IR builder can constant-fold the null check if the pointer points to
712	// a constant.
713	IsGuaranteedNonNull = IsNonNull == True;
714
715	// Skip the null check if the pointer is known to be non-null.
716	if (!IsGuaranteedNonNull) {
717	if (AllowNullPointers) {
718	// When performing pointer casts, it's OK if the value is null.
719	// Skip the remaining checks in that case.
720	Done = createBasicBlock(name: "null");
721	llvm::BasicBlock *Rest = createBasicBlock(name: "not.null");
722	Builder.CreateCondBr(Cond: IsNonNull, True: Rest, False: Done);
723	EmitBlock(BB: Rest);
724	} else {
725	Checks.push_back(Elt: std::make_pair(x&: IsNonNull, y: SanitizerKind::Null));
726	}
727	}
728	}
729
730	if (SanOpts.has(K: SanitizerKind::ObjectSize) &&
731	!SkippedChecks.has(K: SanitizerKind::ObjectSize) &&
732	!Ty->isIncompleteType()) {
733	uint64_t TySize = CGM.getMinimumObjectSize(Ty).getQuantity();
734	llvm::Value *Size = llvm::ConstantInt::get(Ty: IntPtrTy, V: TySize);
735	if (ArraySize)
736	Size = Builder.CreateMul(LHS: Size, RHS: ArraySize);
737
738	// Degenerate case: new X[0] does not need an objectsize check.
739	llvm::Constant *ConstantSize = dyn_cast<llvm::Constant>(Val: Size);
740	if (!ConstantSize || !ConstantSize->isNullValue()) {
741	// The glvalue must refer to a large enough storage region.
742	// FIXME: If Address Sanitizer is enabled, insert dynamic instrumentation
743	// to check this.
744	// FIXME: Get object address space
745	llvm::Type *Tys[2] = { IntPtrTy, Int8PtrTy };
746	llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::objectsize, Tys);
747	llvm::Value *Min = Builder.getFalse();
748	llvm::Value *NullIsUnknown = Builder.getFalse();
749	llvm::Value *Dynamic = Builder.getFalse();
750	llvm::Value *LargeEnough = Builder.CreateICmpUGE(
751	LHS: Builder.CreateCall(Callee: F, Args: {Ptr, Min, NullIsUnknown, Dynamic}), RHS: Size);
752	Checks.push_back(Elt: std::make_pair(x&: LargeEnough, y: SanitizerKind::ObjectSize));
753	}
754	}
755
756	llvm::MaybeAlign AlignVal;
757	llvm::Value *PtrAsInt = nullptr;
758
759	if (SanOpts.has(K: SanitizerKind::Alignment) &&
760	!SkippedChecks.has(K: SanitizerKind::Alignment)) {
761	AlignVal = Alignment.getAsMaybeAlign();
762	if (!Ty->isIncompleteType() && !AlignVal)
763	AlignVal = CGM.getNaturalTypeAlignment(T: Ty, BaseInfo: nullptr, TBAAInfo: nullptr,
764	/*ForPointeeType=*/forPointeeType: true)
765	.getAsMaybeAlign();
766
767	// The glvalue must be suitably aligned.
768	if (AlignVal && *AlignVal > llvm::Align(1) &&
769	(!PtrToAlloca || PtrToAlloca->getAlign() < *AlignVal)) {
770	PtrAsInt = Builder.CreatePtrToInt(V: Ptr, DestTy: IntPtrTy);
771	llvm::Value *Align = Builder.CreateAnd(
772	LHS: PtrAsInt, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, V: AlignVal->value() - 1));
773	llvm::Value *Aligned =
774	Builder.CreateICmpEQ(LHS: Align, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, V: 0));
775	if (Aligned != True)
776	Checks.push_back(Elt: std::make_pair(x&: Aligned, y: SanitizerKind::Alignment));
777	}
778	}
779
780	if (Checks.size() > 0) {
781	llvm::Constant *StaticData[] = {
782	EmitCheckSourceLocation(Loc), EmitCheckTypeDescriptor(T: Ty),
783	llvm::ConstantInt::get(Ty: Int8Ty, V: AlignVal ? llvm::Log2(A: *AlignVal) : 1),
784	llvm::ConstantInt::get(Ty: Int8Ty, V: TCK)};
785	EmitCheck(Checked: Checks, Check: SanitizerHandler::TypeMismatch, StaticArgs: StaticData,
786	DynamicArgs: PtrAsInt ? PtrAsInt : Ptr);
787	}
788
789	// If possible, check that the vptr indicates that there is a subobject of
790	// type Ty at offset zero within this object.
791	//
792	// C++11 [basic.life]p5,6:
793	// [For storage which does not refer to an object within its lifetime]
794	// The program has undefined behavior if:
795	// -- the [pointer or glvalue] is used to access a non-static data member
796	// or call a non-static member function
797	if (SanOpts.has(K: SanitizerKind::Vptr) &&
798	!SkippedChecks.has(K: SanitizerKind::Vptr) && isVptrCheckRequired(TCK, Ty)) {
799	// Ensure that the pointer is non-null before loading it. If there is no
800	// compile-time guarantee, reuse the run-time null check or emit a new one.
801	if (!IsGuaranteedNonNull) {
802	if (!IsNonNull)
803	IsNonNull = Builder.CreateIsNotNull(Arg: Ptr);
804	if (!Done)
805	Done = createBasicBlock(name: "vptr.null");
806	llvm::BasicBlock *VptrNotNull = createBasicBlock(name: "vptr.not.null");
807	Builder.CreateCondBr(Cond: IsNonNull, True: VptrNotNull, False: Done);
808	EmitBlock(BB: VptrNotNull);
809	}
810
811	// Compute a hash of the mangled name of the type.
812	//
813	// FIXME: This is not guaranteed to be deterministic! Move to a
814	// fingerprinting mechanism once LLVM provides one. For the time
815	// being the implementation happens to be deterministic.
816	SmallString<64> MangledName;
817	llvm::raw_svector_ostream Out(MangledName);
818	CGM.getCXXABI().getMangleContext().mangleCXXRTTI(T: Ty.getUnqualifiedType(),
819	Out);
820
821	// Contained in NoSanitizeList based on the mangled type.
822	if (!CGM.getContext().getNoSanitizeList().containsType(Mask: SanitizerKind::Vptr,
823	MangledTypeName: Out.str())) {
824	llvm::hash_code TypeHash = hash_value(S: Out.str());
825
826	// Load the vptr, and compute hash_16_bytes(TypeHash, vptr).
827	llvm::Value *Low = llvm::ConstantInt::get(Ty: Int64Ty, V: TypeHash);
828	Address VPtrAddr(Ptr, IntPtrTy, getPointerAlign());
829	llvm::Value *VPtrVal = Builder.CreateLoad(Addr: VPtrAddr);
830	llvm::Value *High = Builder.CreateZExt(V: VPtrVal, DestTy: Int64Ty);
831
832	llvm::Value *Hash = emitHash16Bytes(Builder, Low, High);
833	Hash = Builder.CreateTrunc(V: Hash, DestTy: IntPtrTy);
834
835	// Look the hash up in our cache.
836	const int CacheSize = 128;
837	llvm::Type *HashTable = llvm::ArrayType::get(ElementType: IntPtrTy, NumElements: CacheSize);
838	llvm::Value *Cache = CGM.CreateRuntimeVariable(Ty: HashTable,
839	Name: "__ubsan_vptr_type_cache");
840	llvm::Value *Slot = Builder.CreateAnd(LHS: Hash,
841	RHS: llvm::ConstantInt::get(Ty: IntPtrTy,
842	V: CacheSize-1));
843	llvm::Value *Indices[] = { Builder.getInt32(C: 0), Slot };
844	llvm::Value *CacheVal = Builder.CreateAlignedLoad(
845	IntPtrTy, Builder.CreateInBoundsGEP(Ty: HashTable, Ptr: Cache, IdxList: Indices),
846	getPointerAlign());
847
848	// If the hash isn't in the cache, call a runtime handler to perform the
849	// hard work of checking whether the vptr is for an object of the right
850	// type. This will either fill in the cache and return, or produce a
851	// diagnostic.
852	llvm::Value *EqualHash = Builder.CreateICmpEQ(LHS: CacheVal, RHS: Hash);
853	llvm::Constant *StaticData[] = {
854	EmitCheckSourceLocation(Loc),
855	EmitCheckTypeDescriptor(T: Ty),
856	CGM.GetAddrOfRTTIDescriptor(Ty: Ty.getUnqualifiedType()),
857	llvm::ConstantInt::get(Ty: Int8Ty, V: TCK)
858	};
859	llvm::Value *DynamicData[] = { Ptr, Hash };
860	EmitCheck(Checked: std::make_pair(x&: EqualHash, y: SanitizerKind::Vptr),
861	Check: SanitizerHandler::DynamicTypeCacheMiss, StaticArgs: StaticData,
862	DynamicArgs: DynamicData);
863	}
864	}
865
866	if (Done) {
867	Builder.CreateBr(Dest: Done);
868	EmitBlock(BB: Done);
869	}
870	}
871
872	llvm::Value *CodeGenFunction::LoadPassedObjectSize(const Expr *E,
873	QualType EltTy) {
874	ASTContext &C = getContext();
875	uint64_t EltSize = C.getTypeSizeInChars(T: EltTy).getQuantity();
876	if (!EltSize)
877	return nullptr;
878
879	auto *ArrayDeclRef = dyn_cast<DeclRefExpr>(Val: E->IgnoreParenImpCasts());
880	if (!ArrayDeclRef)
881	return nullptr;
882
883	auto *ParamDecl = dyn_cast<ParmVarDecl>(Val: ArrayDeclRef->getDecl());
884	if (!ParamDecl)
885	return nullptr;
886
887	auto *POSAttr = ParamDecl->getAttr<PassObjectSizeAttr>();
888	if (!POSAttr)
889	return nullptr;
890
891	// Don't load the size if it's a lower bound.
892	int POSType = POSAttr->getType();
893	if (POSType != 0 && POSType != 1)
894	return nullptr;
895
896	// Find the implicit size parameter.
897	auto PassedSizeIt = SizeArguments.find(Val: ParamDecl);
898	if (PassedSizeIt == SizeArguments.end())
899	return nullptr;
900
901	const ImplicitParamDecl *PassedSizeDecl = PassedSizeIt->second;
902	assert(LocalDeclMap.count(PassedSizeDecl) && "Passed size not loadable");
903	Address AddrOfSize = LocalDeclMap.find(PassedSizeDecl)->second;
904	llvm::Value *SizeInBytes = EmitLoadOfScalar(Addr: AddrOfSize, /*Volatile=*/false,
905	Ty: C.getSizeType(), Loc: E->getExprLoc());
906	llvm::Value *SizeOfElement =
907	llvm::ConstantInt::get(Ty: SizeInBytes->getType(), V: EltSize);
908	return Builder.CreateUDiv(LHS: SizeInBytes, RHS: SizeOfElement);
909	}
910
911	/// If Base is known to point to the start of an array, return the length of
912	/// that array. Return 0 if the length cannot be determined.
913	static llvm::Value *getArrayIndexingBound(CodeGenFunction &CGF,
914	const Expr *Base,
915	QualType &IndexedType,
916	LangOptions::StrictFlexArraysLevelKind
917	StrictFlexArraysLevel) {
918	// For the vector indexing extension, the bound is the number of elements.
919	if (const VectorType *VT = Base->getType()->getAs<VectorType>()) {
920	IndexedType = Base->getType();
921	return CGF.Builder.getInt32(C: VT->getNumElements());
922	}
923
924	Base = Base->IgnoreParens();
925
926	if (const auto *CE = dyn_cast<CastExpr>(Val: Base)) {
927	if (CE->getCastKind() == CK_ArrayToPointerDecay &&
928	!CE->getSubExpr()->isFlexibleArrayMemberLike(Context&: CGF.getContext(),
929	StrictFlexArraysLevel)) {
930	CodeGenFunction::SanitizerScope SanScope(&CGF);
931
932	IndexedType = CE->getSubExpr()->getType();
933	const ArrayType *AT = IndexedType->castAsArrayTypeUnsafe();
934	if (const auto *CAT = dyn_cast<ConstantArrayType>(AT))
935	return CGF.Builder.getInt(AI: CAT->getSize());
936
937	if (const auto *VAT = dyn_cast<VariableArrayType>(AT))
938	return CGF.getVLASize(VAT).NumElts;
939	// Ignore pass_object_size here. It's not applicable on decayed pointers.
940	}
941	}
942
943	CodeGenFunction::SanitizerScope SanScope(&CGF);
944
945	QualType EltTy{Base->getType()->getPointeeOrArrayElementType(), 0};
946	if (llvm::Value *POS = CGF.LoadPassedObjectSize(E: Base, EltTy)) {
947	IndexedType = Base->getType();
948	return POS;
949	}
950
951	return nullptr;
952	}
953
954	namespace {
955
956	/// \p StructAccessBase returns the base \p Expr of a field access. It returns
957	/// either a \p DeclRefExpr, representing the base pointer to the struct, i.e.:
958	///
959	/// p in p-> a.b.c
960	///
961	/// or a \p MemberExpr, if the \p MemberExpr has the \p RecordDecl we're
962	/// looking for:
963	///
964	/// struct s {
965	/// struct s *ptr;
966	/// int count;
967	/// char array[] __attribute__((counted_by(count)));
968	/// };
969	///
970	/// If we have an expression like \p p->ptr->array[index], we want the
971	/// \p MemberExpr for \p p->ptr instead of \p p.
972	class StructAccessBase
973	: public ConstStmtVisitor<StructAccessBase, const Expr *> {
974	const RecordDecl *ExpectedRD;
975
976	bool IsExpectedRecordDecl(const Expr *E) const {
977	QualType Ty = E->getType();
978	if (Ty->isPointerType())
979	Ty = Ty->getPointeeType();
980	return ExpectedRD == Ty->getAsRecordDecl();
981	}
982
983	public:
984	StructAccessBase(const RecordDecl *ExpectedRD) : ExpectedRD(ExpectedRD) {}
985
986	//===--------------------------------------------------------------------===//
987	// Visitor Methods
988	//===--------------------------------------------------------------------===//
989
990	// NOTE: If we build C++ support for counted_by, then we'll have to handle
991	// horrors like this:
992	//
993	// struct S {
994	// int x, y;
995	// int blah[] __attribute__((counted_by(x)));
996	// } s;
997	//
998	// int foo(int index, int val) {
999	// int (S::*IHatePMDs)[] = &S::blah;
1000	// (s.*IHatePMDs)[index] = val;
1001	// }
1002
1003	const Expr *Visit(const Expr *E) {
1004	return ConstStmtVisitor<StructAccessBase, const Expr *>::Visit(E);
1005	}
1006
1007	const Expr *VisitStmt(const Stmt *S) { return nullptr; }
1008
1009	// These are the types we expect to return (in order of most to least
1010	// likely):
1011	//
1012	// 1. DeclRefExpr - This is the expression for the base of the structure.
1013	// It's exactly what we want to build an access to the \p counted_by
1014	// field.
1015	// 2. MemberExpr - This is the expression that has the same \p RecordDecl
1016	// as the flexble array member's lexical enclosing \p RecordDecl. This
1017	// allows us to catch things like: "p->p->array"
1018	// 3. CompoundLiteralExpr - This is for people who create something
1019	// heretical like (struct foo has a flexible array member):
1020	//
1021	// (struct foo){ 1, 2 }.blah[idx];
1022	const Expr *VisitDeclRefExpr(const DeclRefExpr *E) {
1023	return IsExpectedRecordDecl(E) ? E : nullptr;
1024	}
1025	const Expr *VisitMemberExpr(const MemberExpr *E) {
1026	if (IsExpectedRecordDecl(E) && E->isArrow())
1027	return E;
1028	const Expr *Res = Visit(E: E->getBase());
1029	return !Res && IsExpectedRecordDecl(E) ? E : Res;
1030	}
1031	const Expr *VisitCompoundLiteralExpr(const CompoundLiteralExpr *E) {
1032	return IsExpectedRecordDecl(E) ? E : nullptr;
1033	}
1034	const Expr *VisitCallExpr(const CallExpr *E) {
1035	return IsExpectedRecordDecl(E) ? E : nullptr;
1036	}
1037
1038	const Expr *VisitArraySubscriptExpr(const ArraySubscriptExpr *E) {
1039	if (IsExpectedRecordDecl(E))
1040	return E;
1041	return Visit(E: E->getBase());
1042	}
1043	const Expr *VisitCastExpr(const CastExpr *E) {
1044	return Visit(E: E->getSubExpr());
1045	}
1046	const Expr *VisitParenExpr(const ParenExpr *E) {
1047	return Visit(E: E->getSubExpr());
1048	}
1049	const Expr *VisitUnaryAddrOf(const UnaryOperator *E) {
1050	return Visit(E: E->getSubExpr());
1051	}
1052	const Expr *VisitUnaryDeref(const UnaryOperator *E) {
1053	return Visit(E: E->getSubExpr());
1054	}
1055	};
1056
1057	} // end anonymous namespace
1058
1059	using RecIndicesTy =
1060	SmallVector<std::pair<const RecordDecl *, llvm::Value *>, 8>;
1061
1062	static bool getGEPIndicesToField(CodeGenFunction &CGF, const RecordDecl *RD,
1063	const FieldDecl *FD, RecIndicesTy &Indices) {
1064	const CGRecordLayout &Layout = CGF.CGM.getTypes().getCGRecordLayout(RD);
1065	int64_t FieldNo = -1;
1066	for (const Decl *D : RD->decls()) {
1067	if (const auto *Field = dyn_cast<FieldDecl>(D)) {
1068	FieldNo = Layout.getLLVMFieldNo(Field);
1069	if (FD == Field) {
1070	Indices.emplace_back(std::make_pair(RD, CGF.Builder.getInt32(FieldNo)));
1071	return true;
1072	}
1073	}
1074
1075	if (const auto *Record = dyn_cast<RecordDecl>(D)) {
1076	++FieldNo;
1077	if (getGEPIndicesToField(CGF, Record, FD, Indices)) {
1078	if (RD->isUnion())
1079	FieldNo = 0;
1080	Indices.emplace_back(std::make_pair(RD, CGF.Builder.getInt32(FieldNo)));
1081	return true;
1082	}
1083	}
1084	}
1085
1086	return false;
1087	}
1088
1089	/// This method is typically called in contexts where we can't generate
1090	/// side-effects, like in __builtin_dynamic_object_size. When finding
1091	/// expressions, only choose those that have either already been emitted or can
1092	/// be loaded without side-effects.
1093	///
1094	/// - \p FAMDecl: the \p Decl for the flexible array member. It may not be
1095	/// within the top-level struct.
1096	/// - \p CountDecl: must be within the same non-anonymous struct as \p FAMDecl.
1097	llvm::Value *CodeGenFunction::EmitCountedByFieldExpr(
1098	const Expr *Base, const FieldDecl *FAMDecl, const FieldDecl *CountDecl) {
1099	const RecordDecl *RD = CountDecl->getParent()->getOuterLexicalRecordContext();
1100
1101	// Find the base struct expr (i.e. p in p->a.b.c.d).
1102	const Expr *StructBase = StructAccessBase(RD).Visit(E: Base);
1103	if (!StructBase || StructBase->HasSideEffects(Ctx: getContext()))
1104	return nullptr;
1105
1106	llvm::Value *Res = nullptr;
1107	if (const auto *DRE = dyn_cast<DeclRefExpr>(StructBase)) {
1108	Res = EmitDeclRefLValue(E: DRE).getPointer(*this);
1109	Res = Builder.CreateAlignedLoad(ConvertType(DRE->getType()), Res,
1110	getPointerAlign(), "dre.load");
1111	} else if (const MemberExpr *ME = dyn_cast<MemberExpr>(Val: StructBase)) {
1112	LValue LV = EmitMemberExpr(E: ME);
1113	Address Addr = LV.getAddress(CGF&: *this);
1114	Res = Addr.getPointer();
1115	} else if (StructBase->getType()->isPointerType()) {
1116	LValueBaseInfo BaseInfo;
1117	TBAAAccessInfo TBAAInfo;
1118	Address Addr = EmitPointerWithAlignment(Addr: StructBase, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
1119	Res = Addr.getPointer();
1120	} else {
1121	return nullptr;
1122	}
1123
1124	llvm::Value *Zero = Builder.getInt32(C: 0);
1125	RecIndicesTy Indices;
1126
1127	getGEPIndicesToField(CGF&: *this, RD, FD: CountDecl, Indices);
1128
1129	for (auto I = Indices.rbegin(), E = Indices.rend(); I != E; ++I)
1130	Res = Builder.CreateInBoundsGEP(
1131	Ty: ConvertType(T: QualType(I->first->getTypeForDecl(), 0)), Ptr: Res,
1132	IdxList: {Zero, I->second}, Name: "..counted_by.gep");
1133
1134	return Builder.CreateAlignedLoad(ConvertType(CountDecl->getType()), Res,
1135	getIntAlign(), "..counted_by.load");
1136	}
1137
1138	const FieldDecl *CodeGenFunction::FindCountedByField(const FieldDecl *FD) {
1139	if (!FD || !FD->hasAttr<CountedByAttr>())
1140	return nullptr;
1141
1142	const auto *CBA = FD->getAttr<CountedByAttr>();
1143	if (!CBA)
1144	return nullptr;
1145
1146	auto GetNonAnonStructOrUnion =
1147	[](const RecordDecl *RD) -> const RecordDecl * {
1148	while (RD && RD->isAnonymousStructOrUnion()) {
1149	const auto *R = dyn_cast<RecordDecl>(RD->getDeclContext());
1150	if (!R)
1151	return nullptr;
1152	RD = R;
1153	}
1154	return RD;
1155	};
1156	const RecordDecl *EnclosingRD = GetNonAnonStructOrUnion(FD->getParent());
1157	if (!EnclosingRD)
1158	return nullptr;
1159
1160	DeclarationName DName(CBA->getCountedByField());
1161	DeclContext::lookup_result Lookup = EnclosingRD->lookup(DName);
1162
1163	if (Lookup.empty())
1164	return nullptr;
1165
1166	const NamedDecl *ND = Lookup.front();
1167	if (const auto *IFD = dyn_cast<IndirectFieldDecl>(ND))
1168	ND = IFD->getAnonField();
1169
1170	return dyn_cast<FieldDecl>(Val: ND);
1171	}
1172
1173	void CodeGenFunction::EmitBoundsCheck(const Expr *E, const Expr *Base,
1174	llvm::Value *Index, QualType IndexType,
1175	bool Accessed) {
1176	assert(SanOpts.has(SanitizerKind::ArrayBounds) &&
1177	"should not be called unless adding bounds checks");
1178	const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
1179	getLangOpts().getStrictFlexArraysLevel();
1180	QualType IndexedType;
1181	llvm::Value *Bound =
1182	getArrayIndexingBound(CGF&: *this, Base, IndexedType, StrictFlexArraysLevel);
1183
1184	EmitBoundsCheckImpl(E, Bound, Index, IndexType, IndexedType, Accessed);
1185	}
1186
1187	void CodeGenFunction::EmitBoundsCheckImpl(const Expr *E, llvm::Value *Bound,
1188	llvm::Value *Index,
1189	QualType IndexType,
1190	QualType IndexedType, bool Accessed) {
1191	if (!Bound)
1192	return;
1193
1194	SanitizerScope SanScope(this);
1195
1196	bool IndexSigned = IndexType->isSignedIntegerOrEnumerationType();
1197	llvm::Value *IndexVal = Builder.CreateIntCast(V: Index, DestTy: SizeTy, isSigned: IndexSigned);
1198	llvm::Value *BoundVal = Builder.CreateIntCast(V: Bound, DestTy: SizeTy, isSigned: false);
1199
1200	llvm::Constant *StaticData[] = {
1201	EmitCheckSourceLocation(Loc: E->getExprLoc()),
1202	EmitCheckTypeDescriptor(T: IndexedType),
1203	EmitCheckTypeDescriptor(T: IndexType)
1204	};
1205	llvm::Value *Check = Accessed ? Builder.CreateICmpULT(LHS: IndexVal, RHS: BoundVal)
1206	: Builder.CreateICmpULE(LHS: IndexVal, RHS: BoundVal);
1207	EmitCheck(Checked: std::make_pair(x&: Check, y: SanitizerKind::ArrayBounds),
1208	Check: SanitizerHandler::OutOfBounds, StaticArgs: StaticData, DynamicArgs: Index);
1209	}
1210
1211	CodeGenFunction::ComplexPairTy CodeGenFunction::
1212	EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,
1213	bool isInc, bool isPre) {
1214	ComplexPairTy InVal = EmitLoadOfComplex(src: LV, loc: E->getExprLoc());
1215
1216	llvm::Value *NextVal;
1217	if (isa<llvm::IntegerType>(Val: InVal.first->getType())) {
1218	uint64_t AmountVal = isInc ? 1 : -1;
1219	NextVal = llvm::ConstantInt::get(Ty: InVal.first->getType(), V: AmountVal, IsSigned: true);
1220
1221	// Add the inc/dec to the real part.
1222	NextVal = Builder.CreateAdd(LHS: InVal.first, RHS: NextVal, Name: isInc ? "inc" : "dec");
1223	} else {
1224	QualType ElemTy = E->getType()->castAs<ComplexType>()->getElementType();
1225	llvm::APFloat FVal(getContext().getFloatTypeSemantics(T: ElemTy), 1);
1226	if (!isInc)
1227	FVal.changeSign();
1228	NextVal = llvm::ConstantFP::get(Context&: getLLVMContext(), V: FVal);
1229
1230	// Add the inc/dec to the real part.
1231	NextVal = Builder.CreateFAdd(L: InVal.first, R: NextVal, Name: isInc ? "inc" : "dec");
1232	}
1233
1234	ComplexPairTy IncVal(NextVal, InVal.second);
1235
1236	// Store the updated result through the lvalue.
1237	EmitStoreOfComplex(V: IncVal, dest: LV, /*init*/ isInit: false);
1238	if (getLangOpts().OpenMP)
1239	CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF&: *this,
1240	LHS: E->getSubExpr());
1241
1242	// If this is a postinc, return the value read from memory, otherwise use the
1243	// updated value.
1244	return isPre ? IncVal : InVal;
1245	}
1246
1247	void CodeGenModule::EmitExplicitCastExprType(const ExplicitCastExpr *E,
1248	CodeGenFunction *CGF) {
1249	// Bind VLAs in the cast type.
1250	if (CGF && E->getType()->isVariablyModifiedType())
1251	CGF->EmitVariablyModifiedType(Ty: E->getType());
1252
1253	if (CGDebugInfo *DI = getModuleDebugInfo())
1254	DI->EmitExplicitCastType(Ty: E->getType());
1255	}
1256
1257	//===----------------------------------------------------------------------===//
1258	// LValue Expression Emission
1259	//===----------------------------------------------------------------------===//
1260
1261	static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo,
1262	TBAAAccessInfo *TBAAInfo,
1263	KnownNonNull_t IsKnownNonNull,
1264	CodeGenFunction &CGF) {
1265	// We allow this with ObjC object pointers because of fragile ABIs.
1266	assert(E->getType()->isPointerType() ||
1267	E->getType()->isObjCObjectPointerType());
1268	E = E->IgnoreParens();
1269
1270	// Casts:
1271	if (const CastExpr *CE = dyn_cast<CastExpr>(Val: E)) {
1272	if (const auto *ECE = dyn_cast<ExplicitCastExpr>(Val: CE))
1273	CGF.CGM.EmitExplicitCastExprType(E: ECE, CGF: &CGF);
1274
1275	switch (CE->getCastKind()) {
1276	// Non-converting casts (but not C's implicit conversion from void*).
1277	case CK_BitCast:
1278	case CK_NoOp:
1279	case CK_AddressSpaceConversion:
1280	if (auto PtrTy = CE->getSubExpr()->getType()->getAs<PointerType>()) {
1281	if (PtrTy->getPointeeType()->isVoidType())
1282	break;
1283
1284	LValueBaseInfo InnerBaseInfo;
1285	TBAAAccessInfo InnerTBAAInfo;
1286	Address Addr = CGF.EmitPointerWithAlignment(
1287	Addr: CE->getSubExpr(), BaseInfo: &InnerBaseInfo, TBAAInfo: &InnerTBAAInfo, IsKnownNonNull);
1288	if (BaseInfo) *BaseInfo = InnerBaseInfo;
1289	if (TBAAInfo) *TBAAInfo = InnerTBAAInfo;
1290
1291	if (isa<ExplicitCastExpr>(Val: CE)) {
1292	LValueBaseInfo TargetTypeBaseInfo;
1293	TBAAAccessInfo TargetTypeTBAAInfo;
1294	CharUnits Align = CGF.CGM.getNaturalPointeeTypeAlignment(
1295	T: E->getType(), BaseInfo: &TargetTypeBaseInfo, TBAAInfo: &TargetTypeTBAAInfo);
1296	if (TBAAInfo)
1297	*TBAAInfo =
1298	CGF.CGM.mergeTBAAInfoForCast(SourceInfo: *TBAAInfo, TargetInfo: TargetTypeTBAAInfo);
1299	// If the source l-value is opaque, honor the alignment of the
1300	// casted-to type.
1301	if (InnerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) {
1302	if (BaseInfo)
1303	BaseInfo->mergeForCast(Info: TargetTypeBaseInfo);
1304	Addr = Address(Addr.getPointer(), Addr.getElementType(), Align,
1305	IsKnownNonNull);
1306	}
1307	}
1308
1309	if (CGF.SanOpts.has(K: SanitizerKind::CFIUnrelatedCast) &&
1310	CE->getCastKind() == CK_BitCast) {
1311	if (auto PT = E->getType()->getAs<PointerType>())
1312	CGF.EmitVTablePtrCheckForCast(T: PT->getPointeeType(), Derived: Addr,
1313	/*MayBeNull=*/true,
1314	TCK: CodeGenFunction::CFITCK_UnrelatedCast,
1315	Loc: CE->getBeginLoc());
1316	}
1317
1318	llvm::Type *ElemTy =
1319	CGF.ConvertTypeForMem(T: E->getType()->getPointeeType());
1320	Addr = Addr.withElementType(ElemTy);
1321	if (CE->getCastKind() == CK_AddressSpaceConversion)
1322	Addr = CGF.Builder.CreateAddrSpaceCast(Addr,
1323	Ty: CGF.ConvertType(T: E->getType()));
1324	return Addr;
1325	}
1326	break;
1327
1328	// Array-to-pointer decay.
1329	case CK_ArrayToPointerDecay:
1330	return CGF.EmitArrayToPointerDecay(Array: CE->getSubExpr(), BaseInfo, TBAAInfo);
1331
1332	// Derived-to-base conversions.
1333	case CK_UncheckedDerivedToBase:
1334	case CK_DerivedToBase: {
1335	// TODO: Support accesses to members of base classes in TBAA. For now, we
1336	// conservatively pretend that the complete object is of the base class
1337	// type.
1338	if (TBAAInfo)
1339	*TBAAInfo = CGF.CGM.getTBAAAccessInfo(AccessType: E->getType());
1340	Address Addr = CGF.EmitPointerWithAlignment(
1341	Addr: CE->getSubExpr(), BaseInfo, TBAAInfo: nullptr,
1342	IsKnownNonNull: (KnownNonNull_t)(IsKnownNonNull ||
1343	CE->getCastKind() == CK_UncheckedDerivedToBase));
1344	auto Derived = CE->getSubExpr()->getType()->getPointeeCXXRecordDecl();
1345	return CGF.GetAddressOfBaseClass(
1346	Value: Addr, Derived, PathBegin: CE->path_begin(), PathEnd: CE->path_end(),
1347	NullCheckValue: CGF.ShouldNullCheckClassCastValue(Cast: CE), Loc: CE->getExprLoc());
1348	}
1349
1350	// TODO: Is there any reason to treat base-to-derived conversions
1351	// specially?
1352	default:
1353	break;
1354	}
1355	}
1356
1357	// Unary &.
1358	if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(Val: E)) {
1359	if (UO->getOpcode() == UO_AddrOf) {
1360	LValue LV = CGF.EmitLValue(E: UO->getSubExpr(), IsKnownNonNull);
1361	if (BaseInfo) *BaseInfo = LV.getBaseInfo();
1362	if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo();
1363	return LV.getAddress(CGF);
1364	}
1365	}
1366
1367	// std::addressof and variants.
1368	if (auto *Call = dyn_cast<CallExpr>(Val: E)) {
1369	switch (Call->getBuiltinCallee()) {
1370	default:
1371	break;
1372	case Builtin::BIaddressof:
1373	case Builtin::BI__addressof:
1374	case Builtin::BI__builtin_addressof: {
1375	LValue LV = CGF.EmitLValue(E: Call->getArg(Arg: 0), IsKnownNonNull);
1376	if (BaseInfo) *BaseInfo = LV.getBaseInfo();
1377	if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo();
1378	return LV.getAddress(CGF);
1379	}
1380	}
1381	}
1382
1383	// TODO: conditional operators, comma.
1384
1385	// Otherwise, use the alignment of the type.
1386	CharUnits Align =
1387	CGF.CGM.getNaturalPointeeTypeAlignment(T: E->getType(), BaseInfo, TBAAInfo);
1388	llvm::Type *ElemTy = CGF.ConvertTypeForMem(T: E->getType()->getPointeeType());
1389	return Address(CGF.EmitScalarExpr(E), ElemTy, Align, IsKnownNonNull);
1390	}
1391
1392	/// EmitPointerWithAlignment - Given an expression of pointer type, try to
1393	/// derive a more accurate bound on the alignment of the pointer.
1394	Address CodeGenFunction::EmitPointerWithAlignment(
1395	const Expr *E, LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo,
1396	KnownNonNull_t IsKnownNonNull) {
1397	Address Addr =
1398	::EmitPointerWithAlignment(E, BaseInfo, TBAAInfo, IsKnownNonNull, CGF&: *this);
1399	if (IsKnownNonNull && !Addr.isKnownNonNull())
1400	Addr.setKnownNonNull();
1401	return Addr;
1402	}
1403
1404	llvm::Value *CodeGenFunction::EmitNonNullRValueCheck(RValue RV, QualType T) {
1405	llvm::Value *V = RV.getScalarVal();
1406	if (auto MPT = T->getAs<MemberPointerType>())
1407	return CGM.getCXXABI().EmitMemberPointerIsNotNull(CGF&: *this, MemPtr: V, MPT);
1408	return Builder.CreateICmpNE(LHS: V, RHS: llvm::Constant::getNullValue(Ty: V->getType()));
1409	}
1410
1411	RValue CodeGenFunction::GetUndefRValue(QualType Ty) {
1412	if (Ty->isVoidType())
1413	return RValue::get(V: nullptr);
1414
1415	switch (getEvaluationKind(T: Ty)) {
1416	case TEK_Complex: {
1417	llvm::Type *EltTy =
1418	ConvertType(T: Ty->castAs<ComplexType>()->getElementType());
1419	llvm::Value *U = llvm::UndefValue::get(T: EltTy);
1420	return RValue::getComplex(C: std::make_pair(x&: U, y&: U));
1421	}
1422
1423	// If this is a use of an undefined aggregate type, the aggregate must have an
1424	// identifiable address. Just because the contents of the value are undefined
1425	// doesn't mean that the address can't be taken and compared.
1426	case TEK_Aggregate: {
1427	Address DestPtr = CreateMemTemp(Ty, Name: "undef.agg.tmp");
1428	return RValue::getAggregate(addr: DestPtr);
1429	}
1430
1431	case TEK_Scalar:
1432	return RValue::get(V: llvm::UndefValue::get(T: ConvertType(T: Ty)));
1433	}
1434	llvm_unreachable("bad evaluation kind");
1435	}
1436
1437	RValue CodeGenFunction::EmitUnsupportedRValue(const Expr *E,
1438	const char *Name) {
1439	ErrorUnsupported(E, Name);
1440	return GetUndefRValue(Ty: E->getType());
1441	}
1442
1443	LValue CodeGenFunction::EmitUnsupportedLValue(const Expr *E,
1444	const char *Name) {
1445	ErrorUnsupported(E, Name);
1446	llvm::Type *ElTy = ConvertType(T: E->getType());
1447	llvm::Type *Ty = UnqualPtrTy;
1448	return MakeAddrLValue(
1449	Addr: Address(llvm::UndefValue::get(T: Ty), ElTy, CharUnits::One()), T: E->getType());
1450	}
1451
1452	bool CodeGenFunction::IsWrappedCXXThis(const Expr *Obj) {
1453	const Expr *Base = Obj;
1454	while (!isa<CXXThisExpr>(Val: Base)) {
1455	// The result of a dynamic_cast can be null.
1456	if (isa<CXXDynamicCastExpr>(Val: Base))
1457	return false;
1458
1459	if (const auto *CE = dyn_cast<CastExpr>(Val: Base)) {
1460	Base = CE->getSubExpr();
1461	} else if (const auto *PE = dyn_cast<ParenExpr>(Val: Base)) {
1462	Base = PE->getSubExpr();
1463	} else if (const auto *UO = dyn_cast<UnaryOperator>(Val: Base)) {
1464	if (UO->getOpcode() == UO_Extension)
1465	Base = UO->getSubExpr();
1466	else
1467	return false;
1468	} else {
1469	return false;
1470	}
1471	}
1472	return true;
1473	}
1474
1475	LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) {
1476	LValue LV;
1477	if (SanOpts.has(K: SanitizerKind::ArrayBounds) && isa<ArraySubscriptExpr>(Val: E))
1478	LV = EmitArraySubscriptExpr(E: cast<ArraySubscriptExpr>(Val: E), /*Accessed*/true);
1479	else
1480	LV = EmitLValue(E);
1481	if (!isa<DeclRefExpr>(Val: E) && !LV.isBitField() && LV.isSimple()) {
1482	SanitizerSet SkippedChecks;
1483	if (const auto *ME = dyn_cast<MemberExpr>(Val: E)) {
1484	bool IsBaseCXXThis = IsWrappedCXXThis(Obj: ME->getBase());
1485	if (IsBaseCXXThis)
1486	SkippedChecks.set(K: SanitizerKind::Alignment, Value: true);
1487	if (IsBaseCXXThis || isa<DeclRefExpr>(Val: ME->getBase()))
1488	SkippedChecks.set(K: SanitizerKind::Null, Value: true);
1489	}
1490	EmitTypeCheck(TCK, Loc: E->getExprLoc(), Ptr: LV.getPointer(CGF&: *this), Ty: E->getType(),
1491	Alignment: LV.getAlignment(), SkippedChecks);
1492	}
1493	return LV;
1494	}
1495
1496	/// EmitLValue - Emit code to compute a designator that specifies the location
1497	/// of the expression.
1498	///
1499	/// This can return one of two things: a simple address or a bitfield reference.
1500	/// In either case, the LLVM Value* in the LValue structure is guaranteed to be
1501	/// an LLVM pointer type.
1502	///
1503	/// If this returns a bitfield reference, nothing about the pointee type of the
1504	/// LLVM value is known: For example, it may not be a pointer to an integer.
1505	///
1506	/// If this returns a normal address, and if the lvalue's C type is fixed size,
1507	/// this method guarantees that the returned pointer type will point to an LLVM
1508	/// type of the same size of the lvalue's type. If the lvalue has a variable
1509	/// length type, this is not possible.
1510	///
1511	LValue CodeGenFunction::EmitLValue(const Expr *E,
1512	KnownNonNull_t IsKnownNonNull) {
1513	LValue LV = EmitLValueHelper(E, IsKnownNonNull);
1514	if (IsKnownNonNull && !LV.isKnownNonNull())
1515	LV.setKnownNonNull();
1516	return LV;
1517	}
1518
1519	static QualType getConstantExprReferredType(const FullExpr *E,
1520	const ASTContext &Ctx) {
1521	const Expr *SE = E->getSubExpr()->IgnoreImplicit();
1522	if (isa<OpaqueValueExpr>(Val: SE))
1523	return SE->getType();
1524	return cast<CallExpr>(Val: SE)->getCallReturnType(Ctx)->getPointeeType();
1525	}
1526
1527	LValue CodeGenFunction::EmitLValueHelper(const Expr *E,
1528	KnownNonNull_t IsKnownNonNull) {
1529	ApplyDebugLocation DL(*this, E);
1530	switch (E->getStmtClass()) {
1531	default: return EmitUnsupportedLValue(E, Name: "l-value expression");
1532
1533	case Expr::ObjCPropertyRefExprClass:
1534	llvm_unreachable("cannot emit a property reference directly");
1535
1536	case Expr::ObjCSelectorExprClass:
1537	return EmitObjCSelectorLValue(E: cast<ObjCSelectorExpr>(Val: E));
1538	case Expr::ObjCIsaExprClass:
1539	return EmitObjCIsaExpr(E: cast<ObjCIsaExpr>(Val: E));
1540	case Expr::BinaryOperatorClass:
1541	return EmitBinaryOperatorLValue(E: cast<BinaryOperator>(Val: E));
1542	case Expr::CompoundAssignOperatorClass: {
1543	QualType Ty = E->getType();
1544	if (const AtomicType *AT = Ty->getAs<AtomicType>())
1545	Ty = AT->getValueType();
1546	if (!Ty->isAnyComplexType())
1547	return EmitCompoundAssignmentLValue(E: cast<CompoundAssignOperator>(Val: E));
1548	return EmitComplexCompoundAssignmentLValue(E: cast<CompoundAssignOperator>(Val: E));
1549	}
1550	case Expr::CallExprClass:
1551	case Expr::CXXMemberCallExprClass:
1552	case Expr::CXXOperatorCallExprClass:
1553	case Expr::UserDefinedLiteralClass:
1554	return EmitCallExprLValue(E: cast<CallExpr>(Val: E));
1555	case Expr::CXXRewrittenBinaryOperatorClass:
1556	return EmitLValue(E: cast<CXXRewrittenBinaryOperator>(Val: E)->getSemanticForm(),
1557	IsKnownNonNull);
1558	case Expr::VAArgExprClass:
1559	return EmitVAArgExprLValue(E: cast<VAArgExpr>(Val: E));
1560	case Expr::DeclRefExprClass:
1561	return EmitDeclRefLValue(E: cast<DeclRefExpr>(Val: E));
1562	case Expr::ConstantExprClass: {
1563	const ConstantExpr *CE = cast<ConstantExpr>(Val: E);
1564	if (llvm::Value *Result = ConstantEmitter(*this).tryEmitConstantExpr(CE)) {
1565	QualType RetType = getConstantExprReferredType(CE, getContext());
1566	return MakeNaturalAlignAddrLValue(V: Result, T: RetType);
1567	}
1568	return EmitLValue(E: cast<ConstantExpr>(Val: E)->getSubExpr(), IsKnownNonNull);
1569	}
1570	case Expr::ParenExprClass:
1571	return EmitLValue(E: cast<ParenExpr>(Val: E)->getSubExpr(), IsKnownNonNull);
1572	case Expr::GenericSelectionExprClass:
1573	return EmitLValue(E: cast<GenericSelectionExpr>(Val: E)->getResultExpr(),
1574	IsKnownNonNull);
1575	case Expr::PredefinedExprClass:
1576	return EmitPredefinedLValue(E: cast<PredefinedExpr>(Val: E));
1577	case Expr::StringLiteralClass:
1578	return EmitStringLiteralLValue(E: cast<StringLiteral>(Val: E));
1579	case Expr::ObjCEncodeExprClass:
1580	return EmitObjCEncodeExprLValue(E: cast<ObjCEncodeExpr>(Val: E));
1581	case Expr::PseudoObjectExprClass:
1582	return EmitPseudoObjectLValue(e: cast<PseudoObjectExpr>(Val: E));
1583	case Expr::InitListExprClass:
1584	return EmitInitListLValue(E: cast<InitListExpr>(Val: E));
1585	case Expr::CXXTemporaryObjectExprClass:
1586	case Expr::CXXConstructExprClass:
1587	return EmitCXXConstructLValue(E: cast<CXXConstructExpr>(Val: E));
1588	case Expr::CXXBindTemporaryExprClass:
1589	return EmitCXXBindTemporaryLValue(E: cast<CXXBindTemporaryExpr>(Val: E));
1590	case Expr::CXXUuidofExprClass:
1591	return EmitCXXUuidofLValue(E: cast<CXXUuidofExpr>(Val: E));
1592	case Expr::LambdaExprClass:
1593	return EmitAggExprToLValue(E);
1594
1595	case Expr::ExprWithCleanupsClass: {
1596	const auto *cleanups = cast<ExprWithCleanups>(Val: E);
1597	RunCleanupsScope Scope(*this);
1598	LValue LV = EmitLValue(E: cleanups->getSubExpr(), IsKnownNonNull);
1599	if (LV.isSimple()) {
1600	// Defend against branches out of gnu statement expressions surrounded by
1601	// cleanups.
1602	Address Addr = LV.getAddress(CGF&: *this);
1603	llvm::Value *V = Addr.getPointer();
1604	Scope.ForceCleanup(ValuesToReload: {&V});
1605	return LValue::MakeAddr(address: Addr.withPointer(NewPointer: V, IsKnownNonNull: Addr.isKnownNonNull()),
1606	type: LV.getType(), Context&: getContext(), BaseInfo: LV.getBaseInfo(),
1607	TBAAInfo: LV.getTBAAInfo());
1608	}
1609	// FIXME: Is it possible to create an ExprWithCleanups that produces a
1610	// bitfield lvalue or some other non-simple lvalue?
1611	return LV;
1612	}
1613
1614	case Expr::CXXDefaultArgExprClass: {
1615	auto *DAE = cast<CXXDefaultArgExpr>(Val: E);
1616	CXXDefaultArgExprScope Scope(*this, DAE);
1617	return EmitLValue(E: DAE->getExpr(), IsKnownNonNull);
1618	}
1619	case Expr::CXXDefaultInitExprClass: {
1620	auto *DIE = cast<CXXDefaultInitExpr>(Val: E);
1621	CXXDefaultInitExprScope Scope(*this, DIE);
1622	return EmitLValue(E: DIE->getExpr(), IsKnownNonNull);
1623	}
1624	case Expr::CXXTypeidExprClass:
1625	return EmitCXXTypeidLValue(E: cast<CXXTypeidExpr>(Val: E));
1626
1627	case Expr::ObjCMessageExprClass:
1628	return EmitObjCMessageExprLValue(E: cast<ObjCMessageExpr>(Val: E));
1629	case Expr::ObjCIvarRefExprClass:
1630	return EmitObjCIvarRefLValue(E: cast<ObjCIvarRefExpr>(Val: E));
1631	case Expr::StmtExprClass:
1632	return EmitStmtExprLValue(E: cast<StmtExpr>(Val: E));
1633	case Expr::UnaryOperatorClass:
1634	return EmitUnaryOpLValue(E: cast<UnaryOperator>(Val: E));
1635	case Expr::ArraySubscriptExprClass:
1636	return EmitArraySubscriptExpr(E: cast<ArraySubscriptExpr>(Val: E));
1637	case Expr::MatrixSubscriptExprClass:
1638	return EmitMatrixSubscriptExpr(E: cast<MatrixSubscriptExpr>(Val: E));
1639	case Expr::OMPArraySectionExprClass:
1640	return EmitOMPArraySectionExpr(E: cast<OMPArraySectionExpr>(Val: E));
1641	case Expr::ExtVectorElementExprClass:
1642	return EmitExtVectorElementExpr(E: cast<ExtVectorElementExpr>(Val: E));
1643	case Expr::CXXThisExprClass:
1644	return MakeAddrLValue(Addr: LoadCXXThisAddress(), T: E->getType());
1645	case Expr::MemberExprClass:
1646	return EmitMemberExpr(E: cast<MemberExpr>(Val: E));
1647	case Expr::CompoundLiteralExprClass:
1648	return EmitCompoundLiteralLValue(E: cast<CompoundLiteralExpr>(Val: E));
1649	case Expr::ConditionalOperatorClass:
1650	return EmitConditionalOperatorLValue(cast<ConditionalOperator>(Val: E));
1651	case Expr::BinaryConditionalOperatorClass:
1652	return EmitConditionalOperatorLValue(cast<BinaryConditionalOperator>(Val: E));
1653	case Expr::ChooseExprClass:
1654	return EmitLValue(E: cast<ChooseExpr>(Val: E)->getChosenSubExpr(), IsKnownNonNull);
1655	case Expr::OpaqueValueExprClass:
1656	return EmitOpaqueValueLValue(e: cast<OpaqueValueExpr>(Val: E));
1657	case Expr::SubstNonTypeTemplateParmExprClass:
1658	return EmitLValue(E: cast<SubstNonTypeTemplateParmExpr>(Val: E)->getReplacement(),
1659	IsKnownNonNull);
1660	case Expr::ImplicitCastExprClass:
1661	case Expr::CStyleCastExprClass:
1662	case Expr::CXXFunctionalCastExprClass:
1663	case Expr::CXXStaticCastExprClass:
1664	case Expr::CXXDynamicCastExprClass:
1665	case Expr::CXXReinterpretCastExprClass:
1666	case Expr::CXXConstCastExprClass:
1667	case Expr::CXXAddrspaceCastExprClass:
1668	case Expr::ObjCBridgedCastExprClass:
1669	return EmitCastLValue(E: cast<CastExpr>(Val: E));
1670
1671	case Expr::MaterializeTemporaryExprClass:
1672	return EmitMaterializeTemporaryExpr(M: cast<MaterializeTemporaryExpr>(Val: E));
1673
1674	case Expr::CoawaitExprClass:
1675	return EmitCoawaitLValue(E: cast<CoawaitExpr>(Val: E));
1676	case Expr::CoyieldExprClass:
1677	return EmitCoyieldLValue(E: cast<CoyieldExpr>(Val: E));
1678	case Expr::PackIndexingExprClass:
1679	return EmitLValue(E: cast<PackIndexingExpr>(Val: E)->getSelectedExpr());
1680	}
1681	}
1682
1683	/// Given an object of the given canonical type, can we safely copy a
1684	/// value out of it based on its initializer?
1685	static bool isConstantEmittableObjectType(QualType type) {
1686	assert(type.isCanonical());
1687	assert(!type->isReferenceType());
1688
1689	// Must be const-qualified but non-volatile.
1690	Qualifiers qs = type.getLocalQualifiers();
1691	if (!qs.hasConst() || qs.hasVolatile()) return false;
1692
1693	// Otherwise, all object types satisfy this except C++ classes with
1694	// mutable subobjects or non-trivial copy/destroy behavior.
1695	if (const auto *RT = dyn_cast<RecordType>(Val&: type))
1696	if (const auto *RD = dyn_cast<CXXRecordDecl>(Val: RT->getDecl()))
1697	if (RD->hasMutableFields() || !RD->isTrivial())
1698	return false;
1699
1700	return true;
1701	}
1702
1703	/// Can we constant-emit a load of a reference to a variable of the
1704	/// given type? This is different from predicates like
1705	/// Decl::mightBeUsableInConstantExpressions because we do want it to apply
1706	/// in situations that don't necessarily satisfy the language's rules
1707	/// for this (e.g. C++'s ODR-use rules). For example, we want to able
1708	/// to do this with const float variables even if those variables
1709	/// aren't marked 'constexpr'.
1710	enum ConstantEmissionKind {
1711	CEK_None,
1712	CEK_AsReferenceOnly,
1713	CEK_AsValueOrReference,
1714	CEK_AsValueOnly
1715	};
1716	static ConstantEmissionKind checkVarTypeForConstantEmission(QualType type) {
1717	type = type.getCanonicalType();
1718	if (const auto *ref = dyn_cast<ReferenceType>(Val&: type)) {
1719	if (isConstantEmittableObjectType(type: ref->getPointeeType()))
1720	return CEK_AsValueOrReference;
1721	return CEK_AsReferenceOnly;
1722	}
1723	if (isConstantEmittableObjectType(type))
1724	return CEK_AsValueOnly;
1725	return CEK_None;
1726	}
1727
1728	/// Try to emit a reference to the given value without producing it as
1729	/// an l-value. This is just an optimization, but it avoids us needing
1730	/// to emit global copies of variables if they're named without triggering
1731	/// a formal use in a context where we can't emit a direct reference to them,
1732	/// for instance if a block or lambda or a member of a local class uses a
1733	/// const int variable or constexpr variable from an enclosing function.
1734	CodeGenFunction::ConstantEmission
1735	CodeGenFunction::tryEmitAsConstant(DeclRefExpr *refExpr) {
1736	ValueDecl *value = refExpr->getDecl();
1737
1738	// The value needs to be an enum constant or a constant variable.
1739	ConstantEmissionKind CEK;
1740	if (isa<ParmVarDecl>(Val: value)) {
1741	CEK = CEK_None;
1742	} else if (auto *var = dyn_cast<VarDecl>(Val: value)) {
1743	CEK = checkVarTypeForConstantEmission(var->getType());
1744	} else if (isa<EnumConstantDecl>(Val: value)) {
1745	CEK = CEK_AsValueOnly;
1746	} else {
1747	CEK = CEK_None;
1748	}
1749	if (CEK == CEK_None) return ConstantEmission();
1750
1751	Expr::EvalResult result;
1752	bool resultIsReference;
1753	QualType resultType;
1754
1755	// It's best to evaluate all the way as an r-value if that's permitted.
1756	if (CEK != CEK_AsReferenceOnly &&
1757	refExpr->EvaluateAsRValue(result, getContext())) {
1758	resultIsReference = false;
1759	resultType = refExpr->getType();
1760
1761	// Otherwise, try to evaluate as an l-value.
1762	} else if (CEK != CEK_AsValueOnly &&
1763	refExpr->EvaluateAsLValue(result, getContext())) {
1764	resultIsReference = true;
1765	resultType = value->getType();
1766
1767	// Failure.
1768	} else {
1769	return ConstantEmission();
1770	}
1771
1772	// In any case, if the initializer has side-effects, abandon ship.
1773	if (result.HasSideEffects)
1774	return ConstantEmission();
1775
1776	// In CUDA/HIP device compilation, a lambda may capture a reference variable
1777	// referencing a global host variable by copy. In this case the lambda should
1778	// make a copy of the value of the global host variable. The DRE of the
1779	// captured reference variable cannot be emitted as load from the host
1780	// global variable as compile time constant, since the host variable is not
1781	// accessible on device. The DRE of the captured reference variable has to be
1782	// loaded from captures.
1783	if (CGM.getLangOpts().CUDAIsDevice && result.Val.isLValue() &&
1784	refExpr->refersToEnclosingVariableOrCapture()) {
1785	auto *MD = dyn_cast_or_null<CXXMethodDecl>(Val: CurCodeDecl);
1786	if (MD && MD->getParent()->isLambda() &&
1787	MD->getOverloadedOperator() == OO_Call) {
1788	const APValue::LValueBase &base = result.Val.getLValueBase();
1789	if (const ValueDecl *D = base.dyn_cast<const ValueDecl *>()) {
1790	if (const VarDecl *VD = dyn_cast<const VarDecl>(Val: D)) {
1791	if (!VD->hasAttr<CUDADeviceAttr>()) {
1792	return ConstantEmission();
1793	}
1794	}
1795	}
1796	}
1797	}
1798
1799	// Emit as a constant.
1800	auto C = ConstantEmitter(*this).emitAbstract(loc: refExpr->getLocation(),
1801	value: result.Val, T: resultType);
1802
1803	// Make sure we emit a debug reference to the global variable.
1804	// This should probably fire even for
1805	if (isa<VarDecl>(Val: value)) {
1806	if (!getContext().DeclMustBeEmitted(cast<VarDecl>(Val: value)))
1807	EmitDeclRefExprDbgValue(E: refExpr, Init: result.Val);
1808	} else {
1809	assert(isa<EnumConstantDecl>(value));
1810	EmitDeclRefExprDbgValue(E: refExpr, Init: result.Val);
1811	}
1812
1813	// If we emitted a reference constant, we need to dereference that.
1814	if (resultIsReference)
1815	return ConstantEmission::forReference(C);
1816
1817	return ConstantEmission::forValue(C);
1818	}
1819
1820	static DeclRefExpr *tryToConvertMemberExprToDeclRefExpr(CodeGenFunction &CGF,
1821	const MemberExpr *ME) {
1822	if (auto *VD = dyn_cast<VarDecl>(Val: ME->getMemberDecl())) {
1823	// Try to emit static variable member expressions as DREs.
1824	return DeclRefExpr::Create(
1825	CGF.getContext(), NestedNameSpecifierLoc(), SourceLocation(), VD,
1826	/*RefersToEnclosingVariableOrCapture=*/false, ME->getExprLoc(),
1827	ME->getType(), ME->getValueKind(), nullptr, nullptr, ME->isNonOdrUse());
1828	}
1829	return nullptr;
1830	}
1831
1832	CodeGenFunction::ConstantEmission
1833	CodeGenFunction::tryEmitAsConstant(const MemberExpr *ME) {
1834	if (DeclRefExpr *DRE = tryToConvertMemberExprToDeclRefExpr(CGF&: *this, ME))
1835	return tryEmitAsConstant(refExpr: DRE);
1836	return ConstantEmission();
1837	}
1838
1839	llvm::Value *CodeGenFunction::emitScalarConstant(
1840	const CodeGenFunction::ConstantEmission &Constant, Expr *E) {
1841	assert(Constant && "not a constant");
1842	if (Constant.isReference())
1843	return EmitLoadOfLValue(V: Constant.getReferenceLValue(CGF&: *this, refExpr: E),
1844	Loc: E->getExprLoc())
1845	.getScalarVal();
1846	return Constant.getValue();
1847	}
1848
1849	llvm::Value *CodeGenFunction::EmitLoadOfScalar(LValue lvalue,
1850	SourceLocation Loc) {
1851	return EmitLoadOfScalar(Addr: lvalue.getAddress(CGF&: *this), Volatile: lvalue.isVolatile(),
1852	Ty: lvalue.getType(), Loc, BaseInfo: lvalue.getBaseInfo(),
1853	TBAAInfo: lvalue.getTBAAInfo(), isNontemporal: lvalue.isNontemporal());
1854	}
1855
1856	static bool hasBooleanRepresentation(QualType Ty) {
1857	if (Ty->isBooleanType())
1858	return true;
1859
1860	if (const EnumType *ET = Ty->getAs<EnumType>())
1861	return ET->getDecl()->getIntegerType()->isBooleanType();
1862
1863	if (const AtomicType *AT = Ty->getAs<AtomicType>())
1864	return hasBooleanRepresentation(Ty: AT->getValueType());
1865
1866	return false;
1867	}
1868
1869	static bool getRangeForType(CodeGenFunction &CGF, QualType Ty,
1870	llvm::APInt &Min, llvm::APInt &End,
1871	bool StrictEnums, bool IsBool) {
1872	const EnumType *ET = Ty->getAs<EnumType>();
1873	bool IsRegularCPlusPlusEnum = CGF.getLangOpts().CPlusPlus && StrictEnums &&
1874	ET && !ET->getDecl()->isFixed();
1875	if (!IsBool && !IsRegularCPlusPlusEnum)
1876	return false;
1877
1878	if (IsBool) {
1879	Min = llvm::APInt(CGF.getContext().getTypeSize(T: Ty), 0);
1880	End = llvm::APInt(CGF.getContext().getTypeSize(T: Ty), 2);
1881	} else {
1882	const EnumDecl *ED = ET->getDecl();
1883	ED->getValueRange(Max&: End, Min);
1884	}
1885	return true;
1886	}
1887
1888	llvm::MDNode *CodeGenFunction::getRangeForLoadFromType(QualType Ty) {
1889	llvm::APInt Min, End;
1890	if (!getRangeForType(CGF&: *this, Ty, Min, End, StrictEnums: CGM.getCodeGenOpts().StrictEnums,
1891	IsBool: hasBooleanRepresentation(Ty)))
1892	return nullptr;
1893
1894	llvm::MDBuilder MDHelper(getLLVMContext());
1895	return MDHelper.createRange(Lo: Min, Hi: End);
1896	}
1897
1898	bool CodeGenFunction::EmitScalarRangeCheck(llvm::Value *Value, QualType Ty,
1899	SourceLocation Loc) {
1900	bool HasBoolCheck = SanOpts.has(K: SanitizerKind::Bool);
1901	bool HasEnumCheck = SanOpts.has(K: SanitizerKind::Enum);
1902	if (!HasBoolCheck && !HasEnumCheck)
1903	return false;
1904
1905	bool IsBool = hasBooleanRepresentation(Ty) ||
1906	NSAPI(CGM.getContext()).isObjCBOOLType(T: Ty);
1907	bool NeedsBoolCheck = HasBoolCheck && IsBool;
1908	bool NeedsEnumCheck = HasEnumCheck && Ty->getAs<EnumType>();
1909	if (!NeedsBoolCheck && !NeedsEnumCheck)
1910	return false;
1911
1912	// Single-bit booleans don't need to be checked. Special-case this to avoid
1913	// a bit width mismatch when handling bitfield values. This is handled by
1914	// EmitFromMemory for the non-bitfield case.
1915	if (IsBool &&
1916	cast<llvm::IntegerType>(Val: Value->getType())->getBitWidth() == 1)
1917	return false;
1918
1919	llvm::APInt Min, End;
1920	if (!getRangeForType(CGF&: *this, Ty, Min, End, /*StrictEnums=*/true, IsBool))
1921	return true;
1922
1923	auto &Ctx = getLLVMContext();
1924	SanitizerScope SanScope(this);
1925	llvm::Value *Check;
1926	--End;
1927	if (!Min) {
1928	Check = Builder.CreateICmpULE(LHS: Value, RHS: llvm::ConstantInt::get(Context&: Ctx, V: End));
1929	} else {
1930	llvm::Value *Upper =
1931	Builder.CreateICmpSLE(LHS: Value, RHS: llvm::ConstantInt::get(Context&: Ctx, V: End));
1932	llvm::Value *Lower =
1933	Builder.CreateICmpSGE(LHS: Value, RHS: llvm::ConstantInt::get(Context&: Ctx, V: Min));
1934	Check = Builder.CreateAnd(LHS: Upper, RHS: Lower);
1935	}
1936	llvm::Constant *StaticArgs[] = {EmitCheckSourceLocation(Loc),
1937	EmitCheckTypeDescriptor(T: Ty)};
1938	SanitizerMask Kind =
1939	NeedsEnumCheck ? SanitizerKind::Enum : SanitizerKind::Bool;
1940	EmitCheck(Checked: std::make_pair(x&: Check, y&: Kind), Check: SanitizerHandler::LoadInvalidValue,
1941	StaticArgs, DynamicArgs: EmitCheckValue(V: Value));
1942	return true;
1943	}
1944
1945	llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile,
1946	QualType Ty,
1947	SourceLocation Loc,
1948	LValueBaseInfo BaseInfo,
1949	TBAAAccessInfo TBAAInfo,
1950	bool isNontemporal) {
1951	if (auto *GV = dyn_cast<llvm::GlobalValue>(Val: Addr.getPointer()))
1952	if (GV->isThreadLocal())
1953	Addr = Addr.withPointer(NewPointer: Builder.CreateThreadLocalAddress(Ptr: GV),
1954	IsKnownNonNull: NotKnownNonNull);
1955
1956	if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {
1957	// Boolean vectors use `iN` as storage type.
1958	if (ClangVecTy->isExtVectorBoolType()) {
1959	llvm::Type *ValTy = ConvertType(T: Ty);
1960	unsigned ValNumElems =
1961	cast<llvm::FixedVectorType>(Val: ValTy)->getNumElements();
1962	// Load the `iP` storage object (P is the padded vector size).
1963	auto *RawIntV = Builder.CreateLoad(Addr, IsVolatile: Volatile, Name: "load_bits");
1964	const auto *RawIntTy = RawIntV->getType();
1965	assert(RawIntTy->isIntegerTy() && "compressed iN storage for bitvectors");
1966	// Bitcast iP --> <P x i1>.
1967	auto *PaddedVecTy = llvm::FixedVectorType::get(
1968	ElementType: Builder.getInt1Ty(), NumElts: RawIntTy->getPrimitiveSizeInBits());
1969	llvm::Value *V = Builder.CreateBitCast(V: RawIntV, DestTy: PaddedVecTy);
1970	// Shuffle <P x i1> --> <N x i1> (N is the actual bit size).
1971	V = emitBoolVecConversion(SrcVec: V, NumElementsDst: ValNumElems, Name: "extractvec");
1972
1973	return EmitFromMemory(Value: V, Ty);
1974	}
1975
1976	// Handle vectors of size 3 like size 4 for better performance.
1977	const llvm::Type *EltTy = Addr.getElementType();
1978	const auto *VTy = cast<llvm::FixedVectorType>(Val: EltTy);
1979
1980	if (!CGM.getCodeGenOpts().PreserveVec3Type && VTy->getNumElements() == 3) {
1981
1982	llvm::VectorType *vec4Ty =
1983	llvm::FixedVectorType::get(ElementType: VTy->getElementType(), NumElts: 4);
1984	Address Cast = Addr.withElementType(ElemTy: vec4Ty);
1985	// Now load value.
1986	llvm::Value *V = Builder.CreateLoad(Addr: Cast, IsVolatile: Volatile, Name: "loadVec4");
1987
1988	// Shuffle vector to get vec3.
1989	V = Builder.CreateShuffleVector(V, Mask: ArrayRef<int>{0, 1, 2}, Name: "extractVec");
1990	return EmitFromMemory(Value: V, Ty);
1991	}
1992	}
1993
1994	// Atomic operations have to be done on integral types.
1995	LValue AtomicLValue =
1996	LValue::MakeAddr(address: Addr, type: Ty, Context&: getContext(), BaseInfo, TBAAInfo);
1997	if (Ty->isAtomicType() || LValueIsSuitableForInlineAtomic(Src: AtomicLValue)) {
1998	return EmitAtomicLoad(LV: AtomicLValue, SL: Loc).getScalarVal();
1999	}
2000
2001	llvm::LoadInst *Load = Builder.CreateLoad(Addr, IsVolatile: Volatile);
2002	if (isNontemporal) {
2003	llvm::MDNode *Node = llvm::MDNode::get(
2004	Context&: Load->getContext(), MDs: llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: 1)));
2005	Load->setMetadata(KindID: llvm::LLVMContext::MD_nontemporal, Node);
2006	}
2007
2008	CGM.DecorateInstructionWithTBAA(Inst: Load, TBAAInfo);
2009
2010	if (EmitScalarRangeCheck(Value: Load, Ty, Loc)) {
2011	// In order to prevent the optimizer from throwing away the check, don't
2012	// attach range metadata to the load.
2013	} else if (CGM.getCodeGenOpts().OptimizationLevel > 0)
2014	if (llvm::MDNode *RangeInfo = getRangeForLoadFromType(Ty)) {
2015	Load->setMetadata(KindID: llvm::LLVMContext::MD_range, Node: RangeInfo);
2016	Load->setMetadata(KindID: llvm::LLVMContext::MD_noundef,
2017	Node: llvm::MDNode::get(Context&: getLLVMContext(), MDs: std::nullopt));
2018	}
2019
2020	return EmitFromMemory(Value: Load, Ty);
2021	}
2022
2023	llvm::Value *CodeGenFunction::EmitToMemory(llvm::Value *Value, QualType Ty) {
2024	// Bool has a different representation in memory than in registers.
2025	if (hasBooleanRepresentation(Ty)) {
2026	// This should really always be an i1, but sometimes it's already
2027	// an i8, and it's awkward to track those cases down.
2028	if (Value->getType()->isIntegerTy(Bitwidth: 1))
2029	return Builder.CreateZExt(V: Value, DestTy: ConvertTypeForMem(T: Ty), Name: "frombool");
2030	assert(Value->getType()->isIntegerTy(getContext().getTypeSize(Ty)) &&
2031	"wrong value rep of bool");
2032	}
2033
2034	return Value;
2035	}
2036
2037	llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) {
2038	// Bool has a different representation in memory than in registers.
2039	if (hasBooleanRepresentation(Ty)) {
2040	assert(Value->getType()->isIntegerTy(getContext().getTypeSize(Ty)) &&
2041	"wrong value rep of bool");
2042	return Builder.CreateTrunc(V: Value, DestTy: Builder.getInt1Ty(), Name: "tobool");
2043	}
2044	if (Ty->isExtVectorBoolType()) {
2045	const auto *RawIntTy = Value->getType();
2046	// Bitcast iP --> <P x i1>.
2047	auto *PaddedVecTy = llvm::FixedVectorType::get(
2048	ElementType: Builder.getInt1Ty(), NumElts: RawIntTy->getPrimitiveSizeInBits());
2049	auto *V = Builder.CreateBitCast(V: Value, DestTy: PaddedVecTy);
2050	// Shuffle <P x i1> --> <N x i1> (N is the actual bit size).
2051	llvm::Type *ValTy = ConvertType(T: Ty);
2052	unsigned ValNumElems = cast<llvm::FixedVectorType>(Val: ValTy)->getNumElements();
2053	return emitBoolVecConversion(SrcVec: V, NumElementsDst: ValNumElems, Name: "extractvec");
2054	}
2055
2056	return Value;
2057	}
2058
2059	// Convert the pointer of \p Addr to a pointer to a vector (the value type of
2060	// MatrixType), if it points to a array (the memory type of MatrixType).
2061	static Address MaybeConvertMatrixAddress(Address Addr, CodeGenFunction &CGF,
2062	bool IsVector = true) {
2063	auto *ArrayTy = dyn_cast<llvm::ArrayType>(Val: Addr.getElementType());
2064	if (ArrayTy && IsVector) {
2065	auto *VectorTy = llvm::FixedVectorType::get(ElementType: ArrayTy->getElementType(),
2066	NumElts: ArrayTy->getNumElements());
2067
2068	return Addr.withElementType(ElemTy: VectorTy);
2069	}
2070	auto *VectorTy = dyn_cast<llvm::VectorType>(Val: Addr.getElementType());
2071	if (VectorTy && !IsVector) {
2072	auto *ArrayTy = llvm::ArrayType::get(
2073	ElementType: VectorTy->getElementType(),
2074	NumElements: cast<llvm::FixedVectorType>(Val: VectorTy)->getNumElements());
2075
2076	return Addr.withElementType(ElemTy: ArrayTy);
2077	}
2078
2079	return Addr;
2080	}
2081
2082	// Emit a store of a matrix LValue. This may require casting the original
2083	// pointer to memory address (ArrayType) to a pointer to the value type
2084	// (VectorType).
2085	static void EmitStoreOfMatrixScalar(llvm::Value *value, LValue lvalue,
2086	bool isInit, CodeGenFunction &CGF) {
2087	Address Addr = MaybeConvertMatrixAddress(Addr: lvalue.getAddress(CGF), CGF,
2088	IsVector: value->getType()->isVectorTy());
2089	CGF.EmitStoreOfScalar(Value: value, Addr, Volatile: lvalue.isVolatile(), Ty: lvalue.getType(),
2090	BaseInfo: lvalue.getBaseInfo(), TBAAInfo: lvalue.getTBAAInfo(), isInit,
2091	isNontemporal: lvalue.isNontemporal());
2092	}
2093
2094	void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr,
2095	bool Volatile, QualType Ty,
2096	LValueBaseInfo BaseInfo,
2097	TBAAAccessInfo TBAAInfo,
2098	bool isInit, bool isNontemporal) {
2099	if (auto *GV = dyn_cast<llvm::GlobalValue>(Val: Addr.getPointer()))
2100	if (GV->isThreadLocal())
2101	Addr = Addr.withPointer(NewPointer: Builder.CreateThreadLocalAddress(Ptr: GV),
2102	IsKnownNonNull: NotKnownNonNull);
2103
2104	llvm::Type *SrcTy = Value->getType();
2105	if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {
2106	auto *VecTy = dyn_cast<llvm::FixedVectorType>(Val: SrcTy);
2107	if (VecTy && ClangVecTy->isExtVectorBoolType()) {
2108	auto *MemIntTy = cast<llvm::IntegerType>(Val: Addr.getElementType());
2109	// Expand to the memory bit width.
2110	unsigned MemNumElems = MemIntTy->getPrimitiveSizeInBits();
2111	// <N x i1> --> <P x i1>.
2112	Value = emitBoolVecConversion(SrcVec: Value, NumElementsDst: MemNumElems, Name: "insertvec");
2113	// <P x i1> --> iP.
2114	Value = Builder.CreateBitCast(V: Value, DestTy: MemIntTy);
2115	} else if (!CGM.getCodeGenOpts().PreserveVec3Type) {
2116	// Handle vec3 special.
2117	if (VecTy && cast<llvm::FixedVectorType>(Val: VecTy)->getNumElements() == 3) {
2118	// Our source is a vec3, do a shuffle vector to make it a vec4.
2119	Value = Builder.CreateShuffleVector(V: Value, Mask: ArrayRef<int>{0, 1, 2, -1},
2120	Name: "extractVec");
2121	SrcTy = llvm::FixedVectorType::get(ElementType: VecTy->getElementType(), NumElts: 4);
2122	}
2123	if (Addr.getElementType() != SrcTy) {
2124	Addr = Addr.withElementType(ElemTy: SrcTy);
2125	}
2126	}
2127	}
2128
2129	Value = EmitToMemory(Value, Ty);
2130
2131	LValue AtomicLValue =
2132	LValue::MakeAddr(address: Addr, type: Ty, Context&: getContext(), BaseInfo, TBAAInfo);
2133	if (Ty->isAtomicType() ||
2134	(!isInit && LValueIsSuitableForInlineAtomic(Src: AtomicLValue))) {
2135	EmitAtomicStore(rvalue: RValue::get(V: Value), lvalue: AtomicLValue, isInit);
2136	return;
2137	}
2138
2139	llvm::StoreInst *Store = Builder.CreateStore(Val: Value, Addr, IsVolatile: Volatile);
2140	if (isNontemporal) {
2141	llvm::MDNode *Node =
2142	llvm::MDNode::get(Context&: Store->getContext(),
2143	MDs: llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: 1)));
2144	Store->setMetadata(KindID: llvm::LLVMContext::MD_nontemporal, Node);
2145	}
2146
2147	CGM.DecorateInstructionWithTBAA(Inst: Store, TBAAInfo);
2148	}
2149
2150	void CodeGenFunction::EmitStoreOfScalar(llvm::Value *value, LValue lvalue,
2151	bool isInit) {
2152	if (lvalue.getType()->isConstantMatrixType()) {
2153	EmitStoreOfMatrixScalar(value, lvalue, isInit, CGF&: *this);
2154	return;
2155	}
2156
2157	EmitStoreOfScalar(Value: value, Addr: lvalue.getAddress(CGF&: *this), Volatile: lvalue.isVolatile(),
2158	Ty: lvalue.getType(), BaseInfo: lvalue.getBaseInfo(),
2159	TBAAInfo: lvalue.getTBAAInfo(), isInit, isNontemporal: lvalue.isNontemporal());
2160	}
2161
2162	// Emit a load of a LValue of matrix type. This may require casting the pointer
2163	// to memory address (ArrayType) to a pointer to the value type (VectorType).
2164	static RValue EmitLoadOfMatrixLValue(LValue LV, SourceLocation Loc,
2165	CodeGenFunction &CGF) {
2166	assert(LV.getType()->isConstantMatrixType());
2167	Address Addr = MaybeConvertMatrixAddress(Addr: LV.getAddress(CGF), CGF);
2168	LV.setAddress(Addr);
2169	return RValue::get(V: CGF.EmitLoadOfScalar(lvalue: LV, Loc));
2170	}
2171
2172	/// EmitLoadOfLValue - Given an expression that represents a value lvalue, this
2173	/// method emits the address of the lvalue, then loads the result as an rvalue,
2174	/// returning the rvalue.
2175	RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, SourceLocation Loc) {
2176	if (LV.isObjCWeak()) {
2177	// load of a __weak object.
2178	Address AddrWeakObj = LV.getAddress(CGF&: *this);
2179	return RValue::get(V: CGM.getObjCRuntime().EmitObjCWeakRead(CGF&: *this,
2180	AddrWeakObj));
2181	}
2182	if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak) {
2183	// In MRC mode, we do a load+autorelease.
2184	if (!getLangOpts().ObjCAutoRefCount) {
2185	return RValue::get(V: EmitARCLoadWeak(addr: LV.getAddress(CGF&: *this)));
2186	}
2187
2188	// In ARC mode, we load retained and then consume the value.
2189	llvm::Value *Object = EmitARCLoadWeakRetained(addr: LV.getAddress(CGF&: *this));
2190	Object = EmitObjCConsumeObject(T: LV.getType(), Ptr: Object);
2191	return RValue::get(V: Object);
2192	}
2193
2194	if (LV.isSimple()) {
2195	assert(!LV.getType()->isFunctionType());
2196
2197	if (LV.getType()->isConstantMatrixType())
2198	return EmitLoadOfMatrixLValue(LV, Loc, CGF&: *this);
2199
2200	// Everything needs a load.
2201	return RValue::get(V: EmitLoadOfScalar(lvalue: LV, Loc));
2202	}
2203
2204	if (LV.isVectorElt()) {
2205	llvm::LoadInst *Load = Builder.CreateLoad(Addr: LV.getVectorAddress(),
2206	IsVolatile: LV.isVolatileQualified());
2207	return RValue::get(V: Builder.CreateExtractElement(Vec: Load, Idx: LV.getVectorIdx(),
2208	Name: "vecext"));
2209	}
2210
2211	// If this is a reference to a subset of the elements of a vector, either
2212	// shuffle the input or extract/insert them as appropriate.
2213	if (LV.isExtVectorElt()) {
2214	return EmitLoadOfExtVectorElementLValue(V: LV);
2215	}
2216
2217	// Global Register variables always invoke intrinsics
2218	if (LV.isGlobalReg())
2219	return EmitLoadOfGlobalRegLValue(LV);
2220
2221	if (LV.isMatrixElt()) {
2222	llvm::Value *Idx = LV.getMatrixIdx();
2223	if (CGM.getCodeGenOpts().OptimizationLevel > 0) {
2224	const auto *const MatTy = LV.getType()->castAs<ConstantMatrixType>();
2225	llvm::MatrixBuilder MB(Builder);
2226	MB.CreateIndexAssumption(Idx, NumElements: MatTy->getNumElementsFlattened());
2227	}
2228	llvm::LoadInst *Load =
2229	Builder.CreateLoad(Addr: LV.getMatrixAddress(), IsVolatile: LV.isVolatileQualified());
2230	return RValue::get(V: Builder.CreateExtractElement(Vec: Load, Idx, Name: "matrixext"));
2231	}
2232
2233	assert(LV.isBitField() && "Unknown LValue type!");
2234	return EmitLoadOfBitfieldLValue(LV, Loc);
2235	}
2236
2237	RValue CodeGenFunction::EmitLoadOfBitfieldLValue(LValue LV,
2238	SourceLocation Loc) {
2239	const CGBitFieldInfo &Info = LV.getBitFieldInfo();
2240
2241	// Get the output type.
2242	llvm::Type *ResLTy = ConvertType(T: LV.getType());
2243
2244	Address Ptr = LV.getBitFieldAddress();
2245	llvm::Value *Val =
2246	Builder.CreateLoad(Addr: Ptr, IsVolatile: LV.isVolatileQualified(), Name: "bf.load");
2247
2248	bool UseVolatile = LV.isVolatileQualified() &&
2249	Info.VolatileStorageSize != 0 && isAAPCS(TargetInfo: CGM.getTarget());
2250	const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset;
2251	const unsigned StorageSize =
2252	UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
2253	if (Info.IsSigned) {
2254	assert(static_cast<unsigned>(Offset + Info.Size) <= StorageSize);
2255	unsigned HighBits = StorageSize - Offset - Info.Size;
2256	if (HighBits)
2257	Val = Builder.CreateShl(LHS: Val, RHS: HighBits, Name: "bf.shl");
2258	if (Offset + HighBits)
2259	Val = Builder.CreateAShr(LHS: Val, RHS: Offset + HighBits, Name: "bf.ashr");
2260	} else {
2261	if (Offset)
2262	Val = Builder.CreateLShr(LHS: Val, RHS: Offset, Name: "bf.lshr");
2263	if (static_cast<unsigned>(Offset) + Info.Size < StorageSize)
2264	Val = Builder.CreateAnd(
2265	LHS: Val, RHS: llvm::APInt::getLowBitsSet(numBits: StorageSize, loBitsSet: Info.Size), Name: "bf.clear");
2266	}
2267	Val = Builder.CreateIntCast(V: Val, DestTy: ResLTy, isSigned: Info.IsSigned, Name: "bf.cast");
2268	EmitScalarRangeCheck(Value: Val, Ty: LV.getType(), Loc);
2269	return RValue::get(V: Val);
2270	}
2271
2272	// If this is a reference to a subset of the elements of a vector, create an
2273	// appropriate shufflevector.
2274	RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV) {
2275	llvm::Value *Vec = Builder.CreateLoad(Addr: LV.getExtVectorAddress(),
2276	IsVolatile: LV.isVolatileQualified());
2277
2278	// HLSL allows treating scalars as one-element vectors. Converting the scalar
2279	// IR value to a vector here allows the rest of codegen to behave as normal.
2280	if (getLangOpts().HLSL && !Vec->getType()->isVectorTy()) {
2281	llvm::Type *DstTy = llvm::FixedVectorType::get(ElementType: Vec->getType(), NumElts: 1);
2282	llvm::Value *Zero = llvm::Constant::getNullValue(Ty: CGM.Int64Ty);
2283	Vec = Builder.CreateInsertElement(VecTy: DstTy, NewElt: Vec, Idx: Zero, Name: "cast.splat");
2284	}
2285
2286	const llvm::Constant *Elts = LV.getExtVectorElts();
2287
2288	// If the result of the expression is a non-vector type, we must be extracting
2289	// a single element. Just codegen as an extractelement.
2290	const VectorType *ExprVT = LV.getType()->getAs<VectorType>();
2291	if (!ExprVT) {
2292	unsigned InIdx = getAccessedFieldNo(Idx: 0, Elts);
2293	llvm::Value *Elt = llvm::ConstantInt::get(Ty: SizeTy, V: InIdx);
2294	return RValue::get(V: Builder.CreateExtractElement(Vec, Idx: Elt));
2295	}
2296
2297	// Always use shuffle vector to try to retain the original program structure
2298	unsigned NumResultElts = ExprVT->getNumElements();
2299
2300	SmallVector<int, 4> Mask;
2301	for (unsigned i = 0; i != NumResultElts; ++i)
2302	Mask.push_back(Elt: getAccessedFieldNo(Idx: i, Elts));
2303
2304	Vec = Builder.CreateShuffleVector(V: Vec, Mask);
2305	return RValue::get(V: Vec);
2306	}
2307
2308	/// Generates lvalue for partial ext_vector access.
2309	Address CodeGenFunction::EmitExtVectorElementLValue(LValue LV) {
2310	Address VectorAddress = LV.getExtVectorAddress();
2311	QualType EQT = LV.getType()->castAs<VectorType>()->getElementType();
2312	llvm::Type *VectorElementTy = CGM.getTypes().ConvertType(T: EQT);
2313
2314	Address CastToPointerElement = VectorAddress.withElementType(ElemTy: VectorElementTy);
2315
2316	const llvm::Constant *Elts = LV.getExtVectorElts();
2317	unsigned ix = getAccessedFieldNo(Idx: 0, Elts);
2318
2319	Address VectorBasePtrPlusIx =
2320	Builder.CreateConstInBoundsGEP(Addr: CastToPointerElement, Index: ix,
2321	Name: "vector.elt");
2322
2323	return VectorBasePtrPlusIx;
2324	}
2325
2326	/// Load of global gamed gegisters are always calls to intrinsics.
2327	RValue CodeGenFunction::EmitLoadOfGlobalRegLValue(LValue LV) {
2328	assert((LV.getType()->isIntegerType() || LV.getType()->isPointerType()) &&
2329	"Bad type for register variable");
2330	llvm::MDNode *RegName = cast<llvm::MDNode>(
2331	Val: cast<llvm::MetadataAsValue>(Val: LV.getGlobalReg())->getMetadata());
2332
2333	// We accept integer and pointer types only
2334	llvm::Type *OrigTy = CGM.getTypes().ConvertType(T: LV.getType());
2335	llvm::Type *Ty = OrigTy;
2336	if (OrigTy->isPointerTy())
2337	Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy);
2338	llvm::Type *Types[] = { Ty };
2339
2340	llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::read_register, Types);
2341	llvm::Value *Call = Builder.CreateCall(
2342	Callee: F, Args: llvm::MetadataAsValue::get(Context&: Ty->getContext(), MD: RegName));
2343	if (OrigTy->isPointerTy())
2344	Call = Builder.CreateIntToPtr(V: Call, DestTy: OrigTy);
2345	return RValue::get(V: Call);
2346	}
2347
2348	/// EmitStoreThroughLValue - Store the specified rvalue into the specified
2349	/// lvalue, where both are guaranteed to the have the same type, and that type
2350	/// is 'Ty'.
2351	void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst,
2352	bool isInit) {
2353	if (!Dst.isSimple()) {
2354	if (Dst.isVectorElt()) {
2355	// Read/modify/write the vector, inserting the new element.
2356	llvm::Value *Vec = Builder.CreateLoad(Addr: Dst.getVectorAddress(),
2357	IsVolatile: Dst.isVolatileQualified());
2358	auto *IRStoreTy = dyn_cast<llvm::IntegerType>(Val: Vec->getType());
2359	if (IRStoreTy) {
2360	auto *IRVecTy = llvm::FixedVectorType::get(
2361	ElementType: Builder.getInt1Ty(), NumElts: IRStoreTy->getPrimitiveSizeInBits());
2362	Vec = Builder.CreateBitCast(V: Vec, DestTy: IRVecTy);
2363	// iN --> <N x i1>.
2364	}
2365	Vec = Builder.CreateInsertElement(Vec, NewElt: Src.getScalarVal(),
2366	Idx: Dst.getVectorIdx(), Name: "vecins");
2367	if (IRStoreTy) {
2368	// <N x i1> --> <iN>.
2369	Vec = Builder.CreateBitCast(V: Vec, DestTy: IRStoreTy);
2370	}
2371	Builder.CreateStore(Val: Vec, Addr: Dst.getVectorAddress(),
2372	IsVolatile: Dst.isVolatileQualified());
2373	return;
2374	}
2375
2376	// If this is an update of extended vector elements, insert them as
2377	// appropriate.
2378	if (Dst.isExtVectorElt())
2379	return EmitStoreThroughExtVectorComponentLValue(Src, Dst);
2380
2381	if (Dst.isGlobalReg())
2382	return EmitStoreThroughGlobalRegLValue(Src, Dst);
2383
2384	if (Dst.isMatrixElt()) {
2385	llvm::Value *Idx = Dst.getMatrixIdx();
2386	if (CGM.getCodeGenOpts().OptimizationLevel > 0) {
2387	const auto *const MatTy = Dst.getType()->castAs<ConstantMatrixType>();
2388	llvm::MatrixBuilder MB(Builder);
2389	MB.CreateIndexAssumption(Idx, NumElements: MatTy->getNumElementsFlattened());
2390	}
2391	llvm::Instruction *Load = Builder.CreateLoad(Addr: Dst.getMatrixAddress());
2392	llvm::Value *Vec =
2393	Builder.CreateInsertElement(Vec: Load, NewElt: Src.getScalarVal(), Idx, Name: "matins");
2394	Builder.CreateStore(Val: Vec, Addr: Dst.getMatrixAddress(),
2395	IsVolatile: Dst.isVolatileQualified());
2396	return;
2397	}
2398
2399	assert(Dst.isBitField() && "Unknown LValue type");
2400	return EmitStoreThroughBitfieldLValue(Src, Dst);
2401	}
2402
2403	// There's special magic for assigning into an ARC-qualified l-value.
2404	if (Qualifiers::ObjCLifetime Lifetime = Dst.getQuals().getObjCLifetime()) {
2405	switch (Lifetime) {
2406	case Qualifiers::OCL_None:
2407	llvm_unreachable("present but none");
2408
2409	case Qualifiers::OCL_ExplicitNone:
2410	// nothing special
2411	break;
2412
2413	case Qualifiers::OCL_Strong:
2414	if (isInit) {
2415	Src = RValue::get(V: EmitARCRetain(type: Dst.getType(), value: Src.getScalarVal()));
2416	break;
2417	}
2418	EmitARCStoreStrong(lvalue: Dst, value: Src.getScalarVal(), /*ignore*/ resultIgnored: true);
2419	return;
2420
2421	case Qualifiers::OCL_Weak:
2422	if (isInit)
2423	// Initialize and then skip the primitive store.
2424	EmitARCInitWeak(addr: Dst.getAddress(CGF&: *this), value: Src.getScalarVal());
2425	else
2426	EmitARCStoreWeak(addr: Dst.getAddress(CGF&: *this), value: Src.getScalarVal(),
2427	/*ignore*/ ignored: true);
2428	return;
2429
2430	case Qualifiers::OCL_Autoreleasing:
2431	Src = RValue::get(V: EmitObjCExtendObjectLifetime(T: Dst.getType(),
2432	Ptr: Src.getScalarVal()));
2433	// fall into the normal path
2434	break;
2435	}
2436	}
2437
2438	if (Dst.isObjCWeak() && !Dst.isNonGC()) {
2439	// load of a __weak object.
2440	Address LvalueDst = Dst.getAddress(CGF&: *this);
2441	llvm::Value *src = Src.getScalarVal();
2442	CGM.getObjCRuntime().EmitObjCWeakAssign(CGF&: *this, src, dest: LvalueDst);
2443	return;
2444	}
2445
2446	if (Dst.isObjCStrong() && !Dst.isNonGC()) {
2447	// load of a __strong object.
2448	Address LvalueDst = Dst.getAddress(CGF&: *this);
2449	llvm::Value *src = Src.getScalarVal();
2450	if (Dst.isObjCIvar()) {
2451	assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL");
2452	llvm::Type *ResultType = IntPtrTy;
2453	Address dst = EmitPointerWithAlignment(E: Dst.getBaseIvarExp());
2454	llvm::Value *RHS = dst.getPointer();
2455	RHS = Builder.CreatePtrToInt(V: RHS, DestTy: ResultType, Name: "sub.ptr.rhs.cast");
2456	llvm::Value *LHS =
2457	Builder.CreatePtrToInt(V: LvalueDst.getPointer(), DestTy: ResultType,
2458	Name: "sub.ptr.lhs.cast");
2459	llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, Name: "ivar.offset");
2460	CGM.getObjCRuntime().EmitObjCIvarAssign(CGF&: *this, src, dest: dst,
2461	ivarOffset: BytesBetween);
2462	} else if (Dst.isGlobalObjCRef()) {
2463	CGM.getObjCRuntime().EmitObjCGlobalAssign(CGF&: *this, src, dest: LvalueDst,
2464	threadlocal: Dst.isThreadLocalRef());
2465	}
2466	else
2467	CGM.getObjCRuntime().EmitObjCStrongCastAssign(CGF&: *this, src, dest: LvalueDst);
2468	return;
2469	}
2470
2471	assert(Src.isScalar() && "Can't emit an agg store with this method");
2472	EmitStoreOfScalar(value: Src.getScalarVal(), lvalue: Dst, isInit);
2473	}
2474
2475	void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
2476	llvm::Value **Result) {
2477	const CGBitFieldInfo &Info = Dst.getBitFieldInfo();
2478	llvm::Type *ResLTy = ConvertTypeForMem(T: Dst.getType());
2479	Address Ptr = Dst.getBitFieldAddress();
2480
2481	// Get the source value, truncated to the width of the bit-field.
2482	llvm::Value *SrcVal = Src.getScalarVal();
2483
2484	// Cast the source to the storage type and shift it into place.
2485	SrcVal = Builder.CreateIntCast(V: SrcVal, DestTy: Ptr.getElementType(),
2486	/*isSigned=*/false);
2487	llvm::Value *MaskedVal = SrcVal;
2488
2489	const bool UseVolatile =
2490	CGM.getCodeGenOpts().AAPCSBitfieldWidth && Dst.isVolatileQualified() &&
2491	Info.VolatileStorageSize != 0 && isAAPCS(TargetInfo: CGM.getTarget());
2492	const unsigned StorageSize =
2493	UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
2494	const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset;
2495	// See if there are other bits in the bitfield's storage we'll need to load
2496	// and mask together with source before storing.
2497	if (StorageSize != Info.Size) {
2498	assert(StorageSize > Info.Size && "Invalid bitfield size.");
2499	llvm::Value *Val =
2500	Builder.CreateLoad(Addr: Ptr, IsVolatile: Dst.isVolatileQualified(), Name: "bf.load");
2501
2502	// Mask the source value as needed.
2503	if (!hasBooleanRepresentation(Ty: Dst.getType()))
2504	SrcVal = Builder.CreateAnd(
2505	LHS: SrcVal, RHS: llvm::APInt::getLowBitsSet(numBits: StorageSize, loBitsSet: Info.Size),
2506	Name: "bf.value");
2507	MaskedVal = SrcVal;
2508	if (Offset)
2509	SrcVal = Builder.CreateShl(LHS: SrcVal, RHS: Offset, Name: "bf.shl");
2510
2511	// Mask out the original value.
2512	Val = Builder.CreateAnd(
2513	LHS: Val, RHS: ~llvm::APInt::getBitsSet(numBits: StorageSize, loBit: Offset, hiBit: Offset + Info.Size),
2514	Name: "bf.clear");
2515
2516	// Or together the unchanged values and the source value.
2517	SrcVal = Builder.CreateOr(LHS: Val, RHS: SrcVal, Name: "bf.set");
2518	} else {
2519	assert(Offset == 0);
2520	// According to the AACPS:
2521	// When a volatile bit-field is written, and its container does not overlap
2522	// with any non-bit-field member, its container must be read exactly once
2523	// and written exactly once using the access width appropriate to the type
2524	// of the container. The two accesses are not atomic.
2525	if (Dst.isVolatileQualified() && isAAPCS(TargetInfo: CGM.getTarget()) &&
2526	CGM.getCodeGenOpts().ForceAAPCSBitfieldLoad)
2527	Builder.CreateLoad(Addr: Ptr, IsVolatile: true, Name: "bf.load");
2528	}
2529
2530	// Write the new value back out.
2531	Builder.CreateStore(Val: SrcVal, Addr: Ptr, IsVolatile: Dst.isVolatileQualified());
2532
2533	// Return the new value of the bit-field, if requested.
2534	if (Result) {
2535	llvm::Value *ResultVal = MaskedVal;
2536
2537	// Sign extend the value if needed.
2538	if (Info.IsSigned) {
2539	assert(Info.Size <= StorageSize);
2540	unsigned HighBits = StorageSize - Info.Size;
2541	if (HighBits) {
2542	ResultVal = Builder.CreateShl(LHS: ResultVal, RHS: HighBits, Name: "bf.result.shl");
2543	ResultVal = Builder.CreateAShr(LHS: ResultVal, RHS: HighBits, Name: "bf.result.ashr");
2544	}
2545	}
2546
2547	ResultVal = Builder.CreateIntCast(V: ResultVal, DestTy: ResLTy, isSigned: Info.IsSigned,
2548	Name: "bf.result.cast");
2549	*Result = EmitFromMemory(Value: ResultVal, Ty: Dst.getType());
2550	}
2551	}
2552
2553	void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
2554	LValue Dst) {
2555	// HLSL allows storing to scalar values through ExtVector component LValues.
2556	// To support this we need to handle the case where the destination address is
2557	// a scalar.
2558	Address DstAddr = Dst.getExtVectorAddress();
2559	if (!DstAddr.getElementType()->isVectorTy()) {
2560	assert(!Dst.getType()->isVectorType() &&
2561	"this should only occur for non-vector l-values");
2562	Builder.CreateStore(Val: Src.getScalarVal(), Addr: DstAddr, IsVolatile: Dst.isVolatileQualified());
2563	return;
2564	}
2565
2566	// This access turns into a read/modify/write of the vector. Load the input
2567	// value now.
2568	llvm::Value *Vec = Builder.CreateLoad(Addr: DstAddr, IsVolatile: Dst.isVolatileQualified());
2569	const llvm::Constant *Elts = Dst.getExtVectorElts();
2570
2571	llvm::Value *SrcVal = Src.getScalarVal();
2572
2573	if (const VectorType *VTy = Dst.getType()->getAs<VectorType>()) {
2574	unsigned NumSrcElts = VTy->getNumElements();
2575	unsigned NumDstElts =
2576	cast<llvm::FixedVectorType>(Val: Vec->getType())->getNumElements();
2577	if (NumDstElts == NumSrcElts) {
2578	// Use shuffle vector is the src and destination are the same number of
2579	// elements and restore the vector mask since it is on the side it will be
2580	// stored.
2581	SmallVector<int, 4> Mask(NumDstElts);
2582	for (unsigned i = 0; i != NumSrcElts; ++i)
2583	Mask[getAccessedFieldNo(Idx: i, Elts)] = i;
2584
2585	Vec = Builder.CreateShuffleVector(V: SrcVal, Mask);
2586	} else if (NumDstElts > NumSrcElts) {
2587	// Extended the source vector to the same length and then shuffle it
2588	// into the destination.
2589	// FIXME: since we're shuffling with undef, can we just use the indices
2590	// into that? This could be simpler.
2591	SmallVector<int, 4> ExtMask;
2592	for (unsigned i = 0; i != NumSrcElts; ++i)
2593	ExtMask.push_back(Elt: i);
2594	ExtMask.resize(N: NumDstElts, NV: -1);
2595	llvm::Value *ExtSrcVal = Builder.CreateShuffleVector(V: SrcVal, Mask: ExtMask);
2596	// build identity
2597	SmallVector<int, 4> Mask;
2598	for (unsigned i = 0; i != NumDstElts; ++i)
2599	Mask.push_back(Elt: i);
2600
2601	// When the vector size is odd and .odd or .hi is used, the last element
2602	// of the Elts constant array will be one past the size of the vector.
2603	// Ignore the last element here, if it is greater than the mask size.
2604	if (getAccessedFieldNo(Idx: NumSrcElts - 1, Elts) == Mask.size())
2605	NumSrcElts--;
2606
2607	// modify when what gets shuffled in
2608	for (unsigned i = 0; i != NumSrcElts; ++i)
2609	Mask[getAccessedFieldNo(Idx: i, Elts)] = i + NumDstElts;
2610	Vec = Builder.CreateShuffleVector(V1: Vec, V2: ExtSrcVal, Mask);
2611	} else {
2612	// We should never shorten the vector
2613	llvm_unreachable("unexpected shorten vector length");
2614	}
2615	} else {
2616	// If the Src is a scalar (not a vector), and the target is a vector it must
2617	// be updating one element.
2618	unsigned InIdx = getAccessedFieldNo(Idx: 0, Elts);
2619	llvm::Value *Elt = llvm::ConstantInt::get(Ty: SizeTy, V: InIdx);
2620	Vec = Builder.CreateInsertElement(Vec, NewElt: SrcVal, Idx: Elt);
2621	}
2622
2623	Builder.CreateStore(Val: Vec, Addr: Dst.getExtVectorAddress(),
2624	IsVolatile: Dst.isVolatileQualified());
2625	}
2626
2627	/// Store of global named registers are always calls to intrinsics.
2628	void CodeGenFunction::EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst) {
2629	assert((Dst.getType()->isIntegerType() || Dst.getType()->isPointerType()) &&
2630	"Bad type for register variable");
2631	llvm::MDNode *RegName = cast<llvm::MDNode>(
2632	Val: cast<llvm::MetadataAsValue>(Val: Dst.getGlobalReg())->getMetadata());
2633	assert(RegName && "Register LValue is not metadata");
2634
2635	// We accept integer and pointer types only
2636	llvm::Type *OrigTy = CGM.getTypes().ConvertType(T: Dst.getType());
2637	llvm::Type *Ty = OrigTy;
2638	if (OrigTy->isPointerTy())
2639	Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy);
2640	llvm::Type *Types[] = { Ty };
2641
2642	llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::write_register, Types);
2643	llvm::Value *Value = Src.getScalarVal();
2644	if (OrigTy->isPointerTy())
2645	Value = Builder.CreatePtrToInt(V: Value, DestTy: Ty);
2646	Builder.CreateCall(
2647	Callee: F, Args: {llvm::MetadataAsValue::get(Context&: Ty->getContext(), MD: RegName), Value});
2648	}
2649
2650	// setObjCGCLValueClass - sets class of the lvalue for the purpose of
2651	// generating write-barries API. It is currently a global, ivar,
2652	// or neither.
2653	static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,
2654	LValue &LV,
2655	bool IsMemberAccess=false) {
2656	if (Ctx.getLangOpts().getGC() == LangOptions::NonGC)
2657	return;
2658
2659	if (isa<ObjCIvarRefExpr>(Val: E)) {
2660	QualType ExpTy = E->getType();
2661	if (IsMemberAccess && ExpTy->isPointerType()) {
2662	// If ivar is a structure pointer, assigning to field of
2663	// this struct follows gcc's behavior and makes it a non-ivar
2664	// writer-barrier conservatively.
2665	ExpTy = ExpTy->castAs<PointerType>()->getPointeeType();
2666	if (ExpTy->isRecordType()) {
2667	LV.setObjCIvar(false);
2668	return;
2669	}
2670	}
2671	LV.setObjCIvar(true);
2672	auto *Exp = cast<ObjCIvarRefExpr>(Val: const_cast<Expr *>(E));
2673	LV.setBaseIvarExp(Exp->getBase());
2674	LV.setObjCArray(E->getType()->isArrayType());
2675	return;
2676	}
2677
2678	if (const auto *Exp = dyn_cast<DeclRefExpr>(Val: E)) {
2679	if (const auto *VD = dyn_cast<VarDecl>(Val: Exp->getDecl())) {
2680	if (VD->hasGlobalStorage()) {
2681	LV.setGlobalObjCRef(true);
2682	LV.setThreadLocalRef(VD->getTLSKind() != VarDecl::TLS_None);
2683	}
2684	}
2685	LV.setObjCArray(E->getType()->isArrayType());
2686	return;
2687	}
2688
2689	if (const auto *Exp = dyn_cast<UnaryOperator>(Val: E)) {
2690	setObjCGCLValueClass(Ctx, E: Exp->getSubExpr(), LV, IsMemberAccess);
2691	return;
2692	}
2693
2694	if (const auto *Exp = dyn_cast<ParenExpr>(Val: E)) {
2695	setObjCGCLValueClass(Ctx, E: Exp->getSubExpr(), LV, IsMemberAccess);
2696	if (LV.isObjCIvar()) {
2697	// If cast is to a structure pointer, follow gcc's behavior and make it
2698	// a non-ivar write-barrier.
2699	QualType ExpTy = E->getType();
2700	if (ExpTy->isPointerType())
2701	ExpTy = ExpTy->castAs<PointerType>()->getPointeeType();
2702	if (ExpTy->isRecordType())
2703	LV.setObjCIvar(false);
2704	}
2705	return;
2706	}
2707
2708	if (const auto *Exp = dyn_cast<GenericSelectionExpr>(Val: E)) {
2709	setObjCGCLValueClass(Ctx, E: Exp->getResultExpr(), LV);
2710	return;
2711	}
2712
2713	if (const auto *Exp = dyn_cast<ImplicitCastExpr>(Val: E)) {
2714	setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2715	return;
2716	}
2717
2718	if (const auto *Exp = dyn_cast<CStyleCastExpr>(Val: E)) {
2719	setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2720	return;
2721	}
2722
2723	if (const auto *Exp = dyn_cast<ObjCBridgedCastExpr>(Val: E)) {
2724	setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2725	return;
2726	}
2727
2728	if (const auto *Exp = dyn_cast<ArraySubscriptExpr>(Val: E)) {
2729	setObjCGCLValueClass(Ctx, E: Exp->getBase(), LV);
2730	if (LV.isObjCIvar() && !LV.isObjCArray())
2731	// Using array syntax to assigning to what an ivar points to is not
2732	// same as assigning to the ivar itself. {id *Names;} Names[i] = 0;
2733	LV.setObjCIvar(false);
2734	else if (LV.isGlobalObjCRef() && !LV.isObjCArray())
2735	// Using array syntax to assigning to what global points to is not
2736	// same as assigning to the global itself. {id *G;} G[i] = 0;
2737	LV.setGlobalObjCRef(false);
2738	return;
2739	}
2740
2741	if (const auto *Exp = dyn_cast<MemberExpr>(Val: E)) {
2742	setObjCGCLValueClass(Ctx, E: Exp->getBase(), LV, IsMemberAccess: true);
2743	// We don't know if member is an 'ivar', but this flag is looked at
2744	// only in the context of LV.isObjCIvar().
2745	LV.setObjCArray(E->getType()->isArrayType());
2746	return;
2747	}
2748	}
2749
2750	static LValue EmitThreadPrivateVarDeclLValue(
2751	CodeGenFunction &CGF, const VarDecl *VD, QualType T, Address Addr,
2752	llvm::Type *RealVarTy, SourceLocation Loc) {
2753	if (CGF.CGM.getLangOpts().OpenMPIRBuilder)
2754	Addr = CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate(
2755	CGF, VD, VDAddr: Addr, Loc);
2756	else
2757	Addr =
2758	CGF.CGM.getOpenMPRuntime().getAddrOfThreadPrivate(CGF, VD, VDAddr: Addr, Loc);
2759
2760	Addr = Addr.withElementType(ElemTy: RealVarTy);
2761	return CGF.MakeAddrLValue(Addr, T, Source: AlignmentSource::Decl);
2762	}
2763
2764	static Address emitDeclTargetVarDeclLValue(CodeGenFunction &CGF,
2765	const VarDecl *VD, QualType T) {
2766	std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2767	OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2768	// Return an invalid address if variable is MT_To (or MT_Enter starting with
2769	// OpenMP 5.2) and unified memory is not enabled. For all other cases: MT_Link
2770	// and MT_To (or MT_Enter) with unified memory, return a valid address.
2771	if (!Res || ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
2772	*Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
2773	!CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory()))
2774	return Address::invalid();
2775	assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
2776	((*Res == OMPDeclareTargetDeclAttr::MT_To ||
2777	*Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
2778	CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) &&
2779	"Expected link clause OR to clause with unified memory enabled.");
2780	QualType PtrTy = CGF.getContext().getPointerType(VD->getType());
2781	Address Addr = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
2782	return CGF.EmitLoadOfPointer(Ptr: Addr, PtrTy: PtrTy->castAs<PointerType>());
2783	}
2784
2785	Address
2786	CodeGenFunction::EmitLoadOfReference(LValue RefLVal,
2787	LValueBaseInfo *PointeeBaseInfo,
2788	TBAAAccessInfo *PointeeTBAAInfo) {
2789	llvm::LoadInst *Load =
2790	Builder.CreateLoad(Addr: RefLVal.getAddress(CGF&: *this), IsVolatile: RefLVal.isVolatile());
2791	CGM.DecorateInstructionWithTBAA(Inst: Load, TBAAInfo: RefLVal.getTBAAInfo());
2792
2793	QualType PointeeType = RefLVal.getType()->getPointeeType();
2794	CharUnits Align = CGM.getNaturalTypeAlignment(
2795	T: PointeeType, BaseInfo: PointeeBaseInfo, TBAAInfo: PointeeTBAAInfo,
2796	/* forPointeeType= */ true);
2797	return Address(Load, ConvertTypeForMem(T: PointeeType), Align);
2798	}
2799
2800	LValue CodeGenFunction::EmitLoadOfReferenceLValue(LValue RefLVal) {
2801	LValueBaseInfo PointeeBaseInfo;
2802	TBAAAccessInfo PointeeTBAAInfo;
2803	Address PointeeAddr = EmitLoadOfReference(RefLVal, PointeeBaseInfo: &PointeeBaseInfo,
2804	PointeeTBAAInfo: &PointeeTBAAInfo);
2805	return MakeAddrLValue(Addr: PointeeAddr, T: RefLVal.getType()->getPointeeType(),
2806	BaseInfo: PointeeBaseInfo, TBAAInfo: PointeeTBAAInfo);
2807	}
2808
2809	Address CodeGenFunction::EmitLoadOfPointer(Address Ptr,
2810	const PointerType *PtrTy,
2811	LValueBaseInfo *BaseInfo,
2812	TBAAAccessInfo *TBAAInfo) {
2813	llvm::Value *Addr = Builder.CreateLoad(Addr: Ptr);
2814	return Address(Addr, ConvertTypeForMem(T: PtrTy->getPointeeType()),
2815	CGM.getNaturalTypeAlignment(T: PtrTy->getPointeeType(), BaseInfo,
2816	TBAAInfo,
2817	/*forPointeeType=*/true));
2818	}
2819
2820	LValue CodeGenFunction::EmitLoadOfPointerLValue(Address PtrAddr,
2821	const PointerType *PtrTy) {
2822	LValueBaseInfo BaseInfo;
2823	TBAAAccessInfo TBAAInfo;
2824	Address Addr = EmitLoadOfPointer(Ptr: PtrAddr, PtrTy, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
2825	return MakeAddrLValue(Addr, T: PtrTy->getPointeeType(), BaseInfo, TBAAInfo);
2826	}
2827
2828	static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF,
2829	const Expr *E, const VarDecl *VD) {
2830	QualType T = E->getType();
2831
2832	// If it's thread_local, emit a call to its wrapper function instead.
2833	if (VD->getTLSKind() == VarDecl::TLS_Dynamic &&
2834	CGF.CGM.getCXXABI().usesThreadWrapperFunction(VD))
2835	return CGF.CGM.getCXXABI().EmitThreadLocalVarDeclLValue(CGF, VD, LValType: T);
2836	// Check if the variable is marked as declare target with link clause in
2837	// device codegen.
2838	if (CGF.getLangOpts().OpenMPIsTargetDevice) {
2839	Address Addr = emitDeclTargetVarDeclLValue(CGF, VD, T);
2840	if (Addr.isValid())
2841	return CGF.MakeAddrLValue(Addr, T, Source: AlignmentSource::Decl);
2842	}
2843
2844	llvm::Value *V = CGF.CGM.GetAddrOfGlobalVar(D: VD);
2845
2846	if (VD->getTLSKind() != VarDecl::TLS_None)
2847	V = CGF.Builder.CreateThreadLocalAddress(Ptr: V);
2848
2849	llvm::Type *RealVarTy = CGF.getTypes().ConvertTypeForMem(T: VD->getType());
2850	CharUnits Alignment = CGF.getContext().getDeclAlign(VD);
2851	Address Addr(V, RealVarTy, Alignment);
2852	// Emit reference to the private copy of the variable if it is an OpenMP
2853	// threadprivate variable.
2854	if (CGF.getLangOpts().OpenMP && !CGF.getLangOpts().OpenMPSimd &&
2855	VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
2856	return EmitThreadPrivateVarDeclLValue(CGF, VD, T, Addr, RealVarTy,
2857	Loc: E->getExprLoc());
2858	}
2859	LValue LV = VD->getType()->isReferenceType() ?
2860	CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(),
2861	AlignmentSource::Decl) :
2862	CGF.MakeAddrLValue(Addr, T, Source: AlignmentSource::Decl);
2863	setObjCGCLValueClass(Ctx: CGF.getContext(), E, LV);
2864	return LV;
2865	}
2866
2867	static llvm::Constant *EmitFunctionDeclPointer(CodeGenModule &CGM,
2868	GlobalDecl GD) {
2869	const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
2870	if (FD->hasAttr<WeakRefAttr>()) {
2871	ConstantAddress aliasee = CGM.GetWeakRefReference(FD);
2872	return aliasee.getPointer();
2873	}
2874
2875	llvm::Constant *V = CGM.GetAddrOfFunction(GD);
2876	return V;
2877	}
2878
2879	static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF, const Expr *E,
2880	GlobalDecl GD) {
2881	const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
2882	llvm::Value *V = EmitFunctionDeclPointer(CGM&: CGF.CGM, GD);
2883	CharUnits Alignment = CGF.getContext().getDeclAlign(FD);
2884	return CGF.MakeAddrLValue(V, T: E->getType(), Alignment,
2885	Source: AlignmentSource::Decl);
2886	}
2887
2888	static LValue EmitCapturedFieldLValue(CodeGenFunction &CGF, const FieldDecl *FD,
2889	llvm::Value *ThisValue) {
2890
2891	return CGF.EmitLValueForLambdaField(Field: FD, ThisValue);
2892	}
2893
2894	/// Named Registers are named metadata pointing to the register name
2895	/// which will be read from/written to as an argument to the intrinsic
2896	/// @llvm.read/write_register.
2897	/// So far, only the name is being passed down, but other options such as
2898	/// register type, allocation type or even optimization options could be
2899	/// passed down via the metadata node.
2900	static LValue EmitGlobalNamedRegister(const VarDecl *VD, CodeGenModule &CGM) {
2901	SmallString<64> Name("llvm.named.register.");
2902	AsmLabelAttr *Asm = VD->getAttr<AsmLabelAttr>();
2903	assert(Asm->getLabel().size() < 64-Name.size() &&
2904	"Register name too big");
2905	Name.append(Asm->getLabel());
2906	llvm::NamedMDNode *M =
2907	CGM.getModule().getOrInsertNamedMetadata(Name);
2908	if (M->getNumOperands() == 0) {
2909	llvm::MDString *Str = llvm::MDString::get(CGM.getLLVMContext(),
2910	Asm->getLabel());
2911	llvm::Metadata *Ops[] = {Str};
2912	M->addOperand(M: llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: Ops));
2913	}
2914
2915	CharUnits Alignment = CGM.getContext().getDeclAlign(VD);
2916
2917	llvm::Value *Ptr =
2918	llvm::MetadataAsValue::get(Context&: CGM.getLLVMContext(), MD: M->getOperand(i: 0));
2919	return LValue::MakeGlobalReg(V: Ptr, alignment: Alignment, type: VD->getType());
2920	}
2921
2922	/// Determine whether we can emit a reference to \p VD from the current
2923	/// context, despite not necessarily having seen an odr-use of the variable in
2924	/// this context.
2925	static bool canEmitSpuriousReferenceToVariable(CodeGenFunction &CGF,
2926	const DeclRefExpr *E,
2927	const VarDecl *VD) {
2928	// For a variable declared in an enclosing scope, do not emit a spurious
2929	// reference even if we have a capture, as that will emit an unwarranted
2930	// reference to our capture state, and will likely generate worse code than
2931	// emitting a local copy.
2932	if (E->refersToEnclosingVariableOrCapture())
2933	return false;
2934
2935	// For a local declaration declared in this function, we can always reference
2936	// it even if we don't have an odr-use.
2937	if (VD->hasLocalStorage()) {
2938	return VD->getDeclContext() ==
2939	dyn_cast_or_null<DeclContext>(Val: CGF.CurCodeDecl);
2940	}
2941
2942	// For a global declaration, we can emit a reference to it if we know
2943	// for sure that we are able to emit a definition of it.
2944	VD = VD->getDefinition(C&: CGF.getContext());
2945	if (!VD)
2946	return false;
2947
2948	// Don't emit a spurious reference if it might be to a variable that only
2949	// exists on a different device / target.
2950	// FIXME: This is unnecessarily broad. Check whether this would actually be a
2951	// cross-target reference.
2952	if (CGF.getLangOpts().OpenMP || CGF.getLangOpts().CUDA ||
2953	CGF.getLangOpts().OpenCL) {
2954	return false;
2955	}
2956
2957	// We can emit a spurious reference only if the linkage implies that we'll
2958	// be emitting a non-interposable symbol that will be retained until link
2959	// time.
2960	switch (CGF.CGM.getLLVMLinkageVarDefinition(VD)) {
2961	case llvm::GlobalValue::ExternalLinkage:
2962	case llvm::GlobalValue::LinkOnceODRLinkage:
2963	case llvm::GlobalValue::WeakODRLinkage:
2964	case llvm::GlobalValue::InternalLinkage:
2965	case llvm::GlobalValue::PrivateLinkage:
2966	return true;
2967	default:
2968	return false;
2969	}
2970	}
2971
2972	LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
2973	const NamedDecl *ND = E->getDecl();
2974	QualType T = E->getType();
2975
2976	assert(E->isNonOdrUse() != NOUR_Unevaluated &&
2977	"should not emit an unevaluated operand");
2978
2979	if (const auto *VD = dyn_cast<VarDecl>(ND)) {
2980	// Global Named registers access via intrinsics only
2981	if (VD->getStorageClass() == SC_Register &&
2982	VD->hasAttr<AsmLabelAttr>() && !VD->isLocalVarDecl())
2983	return EmitGlobalNamedRegister(VD, CGM);
2984
2985	// If this DeclRefExpr does not constitute an odr-use of the variable,
2986	// we're not permitted to emit a reference to it in general, and it might
2987	// not be captured if capture would be necessary for a use. Emit the
2988	// constant value directly instead.
2989	if (E->isNonOdrUse() == NOUR_Constant &&
2990	(VD->getType()->isReferenceType() ||
2991	!canEmitSpuriousReferenceToVariable(*this, E, VD))) {
2992	VD->getAnyInitializer(VD);
2993	llvm::Constant *Val = ConstantEmitter(*this).emitAbstract(
2994	E->getLocation(), *VD->evaluateValue(), VD->getType());
2995	assert(Val && "failed to emit constant expression");
2996
2997	Address Addr = Address::invalid();
2998	if (!VD->getType()->isReferenceType()) {
2999	// Spill the constant value to a global.
3000	Addr = CGM.createUnnamedGlobalFrom(D: *VD, Constant: Val,
3001	Align: getContext().getDeclAlign(D: VD));
3002	llvm::Type *VarTy = getTypes().ConvertTypeForMem(T: VD->getType());
3003	auto *PTy = llvm::PointerType::get(
3004	VarTy, getTypes().getTargetAddressSpace(T: VD->getType()));
3005	Addr = Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, PTy, VarTy);
3006	} else {
3007	// Should we be using the alignment of the constant pointer we emitted?
3008	CharUnits Alignment =
3009	CGM.getNaturalTypeAlignment(T: E->getType(),
3010	/* BaseInfo= */ nullptr,
3011	/* TBAAInfo= */ nullptr,
3012	/* forPointeeType= */ true);
3013	Addr = Address(Val, ConvertTypeForMem(T: E->getType()), Alignment);
3014	}
3015	return MakeAddrLValue(Addr, T, Source: AlignmentSource::Decl);
3016	}
3017
3018	// FIXME: Handle other kinds of non-odr-use DeclRefExprs.
3019
3020	// Check for captured variables.
3021	if (E->refersToEnclosingVariableOrCapture()) {
3022	VD = VD->getCanonicalDecl();
3023	if (auto *FD = LambdaCaptureFields.lookup(VD))
3024	return EmitCapturedFieldLValue(*this, FD, CXXABIThisValue);
3025	if (CapturedStmtInfo) {
3026	auto I = LocalDeclMap.find(VD);
3027	if (I != LocalDeclMap.end()) {
3028	LValue CapLVal;
3029	if (VD->getType()->isReferenceType())
3030	CapLVal = EmitLoadOfReferenceLValue(I->second, VD->getType(),
3031	AlignmentSource::Decl);
3032	else
3033	CapLVal = MakeAddrLValue(I->second, T);
3034	// Mark lvalue as nontemporal if the variable is marked as nontemporal
3035	// in simd context.
3036	if (getLangOpts().OpenMP &&
3037	CGM.getOpenMPRuntime().isNontemporalDecl(VD: VD))
3038	CapLVal.setNontemporal(/*Value=*/true);
3039	return CapLVal;
3040	}
3041	LValue CapLVal =
3042	EmitCapturedFieldLValue(*this, CapturedStmtInfo->lookup(VD: VD),
3043	CapturedStmtInfo->getContextValue());
3044	Address LValueAddress = CapLVal.getAddress(CGF&: *this);
3045	CapLVal = MakeAddrLValue(
3046	Addr: Address(LValueAddress.getPointer(), LValueAddress.getElementType(),
3047	getContext().getDeclAlign(D: VD)),
3048	T: CapLVal.getType(), BaseInfo: LValueBaseInfo(AlignmentSource::Decl),
3049	TBAAInfo: CapLVal.getTBAAInfo());
3050	// Mark lvalue as nontemporal if the variable is marked as nontemporal
3051	// in simd context.
3052	if (getLangOpts().OpenMP &&
3053	CGM.getOpenMPRuntime().isNontemporalDecl(VD: VD))
3054	CapLVal.setNontemporal(/*Value=*/true);
3055	return CapLVal;
3056	}
3057
3058	assert(isa<BlockDecl>(CurCodeDecl));
3059	Address addr = GetAddrOfBlockDecl(var: VD);
3060	return MakeAddrLValue(Addr: addr, T, Source: AlignmentSource::Decl);
3061	}
3062	}
3063
3064	// FIXME: We should be able to assert this for FunctionDecls as well!
3065	// FIXME: We should be able to assert this for all DeclRefExprs, not just
3066	// those with a valid source location.
3067	assert((ND->isUsed(false) || !isa<VarDecl>(ND) || E->isNonOdrUse() ||
3068	!E->getLocation().isValid()) &&
3069	"Should not use decl without marking it used!");
3070
3071	if (ND->hasAttr<WeakRefAttr>()) {
3072	const auto *VD = cast<ValueDecl>(Val: ND);
3073	ConstantAddress Aliasee = CGM.GetWeakRefReference(VD: VD);
3074	return MakeAddrLValue(Addr: Aliasee, T, Source: AlignmentSource::Decl);
3075	}
3076
3077	if (const auto *VD = dyn_cast<VarDecl>(ND)) {
3078	// Check if this is a global variable.
3079	if (VD->hasLinkage() || VD->isStaticDataMember())
3080	return EmitGlobalVarDeclLValue(*this, E, VD);
3081
3082	Address addr = Address::invalid();
3083
3084	// The variable should generally be present in the local decl map.
3085	auto iter = LocalDeclMap.find(VD);
3086	if (iter != LocalDeclMap.end()) {
3087	addr = iter->second;
3088
3089	// Otherwise, it might be static local we haven't emitted yet for
3090	// some reason; most likely, because it's in an outer function.
3091	} else if (VD->isStaticLocal()) {
3092	llvm::Constant *var = CGM.getOrCreateStaticVarDecl(
3093	D: *VD, Linkage: CGM.getLLVMLinkageVarDefinition(VD: VD));
3094	addr = Address(
3095	var, ConvertTypeForMem(T: VD->getType()), getContext().getDeclAlign(D: VD));
3096
3097	// No other cases for now.
3098	} else {
3099	llvm_unreachable("DeclRefExpr for Decl not entered in LocalDeclMap?");
3100	}
3101
3102	// Handle threadlocal function locals.
3103	if (VD->getTLSKind() != VarDecl::TLS_None)
3104	addr = addr.withPointer(
3105	NewPointer: Builder.CreateThreadLocalAddress(Ptr: addr.getPointer()), IsKnownNonNull: NotKnownNonNull);
3106
3107	// Check for OpenMP threadprivate variables.
3108	if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd &&
3109	VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
3110	return EmitThreadPrivateVarDeclLValue(
3111	*this, VD, T, addr, getTypes().ConvertTypeForMem(T: VD->getType()),
3112	E->getExprLoc());
3113	}
3114
3115	// Drill into block byref variables.
3116	bool isBlockByref = VD->isEscapingByref();
3117	if (isBlockByref) {
3118	addr = emitBlockByrefAddress(addr, VD);
3119	}
3120
3121	// Drill into reference types.
3122	LValue LV = VD->getType()->isReferenceType() ?
3123	EmitLoadOfReferenceLValue(addr, VD->getType(), AlignmentSource::Decl) :
3124	MakeAddrLValue(Addr: addr, T, Source: AlignmentSource::Decl);
3125
3126	bool isLocalStorage = VD->hasLocalStorage();
3127
3128	bool NonGCable = isLocalStorage &&
3129	!VD->getType()->isReferenceType() &&
3130	!isBlockByref;
3131	if (NonGCable) {
3132	LV.getQuals().removeObjCGCAttr();
3133	LV.setNonGC(true);
3134	}
3135
3136	bool isImpreciseLifetime =
3137	(isLocalStorage && !VD->hasAttr<ObjCPreciseLifetimeAttr>());
3138	if (isImpreciseLifetime)
3139	LV.setARCPreciseLifetime(ARCImpreciseLifetime);
3140	setObjCGCLValueClass(getContext(), E, LV);
3141	return LV;
3142	}
3143
3144	if (const auto *FD = dyn_cast<FunctionDecl>(ND)) {
3145	LValue LV = EmitFunctionDeclLValue(*this, E, FD);
3146
3147	// Emit debuginfo for the function declaration if the target wants to.
3148	if (getContext().getTargetInfo().allowDebugInfoForExternalRef()) {
3149	if (CGDebugInfo *DI = CGM.getModuleDebugInfo()) {
3150	auto *Fn =
3151	cast<llvm::Function>(Val: LV.getPointer(CGF&: *this)->stripPointerCasts());
3152	if (!Fn->getSubprogram())
3153	DI->EmitFunctionDecl(GD: FD, Loc: FD->getLocation(), FnType: T, Fn: Fn);
3154	}
3155	}
3156
3157	return LV;
3158	}
3159
3160	// FIXME: While we're emitting a binding from an enclosing scope, all other
3161	// DeclRefExprs we see should be implicitly treated as if they also refer to
3162	// an enclosing scope.
3163	if (const auto *BD = dyn_cast<BindingDecl>(ND)) {
3164	if (E->refersToEnclosingVariableOrCapture()) {
3165	auto *FD = LambdaCaptureFields.lookup(Val: BD);
3166	return EmitCapturedFieldLValue(*this, FD, CXXABIThisValue);
3167	}
3168	return EmitLValue(E: BD->getBinding());
3169	}
3170
3171	// We can form DeclRefExprs naming GUID declarations when reconstituting
3172	// non-type template parameters into expressions.
3173	if (const auto *GD = dyn_cast<MSGuidDecl>(ND))
3174	return MakeAddrLValue(CGM.GetAddrOfMSGuidDecl(GD: GD), T,
3175	AlignmentSource::Decl);
3176
3177	if (const auto *TPO = dyn_cast<TemplateParamObjectDecl>(ND)) {
3178	auto ATPO = CGM.GetAddrOfTemplateParamObject(TPO: TPO);
3179	auto AS = getLangASFromTargetAS(ATPO.getAddressSpace());
3180
3181	if (AS != T.getAddressSpace()) {
3182	auto TargetAS = getContext().getTargetAddressSpace(AS: T.getAddressSpace());
3183	auto PtrTy = ATPO.getElementType()->getPointerTo(TargetAS);
3184	auto ASC = getTargetHooks().performAddrSpaceCast(
3185	CGM, ATPO.getPointer(), AS, T.getAddressSpace(), PtrTy);
3186	ATPO = ConstantAddress(ASC, ATPO.getElementType(), ATPO.getAlignment());
3187	}
3188
3189	return MakeAddrLValue(ATPO, T, AlignmentSource::Decl);
3190	}
3191
3192	llvm_unreachable("Unhandled DeclRefExpr");
3193	}
3194
3195	LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) {
3196	// __extension__ doesn't affect lvalue-ness.
3197	if (E->getOpcode() == UO_Extension)
3198	return EmitLValue(E: E->getSubExpr());
3199
3200	QualType ExprTy = getContext().getCanonicalType(T: E->getSubExpr()->getType());
3201	switch (E->getOpcode()) {
3202	default: llvm_unreachable("Unknown unary operator lvalue!");
3203	case UO_Deref: {
3204	QualType T = E->getSubExpr()->getType()->getPointeeType();
3205	assert(!T.isNull() && "CodeGenFunction::EmitUnaryOpLValue: Illegal type");
3206
3207	LValueBaseInfo BaseInfo;
3208	TBAAAccessInfo TBAAInfo;
3209	Address Addr = EmitPointerWithAlignment(E: E->getSubExpr(), BaseInfo: &BaseInfo,
3210	TBAAInfo: &TBAAInfo);
3211	LValue LV = MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
3212	LV.getQuals().setAddressSpace(ExprTy.getAddressSpace());
3213
3214	// We should not generate __weak write barrier on indirect reference
3215	// of a pointer to object; as in void foo (__weak id *param); *param = 0;
3216	// But, we continue to generate __strong write barrier on indirect write
3217	// into a pointer to object.
3218	if (getLangOpts().ObjC &&
3219	getLangOpts().getGC() != LangOptions::NonGC &&
3220	LV.isObjCWeak())
3221	LV.setNonGC(!E->isOBJCGCCandidate(getContext()));
3222	return LV;
3223	}
3224	case UO_Real:
3225	case UO_Imag: {
3226	LValue LV = EmitLValue(E: E->getSubExpr());
3227	assert(LV.isSimple() && "real/imag on non-ordinary l-value");
3228
3229	// __real is valid on scalars. This is a faster way of testing that.
3230	// __imag can only produce an rvalue on scalars.
3231	if (E->getOpcode() == UO_Real &&
3232	!LV.getAddress(CGF&: *this).getElementType()->isStructTy()) {
3233	assert(E->getSubExpr()->getType()->isArithmeticType());
3234	return LV;
3235	}
3236
3237	QualType T = ExprTy->castAs<ComplexType>()->getElementType();
3238
3239	Address Component =
3240	(E->getOpcode() == UO_Real
3241	? emitAddrOfRealComponent(complex: LV.getAddress(CGF&: *this), complexType: LV.getType())
3242	: emitAddrOfImagComponent(complex: LV.getAddress(CGF&: *this), complexType: LV.getType()));
3243	LValue ElemLV = MakeAddrLValue(Addr: Component, T, BaseInfo: LV.getBaseInfo(),
3244	TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: T));
3245	ElemLV.getQuals().addQualifiers(Q: LV.getQuals());
3246	return ElemLV;
3247	}
3248	case UO_PreInc:
3249	case UO_PreDec: {
3250	LValue LV = EmitLValue(E: E->getSubExpr());
3251	bool isInc = E->getOpcode() == UO_PreInc;
3252
3253	if (E->getType()->isAnyComplexType())
3254	EmitComplexPrePostIncDec(E, LV, isInc, isPre: true/*isPre*/);
3255	else
3256	EmitScalarPrePostIncDec(E, LV, isInc, isPre: true/*isPre*/);
3257	return LV;
3258	}
3259	}
3260	}
3261
3262	LValue CodeGenFunction::EmitStringLiteralLValue(const StringLiteral *E) {
3263	return MakeAddrLValue(CGM.GetAddrOfConstantStringFromLiteral(S: E),
3264	E->getType(), AlignmentSource::Decl);
3265	}
3266
3267	LValue CodeGenFunction::EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E) {
3268	return MakeAddrLValue(CGM.GetAddrOfConstantStringFromObjCEncode(E),
3269	E->getType(), AlignmentSource::Decl);
3270	}
3271
3272	LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) {
3273	auto SL = E->getFunctionName();
3274	assert(SL != nullptr && "No StringLiteral name in PredefinedExpr");
3275	StringRef FnName = CurFn->getName();
3276	if (FnName.starts_with(Prefix: "\01"))
3277	FnName = FnName.substr(Start: 1);
3278	StringRef NameItems[] = {
3279	PredefinedExpr::getIdentKindName(IK: E->getIdentKind()), FnName};
3280	std::string GVName = llvm::join(Begin: NameItems, End: NameItems + 2, Separator: ".");
3281	if (auto *BD = dyn_cast_or_null<BlockDecl>(Val: CurCodeDecl)) {
3282	std::string Name = std::string(SL->getString());
3283	if (!Name.empty()) {
3284	unsigned Discriminator =
3285	CGM.getCXXABI().getMangleContext().getBlockId(BD, Local: true);
3286	if (Discriminator)
3287	Name += "_" + Twine(Discriminator + 1).str();
3288	auto C = CGM.GetAddrOfConstantCString(Str: Name, GlobalName: GVName.c_str());
3289	return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl);
3290	} else {
3291	auto C =
3292	CGM.GetAddrOfConstantCString(Str: std::string(FnName), GlobalName: GVName.c_str());
3293	return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl);
3294	}
3295	}
3296	auto C = CGM.GetAddrOfConstantStringFromLiteral(S: SL, Name: GVName);
3297	return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl);
3298	}
3299
3300	/// Emit a type description suitable for use by a runtime sanitizer library. The
3301	/// format of a type descriptor is
3302	///
3303	/// \code
3304	/// { i16 TypeKind, i16 TypeInfo }
3305	/// \endcode
3306	///
3307	/// followed by an array of i8 containing the type name. TypeKind is 0 for an
3308	/// integer, 1 for a floating point value, and -1 for anything else.
3309	llvm::Constant *CodeGenFunction::EmitCheckTypeDescriptor(QualType T) {
3310	// Only emit each type's descriptor once.
3311	if (llvm::Constant *C = CGM.getTypeDescriptorFromMap(Ty: T))
3312	return C;
3313
3314	uint16_t TypeKind = -1;
3315	uint16_t TypeInfo = 0;
3316
3317	if (T->isIntegerType()) {
3318	TypeKind = 0;
3319	TypeInfo = (llvm::Log2_32(Value: getContext().getTypeSize(T)) << 1) |
3320	(T->isSignedIntegerType() ? 1 : 0);
3321	} else if (T->isFloatingType()) {
3322	TypeKind = 1;
3323	TypeInfo = getContext().getTypeSize(T);
3324	}
3325
3326	// Format the type name as if for a diagnostic, including quotes and
3327	// optionally an 'aka'.
3328	SmallString<32> Buffer;
3329	CGM.getDiags().ConvertArgToString(
3330	Kind: DiagnosticsEngine::ak_qualtype, Val: (intptr_t)T.getAsOpaquePtr(), Modifier: StringRef(),
3331	Argument: StringRef(), PrevArgs: std::nullopt, Output&: Buffer, QualTypeVals: std::nullopt);
3332
3333	llvm::Constant *Components[] = {
3334	Builder.getInt16(C: TypeKind), Builder.getInt16(C: TypeInfo),
3335	llvm::ConstantDataArray::getString(Context&: getLLVMContext(), Initializer: Buffer)
3336	};
3337	llvm::Constant *Descriptor = llvm::ConstantStruct::getAnon(V: Components);
3338
3339	auto *GV = new llvm::GlobalVariable(
3340	CGM.getModule(), Descriptor->getType(),
3341	/*isConstant=*/true, llvm::GlobalVariable::PrivateLinkage, Descriptor);
3342	GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3343	CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV);
3344
3345	// Remember the descriptor for this type.
3346	CGM.setTypeDescriptorInMap(Ty: T, C: GV);
3347
3348	return GV;
3349	}
3350
3351	llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V) {
3352	llvm::Type *TargetTy = IntPtrTy;
3353
3354	if (V->getType() == TargetTy)
3355	return V;
3356
3357	// Floating-point types which fit into intptr_t are bitcast to integers
3358	// and then passed directly (after zero-extension, if necessary).
3359	if (V->getType()->isFloatingPointTy()) {
3360	unsigned Bits = V->getType()->getPrimitiveSizeInBits().getFixedValue();
3361	if (Bits <= TargetTy->getIntegerBitWidth())
3362	V = Builder.CreateBitCast(V, DestTy: llvm::Type::getIntNTy(C&: getLLVMContext(),
3363	N: Bits));
3364	}
3365
3366	// Integers which fit in intptr_t are zero-extended and passed directly.
3367	if (V->getType()->isIntegerTy() &&
3368	V->getType()->getIntegerBitWidth() <= TargetTy->getIntegerBitWidth())
3369	return Builder.CreateZExt(V, DestTy: TargetTy);
3370
3371	// Pointers are passed directly, everything else is passed by address.
3372	if (!V->getType()->isPointerTy()) {
3373	Address Ptr = CreateDefaultAlignTempAlloca(Ty: V->getType());
3374	Builder.CreateStore(Val: V, Addr: Ptr);
3375	V = Ptr.getPointer();
3376	}
3377	return Builder.CreatePtrToInt(V, DestTy: TargetTy);
3378	}
3379
3380	/// Emit a representation of a SourceLocation for passing to a handler
3381	/// in a sanitizer runtime library. The format for this data is:
3382	/// \code
3383	/// struct SourceLocation {
3384	/// const char *Filename;
3385	/// int32_t Line, Column;
3386	/// };
3387	/// \endcode
3388	/// For an invalid SourceLocation, the Filename pointer is null.
3389	llvm::Constant *CodeGenFunction::EmitCheckSourceLocation(SourceLocation Loc) {
3390	llvm::Constant *Filename;
3391	int Line, Column;
3392
3393	PresumedLoc PLoc = getContext().getSourceManager().getPresumedLoc(Loc);
3394	if (PLoc.isValid()) {
3395	StringRef FilenameString = PLoc.getFilename();
3396
3397	int PathComponentsToStrip =
3398	CGM.getCodeGenOpts().EmitCheckPathComponentsToStrip;
3399	if (PathComponentsToStrip < 0) {
3400	assert(PathComponentsToStrip != INT_MIN);
3401	int PathComponentsToKeep = -PathComponentsToStrip;
3402	auto I = llvm::sys::path::rbegin(path: FilenameString);
3403	auto E = llvm::sys::path::rend(path: FilenameString);
3404	while (I != E && --PathComponentsToKeep)
3405	++I;
3406
3407	FilenameString = FilenameString.substr(Start: I - E);
3408	} else if (PathComponentsToStrip > 0) {
3409	auto I = llvm::sys::path::begin(path: FilenameString);
3410	auto E = llvm::sys::path::end(path: FilenameString);
3411	while (I != E && PathComponentsToStrip--)
3412	++I;
3413
3414	if (I != E)
3415	FilenameString =
3416	FilenameString.substr(Start: I - llvm::sys::path::begin(path: FilenameString));
3417	else
3418	FilenameString = llvm::sys::path::filename(path: FilenameString);
3419	}
3420
3421	auto FilenameGV =
3422	CGM.GetAddrOfConstantCString(Str: std::string(FilenameString), GlobalName: ".src");
3423	CGM.getSanitizerMetadata()->disableSanitizerForGlobal(
3424	GV: cast<llvm::GlobalVariable>(
3425	Val: FilenameGV.getPointer()->stripPointerCasts()));
3426	Filename = FilenameGV.getPointer();
3427	Line = PLoc.getLine();
3428	Column = PLoc.getColumn();
3429	} else {
3430	Filename = llvm::Constant::getNullValue(Ty: Int8PtrTy);
3431	Line = Column = 0;
3432	}
3433
3434	llvm::Constant *Data[] = {Filename, Builder.getInt32(C: Line),
3435	Builder.getInt32(C: Column)};
3436
3437	return llvm::ConstantStruct::getAnon(V: Data);
3438	}
3439
3440	namespace {
3441	/// Specify under what conditions this check can be recovered
3442	enum class CheckRecoverableKind {
3443	/// Always terminate program execution if this check fails.
3444	Unrecoverable,
3445	/// Check supports recovering, runtime has both fatal (noreturn) and
3446	/// non-fatal handlers for this check.
3447	Recoverable,
3448	/// Runtime conditionally aborts, always need to support recovery.
3449	AlwaysRecoverable
3450	};
3451	}
3452
3453	static CheckRecoverableKind getRecoverableKind(SanitizerMask Kind) {
3454	assert(Kind.countPopulation() == 1);
3455	if (Kind == SanitizerKind::Vptr)
3456	return CheckRecoverableKind::AlwaysRecoverable;
3457	else if (Kind == SanitizerKind::Return || Kind == SanitizerKind::Unreachable)
3458	return CheckRecoverableKind::Unrecoverable;
3459	else
3460	return CheckRecoverableKind::Recoverable;
3461	}
3462
3463	namespace {
3464	struct SanitizerHandlerInfo {
3465	char const *const Name;
3466	unsigned Version;
3467	};
3468	}
3469
3470	const SanitizerHandlerInfo SanitizerHandlers[] = {
3471	#define SANITIZER_CHECK(Enum, Name, Version) {#Name, Version},
3472	LIST_SANITIZER_CHECKS
3473	#undef SANITIZER_CHECK
3474	};
3475
3476	static void emitCheckHandlerCall(CodeGenFunction &CGF,
3477	llvm::FunctionType *FnType,
3478	ArrayRef<llvm::Value *> FnArgs,
3479	SanitizerHandler CheckHandler,
3480	CheckRecoverableKind RecoverKind, bool IsFatal,
3481	llvm::BasicBlock *ContBB) {
3482	assert(IsFatal || RecoverKind != CheckRecoverableKind::Unrecoverable);
3483	std::optional<ApplyDebugLocation> DL;
3484	if (!CGF.Builder.getCurrentDebugLocation()) {
3485	// Ensure that the call has at least an artificial debug location.
3486	DL.emplace(args&: CGF, args: SourceLocation());
3487	}
3488	bool NeedsAbortSuffix =
3489	IsFatal && RecoverKind != CheckRecoverableKind::Unrecoverable;
3490	bool MinimalRuntime = CGF.CGM.getCodeGenOpts().SanitizeMinimalRuntime;
3491	const SanitizerHandlerInfo &CheckInfo = SanitizerHandlers[CheckHandler];
3492	const StringRef CheckName = CheckInfo.Name;
3493	std::string FnName = "__ubsan_handle_" + CheckName.str();
3494	if (CheckInfo.Version && !MinimalRuntime)
3495	FnName += "_v" + llvm::utostr(X: CheckInfo.Version);
3496	if (MinimalRuntime)
3497	FnName += "_minimal";
3498	if (NeedsAbortSuffix)
3499	FnName += "_abort";
3500	bool MayReturn =
3501	!IsFatal || RecoverKind == CheckRecoverableKind::AlwaysRecoverable;
3502
3503	llvm::AttrBuilder B(CGF.getLLVMContext());
3504	if (!MayReturn) {
3505	B.addAttribute(llvm::Attribute::NoReturn)
3506	.addAttribute(llvm::Attribute::NoUnwind);
3507	}
3508	B.addUWTableAttr(Kind: llvm::UWTableKind::Default);
3509
3510	llvm::FunctionCallee Fn = CGF.CGM.CreateRuntimeFunction(
3511	Ty: FnType, Name: FnName,
3512	ExtraAttrs: llvm::AttributeList::get(C&: CGF.getLLVMContext(),
3513	Index: llvm::AttributeList::FunctionIndex, B),
3514	/*Local=*/true);
3515	llvm::CallInst *HandlerCall = CGF.EmitNounwindRuntimeCall(callee: Fn, args: FnArgs);
3516	if (!MayReturn) {
3517	HandlerCall->setDoesNotReturn();
3518	CGF.Builder.CreateUnreachable();
3519	} else {
3520	CGF.Builder.CreateBr(Dest: ContBB);
3521	}
3522	}
3523
3524	void CodeGenFunction::EmitCheck(
3525	ArrayRef<std::pair<llvm::Value *, SanitizerMask>> Checked,
3526	SanitizerHandler CheckHandler, ArrayRef<llvm::Constant *> StaticArgs,
3527	ArrayRef<llvm::Value *> DynamicArgs) {
3528	assert(IsSanitizerScope);
3529	assert(Checked.size() > 0);
3530	assert(CheckHandler >= 0 &&
3531	size_t(CheckHandler) < std::size(SanitizerHandlers));
3532	const StringRef CheckName = SanitizerHandlers[CheckHandler].Name;
3533
3534	llvm::Value *FatalCond = nullptr;
3535	llvm::Value *RecoverableCond = nullptr;
3536	llvm::Value *TrapCond = nullptr;
3537	for (int i = 0, n = Checked.size(); i < n; ++i) {
3538	llvm::Value *Check = Checked[i].first;
3539	// -fsanitize-trap= overrides -fsanitize-recover=.
3540	llvm::Value *&Cond =
3541	CGM.getCodeGenOpts().SanitizeTrap.has(K: Checked[i].second)
3542	? TrapCond
3543	: CGM.getCodeGenOpts().SanitizeRecover.has(K: Checked[i].second)
3544	? RecoverableCond
3545	: FatalCond;
3546	Cond = Cond ? Builder.CreateAnd(LHS: Cond, RHS: Check) : Check;
3547	}
3548
3549	if (TrapCond)
3550	EmitTrapCheck(Checked: TrapCond, CheckHandlerID: CheckHandler);
3551	if (!FatalCond && !RecoverableCond)
3552	return;
3553
3554	llvm::Value *JointCond;
3555	if (FatalCond && RecoverableCond)
3556	JointCond = Builder.CreateAnd(LHS: FatalCond, RHS: RecoverableCond);
3557	else
3558	JointCond = FatalCond ? FatalCond : RecoverableCond;
3559	assert(JointCond);
3560
3561	CheckRecoverableKind RecoverKind = getRecoverableKind(Kind: Checked[0].second);
3562	assert(SanOpts.has(Checked[0].second));
3563	#ifndef NDEBUG
3564	for (int i = 1, n = Checked.size(); i < n; ++i) {
3565	assert(RecoverKind == getRecoverableKind(Checked[i].second) &&
3566	"All recoverable kinds in a single check must be same!");
3567	assert(SanOpts.has(Checked[i].second));
3568	}
3569	#endif
3570
3571	llvm::BasicBlock *Cont = createBasicBlock(name: "cont");
3572	llvm::BasicBlock *Handlers = createBasicBlock(name: "handler." + CheckName);
3573	llvm::Instruction *Branch = Builder.CreateCondBr(Cond: JointCond, True: Cont, False: Handlers);
3574	// Give hint that we very much don't expect to execute the handler
3575	// Value chosen to match UR_NONTAKEN_WEIGHT, see BranchProbabilityInfo.cpp
3576	llvm::MDBuilder MDHelper(getLLVMContext());
3577	llvm::MDNode *Node = MDHelper.createBranchWeights(TrueWeight: (1U << 20) - 1, FalseWeight: 1);
3578	Branch->setMetadata(KindID: llvm::LLVMContext::MD_prof, Node);
3579	EmitBlock(BB: Handlers);
3580
3581	// Handler functions take an i8* pointing to the (handler-specific) static
3582	// information block, followed by a sequence of intptr_t arguments
3583	// representing operand values.
3584	SmallVector<llvm::Value *, 4> Args;
3585	SmallVector<llvm::Type *, 4> ArgTypes;
3586	if (!CGM.getCodeGenOpts().SanitizeMinimalRuntime) {
3587	Args.reserve(N: DynamicArgs.size() + 1);
3588	ArgTypes.reserve(N: DynamicArgs.size() + 1);
3589
3590	// Emit handler arguments and create handler function type.
3591	if (!StaticArgs.empty()) {
3592	llvm::Constant *Info = llvm::ConstantStruct::getAnon(V: StaticArgs);
3593	auto *InfoPtr = new llvm::GlobalVariable(
3594	CGM.getModule(), Info->getType(), false,
3595	llvm::GlobalVariable::PrivateLinkage, Info, "", nullptr,
3596	llvm::GlobalVariable::NotThreadLocal,
3597	CGM.getDataLayout().getDefaultGlobalsAddressSpace());
3598	InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3599	CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV: InfoPtr);
3600	Args.push_back(Elt: InfoPtr);
3601	ArgTypes.push_back(Elt: Args.back()->getType());
3602	}
3603
3604	for (size_t i = 0, n = DynamicArgs.size(); i != n; ++i) {
3605	Args.push_back(Elt: EmitCheckValue(V: DynamicArgs[i]));
3606	ArgTypes.push_back(Elt: IntPtrTy);
3607	}
3608	}
3609
3610	llvm::FunctionType *FnType =
3611	llvm::FunctionType::get(Result: CGM.VoidTy, Params: ArgTypes, isVarArg: false);
3612
3613	if (!FatalCond || !RecoverableCond) {
3614	// Simple case: we need to generate a single handler call, either
3615	// fatal, or non-fatal.
3616	emitCheckHandlerCall(CGF&: *this, FnType, FnArgs: Args, CheckHandler, RecoverKind,
3617	IsFatal: (FatalCond != nullptr), ContBB: Cont);
3618	} else {
3619	// Emit two handler calls: first one for set of unrecoverable checks,
3620	// another one for recoverable.
3621	llvm::BasicBlock *NonFatalHandlerBB =
3622	createBasicBlock(name: "non_fatal." + CheckName);
3623	llvm::BasicBlock *FatalHandlerBB = createBasicBlock(name: "fatal." + CheckName);
3624	Builder.CreateCondBr(Cond: FatalCond, True: NonFatalHandlerBB, False: FatalHandlerBB);
3625	EmitBlock(BB: FatalHandlerBB);
3626	emitCheckHandlerCall(CGF&: *this, FnType, FnArgs: Args, CheckHandler, RecoverKind, IsFatal: true,
3627	ContBB: NonFatalHandlerBB);
3628	EmitBlock(BB: NonFatalHandlerBB);
3629	emitCheckHandlerCall(CGF&: *this, FnType, FnArgs: Args, CheckHandler, RecoverKind, IsFatal: false,
3630	ContBB: Cont);
3631	}
3632
3633	EmitBlock(BB: Cont);
3634	}
3635
3636	void CodeGenFunction::EmitCfiSlowPathCheck(
3637	SanitizerMask Kind, llvm::Value *Cond, llvm::ConstantInt *TypeId,
3638	llvm::Value *Ptr, ArrayRef<llvm::Constant *> StaticArgs) {
3639	llvm::BasicBlock *Cont = createBasicBlock(name: "cfi.cont");
3640
3641	llvm::BasicBlock *CheckBB = createBasicBlock(name: "cfi.slowpath");
3642	llvm::BranchInst *BI = Builder.CreateCondBr(Cond, True: Cont, False: CheckBB);
3643
3644	llvm::MDBuilder MDHelper(getLLVMContext());
3645	llvm::MDNode *Node = MDHelper.createBranchWeights(TrueWeight: (1U << 20) - 1, FalseWeight: 1);
3646	BI->setMetadata(KindID: llvm::LLVMContext::MD_prof, Node);
3647
3648	EmitBlock(BB: CheckBB);
3649
3650	bool WithDiag = !CGM.getCodeGenOpts().SanitizeTrap.has(K: Kind);
3651
3652	llvm::CallInst *CheckCall;
3653	llvm::FunctionCallee SlowPathFn;
3654	if (WithDiag) {
3655	llvm::Constant *Info = llvm::ConstantStruct::getAnon(V: StaticArgs);
3656	auto *InfoPtr =
3657	new llvm::GlobalVariable(CGM.getModule(), Info->getType(), false,
3658	llvm::GlobalVariable::PrivateLinkage, Info);
3659	InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3660	CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV: InfoPtr);
3661
3662	SlowPathFn = CGM.getModule().getOrInsertFunction(
3663	Name: "__cfi_slowpath_diag",
3664	T: llvm::FunctionType::get(Result: VoidTy, Params: {Int64Ty, Int8PtrTy, Int8PtrTy},
3665	isVarArg: false));
3666	CheckCall = Builder.CreateCall(Callee: SlowPathFn, Args: {TypeId, Ptr, InfoPtr});
3667	} else {
3668	SlowPathFn = CGM.getModule().getOrInsertFunction(
3669	Name: "__cfi_slowpath",
3670	T: llvm::FunctionType::get(Result: VoidTy, Params: {Int64Ty, Int8PtrTy}, isVarArg: false));
3671	CheckCall = Builder.CreateCall(Callee: SlowPathFn, Args: {TypeId, Ptr});
3672	}
3673
3674	CGM.setDSOLocal(
3675	cast<llvm::GlobalValue>(Val: SlowPathFn.getCallee()->stripPointerCasts()));
3676	CheckCall->setDoesNotThrow();
3677
3678	EmitBlock(BB: Cont);
3679	}
3680
3681	// Emit a stub for __cfi_check function so that the linker knows about this
3682	// symbol in LTO mode.
3683	void CodeGenFunction::EmitCfiCheckStub() {
3684	llvm::Module *M = &CGM.getModule();
3685	auto &Ctx = M->getContext();
3686	llvm::Function *F = llvm::Function::Create(
3687	Ty: llvm::FunctionType::get(Result: VoidTy, Params: {Int64Ty, Int8PtrTy, Int8PtrTy}, isVarArg: false),
3688	Linkage: llvm::GlobalValue::WeakAnyLinkage, N: "__cfi_check", M);
3689	F->setAlignment(llvm::Align(4096));
3690	CGM.setDSOLocal(F);
3691	llvm::BasicBlock *BB = llvm::BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: F);
3692	// CrossDSOCFI pass is not executed if there is no executable code.
3693	SmallVector<llvm::Value*> Args{F->getArg(i: 2), F->getArg(i: 1)};
3694	llvm::CallInst::Create(Func: M->getFunction(Name: "__cfi_check_fail"), Args, NameStr: "", InsertAtEnd: BB);
3695	llvm::ReturnInst::Create(C&: Ctx, retVal: nullptr, InsertAtEnd: BB);
3696	}
3697
3698	// This function is basically a switch over the CFI failure kind, which is
3699	// extracted from CFICheckFailData (1st function argument). Each case is either
3700	// llvm.trap or a call to one of the two runtime handlers, based on
3701	// -fsanitize-trap and -fsanitize-recover settings. Default case (invalid
3702	// failure kind) traps, but this should really never happen. CFICheckFailData
3703	// can be nullptr if the calling module has -fsanitize-trap behavior for this
3704	// check kind; in this case __cfi_check_fail traps as well.
3705	void CodeGenFunction::EmitCfiCheckFail() {
3706	SanitizerScope SanScope(this);
3707	FunctionArgList Args;
3708	ImplicitParamDecl ArgData(getContext(), getContext().VoidPtrTy,
3709	ImplicitParamKind::Other);
3710	ImplicitParamDecl ArgAddr(getContext(), getContext().VoidPtrTy,
3711	ImplicitParamKind::Other);
3712	Args.push_back(&ArgData);
3713	Args.push_back(&ArgAddr);
3714
3715	const CGFunctionInfo &FI =
3716	CGM.getTypes().arrangeBuiltinFunctionDeclaration(getContext().VoidTy, Args);
3717
3718	llvm::Function *F = llvm::Function::Create(
3719	Ty: llvm::FunctionType::get(Result: VoidTy, Params: {VoidPtrTy, VoidPtrTy}, isVarArg: false),
3720	Linkage: llvm::GlobalValue::WeakODRLinkage, N: "__cfi_check_fail", M: &CGM.getModule());
3721
3722	CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F, /*IsThunk=*/false);
3723	CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F);
3724	F->setVisibility(llvm::GlobalValue::HiddenVisibility);
3725
3726	StartFunction(GD: GlobalDecl(), RetTy: CGM.getContext().VoidTy, Fn: F, FnInfo: FI, Args,
3727	Loc: SourceLocation());
3728
3729	// This function is not affected by NoSanitizeList. This function does
3730	// not have a source location, but "src:*" would still apply. Revert any
3731	// changes to SanOpts made in StartFunction.
3732	SanOpts = CGM.getLangOpts().Sanitize;
3733
3734	llvm::Value *Data =
3735	EmitLoadOfScalar(GetAddrOfLocalVar(&ArgData), /*Volatile=*/false,
3736	CGM.getContext().VoidPtrTy, ArgData.getLocation());
3737	llvm::Value *Addr =
3738	EmitLoadOfScalar(GetAddrOfLocalVar(&ArgAddr), /*Volatile=*/false,
3739	CGM.getContext().VoidPtrTy, ArgAddr.getLocation());
3740
3741	// Data == nullptr means the calling module has trap behaviour for this check.
3742	llvm::Value *DataIsNotNullPtr =
3743	Builder.CreateICmpNE(LHS: Data, RHS: llvm::ConstantPointerNull::get(T: Int8PtrTy));
3744	EmitTrapCheck(Checked: DataIsNotNullPtr, CheckHandlerID: SanitizerHandler::CFICheckFail);
3745
3746	llvm::StructType *SourceLocationTy =
3747	llvm::StructType::get(elt1: VoidPtrTy, elts: Int32Ty, elts: Int32Ty);
3748	llvm::StructType *CfiCheckFailDataTy =
3749	llvm::StructType::get(elt1: Int8Ty, elts: SourceLocationTy, elts: VoidPtrTy);
3750
3751	llvm::Value *V = Builder.CreateConstGEP2_32(
3752	Ty: CfiCheckFailDataTy,
3753	Ptr: Builder.CreatePointerCast(V: Data, DestTy: CfiCheckFailDataTy->getPointerTo(AddrSpace: 0)), Idx0: 0,
3754	Idx1: 0);
3755
3756	Address CheckKindAddr(V, Int8Ty, getIntAlign());
3757	llvm::Value *CheckKind = Builder.CreateLoad(Addr: CheckKindAddr);
3758
3759	llvm::Value *AllVtables = llvm::MetadataAsValue::get(
3760	Context&: CGM.getLLVMContext(),
3761	MD: llvm::MDString::get(Context&: CGM.getLLVMContext(), Str: "all-vtables"));
3762	llvm::Value *ValidVtable = Builder.CreateZExt(
3763	Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::type_test),
3764	{Addr, AllVtables}),
3765	IntPtrTy);
3766
3767	const std::pair<int, SanitizerMask> CheckKinds[] = {
3768	{CFITCK_VCall, SanitizerKind::CFIVCall},
3769	{CFITCK_NVCall, SanitizerKind::CFINVCall},
3770	{CFITCK_DerivedCast, SanitizerKind::CFIDerivedCast},
3771	{CFITCK_UnrelatedCast, SanitizerKind::CFIUnrelatedCast},
3772	{CFITCK_ICall, SanitizerKind::CFIICall}};
3773
3774	SmallVector<std::pair<llvm::Value *, SanitizerMask>, 5> Checks;
3775	for (auto CheckKindMaskPair : CheckKinds) {
3776	int Kind = CheckKindMaskPair.first;
3777	SanitizerMask Mask = CheckKindMaskPair.second;
3778	llvm::Value *Cond =
3779	Builder.CreateICmpNE(LHS: CheckKind, RHS: llvm::ConstantInt::get(Ty: Int8Ty, V: Kind));
3780	if (CGM.getLangOpts().Sanitize.has(K: Mask))
3781	EmitCheck(Checked: std::make_pair(x&: Cond, y&: Mask), CheckHandler: SanitizerHandler::CFICheckFail, StaticArgs: {},
3782	DynamicArgs: {Data, Addr, ValidVtable});
3783	else
3784	EmitTrapCheck(Checked: Cond, CheckHandlerID: SanitizerHandler::CFICheckFail);
3785	}
3786
3787	FinishFunction();
3788	// The only reference to this function will be created during LTO link.
3789	// Make sure it survives until then.
3790	CGM.addUsedGlobal(GV: F);
3791	}
3792
3793	void CodeGenFunction::EmitUnreachable(SourceLocation Loc) {
3794	if (SanOpts.has(K: SanitizerKind::Unreachable)) {
3795	SanitizerScope SanScope(this);
3796	EmitCheck(Checked: std::make_pair(x: static_cast<llvm::Value *>(Builder.getFalse()),
3797	y: SanitizerKind::Unreachable),
3798	CheckHandler: SanitizerHandler::BuiltinUnreachable,
3799	StaticArgs: EmitCheckSourceLocation(Loc), DynamicArgs: std::nullopt);
3800	}
3801	Builder.CreateUnreachable();
3802	}
3803
3804	void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked,
3805	SanitizerHandler CheckHandlerID) {
3806	llvm::BasicBlock *Cont = createBasicBlock(name: "cont");
3807
3808	// If we're optimizing, collapse all calls to trap down to just one per
3809	// check-type per function to save on code size.
3810	if ((int)TrapBBs.size() <= CheckHandlerID)
3811	TrapBBs.resize(N: CheckHandlerID + 1);
3812
3813	llvm::BasicBlock *&TrapBB = TrapBBs[CheckHandlerID];
3814
3815	if (!ClSanitizeDebugDeoptimization &&
3816	CGM.getCodeGenOpts().OptimizationLevel && TrapBB &&
3817	(!CurCodeDecl || !CurCodeDecl->hasAttr<OptimizeNoneAttr>())) {
3818	auto Call = TrapBB->begin();
3819	assert(isa<llvm::CallInst>(Call) && "Expected call in trap BB");
3820
3821	Call->applyMergedLocation(LocA: Call->getDebugLoc(),
3822	LocB: Builder.getCurrentDebugLocation());
3823	Builder.CreateCondBr(Cond: Checked, True: Cont, False: TrapBB);
3824	} else {
3825	TrapBB = createBasicBlock(name: "trap");
3826	Builder.CreateCondBr(Cond: Checked, True: Cont, False: TrapBB);
3827	EmitBlock(BB: TrapBB);
3828
3829	llvm::CallInst *TrapCall = Builder.CreateCall(
3830	CGM.getIntrinsic(llvm::Intrinsic::ubsantrap),
3831	llvm::ConstantInt::get(CGM.Int8Ty, ClSanitizeDebugDeoptimization
3832	? TrapBB->getParent()->size()
3833	: CheckHandlerID));
3834
3835	if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {
3836	auto A = llvm::Attribute::get(Context&: getLLVMContext(), Kind: "trap-func-name",
3837	Val: CGM.getCodeGenOpts().TrapFuncName);
3838	TrapCall->addFnAttr(Attr: A);
3839	}
3840	TrapCall->setDoesNotReturn();
3841	TrapCall->setDoesNotThrow();
3842	Builder.CreateUnreachable();
3843	}
3844
3845	EmitBlock(BB: Cont);
3846	}
3847
3848	llvm::CallInst *CodeGenFunction::EmitTrapCall(llvm::Intrinsic::ID IntrID) {
3849	llvm::CallInst *TrapCall =
3850	Builder.CreateCall(Callee: CGM.getIntrinsic(IID: IntrID));
3851
3852	if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {
3853	auto A = llvm::Attribute::get(Context&: getLLVMContext(), Kind: "trap-func-name",
3854	Val: CGM.getCodeGenOpts().TrapFuncName);
3855	TrapCall->addFnAttr(Attr: A);
3856	}
3857
3858	return TrapCall;
3859	}
3860
3861	Address CodeGenFunction::EmitArrayToPointerDecay(const Expr *E,
3862	LValueBaseInfo *BaseInfo,
3863	TBAAAccessInfo *TBAAInfo) {
3864	assert(E->getType()->isArrayType() &&
3865	"Array to pointer decay must have array source type!");
3866
3867	// Expressions of array type can't be bitfields or vector elements.
3868	LValue LV = EmitLValue(E);
3869	Address Addr = LV.getAddress(CGF&: *this);
3870
3871	// If the array type was an incomplete type, we need to make sure
3872	// the decay ends up being the right type.
3873	llvm::Type *NewTy = ConvertType(T: E->getType());
3874	Addr = Addr.withElementType(ElemTy: NewTy);
3875
3876	// Note that VLA pointers are always decayed, so we don't need to do
3877	// anything here.
3878	if (!E->getType()->isVariableArrayType()) {
3879	assert(isa<llvm::ArrayType>(Addr.getElementType()) &&
3880	"Expected pointer to array");
3881	Addr = Builder.CreateConstArrayGEP(Addr, Index: 0, Name: "arraydecay");
3882	}
3883
3884	// The result of this decay conversion points to an array element within the
3885	// base lvalue. However, since TBAA currently does not support representing
3886	// accesses to elements of member arrays, we conservatively represent accesses
3887	// to the pointee object as if it had no any base lvalue specified.
3888	// TODO: Support TBAA for member arrays.
3889	QualType EltType = E->getType()->castAsArrayTypeUnsafe()->getElementType();
3890	if (BaseInfo) *BaseInfo = LV.getBaseInfo();
3891	if (TBAAInfo) *TBAAInfo = CGM.getTBAAAccessInfo(AccessType: EltType);
3892
3893	return Addr.withElementType(ElemTy: ConvertTypeForMem(T: EltType));
3894	}
3895
3896	/// isSimpleArrayDecayOperand - If the specified expr is a simple decay from an
3897	/// array to pointer, return the array subexpression.
3898	static const Expr *isSimpleArrayDecayOperand(const Expr *E) {
3899	// If this isn't just an array->pointer decay, bail out.
3900	const auto *CE = dyn_cast<CastExpr>(Val: E);
3901	if (!CE || CE->getCastKind() != CK_ArrayToPointerDecay)
3902	return nullptr;
3903
3904	// If this is a decay from variable width array, bail out.
3905	const Expr *SubExpr = CE->getSubExpr();
3906	if (SubExpr->getType()->isVariableArrayType())
3907	return nullptr;
3908
3909	return SubExpr;
3910	}
3911
3912	static llvm::Value *emitArraySubscriptGEP(CodeGenFunction &CGF,
3913	llvm::Type *elemType,
3914	llvm::Value *ptr,
3915	ArrayRef<llvm::Value*> indices,
3916	bool inbounds,
3917	bool signedIndices,
3918	SourceLocation loc,
3919	const llvm::Twine &name = "arrayidx") {
3920	if (inbounds) {
3921	return CGF.EmitCheckedInBoundsGEP(ElemTy: elemType, Ptr: ptr, IdxList: indices, SignedIndices: signedIndices,
3922	IsSubtraction: CodeGenFunction::NotSubtraction, Loc: loc,
3923	Name: name);
3924	} else {
3925	return CGF.Builder.CreateGEP(Ty: elemType, Ptr: ptr, IdxList: indices, Name: name);
3926	}
3927	}
3928
3929	static CharUnits getArrayElementAlign(CharUnits arrayAlign,
3930	llvm::Value *idx,
3931	CharUnits eltSize) {
3932	// If we have a constant index, we can use the exact offset of the
3933	// element we're accessing.
3934	if (auto constantIdx = dyn_cast<llvm::ConstantInt>(Val: idx)) {
3935	CharUnits offset = constantIdx->getZExtValue() * eltSize;
3936	return arrayAlign.alignmentAtOffset(offset);
3937
3938	// Otherwise, use the worst-case alignment for any element.
3939	} else {
3940	return arrayAlign.alignmentOfArrayElement(elementSize: eltSize);
3941	}
3942	}
3943
3944	static QualType getFixedSizeElementType(const ASTContext &ctx,
3945	const VariableArrayType *vla) {
3946	QualType eltType;
3947	do {
3948	eltType = vla->getElementType();
3949	} while ((vla = ctx.getAsVariableArrayType(T: eltType)));
3950	return eltType;
3951	}
3952
3953	static bool hasBPFPreserveStaticOffset(const RecordDecl *D) {
3954	return D && D->hasAttr<BPFPreserveStaticOffsetAttr>();
3955	}
3956
3957	static bool hasBPFPreserveStaticOffset(const Expr *E) {
3958	if (!E)
3959	return false;
3960	QualType PointeeType = E->getType()->getPointeeType();
3961	if (PointeeType.isNull())
3962	return false;
3963	if (const auto *BaseDecl = PointeeType->getAsRecordDecl())
3964	return hasBPFPreserveStaticOffset(D: BaseDecl);
3965	return false;
3966	}
3967
3968	// Wraps Addr with a call to llvm.preserve.static.offset intrinsic.
3969	static Address wrapWithBPFPreserveStaticOffset(CodeGenFunction &CGF,
3970	Address &Addr) {
3971	if (!CGF.getTarget().getTriple().isBPF())
3972	return Addr;
3973
3974	llvm::Function *Fn =
3975	CGF.CGM.getIntrinsic(llvm::Intrinsic::preserve_static_offset);
3976	llvm::CallInst *Call = CGF.Builder.CreateCall(Callee: Fn, Args: {Addr.getPointer()});
3977	return Address(Call, Addr.getElementType(), Addr.getAlignment());
3978	}
3979
3980	/// Given an array base, check whether its member access belongs to a record
3981	/// with preserve_access_index attribute or not.
3982	static bool IsPreserveAIArrayBase(CodeGenFunction &CGF, const Expr *ArrayBase) {
3983	if (!ArrayBase || !CGF.getDebugInfo())
3984	return false;
3985
3986	// Only support base as either a MemberExpr or DeclRefExpr.
3987	// DeclRefExpr to cover cases like:
3988	// struct s { int a; int b[10]; };
3989	// struct s *p;
3990	// p[1].a
3991	// p[1] will generate a DeclRefExpr and p[1].a is a MemberExpr.
3992	// p->b[5] is a MemberExpr example.
3993	const Expr *E = ArrayBase->IgnoreImpCasts();
3994	if (const auto *ME = dyn_cast<MemberExpr>(E))
3995	return ME->getMemberDecl()->hasAttr<BPFPreserveAccessIndexAttr>();
3996
3997	if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: E)) {
3998	const auto *VarDef = dyn_cast<VarDecl>(Val: DRE->getDecl());
3999	if (!VarDef)
4000	return false;
4001
4002	const auto *PtrT = VarDef->getType()->getAs<PointerType>();
4003	if (!PtrT)
4004	return false;
4005
4006	const auto *PointeeT = PtrT->getPointeeType()
4007	->getUnqualifiedDesugaredType();
4008	if (const auto *RecT = dyn_cast<RecordType>(PointeeT))
4009	return RecT->getDecl()->hasAttr<BPFPreserveAccessIndexAttr>();
4010	return false;
4011	}
4012
4013	return false;
4014	}
4015
4016	static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr,
4017	ArrayRef<llvm::Value *> indices,
4018	QualType eltType, bool inbounds,
4019	bool signedIndices, SourceLocation loc,
4020	QualType *arrayType = nullptr,
4021	const Expr *Base = nullptr,
4022	const llvm::Twine &name = "arrayidx") {
4023	// All the indices except that last must be zero.
4024	#ifndef NDEBUG
4025	for (auto *idx : indices.drop_back())
4026	assert(isa<llvm::ConstantInt>(idx) &&
4027	cast<llvm::ConstantInt>(idx)->isZero());
4028	#endif
4029
4030	// Determine the element size of the statically-sized base. This is
4031	// the thing that the indices are expressed in terms of.
4032	if (auto vla = CGF.getContext().getAsVariableArrayType(T: eltType)) {
4033	eltType = getFixedSizeElementType(ctx: CGF.getContext(), vla);
4034	}
4035
4036	// We can use that to compute the best alignment of the element.
4037	CharUnits eltSize = CGF.getContext().getTypeSizeInChars(T: eltType);
4038	CharUnits eltAlign =
4039	getArrayElementAlign(arrayAlign: addr.getAlignment(), idx: indices.back(), eltSize);
4040
4041	if (hasBPFPreserveStaticOffset(E: Base))
4042	addr = wrapWithBPFPreserveStaticOffset(CGF, Addr&: addr);
4043
4044	llvm::Value *eltPtr;
4045	auto LastIndex = dyn_cast<llvm::ConstantInt>(Val: indices.back());
4046	if (!LastIndex ||
4047	(!CGF.IsInPreservedAIRegion && !IsPreserveAIArrayBase(CGF, ArrayBase: Base))) {
4048	eltPtr = emitArraySubscriptGEP(
4049	CGF, elemType: addr.getElementType(), ptr: addr.getPointer(), indices, inbounds,
4050	signedIndices, loc, name);
4051	} else {
4052	// Remember the original array subscript for bpf target
4053	unsigned idx = LastIndex->getZExtValue();
4054	llvm::DIType *DbgInfo = nullptr;
4055	if (arrayType)
4056	DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(Ty: *arrayType, Loc: loc);
4057	eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex(ElTy: addr.getElementType(),
4058	Base: addr.getPointer(),
4059	Dimension: indices.size() - 1,
4060	LastIndex: idx, DbgInfo);
4061	}
4062
4063	return Address(eltPtr, CGF.ConvertTypeForMem(T: eltType), eltAlign);
4064	}
4065
4066	/// The offset of a field from the beginning of the record.
4067	static bool getFieldOffsetInBits(CodeGenFunction &CGF, const RecordDecl *RD,
4068	const FieldDecl *FD, int64_t &Offset) {
4069	ASTContext &Ctx = CGF.getContext();
4070	const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(D: RD);
4071	unsigned FieldNo = 0;
4072
4073	for (const Decl *D : RD->decls()) {
4074	if (const auto *Record = dyn_cast<RecordDecl>(D))
4075	if (getFieldOffsetInBits(CGF, Record, FD, Offset)) {
4076	Offset += Layout.getFieldOffset(FieldNo);
4077	return true;
4078	}
4079
4080	if (const auto *Field = dyn_cast<FieldDecl>(D))
4081	if (FD == Field) {
4082	Offset += Layout.getFieldOffset(FieldNo);
4083	return true;
4084	}
4085
4086	if (isa<FieldDecl>(D))
4087	++FieldNo;
4088	}
4089
4090	return false;
4091	}
4092
4093	/// Returns the relative offset difference between \p FD1 and \p FD2.
4094	/// \code
4095	/// offsetof(struct foo, FD1) - offsetof(struct foo, FD2)
4096	/// \endcode
4097	/// Both fields must be within the same struct.
4098	static std::optional<int64_t> getOffsetDifferenceInBits(CodeGenFunction &CGF,
4099	const FieldDecl *FD1,
4100	const FieldDecl *FD2) {
4101	const RecordDecl *FD1OuterRec =
4102	FD1->getParent()->getOuterLexicalRecordContext();
4103	const RecordDecl *FD2OuterRec =
4104	FD2->getParent()->getOuterLexicalRecordContext();
4105
4106	if (FD1OuterRec != FD2OuterRec)
4107	// Fields must be within the same RecordDecl.
4108	return std::optional<int64_t>();
4109
4110	int64_t FD1Offset = 0;
4111	if (!getFieldOffsetInBits(CGF, RD: FD1OuterRec, FD: FD1, Offset&: FD1Offset))
4112	return std::optional<int64_t>();
4113
4114	int64_t FD2Offset = 0;
4115	if (!getFieldOffsetInBits(CGF, RD: FD2OuterRec, FD: FD2, Offset&: FD2Offset))
4116	return std::optional<int64_t>();
4117
4118	return std::make_optional<int64_t>(t: FD1Offset - FD2Offset);
4119	}
4120
4121	LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
4122	bool Accessed) {
4123	// The index must always be an integer, which is not an aggregate. Emit it
4124	// in lexical order (this complexity is, sadly, required by C++17).
4125	llvm::Value *IdxPre =
4126	(E->getLHS() == E->getIdx()) ? EmitScalarExpr(E: E->getIdx()) : nullptr;
4127	bool SignedIndices = false;
4128	auto EmitIdxAfterBase = [&, IdxPre](bool Promote) -> llvm::Value * {
4129	auto *Idx = IdxPre;
4130	if (E->getLHS() != E->getIdx()) {
4131	assert(E->getRHS() == E->getIdx() && "index was neither LHS nor RHS");
4132	Idx = EmitScalarExpr(E: E->getIdx());
4133	}
4134
4135	QualType IdxTy = E->getIdx()->getType();
4136	bool IdxSigned = IdxTy->isSignedIntegerOrEnumerationType();
4137	SignedIndices |= IdxSigned;
4138
4139	if (SanOpts.has(K: SanitizerKind::ArrayBounds))
4140	EmitBoundsCheck(E, E->getBase(), Idx, IdxTy, Accessed);
4141
4142	// Extend or truncate the index type to 32 or 64-bits.
4143	if (Promote && Idx->getType() != IntPtrTy)
4144	Idx = Builder.CreateIntCast(V: Idx, DestTy: IntPtrTy, isSigned: IdxSigned, Name: "idxprom");
4145
4146	return Idx;
4147	};
4148	IdxPre = nullptr;
4149
4150	// If the base is a vector type, then we are forming a vector element lvalue
4151	// with this subscript.
4152	if (E->getBase()->getType()->isVectorType() &&
4153	!isa<ExtVectorElementExpr>(Val: E->getBase())) {
4154	// Emit the vector as an lvalue to get its address.
4155	LValue LHS = EmitLValue(E: E->getBase());
4156	auto *Idx = EmitIdxAfterBase(/*Promote*/false);
4157	assert(LHS.isSimple() && "Can only subscript lvalue vectors here!");
4158	return LValue::MakeVectorElt(vecAddress: LHS.getAddress(CGF&: *this), Idx,
4159	type: E->getBase()->getType(), BaseInfo: LHS.getBaseInfo(),
4160	TBAAInfo: TBAAAccessInfo());
4161	}
4162
4163	// All the other cases basically behave like simple offsetting.
4164
4165	// Handle the extvector case we ignored above.
4166	if (isa<ExtVectorElementExpr>(Val: E->getBase())) {
4167	LValue LV = EmitLValue(E: E->getBase());
4168	auto *Idx = EmitIdxAfterBase(/*Promote*/true);
4169	Address Addr = EmitExtVectorElementLValue(LV);
4170
4171	QualType EltType = LV.getType()->castAs<VectorType>()->getElementType();
4172	Addr = emitArraySubscriptGEP(CGF&: *this, addr: Addr, indices: Idx, eltType: EltType, /*inbounds*/ true,
4173	signedIndices: SignedIndices, loc: E->getExprLoc());
4174	return MakeAddrLValue(Addr, T: EltType, BaseInfo: LV.getBaseInfo(),
4175	TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: EltType));
4176	}
4177
4178	LValueBaseInfo EltBaseInfo;
4179	TBAAAccessInfo EltTBAAInfo;
4180	Address Addr = Address::invalid();
4181	if (const VariableArrayType *vla =
4182	getContext().getAsVariableArrayType(T: E->getType())) {
4183	// The base must be a pointer, which is not an aggregate. Emit
4184	// it. It needs to be emitted first in case it's what captures
4185	// the VLA bounds.
4186	Addr = EmitPointerWithAlignment(E: E->getBase(), BaseInfo: &EltBaseInfo, TBAAInfo: &EltTBAAInfo);
4187	auto *Idx = EmitIdxAfterBase(/*Promote*/true);
4188
4189	// The element count here is the total number of non-VLA elements.
4190	llvm::Value *numElements = getVLASize(vla).NumElts;
4191
4192	// Effectively, the multiply by the VLA size is part of the GEP.
4193	// GEP indexes are signed, and scaling an index isn't permitted to
4194	// signed-overflow, so we use the same semantics for our explicit
4195	// multiply. We suppress this if overflow is not undefined behavior.
4196	if (getLangOpts().isSignedOverflowDefined()) {
4197	Idx = Builder.CreateMul(LHS: Idx, RHS: numElements);
4198	} else {
4199	Idx = Builder.CreateNSWMul(LHS: Idx, RHS: numElements);
4200	}
4201
4202	Addr = emitArraySubscriptGEP(*this, Addr, Idx, vla->getElementType(),
4203	!getLangOpts().isSignedOverflowDefined(),
4204	SignedIndices, E->getExprLoc());
4205
4206	} else if (const ObjCObjectType *OIT = E->getType()->getAs<ObjCObjectType>()){
4207	// Indexing over an interface, as in "NSString *P; P[4];"
4208
4209	// Emit the base pointer.
4210	Addr = EmitPointerWithAlignment(E: E->getBase(), BaseInfo: &EltBaseInfo, TBAAInfo: &EltTBAAInfo);
4211	auto *Idx = EmitIdxAfterBase(/*Promote*/true);
4212
4213	CharUnits InterfaceSize = getContext().getTypeSizeInChars(OIT);
4214	llvm::Value *InterfaceSizeVal =
4215	llvm::ConstantInt::get(Ty: Idx->getType(), V: InterfaceSize.getQuantity());
4216
4217	llvm::Value *ScaledIdx = Builder.CreateMul(LHS: Idx, RHS: InterfaceSizeVal);
4218
4219	// We don't necessarily build correct LLVM struct types for ObjC
4220	// interfaces, so we can't rely on GEP to do this scaling
4221	// correctly, so we need to cast to i8*. FIXME: is this actually
4222	// true? A lot of other things in the fragile ABI would break...
4223	llvm::Type *OrigBaseElemTy = Addr.getElementType();
4224
4225	// Do the GEP.
4226	CharUnits EltAlign =
4227	getArrayElementAlign(arrayAlign: Addr.getAlignment(), idx: Idx, eltSize: InterfaceSize);
4228	llvm::Value *EltPtr =
4229	emitArraySubscriptGEP(CGF&: *this, elemType: Int8Ty, ptr: Addr.getPointer(), indices: ScaledIdx,
4230	inbounds: false, signedIndices: SignedIndices, loc: E->getExprLoc());
4231	Addr = Address(EltPtr, OrigBaseElemTy, EltAlign);
4232	} else if (const Expr *Array = isSimpleArrayDecayOperand(E: E->getBase())) {
4233	// If this is A[i] where A is an array, the frontend will have decayed the
4234	// base to be a ArrayToPointerDecay implicit cast. While correct, it is
4235	// inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
4236	// "gep x, i" here. Emit one "gep A, 0, i".
4237	assert(Array->getType()->isArrayType() &&
4238	"Array to pointer decay must have array source type!");
4239	LValue ArrayLV;
4240	// For simple multidimensional array indexing, set the 'accessed' flag for
4241	// better bounds-checking of the base expression.
4242	if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: Array))
4243	ArrayLV = EmitArraySubscriptExpr(E: ASE, /*Accessed*/ true);
4244	else
4245	ArrayLV = EmitLValue(E: Array);
4246	auto *Idx = EmitIdxAfterBase(/*Promote*/true);
4247
4248	if (SanOpts.has(K: SanitizerKind::ArrayBounds)) {
4249	// If the array being accessed has a "counted_by" attribute, generate
4250	// bounds checking code. The "count" field is at the top level of the
4251	// struct or in an anonymous struct, that's also at the top level. Future
4252	// expansions may allow the "count" to reside at any place in the struct,
4253	// but the value of "counted_by" will be a "simple" path to the count,
4254	// i.e. "a.b.count", so we shouldn't need the full force of EmitLValue or
4255	// similar to emit the correct GEP.
4256	const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
4257	getLangOpts().getStrictFlexArraysLevel();
4258
4259	if (const auto *ME = dyn_cast<MemberExpr>(Val: Array);
4260	ME &&
4261	ME->isFlexibleArrayMemberLike(getContext(), StrictFlexArraysLevel) &&
4262	ME->getMemberDecl()->hasAttr<CountedByAttr>()) {
4263	const FieldDecl *FAMDecl = dyn_cast<FieldDecl>(Val: ME->getMemberDecl());
4264	if (const FieldDecl *CountFD = FindCountedByField(FD: FAMDecl)) {
4265	if (std::optional<int64_t> Diff =
4266	getOffsetDifferenceInBits(CGF&: *this, FD1: CountFD, FD2: FAMDecl)) {
4267	CharUnits OffsetDiff = CGM.getContext().toCharUnitsFromBits(BitSize: *Diff);
4268
4269	// Create a GEP with a byte offset between the FAM and count and
4270	// use that to load the count value.
4271	Addr = Builder.CreatePointerBitCastOrAddrSpaceCast(
4272	Addr: ArrayLV.getAddress(CGF&: *this), Ty: Int8PtrTy, ElementTy: Int8Ty);
4273
4274	llvm::Type *CountTy = ConvertType(CountFD->getType());
4275	llvm::Value *Res = Builder.CreateInBoundsGEP(
4276	Ty: Int8Ty, Ptr: Addr.getPointer(),
4277	IdxList: Builder.getInt32(C: OffsetDiff.getQuantity()), Name: ".counted_by.gep");
4278	Res = Builder.CreateAlignedLoad(CountTy, Res, getIntAlign(),
4279	".counted_by.load");
4280
4281	// Now emit the bounds checking.
4282	EmitBoundsCheckImpl(E, Res, Idx, E->getIdx()->getType(),
4283	Array->getType(), Accessed);
4284	}
4285	}
4286	}
4287	}
4288
4289	// Propagate the alignment from the array itself to the result.
4290	QualType arrayType = Array->getType();
4291	Addr = emitArraySubscriptGEP(
4292	*this, ArrayLV.getAddress(CGF&: *this), {CGM.getSize(numChars: CharUnits::Zero()), Idx},
4293	E->getType(), !getLangOpts().isSignedOverflowDefined(), SignedIndices,
4294	E->getExprLoc(), &arrayType, E->getBase());
4295	EltBaseInfo = ArrayLV.getBaseInfo();
4296	EltTBAAInfo = CGM.getTBAAInfoForSubobject(Base: ArrayLV, AccessType: E->getType());
4297	} else {
4298	// The base must be a pointer; emit it with an estimate of its alignment.
4299	Addr = EmitPointerWithAlignment(E: E->getBase(), BaseInfo: &EltBaseInfo, TBAAInfo: &EltTBAAInfo);
4300	auto *Idx = EmitIdxAfterBase(/*Promote*/true);
4301	QualType ptrType = E->getBase()->getType();
4302	Addr = emitArraySubscriptGEP(*this, Addr, Idx, E->getType(),
4303	!getLangOpts().isSignedOverflowDefined(),
4304	SignedIndices, E->getExprLoc(), &ptrType,
4305	E->getBase());
4306	}
4307
4308	LValue LV = MakeAddrLValue(Addr, E->getType(), EltBaseInfo, EltTBAAInfo);
4309
4310	if (getLangOpts().ObjC &&
4311	getLangOpts().getGC() != LangOptions::NonGC) {
4312	LV.setNonGC(!E->isOBJCGCCandidate(getContext()));
4313	setObjCGCLValueClass(getContext(), E, LV);
4314	}
4315	return LV;
4316	}
4317
4318	LValue CodeGenFunction::EmitMatrixSubscriptExpr(const MatrixSubscriptExpr *E) {
4319	assert(
4320	!E->isIncomplete() &&
4321	"incomplete matrix subscript expressions should be rejected during Sema");
4322	LValue Base = EmitLValue(E: E->getBase());
4323	llvm::Value *RowIdx = EmitScalarExpr(E: E->getRowIdx());
4324	llvm::Value *ColIdx = EmitScalarExpr(E: E->getColumnIdx());
4325	llvm::Value *NumRows = Builder.getIntN(
4326	N: RowIdx->getType()->getScalarSizeInBits(),
4327	C: E->getBase()->getType()->castAs<ConstantMatrixType>()->getNumRows());
4328	llvm::Value *FinalIdx =
4329	Builder.CreateAdd(LHS: Builder.CreateMul(LHS: ColIdx, RHS: NumRows), RHS: RowIdx);
4330	return LValue::MakeMatrixElt(
4331	matAddress: MaybeConvertMatrixAddress(Addr: Base.getAddress(CGF&: *this), CGF&: *this), Idx: FinalIdx,
4332	type: E->getBase()->getType(), BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
4333	}
4334
4335	static Address emitOMPArraySectionBase(CodeGenFunction &CGF, const Expr *Base,
4336	LValueBaseInfo &BaseInfo,
4337	TBAAAccessInfo &TBAAInfo,
4338	QualType BaseTy, QualType ElTy,
4339	bool IsLowerBound) {
4340	LValue BaseLVal;
4341	if (auto *ASE = dyn_cast<OMPArraySectionExpr>(Val: Base->IgnoreParenImpCasts())) {
4342	BaseLVal = CGF.EmitOMPArraySectionExpr(E: ASE, IsLowerBound);
4343	if (BaseTy->isArrayType()) {
4344	Address Addr = BaseLVal.getAddress(CGF);
4345	BaseInfo = BaseLVal.getBaseInfo();
4346
4347	// If the array type was an incomplete type, we need to make sure
4348	// the decay ends up being the right type.
4349	llvm::Type *NewTy = CGF.ConvertType(T: BaseTy);
4350	Addr = Addr.withElementType(ElemTy: NewTy);
4351
4352	// Note that VLA pointers are always decayed, so we don't need to do
4353	// anything here.
4354	if (!BaseTy->isVariableArrayType()) {
4355	assert(isa<llvm::ArrayType>(Addr.getElementType()) &&
4356	"Expected pointer to array");
4357	Addr = CGF.Builder.CreateConstArrayGEP(Addr, Index: 0, Name: "arraydecay");
4358	}
4359
4360	return Addr.withElementType(ElemTy: CGF.ConvertTypeForMem(T: ElTy));
4361	}
4362	LValueBaseInfo TypeBaseInfo;
4363	TBAAAccessInfo TypeTBAAInfo;
4364	CharUnits Align =
4365	CGF.CGM.getNaturalTypeAlignment(T: ElTy, BaseInfo: &TypeBaseInfo, TBAAInfo: &TypeTBAAInfo);
4366	BaseInfo.mergeForCast(Info: TypeBaseInfo);
4367	TBAAInfo = CGF.CGM.mergeTBAAInfoForCast(SourceInfo: TBAAInfo, TargetInfo: TypeTBAAInfo);
4368	return Address(CGF.Builder.CreateLoad(Addr: BaseLVal.getAddress(CGF)),
4369	CGF.ConvertTypeForMem(T: ElTy), Align);
4370	}
4371	return CGF.EmitPointerWithAlignment(E: Base, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
4372	}
4373
4374	LValue CodeGenFunction::EmitOMPArraySectionExpr(const OMPArraySectionExpr *E,
4375	bool IsLowerBound) {
4376	QualType BaseTy = OMPArraySectionExpr::getBaseOriginalType(Base: E->getBase());
4377	QualType ResultExprTy;
4378	if (auto *AT = getContext().getAsArrayType(T: BaseTy))
4379	ResultExprTy = AT->getElementType();
4380	else
4381	ResultExprTy = BaseTy->getPointeeType();
4382	llvm::Value *Idx = nullptr;
4383	if (IsLowerBound || E->getColonLocFirst().isInvalid()) {
4384	// Requesting lower bound or upper bound, but without provided length and
4385	// without ':' symbol for the default length -> length = 1.
4386	// Idx = LowerBound ?: 0;
4387	if (auto *LowerBound = E->getLowerBound()) {
4388	Idx = Builder.CreateIntCast(
4389	V: EmitScalarExpr(E: LowerBound), DestTy: IntPtrTy,
4390	isSigned: LowerBound->getType()->hasSignedIntegerRepresentation());
4391	} else
4392	Idx = llvm::ConstantInt::getNullValue(Ty: IntPtrTy);
4393	} else {
4394	// Try to emit length or lower bound as constant. If this is possible, 1
4395	// is subtracted from constant length or lower bound. Otherwise, emit LLVM
4396	// IR (LB + Len) - 1.
4397	auto &C = CGM.getContext();
4398	auto *Length = E->getLength();
4399	llvm::APSInt ConstLength;
4400	if (Length) {
4401	// Idx = LowerBound + Length - 1;
4402	if (std::optional<llvm::APSInt> CL = Length->getIntegerConstantExpr(Ctx: C)) {
4403	ConstLength = CL->zextOrTrunc(width: PointerWidthInBits);
4404	Length = nullptr;
4405	}
4406	auto *LowerBound = E->getLowerBound();
4407	llvm::APSInt ConstLowerBound(PointerWidthInBits, /*isUnsigned=*/false);
4408	if (LowerBound) {
4409	if (std::optional<llvm::APSInt> LB =
4410	LowerBound->getIntegerConstantExpr(Ctx: C)) {
4411	ConstLowerBound = LB->zextOrTrunc(width: PointerWidthInBits);
4412	LowerBound = nullptr;
4413	}
4414	}
4415	if (!Length)
4416	--ConstLength;
4417	else if (!LowerBound)
4418	--ConstLowerBound;
4419
4420	if (Length || LowerBound) {
4421	auto *LowerBoundVal =
4422	LowerBound
4423	? Builder.CreateIntCast(
4424	V: EmitScalarExpr(E: LowerBound), DestTy: IntPtrTy,
4425	isSigned: LowerBound->getType()->hasSignedIntegerRepresentation())
4426	: llvm::ConstantInt::get(Ty: IntPtrTy, V: ConstLowerBound);
4427	auto *LengthVal =
4428	Length
4429	? Builder.CreateIntCast(
4430	V: EmitScalarExpr(E: Length), DestTy: IntPtrTy,
4431	isSigned: Length->getType()->hasSignedIntegerRepresentation())
4432	: llvm::ConstantInt::get(Ty: IntPtrTy, V: ConstLength);
4433	Idx = Builder.CreateAdd(LHS: LowerBoundVal, RHS: LengthVal, Name: "lb_add_len",
4434	/*HasNUW=*/false,
4435	HasNSW: !getLangOpts().isSignedOverflowDefined());
4436	if (Length && LowerBound) {
4437	Idx = Builder.CreateSub(
4438	LHS: Idx, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, /*V=*/1), Name: "idx_sub_1",
4439	/*HasNUW=*/false, HasNSW: !getLangOpts().isSignedOverflowDefined());
4440	}
4441	} else
4442	Idx = llvm::ConstantInt::get(Ty: IntPtrTy, V: ConstLength + ConstLowerBound);
4443	} else {
4444	// Idx = ArraySize - 1;
4445	QualType ArrayTy = BaseTy->isPointerType()
4446	? E->getBase()->IgnoreParenImpCasts()->getType()
4447	: BaseTy;
4448	if (auto *VAT = C.getAsVariableArrayType(T: ArrayTy)) {
4449	Length = VAT->getSizeExpr();
4450	if (std::optional<llvm::APSInt> L = Length->getIntegerConstantExpr(Ctx: C)) {
4451	ConstLength = *L;
4452	Length = nullptr;
4453	}
4454	} else {
4455	auto *CAT = C.getAsConstantArrayType(T: ArrayTy);
4456	assert(CAT && "unexpected type for array initializer");
4457	ConstLength = CAT->getSize();
4458	}
4459	if (Length) {
4460	auto *LengthVal = Builder.CreateIntCast(
4461	V: EmitScalarExpr(E: Length), DestTy: IntPtrTy,
4462	isSigned: Length->getType()->hasSignedIntegerRepresentation());
4463	Idx = Builder.CreateSub(
4464	LHS: LengthVal, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, /*V=*/1), Name: "len_sub_1",
4465	/*HasNUW=*/false, HasNSW: !getLangOpts().isSignedOverflowDefined());
4466	} else {
4467	ConstLength = ConstLength.zextOrTrunc(width: PointerWidthInBits);
4468	--ConstLength;
4469	Idx = llvm::ConstantInt::get(Ty: IntPtrTy, V: ConstLength);
4470	}
4471	}
4472	}
4473	assert(Idx);
4474
4475	Address EltPtr = Address::invalid();
4476	LValueBaseInfo BaseInfo;
4477	TBAAAccessInfo TBAAInfo;
4478	if (auto *VLA = getContext().getAsVariableArrayType(T: ResultExprTy)) {
4479	// The base must be a pointer, which is not an aggregate. Emit
4480	// it. It needs to be emitted first in case it's what captures
4481	// the VLA bounds.
4482	Address Base =
4483	emitOMPArraySectionBase(*this, E->getBase(), BaseInfo, TBAAInfo,
4484	BaseTy, VLA->getElementType(), IsLowerBound);
4485	// The element count here is the total number of non-VLA elements.
4486	llvm::Value *NumElements = getVLASize(vla: VLA).NumElts;
4487
4488	// Effectively, the multiply by the VLA size is part of the GEP.
4489	// GEP indexes are signed, and scaling an index isn't permitted to
4490	// signed-overflow, so we use the same semantics for our explicit
4491	// multiply. We suppress this if overflow is not undefined behavior.
4492	if (getLangOpts().isSignedOverflowDefined())
4493	Idx = Builder.CreateMul(LHS: Idx, RHS: NumElements);
4494	else
4495	Idx = Builder.CreateNSWMul(LHS: Idx, RHS: NumElements);
4496	EltPtr = emitArraySubscriptGEP(*this, Base, Idx, VLA->getElementType(),
4497	!getLangOpts().isSignedOverflowDefined(),
4498	/*signedIndices=*/false, E->getExprLoc());
4499	} else if (const Expr *Array = isSimpleArrayDecayOperand(E: E->getBase())) {
4500	// If this is A[i] where A is an array, the frontend will have decayed the
4501	// base to be a ArrayToPointerDecay implicit cast. While correct, it is
4502	// inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
4503	// "gep x, i" here. Emit one "gep A, 0, i".
4504	assert(Array->getType()->isArrayType() &&
4505	"Array to pointer decay must have array source type!");
4506	LValue ArrayLV;
4507	// For simple multidimensional array indexing, set the 'accessed' flag for
4508	// better bounds-checking of the base expression.
4509	if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: Array))
4510	ArrayLV = EmitArraySubscriptExpr(E: ASE, /*Accessed*/ true);
4511	else
4512	ArrayLV = EmitLValue(E: Array);
4513
4514	// Propagate the alignment from the array itself to the result.
4515	EltPtr = emitArraySubscriptGEP(
4516	CGF&: *this, addr: ArrayLV.getAddress(CGF&: *this), indices: {CGM.getSize(numChars: CharUnits::Zero()), Idx},
4517	eltType: ResultExprTy, inbounds: !getLangOpts().isSignedOverflowDefined(),
4518	/*signedIndices=*/false, loc: E->getExprLoc());
4519	BaseInfo = ArrayLV.getBaseInfo();
4520	TBAAInfo = CGM.getTBAAInfoForSubobject(Base: ArrayLV, AccessType: ResultExprTy);
4521	} else {
4522	Address Base = emitOMPArraySectionBase(CGF&: *this, Base: E->getBase(), BaseInfo,
4523	TBAAInfo, BaseTy, ElTy: ResultExprTy,
4524	IsLowerBound);
4525	EltPtr = emitArraySubscriptGEP(CGF&: *this, addr: Base, indices: Idx, eltType: ResultExprTy,
4526	inbounds: !getLangOpts().isSignedOverflowDefined(),
4527	/*signedIndices=*/false, loc: E->getExprLoc());
4528	}
4529
4530	return MakeAddrLValue(Addr: EltPtr, T: ResultExprTy, BaseInfo, TBAAInfo);
4531	}
4532
4533	LValue CodeGenFunction::
4534	EmitExtVectorElementExpr(const ExtVectorElementExpr *E) {
4535	// Emit the base vector as an l-value.
4536	LValue Base;
4537
4538	// ExtVectorElementExpr's base can either be a vector or pointer to vector.
4539	if (E->isArrow()) {
4540	// If it is a pointer to a vector, emit the address and form an lvalue with
4541	// it.
4542	LValueBaseInfo BaseInfo;
4543	TBAAAccessInfo TBAAInfo;
4544	Address Ptr = EmitPointerWithAlignment(E: E->getBase(), BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
4545	const auto *PT = E->getBase()->getType()->castAs<PointerType>();
4546	Base = MakeAddrLValue(Addr: Ptr, T: PT->getPointeeType(), BaseInfo, TBAAInfo);
4547	Base.getQuals().removeObjCGCAttr();
4548	} else if (E->getBase()->isGLValue()) {
4549	// Otherwise, if the base is an lvalue ( as in the case of foo.x.x),
4550	// emit the base as an lvalue.
4551	assert(E->getBase()->getType()->isVectorType());
4552	Base = EmitLValue(E: E->getBase());
4553	} else {
4554	// Otherwise, the base is a normal rvalue (as in (V+V).x), emit it as such.
4555	assert(E->getBase()->getType()->isVectorType() &&
4556	"Result must be a vector");
4557	llvm::Value *Vec = EmitScalarExpr(E: E->getBase());
4558
4559	// Store the vector to memory (because LValue wants an address).
4560	Address VecMem = CreateMemTemp(Ty: E->getBase()->getType());
4561	Builder.CreateStore(Val: Vec, Addr: VecMem);
4562	Base = MakeAddrLValue(Addr: VecMem, T: E->getBase()->getType(),
4563	Source: AlignmentSource::Decl);
4564	}
4565
4566	QualType type =
4567	E->getType().withCVRQualifiers(Base.getQuals().getCVRQualifiers());
4568
4569	// Encode the element access list into a vector of unsigned indices.
4570	SmallVector<uint32_t, 4> Indices;
4571	E->getEncodedElementAccess(Elts&: Indices);
4572
4573	if (Base.isSimple()) {
4574	llvm::Constant *CV =
4575	llvm::ConstantDataVector::get(Context&: getLLVMContext(), Elts: Indices);
4576	return LValue::MakeExtVectorElt(vecAddress: Base.getAddress(CGF&: *this), Elts: CV, type,
4577	BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
4578	}
4579	assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");
4580
4581	llvm::Constant *BaseElts = Base.getExtVectorElts();
4582	SmallVector<llvm::Constant *, 4> CElts;
4583
4584	for (unsigned i = 0, e = Indices.size(); i != e; ++i)
4585	CElts.push_back(Elt: BaseElts->getAggregateElement(Elt: Indices[i]));
4586	llvm::Constant *CV = llvm::ConstantVector::get(V: CElts);
4587	return LValue::MakeExtVectorElt(vecAddress: Base.getExtVectorAddress(), Elts: CV, type,
4588	BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
4589	}
4590
4591	LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) {
4592	if (DeclRefExpr *DRE = tryToConvertMemberExprToDeclRefExpr(CGF&: *this, ME: E)) {
4593	EmitIgnoredExpr(E: E->getBase());
4594	return EmitDeclRefLValue(E: DRE);
4595	}
4596
4597	Expr *BaseExpr = E->getBase();
4598	// If this is s.x, emit s as an lvalue. If it is s->x, emit s as a scalar.
4599	LValue BaseLV;
4600	if (E->isArrow()) {
4601	LValueBaseInfo BaseInfo;
4602	TBAAAccessInfo TBAAInfo;
4603	Address Addr = EmitPointerWithAlignment(E: BaseExpr, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
4604	QualType PtrTy = BaseExpr->getType()->getPointeeType();
4605	SanitizerSet SkippedChecks;
4606	bool IsBaseCXXThis = IsWrappedCXXThis(Obj: BaseExpr);
4607	if (IsBaseCXXThis)
4608	SkippedChecks.set(K: SanitizerKind::Alignment, Value: true);
4609	if (IsBaseCXXThis || isa<DeclRefExpr>(Val: BaseExpr))
4610	SkippedChecks.set(K: SanitizerKind::Null, Value: true);
4611	EmitTypeCheck(TCK: TCK_MemberAccess, Loc: E->getExprLoc(), Ptr: Addr.getPointer(), Ty: PtrTy,
4612	/*Alignment=*/CharUnits::Zero(), SkippedChecks);
4613	BaseLV = MakeAddrLValue(Addr, T: PtrTy, BaseInfo, TBAAInfo);
4614	} else
4615	BaseLV = EmitCheckedLValue(E: BaseExpr, TCK: TCK_MemberAccess);
4616
4617	NamedDecl *ND = E->getMemberDecl();
4618	if (auto *Field = dyn_cast<FieldDecl>(ND)) {
4619	LValue LV = EmitLValueForField(Base: BaseLV, Field: Field);
4620	setObjCGCLValueClass(getContext(), E, LV);
4621	if (getLangOpts().OpenMP) {
4622	// If the member was explicitly marked as nontemporal, mark it as
4623	// nontemporal. If the base lvalue is marked as nontemporal, mark access
4624	// to children as nontemporal too.
4625	if ((IsWrappedCXXThis(Obj: BaseExpr) &&
4626	CGM.getOpenMPRuntime().isNontemporalDecl(VD: Field)) ||
4627	BaseLV.isNontemporal())
4628	LV.setNontemporal(/*Value=*/true);
4629	}
4630	return LV;
4631	}
4632
4633	if (const auto *FD = dyn_cast<FunctionDecl>(ND))
4634	return EmitFunctionDeclLValue(*this, E, FD);
4635
4636	llvm_unreachable("Unhandled member declaration!");
4637	}
4638
4639	/// Given that we are currently emitting a lambda, emit an l-value for
4640	/// one of its members.
4641	///
4642	LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field,
4643	llvm::Value *ThisValue) {
4644	bool HasExplicitObjectParameter = false;
4645	if (const auto *MD = dyn_cast_if_present<CXXMethodDecl>(Val: CurCodeDecl)) {
4646	HasExplicitObjectParameter = MD->isExplicitObjectMemberFunction();
4647	assert(MD->getParent()->isLambda());
4648	assert(MD->getParent() == Field->getParent());
4649	}
4650	LValue LambdaLV;
4651	if (HasExplicitObjectParameter) {
4652	const VarDecl *D = cast<CXXMethodDecl>(Val: CurCodeDecl)->getParamDecl(0);
4653	auto It = LocalDeclMap.find(D);
4654	assert(It != LocalDeclMap.end() && "explicit parameter not loaded?");
4655	Address AddrOfExplicitObject = It->getSecond();
4656	if (D->getType()->isReferenceType())
4657	LambdaLV = EmitLoadOfReferenceLValue(AddrOfExplicitObject, D->getType(),
4658	AlignmentSource::Decl);
4659	else
4660	LambdaLV = MakeNaturalAlignAddrLValue(V: AddrOfExplicitObject.getPointer(),
4661	T: D->getType().getNonReferenceType());
4662	} else {
4663	QualType LambdaTagType = getContext().getTagDeclType(Field->getParent());
4664	LambdaLV = MakeNaturalAlignAddrLValue(V: ThisValue, T: LambdaTagType);
4665	}
4666	return EmitLValueForField(Base: LambdaLV, Field);
4667	}
4668
4669	LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field) {
4670	return EmitLValueForLambdaField(Field, ThisValue: CXXABIThisValue);
4671	}
4672
4673	/// Get the field index in the debug info. The debug info structure/union
4674	/// will ignore the unnamed bitfields.
4675	unsigned CodeGenFunction::getDebugInfoFIndex(const RecordDecl *Rec,
4676	unsigned FieldIndex) {
4677	unsigned I = 0, Skipped = 0;
4678
4679	for (auto *F : Rec->getDefinition()->fields()) {
4680	if (I == FieldIndex)
4681	break;
4682	if (F->isUnnamedBitfield())
4683	Skipped++;
4684	I++;
4685	}
4686
4687	return FieldIndex - Skipped;
4688	}
4689
4690	/// Get the address of a zero-sized field within a record. The resulting
4691	/// address doesn't necessarily have the right type.
4692	static Address emitAddrOfZeroSizeField(CodeGenFunction &CGF, Address Base,
4693	const FieldDecl *Field) {
4694	CharUnits Offset = CGF.getContext().toCharUnitsFromBits(
4695	BitSize: CGF.getContext().getFieldOffset(Field));
4696	if (Offset.isZero())
4697	return Base;
4698	Base = Base.withElementType(ElemTy: CGF.Int8Ty);
4699	return CGF.Builder.CreateConstInBoundsByteGEP(Addr: Base, Offset);
4700	}
4701
4702	/// Drill down to the storage of a field without walking into
4703	/// reference types.
4704	///
4705	/// The resulting address doesn't necessarily have the right type.
4706	static Address emitAddrOfFieldStorage(CodeGenFunction &CGF, Address base,
4707	const FieldDecl *field) {
4708	if (field->isZeroSize(Ctx: CGF.getContext()))
4709	return emitAddrOfZeroSizeField(CGF, Base: base, Field: field);
4710
4711	const RecordDecl *rec = field->getParent();
4712
4713	unsigned idx =
4714	CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(FD: field);
4715
4716	return CGF.Builder.CreateStructGEP(base, idx, field->getName());
4717	}
4718
4719	static Address emitPreserveStructAccess(CodeGenFunction &CGF, LValue base,
4720	Address addr, const FieldDecl *field) {
4721	const RecordDecl *rec = field->getParent();
4722	llvm::DIType *DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(
4723	Ty: base.getType(), Loc: rec->getLocation());
4724
4725	unsigned idx =
4726	CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(FD: field);
4727
4728	return CGF.Builder.CreatePreserveStructAccessIndex(
4729	Addr: addr, Index: idx, FieldIndex: CGF.getDebugInfoFIndex(Rec: rec, FieldIndex: field->getFieldIndex()), DbgInfo);
4730	}
4731
4732	static bool hasAnyVptr(const QualType Type, const ASTContext &Context) {
4733	const auto *RD = Type.getTypePtr()->getAsCXXRecordDecl();
4734	if (!RD)
4735	return false;
4736
4737	if (RD->isDynamicClass())
4738	return true;
4739
4740	for (const auto &Base : RD->bases())
4741	if (hasAnyVptr(Type: Base.getType(), Context))
4742	return true;
4743
4744	for (const FieldDecl *Field : RD->fields())
4745	if (hasAnyVptr(Field->getType(), Context))
4746	return true;
4747
4748	return false;
4749	}
4750
4751	LValue CodeGenFunction::EmitLValueForField(LValue base,
4752	const FieldDecl *field) {
4753	LValueBaseInfo BaseInfo = base.getBaseInfo();
4754
4755	if (field->isBitField()) {
4756	const CGRecordLayout &RL =
4757	CGM.getTypes().getCGRecordLayout(field->getParent());
4758	const CGBitFieldInfo &Info = RL.getBitFieldInfo(FD: field);
4759	const bool UseVolatile = isAAPCS(TargetInfo: CGM.getTarget()) &&
4760	CGM.getCodeGenOpts().AAPCSBitfieldWidth &&
4761	Info.VolatileStorageSize != 0 &&
4762	field->getType()
4763	.withCVRQualifiers(base.getVRQualifiers())
4764	.isVolatileQualified();
4765	Address Addr = base.getAddress(CGF&: *this);
4766	unsigned Idx = RL.getLLVMFieldNo(FD: field);
4767	const RecordDecl *rec = field->getParent();
4768	if (hasBPFPreserveStaticOffset(D: rec))
4769	Addr = wrapWithBPFPreserveStaticOffset(CGF&: *this, Addr);
4770	if (!UseVolatile) {
4771	if (!IsInPreservedAIRegion &&
4772	(!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>())) {
4773	if (Idx != 0)
4774	// For structs, we GEP to the field that the record layout suggests.
4775	Addr = Builder.CreateStructGEP(Addr, Idx, field->getName());
4776	} else {
4777	llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateRecordType(
4778	Ty: getContext().getRecordType(Decl: rec), L: rec->getLocation());
4779	Addr = Builder.CreatePreserveStructAccessIndex(
4780	Addr, Index: Idx, FieldIndex: getDebugInfoFIndex(Rec: rec, FieldIndex: field->getFieldIndex()),
4781	DbgInfo);
4782	}
4783	}
4784	const unsigned SS =
4785	UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
4786	// Get the access type.
4787	llvm::Type *FieldIntTy = llvm::Type::getIntNTy(C&: getLLVMContext(), N: SS);
4788	Addr = Addr.withElementType(ElemTy: FieldIntTy);
4789	if (UseVolatile) {
4790	const unsigned VolatileOffset = Info.VolatileStorageOffset.getQuantity();
4791	if (VolatileOffset)
4792	Addr = Builder.CreateConstInBoundsGEP(Addr, Index: VolatileOffset);
4793	}
4794
4795	QualType fieldType =
4796	field->getType().withCVRQualifiers(base.getVRQualifiers());
4797	// TODO: Support TBAA for bit fields.
4798	LValueBaseInfo FieldBaseInfo(BaseInfo.getAlignmentSource());
4799	return LValue::MakeBitfield(Addr, Info, type: fieldType, BaseInfo: FieldBaseInfo,
4800	TBAAInfo: TBAAAccessInfo());
4801	}
4802
4803	// Fields of may-alias structures are may-alias themselves.
4804	// FIXME: this should get propagated down through anonymous structs
4805	// and unions.
4806	QualType FieldType = field->getType();
4807	const RecordDecl *rec = field->getParent();
4808	AlignmentSource BaseAlignSource = BaseInfo.getAlignmentSource();
4809	LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(Source: BaseAlignSource));
4810	TBAAAccessInfo FieldTBAAInfo;
4811	if (base.getTBAAInfo().isMayAlias() ||
4812	rec->hasAttr<MayAliasAttr>() || FieldType->isVectorType()) {
4813	FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
4814	} else if (rec->isUnion()) {
4815	// TODO: Support TBAA for unions.
4816	FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
4817	} else {
4818	// If no base type been assigned for the base access, then try to generate
4819	// one for this base lvalue.
4820	FieldTBAAInfo = base.getTBAAInfo();
4821	if (!FieldTBAAInfo.BaseType) {
4822	FieldTBAAInfo.BaseType = CGM.getTBAABaseTypeInfo(QTy: base.getType());
4823	assert(!FieldTBAAInfo.Offset &&
4824	"Nonzero offset for an access with no base type!");
4825	}
4826
4827	// Adjust offset to be relative to the base type.
4828	const ASTRecordLayout &Layout =
4829	getContext().getASTRecordLayout(D: field->getParent());
4830	unsigned CharWidth = getContext().getCharWidth();
4831	if (FieldTBAAInfo.BaseType)
4832	FieldTBAAInfo.Offset +=
4833	Layout.getFieldOffset(FieldNo: field->getFieldIndex()) / CharWidth;
4834
4835	// Update the final access type and size.
4836	FieldTBAAInfo.AccessType = CGM.getTBAATypeInfo(QTy: FieldType);
4837	FieldTBAAInfo.Size =
4838	getContext().getTypeSizeInChars(T: FieldType).getQuantity();
4839	}
4840
4841	Address addr = base.getAddress(CGF&: *this);
4842	if (hasBPFPreserveStaticOffset(D: rec))
4843	addr = wrapWithBPFPreserveStaticOffset(CGF&: *this, Addr&: addr);
4844	if (auto *ClassDef = dyn_cast<CXXRecordDecl>(Val: rec)) {
4845	if (CGM.getCodeGenOpts().StrictVTablePointers &&
4846	ClassDef->isDynamicClass()) {
4847	// Getting to any field of dynamic object requires stripping dynamic
4848	// information provided by invariant.group. This is because accessing
4849	// fields may leak the real address of dynamic object, which could result
4850	// in miscompilation when leaked pointer would be compared.
4851	auto *stripped = Builder.CreateStripInvariantGroup(Ptr: addr.getPointer());
4852	addr = Address(stripped, addr.getElementType(), addr.getAlignment());
4853	}
4854	}
4855
4856	unsigned RecordCVR = base.getVRQualifiers();
4857	if (rec->isUnion()) {
4858	// For unions, there is no pointer adjustment.
4859	if (CGM.getCodeGenOpts().StrictVTablePointers &&
4860	hasAnyVptr(Type: FieldType, Context: getContext()))
4861	// Because unions can easily skip invariant.barriers, we need to add
4862	// a barrier every time CXXRecord field with vptr is referenced.
4863	addr = Builder.CreateLaunderInvariantGroup(Addr: addr);
4864
4865	if (IsInPreservedAIRegion ||
4866	(getDebugInfo() && rec->hasAttr<BPFPreserveAccessIndexAttr>())) {
4867	// Remember the original union field index
4868	llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(Ty: base.getType(),
4869	Loc: rec->getLocation());
4870	addr = Address(
4871	Builder.CreatePreserveUnionAccessIndex(
4872	Base: addr.getPointer(), FieldIndex: getDebugInfoFIndex(Rec: rec, FieldIndex: field->getFieldIndex()), DbgInfo),
4873	addr.getElementType(), addr.getAlignment());
4874	}
4875
4876	if (FieldType->isReferenceType())
4877	addr = addr.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T: FieldType));
4878	} else {
4879	if (!IsInPreservedAIRegion &&
4880	(!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>()))
4881	// For structs, we GEP to the field that the record layout suggests.
4882	addr = emitAddrOfFieldStorage(CGF&: *this, base: addr, field);
4883	else
4884	// Remember the original struct field index
4885	addr = emitPreserveStructAccess(CGF&: *this, base, addr, field);
4886	}
4887
4888	// If this is a reference field, load the reference right now.
4889	if (FieldType->isReferenceType()) {
4890	LValue RefLVal =
4891	MakeAddrLValue(Addr: addr, T: FieldType, BaseInfo: FieldBaseInfo, TBAAInfo: FieldTBAAInfo);
4892	if (RecordCVR & Qualifiers::Volatile)
4893	RefLVal.getQuals().addVolatile();
4894	addr = EmitLoadOfReference(RefLVal, PointeeBaseInfo: &FieldBaseInfo, PointeeTBAAInfo: &FieldTBAAInfo);
4895
4896	// Qualifiers on the struct don't apply to the referencee.
4897	RecordCVR = 0;
4898	FieldType = FieldType->getPointeeType();
4899	}
4900
4901	// Make sure that the address is pointing to the right type. This is critical
4902	// for both unions and structs.
4903	addr = addr.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T: FieldType));
4904
4905	if (field->hasAttr<AnnotateAttr>())
4906	addr = EmitFieldAnnotations(D: field, V: addr);
4907
4908	LValue LV = MakeAddrLValue(Addr: addr, T: FieldType, BaseInfo: FieldBaseInfo, TBAAInfo: FieldTBAAInfo);
4909	LV.getQuals().addCVRQualifiers(mask: RecordCVR);
4910
4911	// __weak attribute on a field is ignored.
4912	if (LV.getQuals().getObjCGCAttr() == Qualifiers::Weak)
4913	LV.getQuals().removeObjCGCAttr();
4914
4915	return LV;
4916	}
4917
4918	LValue
4919	CodeGenFunction::EmitLValueForFieldInitialization(LValue Base,
4920	const FieldDecl *Field) {
4921	QualType FieldType = Field->getType();
4922
4923	if (!FieldType->isReferenceType())
4924	return EmitLValueForField(base: Base, field: Field);
4925
4926	Address V = emitAddrOfFieldStorage(CGF&: *this, base: Base.getAddress(CGF&: *this), field: Field);
4927
4928	// Make sure that the address is pointing to the right type.
4929	llvm::Type *llvmType = ConvertTypeForMem(T: FieldType);
4930	V = V.withElementType(ElemTy: llvmType);
4931
4932	// TODO: Generate TBAA information that describes this access as a structure
4933	// member access and not just an access to an object of the field's type. This
4934	// should be similar to what we do in EmitLValueForField().
4935	LValueBaseInfo BaseInfo = Base.getBaseInfo();
4936	AlignmentSource FieldAlignSource = BaseInfo.getAlignmentSource();
4937	LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(Source: FieldAlignSource));
4938	return MakeAddrLValue(Addr: V, T: FieldType, BaseInfo: FieldBaseInfo,
4939	TBAAInfo: CGM.getTBAAInfoForSubobject(Base, AccessType: FieldType));
4940	}
4941
4942	LValue CodeGenFunction::EmitCompoundLiteralLValue(const CompoundLiteralExpr *E){
4943	if (E->isFileScope()) {
4944	ConstantAddress GlobalPtr = CGM.GetAddrOfConstantCompoundLiteral(E);
4945	return MakeAddrLValue(GlobalPtr, E->getType(), AlignmentSource::Decl);
4946	}
4947	if (E->getType()->isVariablyModifiedType())
4948	// make sure to emit the VLA size.
4949	EmitVariablyModifiedType(Ty: E->getType());
4950
4951	Address DeclPtr = CreateMemTemp(E->getType(), ".compoundliteral");
4952	const Expr *InitExpr = E->getInitializer();
4953	LValue Result = MakeAddrLValue(DeclPtr, E->getType(), AlignmentSource::Decl);
4954
4955	EmitAnyExprToMem(E: InitExpr, Location: DeclPtr, Quals: E->getType().getQualifiers(),
4956	/*Init*/ IsInit: true);
4957
4958	// Block-scope compound literals are destroyed at the end of the enclosing
4959	// scope in C.
4960	if (!getLangOpts().CPlusPlus)
4961	if (QualType::DestructionKind DtorKind = E->getType().isDestructedType())
4962	pushLifetimeExtendedDestroy(kind: getCleanupKind(kind: DtorKind), addr: DeclPtr,
4963	type: E->getType(), destroyer: getDestroyer(destructionKind: DtorKind),
4964	useEHCleanupForArray: DtorKind & EHCleanup);
4965
4966	return Result;
4967	}
4968
4969	LValue CodeGenFunction::EmitInitListLValue(const InitListExpr *E) {
4970	if (!E->isGLValue())
4971	// Initializing an aggregate temporary in C++11: T{...}.
4972	return EmitAggExprToLValue(E);
4973
4974	// An lvalue initializer list must be initializing a reference.
4975	assert(E->isTransparent() && "non-transparent glvalue init list");
4976	return EmitLValue(E: E->getInit(Init: 0));
4977	}
4978
4979	/// Emit the operand of a glvalue conditional operator. This is either a glvalue
4980	/// or a (possibly-parenthesized) throw-expression. If this is a throw, no
4981	/// LValue is returned and the current block has been terminated.
4982	static std::optional<LValue> EmitLValueOrThrowExpression(CodeGenFunction &CGF,
4983	const Expr *Operand) {
4984	if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(Val: Operand->IgnoreParens())) {
4985	CGF.EmitCXXThrowExpr(E: ThrowExpr, /*KeepInsertionPoint*/false);
4986	return std::nullopt;
4987	}
4988
4989	return CGF.EmitLValue(E: Operand);
4990	}
4991
4992	namespace {
4993	// Handle the case where the condition is a constant evaluatable simple integer,
4994	// which means we don't have to separately handle the true/false blocks.
4995	std::optional<LValue> HandleConditionalOperatorLValueSimpleCase(
4996	CodeGenFunction &CGF, const AbstractConditionalOperator *E) {
4997	const Expr *condExpr = E->getCond();
4998	bool CondExprBool;
4999	if (CGF.ConstantFoldsToSimpleInteger(Cond: condExpr, Result&: CondExprBool)) {
5000	const Expr *Live = E->getTrueExpr(), *Dead = E->getFalseExpr();
5001	if (!CondExprBool)
5002	std::swap(a&: Live, b&: Dead);
5003
5004	if (!CGF.ContainsLabel(Dead)) {
5005	// If the true case is live, we need to track its region.
5006	if (CondExprBool)
5007	CGF.incrementProfileCounter(E);
5008	// If a throw expression we emit it and return an undefined lvalue
5009	// because it can't be used.
5010	if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(Val: Live->IgnoreParens())) {
5011	CGF.EmitCXXThrowExpr(E: ThrowExpr);
5012	llvm::Type *ElemTy = CGF.ConvertType(T: Dead->getType());
5013	llvm::Type *Ty = CGF.UnqualPtrTy;
5014	return CGF.MakeAddrLValue(
5015	Addr: Address(llvm::UndefValue::get(T: Ty), ElemTy, CharUnits::One()),
5016	T: Dead->getType());
5017	}
5018	return CGF.EmitLValue(E: Live);
5019	}
5020	}
5021	return std::nullopt;
5022	}
5023	struct ConditionalInfo {
5024	llvm::BasicBlock *lhsBlock, *rhsBlock;
5025	std::optional<LValue> LHS, RHS;
5026	};
5027
5028	// Create and generate the 3 blocks for a conditional operator.
5029	// Leaves the 'current block' in the continuation basic block.
5030	template<typename FuncTy>
5031	ConditionalInfo EmitConditionalBlocks(CodeGenFunction &CGF,
5032	const AbstractConditionalOperator *E,
5033	const FuncTy &BranchGenFunc) {
5034	ConditionalInfo Info{CGF.createBasicBlock(name: "cond.true"),
5035	CGF.createBasicBlock(name: "cond.false"), std::nullopt,
5036	std::nullopt};
5037	llvm::BasicBlock *endBlock = CGF.createBasicBlock(name: "cond.end");
5038
5039	CodeGenFunction::ConditionalEvaluation eval(CGF);
5040	CGF.EmitBranchOnBoolExpr(Cond: E->getCond(), TrueBlock: Info.lhsBlock, FalseBlock: Info.rhsBlock,
5041	TrueCount: CGF.getProfileCount(E));
5042
5043	// Any temporaries created here are conditional.
5044	CGF.EmitBlock(BB: Info.lhsBlock);
5045	CGF.incrementProfileCounter(E);
5046	eval.begin(CGF);
5047	Info.LHS = BranchGenFunc(CGF, E->getTrueExpr());
5048	eval.end(CGF);
5049	Info.lhsBlock = CGF.Builder.GetInsertBlock();
5050
5051	if (Info.LHS)
5052	CGF.Builder.CreateBr(Dest: endBlock);
5053
5054	// Any temporaries created here are conditional.
5055	CGF.EmitBlock(BB: Info.rhsBlock);
5056	eval.begin(CGF);
5057	Info.RHS = BranchGenFunc(CGF, E->getFalseExpr());
5058	eval.end(CGF);
5059	Info.rhsBlock = CGF.Builder.GetInsertBlock();
5060	CGF.EmitBlock(BB: endBlock);
5061
5062	return Info;
5063	}
5064	} // namespace
5065
5066	void CodeGenFunction::EmitIgnoredConditionalOperator(
5067	const AbstractConditionalOperator *E) {
5068	if (!E->isGLValue()) {
5069	// ?: here should be an aggregate.
5070	assert(hasAggregateEvaluationKind(E->getType()) &&
5071	"Unexpected conditional operator!");
5072	return (void)EmitAggExprToLValue(E);
5073	}
5074
5075	OpaqueValueMapping binding(*this, E);
5076	if (HandleConditionalOperatorLValueSimpleCase(CGF&: *this, E))
5077	return;
5078
5079	EmitConditionalBlocks(CGF&: *this, E, BranchGenFunc: [](CodeGenFunction &CGF, const Expr *E) {
5080	CGF.EmitIgnoredExpr(E);
5081	return LValue{};
5082	});
5083	}
5084	LValue CodeGenFunction::EmitConditionalOperatorLValue(
5085	const AbstractConditionalOperator *expr) {
5086	if (!expr->isGLValue()) {
5087	// ?: here should be an aggregate.
5088	assert(hasAggregateEvaluationKind(expr->getType()) &&
5089	"Unexpected conditional operator!");
5090	return EmitAggExprToLValue(expr);
5091	}
5092
5093	OpaqueValueMapping binding(*this, expr);
5094	if (std::optional<LValue> Res =
5095	HandleConditionalOperatorLValueSimpleCase(CGF&: *this, E: expr))
5096	return *Res;
5097
5098	ConditionalInfo Info = EmitConditionalBlocks(
5099	CGF&: *this, E: expr, BranchGenFunc: [](CodeGenFunction &CGF, const Expr *E) {
5100	return EmitLValueOrThrowExpression(CGF, E);
5101	});
5102
5103	if ((Info.LHS && !Info.LHS->isSimple()) ||
5104	(Info.RHS && !Info.RHS->isSimple()))
5105	return EmitUnsupportedLValue(expr, "conditional operator");
5106
5107	if (Info.LHS && Info.RHS) {
5108	Address lhsAddr = Info.LHS->getAddress(*this);
5109	Address rhsAddr = Info.RHS->getAddress(*this);
5110	llvm::PHINode *phi = Builder.CreatePHI(Ty: lhsAddr.getType(), NumReservedValues: 2, Name: "cond-lvalue");
5111	phi->addIncoming(V: lhsAddr.getPointer(), BB: Info.lhsBlock);
5112	phi->addIncoming(V: rhsAddr.getPointer(), BB: Info.rhsBlock);
5113	Address result(phi, lhsAddr.getElementType(),
5114	std::min(a: lhsAddr.getAlignment(), b: rhsAddr.getAlignment()));
5115	AlignmentSource alignSource =
5116	std::max(Info.LHS->getBaseInfo().getAlignmentSource(),
5117	Info.RHS->getBaseInfo().getAlignmentSource());
5118	TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForConditionalOperator(
5119	InfoA: Info.LHS->getTBAAInfo(), InfoB: Info.RHS->getTBAAInfo());
5120	return MakeAddrLValue(result, expr->getType(), LValueBaseInfo(alignSource),
5121	TBAAInfo);
5122	} else {
5123	assert((Info.LHS || Info.RHS) &&
5124	"both operands of glvalue conditional are throw-expressions?");
5125	return Info.LHS ? *Info.LHS : *Info.RHS;
5126	}
5127	}
5128
5129	/// EmitCastLValue - Casts are never lvalues unless that cast is to a reference
5130	/// type. If the cast is to a reference, we can have the usual lvalue result,
5131	/// otherwise if a cast is needed by the code generator in an lvalue context,
5132	/// then it must mean that we need the address of an aggregate in order to
5133	/// access one of its members. This can happen for all the reasons that casts
5134	/// are permitted with aggregate result, including noop aggregate casts, and
5135	/// cast from scalar to union.
5136	LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
5137	switch (E->getCastKind()) {
5138	case CK_ToVoid:
5139	case CK_BitCast:
5140	case CK_LValueToRValueBitCast:
5141	case CK_ArrayToPointerDecay:
5142	case CK_FunctionToPointerDecay:
5143	case CK_NullToMemberPointer:
5144	case CK_NullToPointer:
5145	case CK_IntegralToPointer:
5146	case CK_PointerToIntegral:
5147	case CK_PointerToBoolean:
5148	case CK_IntegralCast:
5149	case CK_BooleanToSignedIntegral:
5150	case CK_IntegralToBoolean:
5151	case CK_IntegralToFloating:
5152	case CK_FloatingToIntegral:
5153	case CK_FloatingToBoolean:
5154	case CK_FloatingCast:
5155	case CK_FloatingRealToComplex:
5156	case CK_FloatingComplexToReal:
5157	case CK_FloatingComplexToBoolean:
5158	case CK_FloatingComplexCast:
5159	case CK_FloatingComplexToIntegralComplex:
5160	case CK_IntegralRealToComplex:
5161	case CK_IntegralComplexToReal:
5162	case CK_IntegralComplexToBoolean:
5163	case CK_IntegralComplexCast:
5164	case CK_IntegralComplexToFloatingComplex:
5165	case CK_DerivedToBaseMemberPointer:
5166	case CK_BaseToDerivedMemberPointer:
5167	case CK_MemberPointerToBoolean:
5168	case CK_ReinterpretMemberPointer:
5169	case CK_AnyPointerToBlockPointerCast:
5170	case CK_ARCProduceObject:
5171	case CK_ARCConsumeObject:
5172	case CK_ARCReclaimReturnedObject:
5173	case CK_ARCExtendBlockObject:
5174	case CK_CopyAndAutoreleaseBlockObject:
5175	case CK_IntToOCLSampler:
5176	case CK_FloatingToFixedPoint:
5177	case CK_FixedPointToFloating:
5178	case CK_FixedPointCast:
5179	case CK_FixedPointToBoolean:
5180	case CK_FixedPointToIntegral:
5181	case CK_IntegralToFixedPoint:
5182	case CK_MatrixCast:
5183	return EmitUnsupportedLValue(E, "unexpected cast lvalue");
5184
5185	case CK_Dependent:
5186	llvm_unreachable("dependent cast kind in IR gen!");
5187
5188	case CK_BuiltinFnToFnPtr:
5189	llvm_unreachable("builtin functions are handled elsewhere");
5190
5191	// These are never l-values; just use the aggregate emission code.
5192	case CK_NonAtomicToAtomic:
5193	case CK_AtomicToNonAtomic:
5194	return EmitAggExprToLValue(E);
5195
5196	case CK_Dynamic: {
5197	LValue LV = EmitLValue(E: E->getSubExpr());
5198	Address V = LV.getAddress(CGF&: *this);
5199	const auto *DCE = cast<CXXDynamicCastExpr>(Val: E);
5200	return MakeNaturalAlignAddrLValue(V: EmitDynamicCast(V, DCE), T: E->getType());
5201	}
5202
5203	case CK_ConstructorConversion:
5204	case CK_UserDefinedConversion:
5205	case CK_CPointerToObjCPointerCast:
5206	case CK_BlockPointerToObjCPointerCast:
5207	case CK_LValueToRValue:
5208	return EmitLValue(E: E->getSubExpr());
5209
5210	case CK_NoOp: {
5211	// CK_NoOp can model a qualification conversion, which can remove an array
5212	// bound and change the IR type.
5213	// FIXME: Once pointee types are removed from IR, remove this.
5214	LValue LV = EmitLValue(E: E->getSubExpr());
5215	// Propagate the volatile qualifer to LValue, if exist in E.
5216	if (E->changesVolatileQualification())
5217	LV.getQuals() = E->getType().getQualifiers();
5218	if (LV.isSimple()) {
5219	Address V = LV.getAddress(CGF&: *this);
5220	if (V.isValid()) {
5221	llvm::Type *T = ConvertTypeForMem(T: E->getType());
5222	if (V.getElementType() != T)
5223	LV.setAddress(V.withElementType(ElemTy: T));
5224	}
5225	}
5226	return LV;
5227	}
5228
5229	case CK_UncheckedDerivedToBase:
5230	case CK_DerivedToBase: {
5231	const auto *DerivedClassTy =
5232	E->getSubExpr()->getType()->castAs<RecordType>();
5233	auto *DerivedClassDecl = cast<CXXRecordDecl>(Val: DerivedClassTy->getDecl());
5234
5235	LValue LV = EmitLValue(E: E->getSubExpr());
5236	Address This = LV.getAddress(CGF&: *this);
5237
5238	// Perform the derived-to-base conversion
5239	Address Base = GetAddressOfBaseClass(
5240	Value: This, Derived: DerivedClassDecl, PathBegin: E->path_begin(), PathEnd: E->path_end(),
5241	/*NullCheckValue=*/false, Loc: E->getExprLoc());
5242
5243	// TODO: Support accesses to members of base classes in TBAA. For now, we
5244	// conservatively pretend that the complete object is of the base class
5245	// type.
5246	return MakeAddrLValue(Base, E->getType(), LV.getBaseInfo(),
5247	CGM.getTBAAInfoForSubobject(Base: LV, AccessType: E->getType()));
5248	}
5249	case CK_ToUnion:
5250	return EmitAggExprToLValue(E);
5251	case CK_BaseToDerived: {
5252	const auto *DerivedClassTy = E->getType()->castAs<RecordType>();
5253	auto *DerivedClassDecl = cast<CXXRecordDecl>(DerivedClassTy->getDecl());
5254
5255	LValue LV = EmitLValue(E: E->getSubExpr());
5256
5257	// Perform the base-to-derived conversion
5258	Address Derived = GetAddressOfDerivedClass(
5259	Value: LV.getAddress(CGF&: *this), Derived: DerivedClassDecl, PathBegin: E->path_begin(), PathEnd: E->path_end(),
5260	/*NullCheckValue=*/false);
5261
5262	// C++11 [expr.static.cast]p2: Behavior is undefined if a downcast is
5263	// performed and the object is not of the derived type.
5264	if (sanitizePerformTypeCheck())
5265	EmitTypeCheck(TCK: TCK_DowncastReference, Loc: E->getExprLoc(),
5266	Ptr: Derived.getPointer(), Ty: E->getType());
5267
5268	if (SanOpts.has(K: SanitizerKind::CFIDerivedCast))
5269	EmitVTablePtrCheckForCast(T: E->getType(), Derived,
5270	/*MayBeNull=*/false, TCK: CFITCK_DerivedCast,
5271	Loc: E->getBeginLoc());
5272
5273	return MakeAddrLValue(Derived, E->getType(), LV.getBaseInfo(),
5274	CGM.getTBAAInfoForSubobject(Base: LV, AccessType: E->getType()));
5275	}
5276	case CK_LValueBitCast: {
5277	// This must be a reinterpret_cast (or c-style equivalent).
5278	const auto *CE = cast<ExplicitCastExpr>(Val: E);
5279
5280	CGM.EmitExplicitCastExprType(E: CE, CGF: this);
5281	LValue LV = EmitLValue(E: E->getSubExpr());
5282	Address V = LV.getAddress(CGF&: *this).withElementType(
5283	ElemTy: ConvertTypeForMem(T: CE->getTypeAsWritten()->getPointeeType()));
5284
5285	if (SanOpts.has(K: SanitizerKind::CFIUnrelatedCast))
5286	EmitVTablePtrCheckForCast(T: E->getType(), Derived: V,
5287	/*MayBeNull=*/false, TCK: CFITCK_UnrelatedCast,
5288	Loc: E->getBeginLoc());
5289
5290	return MakeAddrLValue(V, E->getType(), LV.getBaseInfo(),
5291	CGM.getTBAAInfoForSubobject(Base: LV, AccessType: E->getType()));
5292	}
5293	case CK_AddressSpaceConversion: {
5294	LValue LV = EmitLValue(E: E->getSubExpr());
5295	QualType DestTy = getContext().getPointerType(E->getType());
5296	llvm::Value *V = getTargetHooks().performAddrSpaceCast(
5297	*this, LV.getPointer(CGF&: *this),
5298	E->getSubExpr()->getType().getAddressSpace(),
5299	E->getType().getAddressSpace(), ConvertType(T: DestTy));
5300	return MakeAddrLValue(Address(V, ConvertTypeForMem(T: E->getType()),
5301	LV.getAddress(CGF&: *this).getAlignment()),
5302	E->getType(), LV.getBaseInfo(), LV.getTBAAInfo());
5303	}
5304	case CK_ObjCObjectLValueCast: {
5305	LValue LV = EmitLValue(E: E->getSubExpr());
5306	Address V = LV.getAddress(CGF&: *this).withElementType(ElemTy: ConvertType(E->getType()));
5307	return MakeAddrLValue(V, E->getType(), LV.getBaseInfo(),
5308	CGM.getTBAAInfoForSubobject(Base: LV, AccessType: E->getType()));
5309	}
5310	case CK_ZeroToOCLOpaqueType:
5311	llvm_unreachable("NULL to OpenCL opaque type lvalue cast is not valid");
5312
5313	case CK_VectorSplat: {
5314	// LValue results of vector splats are only supported in HLSL.
5315	if (!getLangOpts().HLSL)
5316	return EmitUnsupportedLValue(E, "unexpected cast lvalue");
5317	return EmitLValue(E: E->getSubExpr());
5318	}
5319	}
5320
5321	llvm_unreachable("Unhandled lvalue cast kind?");
5322	}
5323
5324	LValue CodeGenFunction::EmitOpaqueValueLValue(const OpaqueValueExpr *e) {
5325	assert(OpaqueValueMappingData::shouldBindAsLValue(e));
5326	return getOrCreateOpaqueLValueMapping(e);
5327	}
5328
5329	LValue
5330	CodeGenFunction::getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e) {
5331	assert(OpaqueValueMapping::shouldBindAsLValue(e));
5332
5333	llvm::DenseMap<const OpaqueValueExpr*,LValue>::iterator
5334	it = OpaqueLValues.find(Val: e);
5335
5336	if (it != OpaqueLValues.end())
5337	return it->second;
5338
5339	assert(e->isUnique() && "LValue for a nonunique OVE hasn't been emitted");
5340	return EmitLValue(E: e->getSourceExpr());
5341	}
5342
5343	RValue
5344	CodeGenFunction::getOrCreateOpaqueRValueMapping(const OpaqueValueExpr *e) {
5345	assert(!OpaqueValueMapping::shouldBindAsLValue(e));
5346
5347	llvm::DenseMap<const OpaqueValueExpr*,RValue>::iterator
5348	it = OpaqueRValues.find(Val: e);
5349
5350	if (it != OpaqueRValues.end())
5351	return it->second;
5352
5353	assert(e->isUnique() && "RValue for a nonunique OVE hasn't been emitted");
5354	return EmitAnyExpr(E: e->getSourceExpr());
5355	}
5356
5357	RValue CodeGenFunction::EmitRValueForField(LValue LV,
5358	const FieldDecl *FD,
5359	SourceLocation Loc) {
5360	QualType FT = FD->getType();
5361	LValue FieldLV = EmitLValueForField(base: LV, field: FD);
5362	switch (getEvaluationKind(T: FT)) {
5363	case TEK_Complex:
5364	return RValue::getComplex(C: EmitLoadOfComplex(src: FieldLV, loc: Loc));
5365	case TEK_Aggregate:
5366	return FieldLV.asAggregateRValue(CGF&: *this);
5367	case TEK_Scalar:
5368	// This routine is used to load fields one-by-one to perform a copy, so
5369	// don't load reference fields.
5370	if (FD->getType()->isReferenceType())
5371	return RValue::get(V: FieldLV.getPointer(CGF&: *this));
5372	// Call EmitLoadOfScalar except when the lvalue is a bitfield to emit a
5373	// primitive load.
5374	if (FieldLV.isBitField())
5375	return EmitLoadOfLValue(LV: FieldLV, Loc);
5376	return RValue::get(V: EmitLoadOfScalar(lvalue: FieldLV, Loc));
5377	}
5378	llvm_unreachable("bad evaluation kind");
5379	}
5380
5381	//===--------------------------------------------------------------------===//
5382	// Expression Emission
5383	//===--------------------------------------------------------------------===//
5384
5385	RValue CodeGenFunction::EmitCallExpr(const CallExpr *E,
5386	ReturnValueSlot ReturnValue) {
5387	// Builtins never have block type.
5388	if (E->getCallee()->getType()->isBlockPointerType())
5389	return EmitBlockCallExpr(E, ReturnValue);
5390
5391	if (const auto *CE = dyn_cast<CXXMemberCallExpr>(Val: E))
5392	return EmitCXXMemberCallExpr(E: CE, ReturnValue);
5393
5394	if (const auto *CE = dyn_cast<CUDAKernelCallExpr>(Val: E))
5395	return EmitCUDAKernelCallExpr(E: CE, ReturnValue);
5396
5397	// A CXXOperatorCallExpr is created even for explicit object methods, but
5398	// these should be treated like static function call.
5399	if (const auto *CE = dyn_cast<CXXOperatorCallExpr>(Val: E))
5400	if (const auto *MD =
5401	dyn_cast_if_present<CXXMethodDecl>(CE->getCalleeDecl());
5402	MD && MD->isImplicitObjectMemberFunction())
5403	return EmitCXXOperatorMemberCallExpr(E: CE, MD: MD, ReturnValue);
5404
5405	CGCallee callee = EmitCallee(E: E->getCallee());
5406
5407	if (callee.isBuiltin()) {
5408	return EmitBuiltinExpr(GD: callee.getBuiltinDecl(), BuiltinID: callee.getBuiltinID(),
5409	E, ReturnValue);
5410	}
5411
5412	if (callee.isPseudoDestructor()) {
5413	return EmitCXXPseudoDestructorExpr(E: callee.getPseudoDestructorExpr());
5414	}
5415
5416	return EmitCall(FnType: E->getCallee()->getType(), Callee: callee, E, ReturnValue);
5417	}
5418
5419	/// Emit a CallExpr without considering whether it might be a subclass.
5420	RValue CodeGenFunction::EmitSimpleCallExpr(const CallExpr *E,
5421	ReturnValueSlot ReturnValue) {
5422	CGCallee Callee = EmitCallee(E: E->getCallee());
5423	return EmitCall(FnType: E->getCallee()->getType(), Callee, E, ReturnValue);
5424	}
5425
5426	// Detect the unusual situation where an inline version is shadowed by a
5427	// non-inline version. In that case we should pick the external one
5428	// everywhere. That's GCC behavior too.
5429	static bool OnlyHasInlineBuiltinDeclaration(const FunctionDecl *FD) {
5430	for (const FunctionDecl *PD = FD; PD; PD = PD->getPreviousDecl())
5431	if (!PD->isInlineBuiltinDeclaration())
5432	return false;
5433	return true;
5434	}
5435
5436	static CGCallee EmitDirectCallee(CodeGenFunction &CGF, GlobalDecl GD) {
5437	const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
5438
5439	if (auto builtinID = FD->getBuiltinID()) {
5440	std::string NoBuiltinFD = ("no-builtin-" + FD->getName()).str();
5441	std::string NoBuiltins = "no-builtins";
5442
5443	StringRef Ident = CGF.CGM.getMangledName(GD);
5444	std::string FDInlineName = (Ident + ".inline").str();
5445
5446	bool IsPredefinedLibFunction =
5447	CGF.getContext().BuiltinInfo.isPredefinedLibFunction(ID: builtinID);
5448	bool HasAttributeNoBuiltin =
5449	CGF.CurFn->getAttributes().hasFnAttr(Kind: NoBuiltinFD) ||
5450	CGF.CurFn->getAttributes().hasFnAttr(Kind: NoBuiltins);
5451
5452	// When directing calling an inline builtin, call it through it's mangled
5453	// name to make it clear it's not the actual builtin.
5454	if (CGF.CurFn->getName() != FDInlineName &&
5455	OnlyHasInlineBuiltinDeclaration(FD)) {
5456	llvm::Constant *CalleePtr = EmitFunctionDeclPointer(CGM&: CGF.CGM, GD);
5457	llvm::Function *Fn = llvm::cast<llvm::Function>(Val: CalleePtr);
5458	llvm::Module *M = Fn->getParent();
5459	llvm::Function *Clone = M->getFunction(Name: FDInlineName);
5460	if (!Clone) {
5461	Clone = llvm::Function::Create(Ty: Fn->getFunctionType(),
5462	Linkage: llvm::GlobalValue::InternalLinkage,
5463	AddrSpace: Fn->getAddressSpace(), N: FDInlineName, M);
5464	Clone->addFnAttr(llvm::Attribute::AlwaysInline);
5465	}
5466	return CGCallee::forDirect(functionPtr: Clone, abstractInfo: GD);
5467	}
5468
5469	// Replaceable builtins provide their own implementation of a builtin. If we
5470	// are in an inline builtin implementation, avoid trivial infinite
5471	// recursion. Honor __attribute__((no_builtin("foo"))) or
5472	// __attribute__((no_builtin)) on the current function unless foo is
5473	// not a predefined library function which means we must generate the
5474	// builtin no matter what.
5475	else if (!IsPredefinedLibFunction || !HasAttributeNoBuiltin)
5476	return CGCallee::forBuiltin(builtinID, builtinDecl: FD);
5477	}
5478
5479	llvm::Constant *CalleePtr = EmitFunctionDeclPointer(CGM&: CGF.CGM, GD);
5480	if (CGF.CGM.getLangOpts().CUDA && !CGF.CGM.getLangOpts().CUDAIsDevice &&
5481	FD->hasAttr<CUDAGlobalAttr>())
5482	CalleePtr = CGF.CGM.getCUDARuntime().getKernelStub(
5483	Handle: cast<llvm::GlobalValue>(Val: CalleePtr->stripPointerCasts()));
5484
5485	return CGCallee::forDirect(functionPtr: CalleePtr, abstractInfo: GD);
5486	}
5487
5488	CGCallee CodeGenFunction::EmitCallee(const Expr *E) {
5489	E = E->IgnoreParens();
5490
5491	// Look through function-to-pointer decay.
5492	if (auto ICE = dyn_cast<ImplicitCastExpr>(Val: E)) {
5493	if (ICE->getCastKind() == CK_FunctionToPointerDecay ||
5494	ICE->getCastKind() == CK_BuiltinFnToFnPtr) {
5495	return EmitCallee(E: ICE->getSubExpr());
5496	}
5497
5498	// Resolve direct calls.
5499	} else if (auto DRE = dyn_cast<DeclRefExpr>(Val: E)) {
5500	if (auto FD = dyn_cast<FunctionDecl>(Val: DRE->getDecl())) {
5501	return EmitDirectCallee(CGF&: *this, GD: FD);
5502	}
5503	} else if (auto ME = dyn_cast<MemberExpr>(Val: E)) {
5504	if (auto FD = dyn_cast<FunctionDecl>(Val: ME->getMemberDecl())) {
5505	EmitIgnoredExpr(E: ME->getBase());
5506	return EmitDirectCallee(CGF&: *this, GD: FD);
5507	}
5508
5509	// Look through template substitutions.
5510	} else if (auto NTTP = dyn_cast<SubstNonTypeTemplateParmExpr>(Val: E)) {
5511	return EmitCallee(E: NTTP->getReplacement());
5512
5513	// Treat pseudo-destructor calls differently.
5514	} else if (auto PDE = dyn_cast<CXXPseudoDestructorExpr>(Val: E)) {
5515	return CGCallee::forPseudoDestructor(E: PDE);
5516	}
5517
5518	// Otherwise, we have an indirect reference.
5519	llvm::Value *calleePtr;
5520	QualType functionType;
5521	if (auto ptrType = E->getType()->getAs<PointerType>()) {
5522	calleePtr = EmitScalarExpr(E);
5523	functionType = ptrType->getPointeeType();
5524	} else {
5525	functionType = E->getType();
5526	calleePtr = EmitLValue(E, IsKnownNonNull: KnownNonNull).getPointer(CGF&: *this);
5527	}
5528	assert(functionType->isFunctionType());
5529
5530	GlobalDecl GD;
5531	if (const auto *VD =
5532	dyn_cast_or_null<VarDecl>(Val: E->getReferencedDeclOfCallee()))
5533	GD = GlobalDecl(VD);
5534
5535	CGCalleeInfo calleeInfo(functionType->getAs<FunctionProtoType>(), GD);
5536	CGCallee callee(calleeInfo, calleePtr);
5537	return callee;
5538	}
5539
5540	LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) {
5541	// Comma expressions just emit their LHS then their RHS as an l-value.
5542	if (E->getOpcode() == BO_Comma) {
5543	EmitIgnoredExpr(E: E->getLHS());
5544	EnsureInsertPoint();
5545	return EmitLValue(E: E->getRHS());
5546	}
5547
5548	if (E->getOpcode() == BO_PtrMemD ||
5549	E->getOpcode() == BO_PtrMemI)
5550	return EmitPointerToDataMemberBinaryExpr(E);
5551
5552	assert(E->getOpcode() == BO_Assign && "unexpected binary l-value");
5553
5554	// Note that in all of these cases, __block variables need the RHS
5555	// evaluated first just in case the variable gets moved by the RHS.
5556
5557	switch (getEvaluationKind(T: E->getType())) {
5558	case TEK_Scalar: {
5559	switch (E->getLHS()->getType().getObjCLifetime()) {
5560	case Qualifiers::OCL_Strong:
5561	return EmitARCStoreStrong(E, /*ignored*/ false).first;
5562
5563	case Qualifiers::OCL_Autoreleasing:
5564	return EmitARCStoreAutoreleasing(e: E).first;
5565
5566	// No reason to do any of these differently.
5567	case Qualifiers::OCL_None:
5568	case Qualifiers::OCL_ExplicitNone:
5569	case Qualifiers::OCL_Weak:
5570	break;
5571	}
5572
5573	RValue RV = EmitAnyExpr(E: E->getRHS());
5574	LValue LV = EmitCheckedLValue(E: E->getLHS(), TCK: TCK_Store);
5575	if (RV.isScalar())
5576	EmitNullabilityCheck(LHS: LV, RHS: RV.getScalarVal(), Loc: E->getExprLoc());
5577	EmitStoreThroughLValue(Src: RV, Dst: LV);
5578	if (getLangOpts().OpenMP)
5579	CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF&: *this,
5580	LHS: E->getLHS());
5581	return LV;
5582	}
5583
5584	case TEK_Complex:
5585	return EmitComplexAssignmentLValue(E);
5586
5587	case TEK_Aggregate:
5588	return EmitAggExprToLValue(E);
5589	}
5590	llvm_unreachable("bad evaluation kind");
5591	}
5592
5593	LValue CodeGenFunction::EmitCallExprLValue(const CallExpr *E) {
5594	RValue RV = EmitCallExpr(E);
5595
5596	if (!RV.isScalar())
5597	return MakeAddrLValue(RV.getAggregateAddress(), E->getType(),
5598	AlignmentSource::Decl);
5599
5600	assert(E->getCallReturnType(getContext())->isReferenceType() &&
5601	"Can't have a scalar return unless the return type is a "
5602	"reference type!");
5603
5604	return MakeNaturalAlignPointeeAddrLValue(V: RV.getScalarVal(), T: E->getType());
5605	}
5606
5607	LValue CodeGenFunction::EmitVAArgExprLValue(const VAArgExpr *E) {
5608	// FIXME: This shouldn't require another copy.
5609	return EmitAggExprToLValue(E);
5610	}
5611
5612	LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) {
5613	assert(E->getType()->getAsCXXRecordDecl()->hasTrivialDestructor()
5614	&& "binding l-value to type which needs a temporary");
5615	AggValueSlot Slot = CreateAggTemp(T: E->getType());
5616	EmitCXXConstructExpr(E, Dest: Slot);
5617	return MakeAddrLValue(Slot.getAddress(), E->getType(), AlignmentSource::Decl);
5618	}
5619
5620	LValue
5621	CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) {
5622	return MakeNaturalAlignAddrLValue(V: EmitCXXTypeidExpr(E), T: E->getType());
5623	}
5624
5625	Address CodeGenFunction::EmitCXXUuidofExpr(const CXXUuidofExpr *E) {
5626	return CGM.GetAddrOfMSGuidDecl(GD: E->getGuidDecl())
5627	.withElementType(ElemTy: ConvertType(E->getType()));
5628	}
5629
5630	LValue CodeGenFunction::EmitCXXUuidofLValue(const CXXUuidofExpr *E) {
5631	return MakeAddrLValue(EmitCXXUuidofExpr(E), E->getType(),
5632	AlignmentSource::Decl);
5633	}
5634
5635	LValue
5636	CodeGenFunction::EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E) {
5637	AggValueSlot Slot = CreateAggTemp(T: E->getType(), Name: "temp.lvalue");
5638	Slot.setExternallyDestructed();
5639	EmitAggExpr(E: E->getSubExpr(), AS: Slot);
5640	EmitCXXTemporary(Temporary: E->getTemporary(), TempType: E->getType(), Ptr: Slot.getAddress());
5641	return MakeAddrLValue(Slot.getAddress(), E->getType(), AlignmentSource::Decl);
5642	}
5643
5644	LValue CodeGenFunction::EmitObjCMessageExprLValue(const ObjCMessageExpr *E) {
5645	RValue RV = EmitObjCMessageExpr(E);
5646
5647	if (!RV.isScalar())
5648	return MakeAddrLValue(RV.getAggregateAddress(), E->getType(),
5649	AlignmentSource::Decl);
5650
5651	assert(E->getMethodDecl()->getReturnType()->isReferenceType() &&
5652	"Can't have a scalar return unless the return type is a "
5653	"reference type!");
5654
5655	return MakeNaturalAlignPointeeAddrLValue(V: RV.getScalarVal(), T: E->getType());
5656	}
5657
5658	LValue CodeGenFunction::EmitObjCSelectorLValue(const ObjCSelectorExpr *E) {
5659	Address V =
5660	CGM.getObjCRuntime().GetAddrOfSelector(CGF&: *this, Sel: E->getSelector());
5661	return MakeAddrLValue(V, E->getType(), AlignmentSource::Decl);
5662	}
5663
5664	llvm::Value *CodeGenFunction::EmitIvarOffset(const ObjCInterfaceDecl *Interface,
5665	const ObjCIvarDecl *Ivar) {
5666	return CGM.getObjCRuntime().EmitIvarOffset(CGF&: *this, Interface, Ivar);
5667	}
5668
5669	llvm::Value *
5670	CodeGenFunction::EmitIvarOffsetAsPointerDiff(const ObjCInterfaceDecl *Interface,
5671	const ObjCIvarDecl *Ivar) {
5672	llvm::Value *OffsetValue = EmitIvarOffset(Interface, Ivar);
5673	QualType PointerDiffType = getContext().getPointerDiffType();
5674	return Builder.CreateZExtOrTrunc(V: OffsetValue,
5675	DestTy: getTypes().ConvertType(T: PointerDiffType));
5676	}
5677
5678	LValue CodeGenFunction::EmitLValueForIvar(QualType ObjectTy,
5679	llvm::Value *BaseValue,
5680	const ObjCIvarDecl *Ivar,
5681	unsigned CVRQualifiers) {
5682	return CGM.getObjCRuntime().EmitObjCValueForIvar(CGF&: *this, ObjectTy, BaseValue,
5683	Ivar, CVRQualifiers);
5684	}
5685
5686	LValue CodeGenFunction::EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E) {
5687	// FIXME: A lot of the code below could be shared with EmitMemberExpr.
5688	llvm::Value *BaseValue = nullptr;
5689	const Expr *BaseExpr = E->getBase();
5690	Qualifiers BaseQuals;
5691	QualType ObjectTy;
5692	if (E->isArrow()) {
5693	BaseValue = EmitScalarExpr(E: BaseExpr);
5694	ObjectTy = BaseExpr->getType()->getPointeeType();
5695	BaseQuals = ObjectTy.getQualifiers();
5696	} else {
5697	LValue BaseLV = EmitLValue(E: BaseExpr);
5698	BaseValue = BaseLV.getPointer(CGF&: *this);
5699	ObjectTy = BaseExpr->getType();
5700	BaseQuals = ObjectTy.getQualifiers();
5701	}
5702
5703	LValue LV =
5704	EmitLValueForIvar(ObjectTy, BaseValue, Ivar: E->getDecl(),
5705	CVRQualifiers: BaseQuals.getCVRQualifiers());
5706	setObjCGCLValueClass(getContext(), E, LV);
5707	return LV;
5708	}
5709
5710	LValue CodeGenFunction::EmitStmtExprLValue(const StmtExpr *E) {
5711	// Can only get l-value for message expression returning aggregate type
5712	RValue RV = EmitAnyExprToTemp(E);
5713	return MakeAddrLValue(RV.getAggregateAddress(), E->getType(),
5714	AlignmentSource::Decl);
5715	}
5716
5717	RValue CodeGenFunction::EmitCall(QualType CalleeType, const CGCallee &OrigCallee,
5718	const CallExpr *E, ReturnValueSlot ReturnValue,
5719	llvm::Value *Chain) {
5720	// Get the actual function type. The callee type will always be a pointer to
5721	// function type or a block pointer type.
5722	assert(CalleeType->isFunctionPointerType() &&
5723	"Call must have function pointer type!");
5724
5725	const Decl *TargetDecl =
5726	OrigCallee.getAbstractInfo().getCalleeDecl().getDecl();
5727
5728	assert((!isa_and_present<FunctionDecl>(TargetDecl) ||
5729	!cast<FunctionDecl>(TargetDecl)->isImmediateFunction()) &&
5730	"trying to emit a call to an immediate function");
5731
5732	CalleeType = getContext().getCanonicalType(T: CalleeType);
5733
5734	auto PointeeType = cast<PointerType>(Val&: CalleeType)->getPointeeType();
5735
5736	CGCallee Callee = OrigCallee;
5737
5738	if (SanOpts.has(K: SanitizerKind::Function) &&
5739	(!TargetDecl || !isa<FunctionDecl>(Val: TargetDecl)) &&
5740	!isa<FunctionNoProtoType>(Val: PointeeType)) {
5741	if (llvm::Constant *PrefixSig =
5742	CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM)) {
5743	SanitizerScope SanScope(this);
5744	auto *TypeHash = getUBSanFunctionTypeHash(T: PointeeType);
5745
5746	llvm::Type *PrefixSigType = PrefixSig->getType();
5747	llvm::StructType *PrefixStructTy = llvm::StructType::get(
5748	Context&: CGM.getLLVMContext(), Elements: {PrefixSigType, Int32Ty}, /*isPacked=*/true);
5749
5750	llvm::Value *CalleePtr = Callee.getFunctionPointer();
5751
5752	// On 32-bit Arm, the low bit of a function pointer indicates whether
5753	// it's using the Arm or Thumb instruction set. The actual first
5754	// instruction lives at the same address either way, so we must clear
5755	// that low bit before using the function address to find the prefix
5756	// structure.
5757	//
5758	// This applies to both Arm and Thumb target triples, because
5759	// either one could be used in an interworking context where it
5760	// might be passed function pointers of both types.
5761	llvm::Value *AlignedCalleePtr;
5762	if (CGM.getTriple().isARM() || CGM.getTriple().isThumb()) {
5763	llvm::Value *CalleeAddress =
5764	Builder.CreatePtrToInt(V: CalleePtr, DestTy: IntPtrTy);
5765	llvm::Value *Mask = llvm::ConstantInt::get(Ty: IntPtrTy, V: ~1);
5766	llvm::Value *AlignedCalleeAddress =
5767	Builder.CreateAnd(LHS: CalleeAddress, RHS: Mask);
5768	AlignedCalleePtr =
5769	Builder.CreateIntToPtr(V: AlignedCalleeAddress, DestTy: CalleePtr->getType());
5770	} else {
5771	AlignedCalleePtr = CalleePtr;
5772	}
5773
5774	llvm::Value *CalleePrefixStruct = AlignedCalleePtr;
5775	llvm::Value *CalleeSigPtr =
5776	Builder.CreateConstGEP2_32(Ty: PrefixStructTy, Ptr: CalleePrefixStruct, Idx0: -1, Idx1: 0);
5777	llvm::Value *CalleeSig =
5778	Builder.CreateAlignedLoad(PrefixSigType, CalleeSigPtr, getIntAlign());
5779	llvm::Value *CalleeSigMatch = Builder.CreateICmpEQ(LHS: CalleeSig, RHS: PrefixSig);
5780
5781	llvm::BasicBlock *Cont = createBasicBlock(name: "cont");
5782	llvm::BasicBlock *TypeCheck = createBasicBlock(name: "typecheck");
5783	Builder.CreateCondBr(Cond: CalleeSigMatch, True: TypeCheck, False: Cont);
5784
5785	EmitBlock(BB: TypeCheck);
5786	llvm::Value *CalleeTypeHash = Builder.CreateAlignedLoad(
5787	Int32Ty,
5788	Builder.CreateConstGEP2_32(Ty: PrefixStructTy, Ptr: CalleePrefixStruct, Idx0: -1, Idx1: 1),
5789	getPointerAlign());
5790	llvm::Value *CalleeTypeHashMatch =
5791	Builder.CreateICmpEQ(LHS: CalleeTypeHash, RHS: TypeHash);
5792	llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc: E->getBeginLoc()),
5793	EmitCheckTypeDescriptor(T: CalleeType)};
5794	EmitCheck(Checked: std::make_pair(x&: CalleeTypeHashMatch, y: SanitizerKind::Function),
5795	CheckHandler: SanitizerHandler::FunctionTypeMismatch, StaticArgs: StaticData,
5796	DynamicArgs: {CalleePtr});
5797
5798	Builder.CreateBr(Dest: Cont);
5799	EmitBlock(BB: Cont);
5800	}
5801	}
5802
5803	const auto *FnType = cast<FunctionType>(Val&: PointeeType);
5804
5805	// If we are checking indirect calls and this call is indirect, check that the
5806	// function pointer is a member of the bit set for the function type.
5807	if (SanOpts.has(K: SanitizerKind::CFIICall) &&
5808	(!TargetDecl || !isa<FunctionDecl>(Val: TargetDecl))) {
5809	SanitizerScope SanScope(this);
5810	EmitSanitizerStatReport(SSK: llvm::SanStat_CFI_ICall);
5811
5812	llvm::Metadata *MD;
5813	if (CGM.getCodeGenOpts().SanitizeCfiICallGeneralizePointers)
5814	MD = CGM.CreateMetadataIdentifierGeneralized(T: QualType(FnType, 0));
5815	else
5816	MD = CGM.CreateMetadataIdentifierForType(T: QualType(FnType, 0));
5817
5818	llvm::Value *TypeId = llvm::MetadataAsValue::get(Context&: getLLVMContext(), MD);
5819
5820	llvm::Value *CalleePtr = Callee.getFunctionPointer();
5821	llvm::Value *TypeTest = Builder.CreateCall(
5822	CGM.getIntrinsic(llvm::Intrinsic::type_test), {CalleePtr, TypeId});
5823
5824	auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD);
5825	llvm::Constant *StaticData[] = {
5826	llvm::ConstantInt::get(Ty: Int8Ty, V: CFITCK_ICall),
5827	EmitCheckSourceLocation(Loc: E->getBeginLoc()),
5828	EmitCheckTypeDescriptor(T: QualType(FnType, 0)),
5829	};
5830	if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && CrossDsoTypeId) {
5831	EmitCfiSlowPathCheck(Kind: SanitizerKind::CFIICall, Cond: TypeTest, TypeId: CrossDsoTypeId,
5832	Ptr: CalleePtr, StaticArgs: StaticData);
5833	} else {
5834	EmitCheck(Checked: std::make_pair(x&: TypeTest, y: SanitizerKind::CFIICall),
5835	CheckHandler: SanitizerHandler::CFICheckFail, StaticArgs: StaticData,
5836	DynamicArgs: {CalleePtr, llvm::UndefValue::get(T: IntPtrTy)});
5837	}
5838	}
5839
5840	CallArgList Args;
5841	if (Chain)
5842	Args.add(rvalue: RValue::get(V: Chain), type: CGM.getContext().VoidPtrTy);
5843
5844	// C++17 requires that we evaluate arguments to a call using assignment syntax
5845	// right-to-left, and that we evaluate arguments to certain other operators
5846	// left-to-right. Note that we allow this to override the order dictated by
5847	// the calling convention on the MS ABI, which means that parameter
5848	// destruction order is not necessarily reverse construction order.
5849	// FIXME: Revisit this based on C++ committee response to unimplementability.
5850	EvaluationOrder Order = EvaluationOrder::Default;
5851	bool StaticOperator = false;
5852	if (auto *OCE = dyn_cast<CXXOperatorCallExpr>(Val: E)) {
5853	if (OCE->isAssignmentOp())
5854	Order = EvaluationOrder::ForceRightToLeft;
5855	else {
5856	switch (OCE->getOperator()) {
5857	case OO_LessLess:
5858	case OO_GreaterGreater:
5859	case OO_AmpAmp:
5860	case OO_PipePipe:
5861	case OO_Comma:
5862	case OO_ArrowStar:
5863	Order = EvaluationOrder::ForceLeftToRight;
5864	break;
5865	default:
5866	break;
5867	}
5868	}
5869
5870	if (const auto *MD =
5871	dyn_cast_if_present<CXXMethodDecl>(OCE->getCalleeDecl());
5872	MD && MD->isStatic())
5873	StaticOperator = true;
5874	}
5875
5876	auto Arguments = E->arguments();
5877	if (StaticOperator) {
5878	// If we're calling a static operator, we need to emit the object argument
5879	// and ignore it.
5880	EmitIgnoredExpr(E: E->getArg(Arg: 0));
5881	Arguments = drop_begin(RangeOrContainer&: Arguments, N: 1);
5882	}
5883	EmitCallArgs(Args, Prototype: dyn_cast<FunctionProtoType>(Val: FnType), ArgRange: Arguments,
5884	AC: E->getDirectCallee(), /*ParamsToSkip=*/0, Order);
5885
5886	const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall(
5887	Args, Ty: FnType, /*ChainCall=*/Chain);
5888
5889	// C99 6.5.2.2p6:
5890	// If the expression that denotes the called function has a type
5891	// that does not include a prototype, [the default argument
5892	// promotions are performed]. If the number of arguments does not
5893	// equal the number of parameters, the behavior is undefined. If
5894	// the function is defined with a type that includes a prototype,
5895	// and either the prototype ends with an ellipsis (, ...) or the
5896	// types of the arguments after promotion are not compatible with
5897	// the types of the parameters, the behavior is undefined. If the
5898	// function is defined with a type that does not include a
5899	// prototype, and the types of the arguments after promotion are
5900	// not compatible with those of the parameters after promotion,
5901	// the behavior is undefined [except in some trivial cases].
5902	// That is, in the general case, we should assume that a call
5903	// through an unprototyped function type works like a *non-variadic*
5904	// call. The way we make this work is to cast to the exact type
5905	// of the promoted arguments.
5906	//
5907	// Chain calls use this same code path to add the invisible chain parameter
5908	// to the function type.
5909	if (isa<FunctionNoProtoType>(Val: FnType) || Chain) {
5910	llvm::Type *CalleeTy = getTypes().GetFunctionType(Info: FnInfo);
5911	int AS = Callee.getFunctionPointer()->getType()->getPointerAddressSpace();
5912	CalleeTy = CalleeTy->getPointerTo(AddrSpace: AS);
5913
5914	llvm::Value *CalleePtr = Callee.getFunctionPointer();
5915	CalleePtr = Builder.CreateBitCast(V: CalleePtr, DestTy: CalleeTy, Name: "callee.knr.cast");
5916	Callee.setFunctionPointer(CalleePtr);
5917	}
5918
5919	// HIP function pointer contains kernel handle when it is used in triple
5920	// chevron. The kernel stub needs to be loaded from kernel handle and used
5921	// as callee.
5922	if (CGM.getLangOpts().HIP && !CGM.getLangOpts().CUDAIsDevice &&
5923	isa<CUDAKernelCallExpr>(Val: E) &&
5924	(!TargetDecl || !isa<FunctionDecl>(Val: TargetDecl))) {
5925	llvm::Value *Handle = Callee.getFunctionPointer();
5926	auto *Stub = Builder.CreateLoad(
5927	Addr: Address(Handle, Handle->getType(), CGM.getPointerAlign()));
5928	Callee.setFunctionPointer(Stub);
5929	}
5930	llvm::CallBase *CallOrInvoke = nullptr;
5931	RValue Call = EmitCall(FnInfo, Callee, ReturnValue, Args, &CallOrInvoke,
5932	E == MustTailCall, E->getExprLoc());
5933
5934	// Generate function declaration DISuprogram in order to be used
5935	// in debug info about call sites.
5936	if (CGDebugInfo *DI = getDebugInfo()) {
5937	if (auto *CalleeDecl = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl)) {
5938	FunctionArgList Args;
5939	QualType ResTy = BuildFunctionArgList(GD: CalleeDecl, Args);
5940	DI->EmitFuncDeclForCallSite(CallOrInvoke,
5941	CalleeType: DI->getFunctionType(FD: CalleeDecl, RetTy: ResTy, Args),
5942	CalleeDecl);
5943	}
5944	}
5945
5946	return Call;
5947	}
5948
5949	LValue CodeGenFunction::
5950	EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E) {
5951	Address BaseAddr = Address::invalid();
5952	if (E->getOpcode() == BO_PtrMemI) {
5953	BaseAddr = EmitPointerWithAlignment(E: E->getLHS());
5954	} else {
5955	BaseAddr = EmitLValue(E: E->getLHS()).getAddress(CGF&: *this);
5956	}
5957
5958	llvm::Value *OffsetV = EmitScalarExpr(E: E->getRHS());
5959	const auto *MPT = E->getRHS()->getType()->castAs<MemberPointerType>();
5960
5961	LValueBaseInfo BaseInfo;
5962	TBAAAccessInfo TBAAInfo;
5963	Address MemberAddr =
5964	EmitCXXMemberDataPointerAddress(E, BaseAddr, OffsetV, MPT, &BaseInfo,
5965	&TBAAInfo);
5966
5967	return MakeAddrLValue(Addr: MemberAddr, T: MPT->getPointeeType(), BaseInfo, TBAAInfo);
5968	}
5969
5970	/// Given the address of a temporary variable, produce an r-value of
5971	/// its type.
5972	RValue CodeGenFunction::convertTempToRValue(Address addr,
5973	QualType type,
5974	SourceLocation loc) {
5975	LValue lvalue = MakeAddrLValue(Addr: addr, T: type, Source: AlignmentSource::Decl);
5976	switch (getEvaluationKind(T: type)) {
5977	case TEK_Complex:
5978	return RValue::getComplex(C: EmitLoadOfComplex(src: lvalue, loc));
5979	case TEK_Aggregate:
5980	return lvalue.asAggregateRValue(CGF&: *this);
5981	case TEK_Scalar:
5982	return RValue::get(V: EmitLoadOfScalar(lvalue, Loc: loc));
5983	}
5984	llvm_unreachable("bad evaluation kind");
5985	}
5986
5987	void CodeGenFunction::SetFPAccuracy(llvm::Value *Val, float Accuracy) {
5988	assert(Val->getType()->isFPOrFPVectorTy());
5989	if (Accuracy == 0.0 || !isa<llvm::Instruction>(Val))
5990	return;
5991
5992	llvm::MDBuilder MDHelper(getLLVMContext());
5993	llvm::MDNode *Node = MDHelper.createFPMath(Accuracy);
5994
5995	cast<llvm::Instruction>(Val)->setMetadata(KindID: llvm::LLVMContext::MD_fpmath, Node);
5996	}
5997
5998	void CodeGenFunction::SetSqrtFPAccuracy(llvm::Value *Val) {
5999	llvm::Type *EltTy = Val->getType()->getScalarType();
6000	if (!EltTy->isFloatTy())
6001	return;
6002
6003	if ((getLangOpts().OpenCL &&
6004	!CGM.getCodeGenOpts().OpenCLCorrectlyRoundedDivSqrt) ||
6005	(getLangOpts().HIP && getLangOpts().CUDAIsDevice &&
6006	!CGM.getCodeGenOpts().HIPCorrectlyRoundedDivSqrt)) {
6007	// OpenCL v1.1 s7.4: minimum accuracy of single precision / is 3ulp
6008	//
6009	// OpenCL v1.2 s5.6.4.2: The -cl-fp32-correctly-rounded-divide-sqrt
6010	// build option allows an application to specify that single precision
6011	// floating-point divide (x/y and 1/x) and sqrt used in the program
6012	// source are correctly rounded.
6013	//
6014	// TODO: CUDA has a prec-sqrt flag
6015	SetFPAccuracy(Val, Accuracy: 3.0f);
6016	}
6017	}
6018
6019	void CodeGenFunction::SetDivFPAccuracy(llvm::Value *Val) {
6020	llvm::Type *EltTy = Val->getType()->getScalarType();
6021	if (!EltTy->isFloatTy())
6022	return;
6023
6024	if ((getLangOpts().OpenCL &&
6025	!CGM.getCodeGenOpts().OpenCLCorrectlyRoundedDivSqrt) ||
6026	(getLangOpts().HIP && getLangOpts().CUDAIsDevice &&
6027	!CGM.getCodeGenOpts().HIPCorrectlyRoundedDivSqrt)) {
6028	// OpenCL v1.1 s7.4: minimum accuracy of single precision / is 2.5ulp
6029	//
6030	// OpenCL v1.2 s5.6.4.2: The -cl-fp32-correctly-rounded-divide-sqrt
6031	// build option allows an application to specify that single precision
6032	// floating-point divide (x/y and 1/x) and sqrt used in the program
6033	// source are correctly rounded.
6034	//
6035	// TODO: CUDA has a prec-div flag
6036	SetFPAccuracy(Val, Accuracy: 2.5f);
6037	}
6038	}
6039
6040	namespace {
6041	struct LValueOrRValue {
6042	LValue LV;
6043	RValue RV;
6044	};
6045	}
6046
6047	static LValueOrRValue emitPseudoObjectExpr(CodeGenFunction &CGF,
6048	const PseudoObjectExpr *E,
6049	bool forLValue,
6050	AggValueSlot slot) {
6051	SmallVector<CodeGenFunction::OpaqueValueMappingData, 4> opaques;
6052
6053	// Find the result expression, if any.
6054	const Expr *resultExpr = E->getResultExpr();
6055	LValueOrRValue result;
6056
6057	for (PseudoObjectExpr::const_semantics_iterator
6058	i = E->semantics_begin(), e = E->semantics_end(); i != e; ++i) {
6059	const Expr *semantic = *i;
6060
6061	// If this semantic expression is an opaque value, bind it
6062	// to the result of its source expression.
6063	if (const auto *ov = dyn_cast<OpaqueValueExpr>(Val: semantic)) {
6064	// Skip unique OVEs.
6065	if (ov->isUnique()) {
6066	assert(ov != resultExpr &&
6067	"A unique OVE cannot be used as the result expression");
6068	continue;
6069	}
6070
6071	// If this is the result expression, we may need to evaluate
6072	// directly into the slot.
6073	typedef CodeGenFunction::OpaqueValueMappingData OVMA;
6074	OVMA opaqueData;
6075	if (ov == resultExpr && ov->isPRValue() && !forLValue &&
6076	CodeGenFunction::hasAggregateEvaluationKind(T: ov->getType())) {
6077	CGF.EmitAggExpr(E: ov->getSourceExpr(), AS: slot);
6078	LValue LV = CGF.MakeAddrLValue(slot.getAddress(), ov->getType(),
6079	AlignmentSource::Decl);
6080	opaqueData = OVMA::bind(CGF, ov, lv: LV);
6081	result.RV = slot.asRValue();
6082
6083	// Otherwise, emit as normal.
6084	} else {
6085	opaqueData = OVMA::bind(CGF, ov, e: ov->getSourceExpr());
6086
6087	// If this is the result, also evaluate the result now.
6088	if (ov == resultExpr) {
6089	if (forLValue)
6090	result.LV = CGF.EmitLValue(ov);
6091	else
6092	result.RV = CGF.EmitAnyExpr(ov, slot);
6093	}
6094	}
6095
6096	opaques.push_back(Elt: opaqueData);
6097
6098	// Otherwise, if the expression is the result, evaluate it
6099	// and remember the result.
6100	} else if (semantic == resultExpr) {
6101	if (forLValue)
6102	result.LV = CGF.EmitLValue(E: semantic);
6103	else
6104	result.RV = CGF.EmitAnyExpr(E: semantic, aggSlot: slot);
6105
6106	// Otherwise, evaluate the expression in an ignored context.
6107	} else {
6108	CGF.EmitIgnoredExpr(E: semantic);
6109	}
6110	}
6111
6112	// Unbind all the opaques now.
6113	for (unsigned i = 0, e = opaques.size(); i != e; ++i)
6114	opaques[i].unbind(CGF);
6115
6116	return result;
6117	}
6118
6119	RValue CodeGenFunction::EmitPseudoObjectRValue(const PseudoObjectExpr *E,
6120	AggValueSlot slot) {
6121	return emitPseudoObjectExpr(CGF&: *this, E, forLValue: false, slot).RV;
6122	}
6123
6124	LValue CodeGenFunction::EmitPseudoObjectLValue(const PseudoObjectExpr *E) {
6125	return emitPseudoObjectExpr(CGF&: *this, E, forLValue: true, slot: AggValueSlot::ignored()).LV;
6126	}
6127