1	//===--- CGExprCXX.cpp - Emit LLVM Code for C++ expressions ---------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This contains code dealing with code generation of C++ expressions
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "CGCUDARuntime.h"
14	#include "CGCXXABI.h"
15	#include "CGDebugInfo.h"
16	#include "CGObjCRuntime.h"
17	#include "CodeGenFunction.h"
18	#include "ConstantEmitter.h"
19	#include "TargetInfo.h"
20	#include "clang/Basic/CodeGenOptions.h"
21	#include "clang/CodeGen/CGFunctionInfo.h"
22	#include "llvm/IR/Intrinsics.h"
23
24	using namespace clang;
25	using namespace CodeGen;
26
27	namespace {
28	struct MemberCallInfo {
29	RequiredArgs ReqArgs;
30	// Number of prefix arguments for the call. Ignores the `this` pointer.
31	unsigned PrefixSize;
32	};
33	}
34
35	static MemberCallInfo
36	commonEmitCXXMemberOrOperatorCall(CodeGenFunction &CGF, GlobalDecl GD,
37	llvm::Value *This, llvm::Value *ImplicitParam,
38	QualType ImplicitParamTy, const CallExpr *CE,
39	CallArgList &Args, CallArgList *RtlArgs) {
40	auto *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
41
42	assert(CE == nullptr || isa<CXXMemberCallExpr>(CE) ||
43	isa<CXXOperatorCallExpr>(CE));
44	assert(MD->isImplicitObjectMemberFunction() &&
45	"Trying to emit a member or operator call expr on a static method!");
46
47	// Push the this ptr.
48	const CXXRecordDecl *RD =
49	CGF.CGM.getCXXABI().getThisArgumentTypeForMethod(GD);
50	Args.add(rvalue: RValue::get(V: This), type: CGF.getTypes().DeriveThisType(RD, MD));
51
52	// If there is an implicit parameter (e.g. VTT), emit it.
53	if (ImplicitParam) {
54	Args.add(rvalue: RValue::get(V: ImplicitParam), type: ImplicitParamTy);
55	}
56
57	const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
58	RequiredArgs required = RequiredArgs::forPrototypePlus(prototype: FPT, additional: Args.size());
59	unsigned PrefixSize = Args.size() - 1;
60
61	// And the rest of the call args.
62	if (RtlArgs) {
63	// Special case: if the caller emitted the arguments right-to-left already
64	// (prior to emitting the *this argument), we're done. This happens for
65	// assignment operators.
66	Args.addFrom(other: *RtlArgs);
67	} else if (CE) {
68	// Special case: skip first argument of CXXOperatorCall (it is "this").
69	unsigned ArgsToSkip = 0;
70	if (const auto *Op = dyn_cast<CXXOperatorCallExpr>(Val: CE)) {
71	if (const auto *M = dyn_cast<CXXMethodDecl>(Op->getCalleeDecl()))
72	ArgsToSkip =
73	static_cast<unsigned>(!M->isExplicitObjectMemberFunction());
74	}
75	CGF.EmitCallArgs(Args, Prototype: FPT, ArgRange: drop_begin(RangeOrContainer: CE->arguments(), N: ArgsToSkip),
76	AC: CE->getDirectCallee());
77	} else {
78	assert(
79	FPT->getNumParams() == 0 &&
80	"No CallExpr specified for function with non-zero number of arguments");
81	}
82	return {.ReqArgs: required, .PrefixSize: PrefixSize};
83	}
84
85	RValue CodeGenFunction::EmitCXXMemberOrOperatorCall(
86	const CXXMethodDecl *MD, const CGCallee &Callee,
87	ReturnValueSlot ReturnValue,
88	llvm::Value *This, llvm::Value *ImplicitParam, QualType ImplicitParamTy,
89	const CallExpr *CE, CallArgList *RtlArgs) {
90	const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
91	CallArgList Args;
92	MemberCallInfo CallInfo = commonEmitCXXMemberOrOperatorCall(
93	*this, MD, This, ImplicitParam, ImplicitParamTy, CE, Args, RtlArgs);
94	auto &FnInfo = CGM.getTypes().arrangeCXXMethodCall(
95	args: Args, type: FPT, required: CallInfo.ReqArgs, numPrefixArgs: CallInfo.PrefixSize);
96	return EmitCall(FnInfo, Callee, ReturnValue, Args, nullptr,
97	CE && CE == MustTailCall,
98	CE ? CE->getExprLoc() : SourceLocation());
99	}
100
101	RValue CodeGenFunction::EmitCXXDestructorCall(
102	GlobalDecl Dtor, const CGCallee &Callee, llvm::Value *This, QualType ThisTy,
103	llvm::Value *ImplicitParam, QualType ImplicitParamTy, const CallExpr *CE) {
104	const CXXMethodDecl *DtorDecl = cast<CXXMethodDecl>(Val: Dtor.getDecl());
105
106	assert(!ThisTy.isNull());
107	assert(ThisTy->getAsCXXRecordDecl() == DtorDecl->getParent() &&
108	"Pointer/Object mixup");
109
110	LangAS SrcAS = ThisTy.getAddressSpace();
111	LangAS DstAS = DtorDecl->getMethodQualifiers().getAddressSpace();
112	if (SrcAS != DstAS) {
113	QualType DstTy = DtorDecl->getThisType();
114	llvm::Type *NewType = CGM.getTypes().ConvertType(T: DstTy);
115	This = getTargetHooks().performAddrSpaceCast(CGF&: *this, V: This, SrcAddr: SrcAS, DestAddr: DstAS,
116	DestTy: NewType);
117	}
118
119	CallArgList Args;
120	commonEmitCXXMemberOrOperatorCall(CGF&: *this, GD: Dtor, This, ImplicitParam,
121	ImplicitParamTy, CE, Args, RtlArgs: nullptr);
122	return EmitCall(CGM.getTypes().arrangeCXXStructorDeclaration(GD: Dtor), Callee,
123	ReturnValueSlot(), Args, nullptr, CE && CE == MustTailCall,
124	CE ? CE->getExprLoc() : SourceLocation{});
125	}
126
127	RValue CodeGenFunction::EmitCXXPseudoDestructorExpr(
128	const CXXPseudoDestructorExpr *E) {
129	QualType DestroyedType = E->getDestroyedType();
130	if (DestroyedType.hasStrongOrWeakObjCLifetime()) {
131	// Automatic Reference Counting:
132	// If the pseudo-expression names a retainable object with weak or
133	// strong lifetime, the object shall be released.
134	Expr *BaseExpr = E->getBase();
135	Address BaseValue = Address::invalid();
136	Qualifiers BaseQuals;
137
138	// If this is s.x, emit s as an lvalue. If it is s->x, emit s as a scalar.
139	if (E->isArrow()) {
140	BaseValue = EmitPointerWithAlignment(Addr: BaseExpr);
141	const auto *PTy = BaseExpr->getType()->castAs<PointerType>();
142	BaseQuals = PTy->getPointeeType().getQualifiers();
143	} else {
144	LValue BaseLV = EmitLValue(E: BaseExpr);
145	BaseValue = BaseLV.getAddress(CGF&: *this);
146	QualType BaseTy = BaseExpr->getType();
147	BaseQuals = BaseTy.getQualifiers();
148	}
149
150	switch (DestroyedType.getObjCLifetime()) {
151	case Qualifiers::OCL_None:
152	case Qualifiers::OCL_ExplicitNone:
153	case Qualifiers::OCL_Autoreleasing:
154	break;
155
156	case Qualifiers::OCL_Strong:
157	EmitARCRelease(value: Builder.CreateLoad(Addr: BaseValue,
158	IsVolatile: DestroyedType.isVolatileQualified()),
159	precise: ARCPreciseLifetime);
160	break;
161
162	case Qualifiers::OCL_Weak:
163	EmitARCDestroyWeak(addr: BaseValue);
164	break;
165	}
166	} else {
167	// C++ [expr.pseudo]p1:
168	// The result shall only be used as the operand for the function call
169	// operator (), and the result of such a call has type void. The only
170	// effect is the evaluation of the postfix-expression before the dot or
171	// arrow.
172	EmitIgnoredExpr(E: E->getBase());
173	}
174
175	return RValue::get(V: nullptr);
176	}
177
178	static CXXRecordDecl *getCXXRecord(const Expr *E) {
179	QualType T = E->getType();
180	if (const PointerType *PTy = T->getAs<PointerType>())
181	T = PTy->getPointeeType();
182	const RecordType *Ty = T->castAs<RecordType>();
183	return cast<CXXRecordDecl>(Val: Ty->getDecl());
184	}
185
186	// Note: This function also emit constructor calls to support a MSVC
187	// extensions allowing explicit constructor function call.
188	RValue CodeGenFunction::EmitCXXMemberCallExpr(const CXXMemberCallExpr *CE,
189	ReturnValueSlot ReturnValue) {
190	const Expr *callee = CE->getCallee()->IgnoreParens();
191
192	if (isa<BinaryOperator>(Val: callee))
193	return EmitCXXMemberPointerCallExpr(E: CE, ReturnValue);
194
195	const MemberExpr *ME = cast<MemberExpr>(Val: callee);
196	const CXXMethodDecl *MD = cast<CXXMethodDecl>(Val: ME->getMemberDecl());
197
198	if (MD->isStatic()) {
199	// The method is static, emit it as we would a regular call.
200	CGCallee callee =
201	CGCallee::forDirect(functionPtr: CGM.GetAddrOfFunction(MD), abstractInfo: GlobalDecl(MD));
202	return EmitCall(getContext().getPointerType(MD->getType()), callee, CE,
203	ReturnValue);
204	}
205
206	bool HasQualifier = ME->hasQualifier();
207	NestedNameSpecifier *Qualifier = HasQualifier ? ME->getQualifier() : nullptr;
208	bool IsArrow = ME->isArrow();
209	const Expr *Base = ME->getBase();
210
211	return EmitCXXMemberOrOperatorMemberCallExpr(
212	CE, MD, ReturnValue, HasQualifier, Qualifier, IsArrow, Base);
213	}
214
215	RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr(
216	const CallExpr *CE, const CXXMethodDecl *MD, ReturnValueSlot ReturnValue,
217	bool HasQualifier, NestedNameSpecifier *Qualifier, bool IsArrow,
218	const Expr *Base) {
219	assert(isa<CXXMemberCallExpr>(CE) || isa<CXXOperatorCallExpr>(CE));
220
221	// Compute the object pointer.
222	bool CanUseVirtualCall = MD->isVirtual() && !HasQualifier;
223
224	const CXXMethodDecl *DevirtualizedMethod = nullptr;
225	if (CanUseVirtualCall &&
226	MD->getDevirtualizedMethod(Base, IsAppleKext: getLangOpts().AppleKext)) {
227	const CXXRecordDecl *BestDynamicDecl = Base->getBestDynamicClassType();
228	DevirtualizedMethod = MD->getCorrespondingMethodInClass(RD: BestDynamicDecl);
229	assert(DevirtualizedMethod);
230	const CXXRecordDecl *DevirtualizedClass = DevirtualizedMethod->getParent();
231	const Expr *Inner = Base->IgnoreParenBaseCasts();
232	if (DevirtualizedMethod->getReturnType().getCanonicalType() !=
233	MD->getReturnType().getCanonicalType())
234	// If the return types are not the same, this might be a case where more
235	// code needs to run to compensate for it. For example, the derived
236	// method might return a type that inherits form from the return
237	// type of MD and has a prefix.
238	// For now we just avoid devirtualizing these covariant cases.
239	DevirtualizedMethod = nullptr;
240	else if (getCXXRecord(E: Inner) == DevirtualizedClass)
241	// If the class of the Inner expression is where the dynamic method
242	// is defined, build the this pointer from it.
243	Base = Inner;
244	else if (getCXXRecord(E: Base) != DevirtualizedClass) {
245	// If the method is defined in a class that is not the best dynamic
246	// one or the one of the full expression, we would have to build
247	// a derived-to-base cast to compute the correct this pointer, but
248	// we don't have support for that yet, so do a virtual call.
249	DevirtualizedMethod = nullptr;
250	}
251	}
252
253	bool TrivialForCodegen =
254	MD->isTrivial() || (MD->isDefaulted() && MD->getParent()->isUnion());
255	bool TrivialAssignment =
256	TrivialForCodegen &&
257	(MD->isCopyAssignmentOperator() || MD->isMoveAssignmentOperator()) &&
258	!MD->getParent()->mayInsertExtraPadding();
259
260	// C++17 demands that we evaluate the RHS of a (possibly-compound) assignment
261	// operator before the LHS.
262	CallArgList RtlArgStorage;
263	CallArgList *RtlArgs = nullptr;
264	LValue TrivialAssignmentRHS;
265	if (auto *OCE = dyn_cast<CXXOperatorCallExpr>(Val: CE)) {
266	if (OCE->isAssignmentOp()) {
267	if (TrivialAssignment) {
268	TrivialAssignmentRHS = EmitLValue(E: CE->getArg(Arg: 1));
269	} else {
270	RtlArgs = &RtlArgStorage;
271	EmitCallArgs(Args&: *RtlArgs, Prototype: MD->getType()->castAs<FunctionProtoType>(),
272	ArgRange: drop_begin(RangeOrContainer: CE->arguments(), N: 1), AC: CE->getDirectCallee(),
273	/*ParamsToSkip*/0, Order: EvaluationOrder::ForceRightToLeft);
274	}
275	}
276	}
277
278	LValue This;
279	if (IsArrow) {
280	LValueBaseInfo BaseInfo;
281	TBAAAccessInfo TBAAInfo;
282	Address ThisValue = EmitPointerWithAlignment(Addr: Base, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
283	This = MakeAddrLValue(Addr: ThisValue, T: Base->getType(), BaseInfo, TBAAInfo);
284	} else {
285	This = EmitLValue(E: Base);
286	}
287
288	if (const CXXConstructorDecl *Ctor = dyn_cast<CXXConstructorDecl>(Val: MD)) {
289	// This is the MSVC p->Ctor::Ctor(...) extension. We assume that's
290	// constructing a new complete object of type Ctor.
291	assert(!RtlArgs);
292	assert(ReturnValue.isNull() && "Constructor shouldn't have return value");
293	CallArgList Args;
294	commonEmitCXXMemberOrOperatorCall(
295	CGF&: *this, GD: {Ctor, Ctor_Complete}, This: This.getPointer(CGF&: *this),
296	/*ImplicitParam=*/nullptr,
297	/*ImplicitParamTy=*/QualType(), CE, Args, RtlArgs: nullptr);
298
299	EmitCXXConstructorCall(Ctor, Ctor_Complete, /*ForVirtualBase=*/false,
300	/*Delegating=*/false, This.getAddress(CGF&: *this), Args,
301	AggValueSlot::DoesNotOverlap, CE->getExprLoc(),
302	/*NewPointerIsChecked=*/false);
303	return RValue::get(V: nullptr);
304	}
305
306	if (TrivialForCodegen) {
307	if (isa<CXXDestructorDecl>(Val: MD))
308	return RValue::get(V: nullptr);
309
310	if (TrivialAssignment) {
311	// We don't like to generate the trivial copy/move assignment operator
312	// when it isn't necessary; just produce the proper effect here.
313	// It's important that we use the result of EmitLValue here rather than
314	// emitting call arguments, in order to preserve TBAA information from
315	// the RHS.
316	LValue RHS = isa<CXXOperatorCallExpr>(Val: CE)
317	? TrivialAssignmentRHS
318	: EmitLValue(*CE->arg_begin());
319	EmitAggregateAssign(Dest: This, Src: RHS, EltTy: CE->getType());
320	return RValue::get(V: This.getPointer(CGF&: *this));
321	}
322
323	assert(MD->getParent()->mayInsertExtraPadding() &&
324	"unknown trivial member function");
325	}
326
327	// Compute the function type we're calling.
328	const CXXMethodDecl *CalleeDecl =
329	DevirtualizedMethod ? DevirtualizedMethod : MD;
330	const CGFunctionInfo *FInfo = nullptr;
331	if (const auto *Dtor = dyn_cast<CXXDestructorDecl>(Val: CalleeDecl))
332	FInfo = &CGM.getTypes().arrangeCXXStructorDeclaration(
333	GD: GlobalDecl(Dtor, Dtor_Complete));
334	else
335	FInfo = &CGM.getTypes().arrangeCXXMethodDeclaration(MD: CalleeDecl);
336
337	llvm::FunctionType *Ty = CGM.getTypes().GetFunctionType(Info: *FInfo);
338
339	// C++11 [class.mfct.non-static]p2:
340	// If a non-static member function of a class X is called for an object that
341	// is not of type X, or of a type derived from X, the behavior is undefined.
342	SourceLocation CallLoc;
343	ASTContext &C = getContext();
344	if (CE)
345	CallLoc = CE->getExprLoc();
346
347	SanitizerSet SkippedChecks;
348	if (const auto *CMCE = dyn_cast<CXXMemberCallExpr>(Val: CE)) {
349	auto *IOA = CMCE->getImplicitObjectArgument();
350	bool IsImplicitObjectCXXThis = IsWrappedCXXThis(E: IOA);
351	if (IsImplicitObjectCXXThis)
352	SkippedChecks.set(K: SanitizerKind::Alignment, Value: true);
353	if (IsImplicitObjectCXXThis || isa<DeclRefExpr>(Val: IOA))
354	SkippedChecks.set(K: SanitizerKind::Null, Value: true);
355	}
356	EmitTypeCheck(TCK: CodeGenFunction::TCK_MemberCall, Loc: CallLoc,
357	V: This.getPointer(CGF&: *this),
358	Type: C.getRecordType(CalleeDecl->getParent()),
359	/*Alignment=*/CharUnits::Zero(), SkippedChecks);
360
361	// C++ [class.virtual]p12:
362	// Explicit qualification with the scope operator (5.1) suppresses the
363	// virtual call mechanism.
364	//
365	// We also don't emit a virtual call if the base expression has a record type
366	// because then we know what the type is.
367	bool UseVirtualCall = CanUseVirtualCall && !DevirtualizedMethod;
368
369	if (const CXXDestructorDecl *Dtor = dyn_cast<CXXDestructorDecl>(Val: CalleeDecl)) {
370	assert(CE->arg_begin() == CE->arg_end() &&
371	"Destructor shouldn't have explicit parameters");
372	assert(ReturnValue.isNull() && "Destructor shouldn't have return value");
373	if (UseVirtualCall) {
374	CGM.getCXXABI().EmitVirtualDestructorCall(CGF&: *this, Dtor, DtorType: Dtor_Complete,
375	This: This.getAddress(CGF&: *this),
376	E: cast<CXXMemberCallExpr>(Val: CE));
377	} else {
378	GlobalDecl GD(Dtor, Dtor_Complete);
379	CGCallee Callee;
380	if (getLangOpts().AppleKext && Dtor->isVirtual() && HasQualifier)
381	Callee = BuildAppleKextVirtualCall(Dtor, Qualifier, Ty);
382	else if (!DevirtualizedMethod)
383	Callee =
384	CGCallee::forDirect(functionPtr: CGM.getAddrOfCXXStructor(GD, FnInfo: FInfo, FnType: Ty), abstractInfo: GD);
385	else {
386	Callee = CGCallee::forDirect(functionPtr: CGM.GetAddrOfFunction(GD, Ty), abstractInfo: GD);
387	}
388
389	QualType ThisTy =
390	IsArrow ? Base->getType()->getPointeeType() : Base->getType();
391	EmitCXXDestructorCall(Dtor: GD, Callee, This: This.getPointer(CGF&: *this), ThisTy,
392	/*ImplicitParam=*/nullptr,
393	/*ImplicitParamTy=*/QualType(), CE);
394	}
395	return RValue::get(V: nullptr);
396	}
397
398	// FIXME: Uses of 'MD' past this point need to be audited. We may need to use
399	// 'CalleeDecl' instead.
400
401	CGCallee Callee;
402	if (UseVirtualCall) {
403	Callee = CGCallee::forVirtual(CE, MD, This.getAddress(CGF&: *this), Ty);
404	} else {
405	if (SanOpts.has(K: SanitizerKind::CFINVCall) &&
406	MD->getParent()->isDynamicClass()) {
407	llvm::Value *VTable;
408	const CXXRecordDecl *RD;
409	std::tie(args&: VTable, args&: RD) = CGM.getCXXABI().LoadVTablePtr(
410	CGF&: *this, This: This.getAddress(CGF&: *this), RD: CalleeDecl->getParent());
411	EmitVTablePtrCheckForCall(RD, VTable, TCK: CFITCK_NVCall, Loc: CE->getBeginLoc());
412	}
413
414	if (getLangOpts().AppleKext && MD->isVirtual() && HasQualifier)
415	Callee = BuildAppleKextVirtualCall(MD, Qual: Qualifier, Ty);
416	else if (!DevirtualizedMethod)
417	Callee =
418	CGCallee::forDirect(functionPtr: CGM.GetAddrOfFunction(MD, Ty), abstractInfo: GlobalDecl(MD));
419	else {
420	Callee =
421	CGCallee::forDirect(functionPtr: CGM.GetAddrOfFunction(DevirtualizedMethod, Ty),
422	abstractInfo: GlobalDecl(DevirtualizedMethod));
423	}
424	}
425
426	if (MD->isVirtual()) {
427	Address NewThisAddr =
428	CGM.getCXXABI().adjustThisArgumentForVirtualFunctionCall(
429	*this, CalleeDecl, This.getAddress(CGF&: *this), UseVirtualCall);
430	This.setAddress(NewThisAddr);
431	}
432
433	return EmitCXXMemberOrOperatorCall(
434	MD: CalleeDecl, Callee, ReturnValue, This: This.getPointer(CGF&: *this),
435	/*ImplicitParam=*/nullptr, ImplicitParamTy: QualType(), CE, RtlArgs);
436	}
437
438	RValue
439	CodeGenFunction::EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E,
440	ReturnValueSlot ReturnValue) {
441	const BinaryOperator *BO =
442	cast<BinaryOperator>(E->getCallee()->IgnoreParens());
443	const Expr *BaseExpr = BO->getLHS();
444	const Expr *MemFnExpr = BO->getRHS();
445
446	const auto *MPT = MemFnExpr->getType()->castAs<MemberPointerType>();
447	const auto *FPT = MPT->getPointeeType()->castAs<FunctionProtoType>();
448	const auto *RD =
449	cast<CXXRecordDecl>(MPT->getClass()->castAs<RecordType>()->getDecl());
450
451	// Emit the 'this' pointer.
452	Address This = Address::invalid();
453	if (BO->getOpcode() == BO_PtrMemI)
454	This = EmitPointerWithAlignment(Addr: BaseExpr, BaseInfo: nullptr, TBAAInfo: nullptr, IsKnownNonNull: KnownNonNull);
455	else
456	This = EmitLValue(E: BaseExpr, IsKnownNonNull: KnownNonNull).getAddress(CGF&: *this);
457
458	EmitTypeCheck(TCK: TCK_MemberCall, Loc: E->getExprLoc(), V: This.getPointer(),
459	Type: QualType(MPT->getClass(), 0));
460
461	// Get the member function pointer.
462	llvm::Value *MemFnPtr = EmitScalarExpr(E: MemFnExpr);
463
464	// Ask the ABI to load the callee. Note that This is modified.
465	llvm::Value *ThisPtrForCall = nullptr;
466	CGCallee Callee =
467	CGM.getCXXABI().EmitLoadOfMemberFunctionPointer(CGF&: *this, E: BO, This,
468	ThisPtrForCall, MemPtr: MemFnPtr, MPT: MPT);
469
470	CallArgList Args;
471
472	QualType ThisType =
473	getContext().getPointerType(getContext().getTagDeclType(Decl: RD));
474
475	// Push the this ptr.
476	Args.add(rvalue: RValue::get(V: ThisPtrForCall), type: ThisType);
477
478	RequiredArgs required = RequiredArgs::forPrototypePlus(FPT, 1);
479
480	// And the rest of the call args
481	EmitCallArgs(Args, Prototype: FPT, ArgRange: E->arguments());
482	return EmitCall(CGM.getTypes().arrangeCXXMethodCall(args: Args, type: FPT, required,
483	/*PrefixSize=*/numPrefixArgs: 0),
484	Callee, ReturnValue, Args, nullptr, E == MustTailCall,
485	E->getExprLoc());
486	}
487
488	RValue
489	CodeGenFunction::EmitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *E,
490	const CXXMethodDecl *MD,
491	ReturnValueSlot ReturnValue) {
492	assert(MD->isImplicitObjectMemberFunction() &&
493	"Trying to emit a member call expr on a static method!");
494	return EmitCXXMemberOrOperatorMemberCallExpr(
495	CE: E, MD, ReturnValue, /*HasQualifier=*/false, /*Qualifier=*/nullptr,
496	/*IsArrow=*/false, Base: E->getArg(0));
497	}
498
499	RValue CodeGenFunction::EmitCUDAKernelCallExpr(const CUDAKernelCallExpr *E,
500	ReturnValueSlot ReturnValue) {
501	return CGM.getCUDARuntime().EmitCUDAKernelCallExpr(CGF&: *this, E, ReturnValue);
502	}
503
504	static void EmitNullBaseClassInitialization(CodeGenFunction &CGF,
505	Address DestPtr,
506	const CXXRecordDecl *Base) {
507	if (Base->isEmpty())
508	return;
509
510	DestPtr = DestPtr.withElementType(ElemTy: CGF.Int8Ty);
511
512	const ASTRecordLayout &Layout = CGF.getContext().getASTRecordLayout(Base);
513	CharUnits NVSize = Layout.getNonVirtualSize();
514
515	// We cannot simply zero-initialize the entire base sub-object if vbptrs are
516	// present, they are initialized by the most derived class before calling the
517	// constructor.
518	SmallVector<std::pair<CharUnits, CharUnits>, 1> Stores;
519	Stores.emplace_back(Args: CharUnits::Zero(), Args&: NVSize);
520
521	// Each store is split by the existence of a vbptr.
522	CharUnits VBPtrWidth = CGF.getPointerSize();
523	std::vector<CharUnits> VBPtrOffsets =
524	CGF.CGM.getCXXABI().getVBPtrOffsets(RD: Base);
525	for (CharUnits VBPtrOffset : VBPtrOffsets) {
526	// Stop before we hit any virtual base pointers located in virtual bases.
527	if (VBPtrOffset >= NVSize)
528	break;
529	std::pair<CharUnits, CharUnits> LastStore = Stores.pop_back_val();
530	CharUnits LastStoreOffset = LastStore.first;
531	CharUnits LastStoreSize = LastStore.second;
532
533	CharUnits SplitBeforeOffset = LastStoreOffset;
534	CharUnits SplitBeforeSize = VBPtrOffset - SplitBeforeOffset;
535	assert(!SplitBeforeSize.isNegative() && "negative store size!");
536	if (!SplitBeforeSize.isZero())
537	Stores.emplace_back(Args&: SplitBeforeOffset, Args&: SplitBeforeSize);
538
539	CharUnits SplitAfterOffset = VBPtrOffset + VBPtrWidth;
540	CharUnits SplitAfterSize = LastStoreSize - SplitAfterOffset;
541	assert(!SplitAfterSize.isNegative() && "negative store size!");
542	if (!SplitAfterSize.isZero())
543	Stores.emplace_back(Args&: SplitAfterOffset, Args&: SplitAfterSize);
544	}
545
546	// If the type contains a pointer to data member we can't memset it to zero.
547	// Instead, create a null constant and copy it to the destination.
548	// TODO: there are other patterns besides zero that we can usefully memset,
549	// like -1, which happens to be the pattern used by member-pointers.
550	// TODO: isZeroInitializable can be over-conservative in the case where a
551	// virtual base contains a member pointer.
552	llvm::Constant *NullConstantForBase = CGF.CGM.EmitNullConstantForBase(Record: Base);
553	if (!NullConstantForBase->isNullValue()) {
554	llvm::GlobalVariable *NullVariable = new llvm::GlobalVariable(
555	CGF.CGM.getModule(), NullConstantForBase->getType(),
556	/*isConstant=*/true, llvm::GlobalVariable::PrivateLinkage,
557	NullConstantForBase, Twine());
558
559	CharUnits Align =
560	std::max(a: Layout.getNonVirtualAlignment(), b: DestPtr.getAlignment());
561	NullVariable->setAlignment(Align.getAsAlign());
562
563	Address SrcPtr(NullVariable, CGF.Int8Ty, Align);
564
565	// Get and call the appropriate llvm.memcpy overload.
566	for (std::pair<CharUnits, CharUnits> Store : Stores) {
567	CharUnits StoreOffset = Store.first;
568	CharUnits StoreSize = Store.second;
569	llvm::Value *StoreSizeVal = CGF.CGM.getSize(numChars: StoreSize);
570	CGF.Builder.CreateMemCpy(
571	Dest: CGF.Builder.CreateConstInBoundsByteGEP(Addr: DestPtr, Offset: StoreOffset),
572	Src: CGF.Builder.CreateConstInBoundsByteGEP(Addr: SrcPtr, Offset: StoreOffset),
573	Size: StoreSizeVal);
574	}
575
576	// Otherwise, just memset the whole thing to zero. This is legal
577	// because in LLVM, all default initializers (other than the ones we just
578	// handled above) are guaranteed to have a bit pattern of all zeros.
579	} else {
580	for (std::pair<CharUnits, CharUnits> Store : Stores) {
581	CharUnits StoreOffset = Store.first;
582	CharUnits StoreSize = Store.second;
583	llvm::Value *StoreSizeVal = CGF.CGM.getSize(numChars: StoreSize);
584	CGF.Builder.CreateMemSet(
585	Dest: CGF.Builder.CreateConstInBoundsByteGEP(Addr: DestPtr, Offset: StoreOffset),
586	Value: CGF.Builder.getInt8(C: 0), Size: StoreSizeVal);
587	}
588	}
589	}
590
591	void
592	CodeGenFunction::EmitCXXConstructExpr(const CXXConstructExpr *E,
593	AggValueSlot Dest) {
594	assert(!Dest.isIgnored() && "Must have a destination!");
595	const CXXConstructorDecl *CD = E->getConstructor();
596
597	// If we require zero initialization before (or instead of) calling the
598	// constructor, as can be the case with a non-user-provided default
599	// constructor, emit the zero initialization now, unless destination is
600	// already zeroed.
601	if (E->requiresZeroInitialization() && !Dest.isZeroed()) {
602	switch (E->getConstructionKind()) {
603	case CXXConstructionKind::Delegating:
604	case CXXConstructionKind::Complete:
605	EmitNullInitialization(DestPtr: Dest.getAddress(), Ty: E->getType());
606	break;
607	case CXXConstructionKind::VirtualBase:
608	case CXXConstructionKind::NonVirtualBase:
609	EmitNullBaseClassInitialization(*this, Dest.getAddress(),
610	CD->getParent());
611	break;
612	}
613	}
614
615	// If this is a call to a trivial default constructor, do nothing.
616	if (CD->isTrivial() && CD->isDefaultConstructor())
617	return;
618
619	// Elide the constructor if we're constructing from a temporary.
620	if (getLangOpts().ElideConstructors && E->isElidable()) {
621	// FIXME: This only handles the simplest case, where the source object
622	// is passed directly as the first argument to the constructor.
623	// This should also handle stepping though implicit casts and
624	// conversion sequences which involve two steps, with a
625	// conversion operator followed by a converting constructor.
626	const Expr *SrcObj = E->getArg(Arg: 0);
627	assert(SrcObj->isTemporaryObject(getContext(), CD->getParent()));
628	assert(
629	getContext().hasSameUnqualifiedType(E->getType(), SrcObj->getType()));
630	EmitAggExpr(E: SrcObj, AS: Dest);
631	return;
632	}
633
634	if (const ArrayType *arrayType
635	= getContext().getAsArrayType(T: E->getType())) {
636	EmitCXXAggrConstructorCall(D: CD, ArrayTy: arrayType, ArrayPtr: Dest.getAddress(), E,
637	NewPointerIsChecked: Dest.isSanitizerChecked());
638	} else {
639	CXXCtorType Type = Ctor_Complete;
640	bool ForVirtualBase = false;
641	bool Delegating = false;
642
643	switch (E->getConstructionKind()) {
644	case CXXConstructionKind::Delegating:
645	// We should be emitting a constructor; GlobalDecl will assert this
646	Type = CurGD.getCtorType();
647	Delegating = true;
648	break;
649
650	case CXXConstructionKind::Complete:
651	Type = Ctor_Complete;
652	break;
653
654	case CXXConstructionKind::VirtualBase:
655	ForVirtualBase = true;
656	[[fallthrough]];
657
658	case CXXConstructionKind::NonVirtualBase:
659	Type = Ctor_Base;
660	}
661
662	// Call the constructor.
663	EmitCXXConstructorCall(D: CD, Type, ForVirtualBase, Delegating, ThisAVS: Dest, E);
664	}
665	}
666
667	void CodeGenFunction::EmitSynthesizedCXXCopyCtor(Address Dest, Address Src,
668	const Expr *Exp) {
669	if (const ExprWithCleanups *E = dyn_cast<ExprWithCleanups>(Val: Exp))
670	Exp = E->getSubExpr();
671	assert(isa<CXXConstructExpr>(Exp) &&
672	"EmitSynthesizedCXXCopyCtor - unknown copy ctor expr");
673	const CXXConstructExpr* E = cast<CXXConstructExpr>(Val: Exp);
674	const CXXConstructorDecl *CD = E->getConstructor();
675	RunCleanupsScope Scope(*this);
676
677	// If we require zero initialization before (or instead of) calling the
678	// constructor, as can be the case with a non-user-provided default
679	// constructor, emit the zero initialization now.
680	// FIXME. Do I still need this for a copy ctor synthesis?
681	if (E->requiresZeroInitialization())
682	EmitNullInitialization(DestPtr: Dest, Ty: E->getType());
683
684	assert(!getContext().getAsConstantArrayType(E->getType())
685	&& "EmitSynthesizedCXXCopyCtor - Copied-in Array");
686	EmitSynthesizedCXXCopyCtorCall(D: CD, This: Dest, Src, E);
687	}
688
689	static CharUnits CalculateCookiePadding(CodeGenFunction &CGF,
690	const CXXNewExpr *E) {
691	if (!E->isArray())
692	return CharUnits::Zero();
693
694	// No cookie is required if the operator new[] being used is the
695	// reserved placement operator new[].
696	if (E->getOperatorNew()->isReservedGlobalPlacementOperator())
697	return CharUnits::Zero();
698
699	return CGF.CGM.getCXXABI().GetArrayCookieSize(expr: E);
700	}
701
702	static llvm::Value *EmitCXXNewAllocSize(CodeGenFunction &CGF,
703	const CXXNewExpr *e,
704	unsigned minElements,
705	llvm::Value *&numElements,
706	llvm::Value *&sizeWithoutCookie) {
707	QualType type = e->getAllocatedType();
708
709	if (!e->isArray()) {
710	CharUnits typeSize = CGF.getContext().getTypeSizeInChars(T: type);
711	sizeWithoutCookie
712	= llvm::ConstantInt::get(Ty: CGF.SizeTy, V: typeSize.getQuantity());
713	return sizeWithoutCookie;
714	}
715
716	// The width of size_t.
717	unsigned sizeWidth = CGF.SizeTy->getBitWidth();
718
719	// Figure out the cookie size.
720	llvm::APInt cookieSize(sizeWidth,
721	CalculateCookiePadding(CGF, E: e).getQuantity());
722
723	// Emit the array size expression.
724	// We multiply the size of all dimensions for NumElements.
725	// e.g for 'int[2][3]', ElemType is 'int' and NumElements is 6.
726	numElements =
727	ConstantEmitter(CGF).tryEmitAbstract(*e->getArraySize(), e->getType());
728	if (!numElements)
729	numElements = CGF.EmitScalarExpr(E: *e->getArraySize());
730	assert(isa<llvm::IntegerType>(numElements->getType()));
731
732	// The number of elements can be have an arbitrary integer type;
733	// essentially, we need to multiply it by a constant factor, add a
734	// cookie size, and verify that the result is representable as a
735	// size_t. That's just a gloss, though, and it's wrong in one
736	// important way: if the count is negative, it's an error even if
737	// the cookie size would bring the total size >= 0.
738	bool isSigned
739	= (*e->getArraySize())->getType()->isSignedIntegerOrEnumerationType();
740	llvm::IntegerType *numElementsType
741	= cast<llvm::IntegerType>(Val: numElements->getType());
742	unsigned numElementsWidth = numElementsType->getBitWidth();
743
744	// Compute the constant factor.
745	llvm::APInt arraySizeMultiplier(sizeWidth, 1);
746	while (const ConstantArrayType *CAT
747	= CGF.getContext().getAsConstantArrayType(T: type)) {
748	type = CAT->getElementType();
749	arraySizeMultiplier *= CAT->getSize();
750	}
751
752	CharUnits typeSize = CGF.getContext().getTypeSizeInChars(T: type);
753	llvm::APInt typeSizeMultiplier(sizeWidth, typeSize.getQuantity());
754	typeSizeMultiplier *= arraySizeMultiplier;
755
756	// This will be a size_t.
757	llvm::Value *size;
758
759	// If someone is doing 'new int[42]' there is no need to do a dynamic check.
760	// Don't bloat the -O0 code.
761	if (llvm::ConstantInt *numElementsC =
762	dyn_cast<llvm::ConstantInt>(Val: numElements)) {
763	const llvm::APInt &count = numElementsC->getValue();
764
765	bool hasAnyOverflow = false;
766
767	// If 'count' was a negative number, it's an overflow.
768	if (isSigned && count.isNegative())
769	hasAnyOverflow = true;
770
771	// We want to do all this arithmetic in size_t. If numElements is
772	// wider than that, check whether it's already too big, and if so,
773	// overflow.
774	else if (numElementsWidth > sizeWidth &&
775	numElementsWidth - sizeWidth > count.countl_zero())
776	hasAnyOverflow = true;
777
778	// Okay, compute a count at the right width.
779	llvm::APInt adjustedCount = count.zextOrTrunc(width: sizeWidth);
780
781	// If there is a brace-initializer, we cannot allocate fewer elements than
782	// there are initializers. If we do, that's treated like an overflow.
783	if (adjustedCount.ult(RHS: minElements))
784	hasAnyOverflow = true;
785
786	// Scale numElements by that. This might overflow, but we don't
787	// care because it only overflows if allocationSize does, too, and
788	// if that overflows then we shouldn't use this.
789	numElements = llvm::ConstantInt::get(Ty: CGF.SizeTy,
790	V: adjustedCount * arraySizeMultiplier);
791
792	// Compute the size before cookie, and track whether it overflowed.
793	bool overflow;
794	llvm::APInt allocationSize
795	= adjustedCount.umul_ov(RHS: typeSizeMultiplier, Overflow&: overflow);
796	hasAnyOverflow |= overflow;
797
798	// Add in the cookie, and check whether it's overflowed.
799	if (cookieSize != 0) {
800	// Save the current size without a cookie. This shouldn't be
801	// used if there was overflow.
802	sizeWithoutCookie = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: allocationSize);
803
804	allocationSize = allocationSize.uadd_ov(RHS: cookieSize, Overflow&: overflow);
805	hasAnyOverflow |= overflow;
806	}
807
808	// On overflow, produce a -1 so operator new will fail.
809	if (hasAnyOverflow) {
810	size = llvm::Constant::getAllOnesValue(Ty: CGF.SizeTy);
811	} else {
812	size = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: allocationSize);
813	}
814
815	// Otherwise, we might need to use the overflow intrinsics.
816	} else {
817	// There are up to five conditions we need to test for:
818	// 1) if isSigned, we need to check whether numElements is negative;
819	// 2) if numElementsWidth > sizeWidth, we need to check whether
820	// numElements is larger than something representable in size_t;
821	// 3) if minElements > 0, we need to check whether numElements is smaller
822	// than that.
823	// 4) we need to compute
824	// sizeWithoutCookie := numElements * typeSizeMultiplier
825	// and check whether it overflows; and
826	// 5) if we need a cookie, we need to compute
827	// size := sizeWithoutCookie + cookieSize
828	// and check whether it overflows.
829
830	llvm::Value *hasOverflow = nullptr;
831
832	// If numElementsWidth > sizeWidth, then one way or another, we're
833	// going to have to do a comparison for (2), and this happens to
834	// take care of (1), too.
835	if (numElementsWidth > sizeWidth) {
836	llvm::APInt threshold =
837	llvm::APInt::getOneBitSet(numBits: numElementsWidth, BitNo: sizeWidth);
838
839	llvm::Value *thresholdV
840	= llvm::ConstantInt::get(Ty: numElementsType, V: threshold);
841
842	hasOverflow = CGF.Builder.CreateICmpUGE(LHS: numElements, RHS: thresholdV);
843	numElements = CGF.Builder.CreateTrunc(V: numElements, DestTy: CGF.SizeTy);
844
845	// Otherwise, if we're signed, we want to sext up to size_t.
846	} else if (isSigned) {
847	if (numElementsWidth < sizeWidth)
848	numElements = CGF.Builder.CreateSExt(V: numElements, DestTy: CGF.SizeTy);
849
850	// If there's a non-1 type size multiplier, then we can do the
851	// signedness check at the same time as we do the multiply
852	// because a negative number times anything will cause an
853	// unsigned overflow. Otherwise, we have to do it here. But at least
854	// in this case, we can subsume the >= minElements check.
855	if (typeSizeMultiplier == 1)
856	hasOverflow = CGF.Builder.CreateICmpSLT(LHS: numElements,
857	RHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: minElements));
858
859	// Otherwise, zext up to size_t if necessary.
860	} else if (numElementsWidth < sizeWidth) {
861	numElements = CGF.Builder.CreateZExt(V: numElements, DestTy: CGF.SizeTy);
862	}
863
864	assert(numElements->getType() == CGF.SizeTy);
865
866	if (minElements) {
867	// Don't allow allocation of fewer elements than we have initializers.
868	if (!hasOverflow) {
869	hasOverflow = CGF.Builder.CreateICmpULT(LHS: numElements,
870	RHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: minElements));
871	} else if (numElementsWidth > sizeWidth) {
872	// The other existing overflow subsumes this check.
873	// We do an unsigned comparison, since any signed value < -1 is
874	// taken care of either above or below.
875	hasOverflow = CGF.Builder.CreateOr(LHS: hasOverflow,
876	RHS: CGF.Builder.CreateICmpULT(LHS: numElements,
877	RHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: minElements)));
878	}
879	}
880
881	size = numElements;
882
883	// Multiply by the type size if necessary. This multiplier
884	// includes all the factors for nested arrays.
885	//
886	// This step also causes numElements to be scaled up by the
887	// nested-array factor if necessary. Overflow on this computation
888	// can be ignored because the result shouldn't be used if
889	// allocation fails.
890	if (typeSizeMultiplier != 1) {
891	llvm::Function *umul_with_overflow
892	= CGF.CGM.getIntrinsic(llvm::Intrinsic::umul_with_overflow, CGF.SizeTy);
893
894	llvm::Value *tsmV =
895	llvm::ConstantInt::get(Ty: CGF.SizeTy, V: typeSizeMultiplier);
896	llvm::Value *result =
897	CGF.Builder.CreateCall(Callee: umul_with_overflow, Args: {size, tsmV});
898
899	llvm::Value *overflowed = CGF.Builder.CreateExtractValue(Agg: result, Idxs: 1);
900	if (hasOverflow)
901	hasOverflow = CGF.Builder.CreateOr(LHS: hasOverflow, RHS: overflowed);
902	else
903	hasOverflow = overflowed;
904
905	size = CGF.Builder.CreateExtractValue(Agg: result, Idxs: 0);
906
907	// Also scale up numElements by the array size multiplier.
908	if (arraySizeMultiplier != 1) {
909	// If the base element type size is 1, then we can re-use the
910	// multiply we just did.
911	if (typeSize.isOne()) {
912	assert(arraySizeMultiplier == typeSizeMultiplier);
913	numElements = size;
914
915	// Otherwise we need a separate multiply.
916	} else {
917	llvm::Value *asmV =
918	llvm::ConstantInt::get(Ty: CGF.SizeTy, V: arraySizeMultiplier);
919	numElements = CGF.Builder.CreateMul(LHS: numElements, RHS: asmV);
920	}
921	}
922	} else {
923	// numElements doesn't need to be scaled.
924	assert(arraySizeMultiplier == 1);
925	}
926
927	// Add in the cookie size if necessary.
928	if (cookieSize != 0) {
929	sizeWithoutCookie = size;
930
931	llvm::Function *uadd_with_overflow
932	= CGF.CGM.getIntrinsic(llvm::Intrinsic::uadd_with_overflow, CGF.SizeTy);
933
934	llvm::Value *cookieSizeV = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: cookieSize);
935	llvm::Value *result =
936	CGF.Builder.CreateCall(Callee: uadd_with_overflow, Args: {size, cookieSizeV});
937
938	llvm::Value *overflowed = CGF.Builder.CreateExtractValue(Agg: result, Idxs: 1);
939	if (hasOverflow)
940	hasOverflow = CGF.Builder.CreateOr(LHS: hasOverflow, RHS: overflowed);
941	else
942	hasOverflow = overflowed;
943
944	size = CGF.Builder.CreateExtractValue(Agg: result, Idxs: 0);
945	}
946
947	// If we had any possibility of dynamic overflow, make a select to
948	// overwrite 'size' with an all-ones value, which should cause
949	// operator new to throw.
950	if (hasOverflow)
951	size = CGF.Builder.CreateSelect(C: hasOverflow,
952	True: llvm::Constant::getAllOnesValue(Ty: CGF.SizeTy),
953	False: size);
954	}
955
956	if (cookieSize == 0)
957	sizeWithoutCookie = size;
958	else
959	assert(sizeWithoutCookie && "didn't set sizeWithoutCookie?");
960
961	return size;
962	}
963
964	static void StoreAnyExprIntoOneUnit(CodeGenFunction &CGF, const Expr *Init,
965	QualType AllocType, Address NewPtr,
966	AggValueSlot::Overlap_t MayOverlap) {
967	// FIXME: Refactor with EmitExprAsInit.
968	switch (CGF.getEvaluationKind(T: AllocType)) {
969	case TEK_Scalar:
970	CGF.EmitScalarInit(init: Init, D: nullptr,
971	lvalue: CGF.MakeAddrLValue(Addr: NewPtr, T: AllocType), capturedByInit: false);
972	return;
973	case TEK_Complex:
974	CGF.EmitComplexExprIntoLValue(E: Init, dest: CGF.MakeAddrLValue(Addr: NewPtr, T: AllocType),
975	/*isInit*/ true);
976	return;
977	case TEK_Aggregate: {
978	AggValueSlot Slot
979	= AggValueSlot::forAddr(addr: NewPtr, quals: AllocType.getQualifiers(),
980	isDestructed: AggValueSlot::IsDestructed,
981	needsGC: AggValueSlot::DoesNotNeedGCBarriers,
982	isAliased: AggValueSlot::IsNotAliased,
983	mayOverlap: MayOverlap, isZeroed: AggValueSlot::IsNotZeroed,
984	isChecked: AggValueSlot::IsSanitizerChecked);
985	CGF.EmitAggExpr(E: Init, AS: Slot);
986	return;
987	}
988	}
989	llvm_unreachable("bad evaluation kind");
990	}
991
992	void CodeGenFunction::EmitNewArrayInitializer(
993	const CXXNewExpr *E, QualType ElementType, llvm::Type *ElementTy,
994	Address BeginPtr, llvm::Value *NumElements,
995	llvm::Value *AllocSizeWithoutCookie) {
996	// If we have a type with trivial initialization and no initializer,
997	// there's nothing to do.
998	if (!E->hasInitializer())
999	return;
1000
1001	Address CurPtr = BeginPtr;
1002
1003	unsigned InitListElements = 0;
1004
1005	const Expr *Init = E->getInitializer();
1006	Address EndOfInit = Address::invalid();
1007	QualType::DestructionKind DtorKind = ElementType.isDestructedType();
1008	EHScopeStack::stable_iterator Cleanup;
1009	llvm::Instruction *CleanupDominator = nullptr;
1010
1011	CharUnits ElementSize = getContext().getTypeSizeInChars(T: ElementType);
1012	CharUnits ElementAlign =
1013	BeginPtr.getAlignment().alignmentOfArrayElement(elementSize: ElementSize);
1014
1015	// Attempt to perform zero-initialization using memset.
1016	auto TryMemsetInitialization = [&]() -> bool {
1017	// FIXME: If the type is a pointer-to-data-member under the Itanium ABI,
1018	// we can initialize with a memset to -1.
1019	if (!CGM.getTypes().isZeroInitializable(T: ElementType))
1020	return false;
1021
1022	// Optimization: since zero initialization will just set the memory
1023	// to all zeroes, generate a single memset to do it in one shot.
1024
1025	// Subtract out the size of any elements we've already initialized.
1026	auto *RemainingSize = AllocSizeWithoutCookie;
1027	if (InitListElements) {
1028	// We know this can't overflow; we check this when doing the allocation.
1029	auto *InitializedSize = llvm::ConstantInt::get(
1030	Ty: RemainingSize->getType(),
1031	V: getContext().getTypeSizeInChars(T: ElementType).getQuantity() *
1032	InitListElements);
1033	RemainingSize = Builder.CreateSub(LHS: RemainingSize, RHS: InitializedSize);
1034	}
1035
1036	// Create the memset.
1037	Builder.CreateMemSet(Dest: CurPtr, Value: Builder.getInt8(C: 0), Size: RemainingSize, IsVolatile: false);
1038	return true;
1039	};
1040
1041	const InitListExpr *ILE = dyn_cast<InitListExpr>(Val: Init);
1042	const CXXParenListInitExpr *CPLIE = nullptr;
1043	const StringLiteral *SL = nullptr;
1044	const ObjCEncodeExpr *OCEE = nullptr;
1045	const Expr *IgnoreParen = nullptr;
1046	if (!ILE) {
1047	IgnoreParen = Init->IgnoreParenImpCasts();
1048	CPLIE = dyn_cast<CXXParenListInitExpr>(Val: IgnoreParen);
1049	SL = dyn_cast<StringLiteral>(Val: IgnoreParen);
1050	OCEE = dyn_cast<ObjCEncodeExpr>(Val: IgnoreParen);
1051	}
1052
1053	// If the initializer is an initializer list, first do the explicit elements.
1054	if (ILE || CPLIE || SL || OCEE) {
1055	// Initializing from a (braced) string literal is a special case; the init
1056	// list element does not initialize a (single) array element.
1057	if ((ILE && ILE->isStringLiteralInit()) || SL || OCEE) {
1058	if (!ILE)
1059	Init = IgnoreParen;
1060	// Initialize the initial portion of length equal to that of the string
1061	// literal. The allocation must be for at least this much; we emitted a
1062	// check for that earlier.
1063	AggValueSlot Slot =
1064	AggValueSlot::forAddr(addr: CurPtr, quals: ElementType.getQualifiers(),
1065	isDestructed: AggValueSlot::IsDestructed,
1066	needsGC: AggValueSlot::DoesNotNeedGCBarriers,
1067	isAliased: AggValueSlot::IsNotAliased,
1068	mayOverlap: AggValueSlot::DoesNotOverlap,
1069	isZeroed: AggValueSlot::IsNotZeroed,
1070	isChecked: AggValueSlot::IsSanitizerChecked);
1071	EmitAggExpr(E: ILE ? ILE->getInit(Init: 0) : Init, AS: Slot);
1072
1073	// Move past these elements.
1074	InitListElements =
1075	cast<ConstantArrayType>(Init->getType()->getAsArrayTypeUnsafe())
1076	->getSize()
1077	.getZExtValue();
1078	CurPtr = Builder.CreateConstInBoundsGEP(
1079	Addr: CurPtr, Index: InitListElements, Name: "string.init.end");
1080
1081	// Zero out the rest, if any remain.
1082	llvm::ConstantInt *ConstNum = dyn_cast<llvm::ConstantInt>(Val: NumElements);
1083	if (!ConstNum || !ConstNum->equalsInt(V: InitListElements)) {
1084	bool OK = TryMemsetInitialization();
1085	(void)OK;
1086	assert(OK && "couldn't memset character type?");
1087	}
1088	return;
1089	}
1090
1091	ArrayRef<const Expr *> InitExprs =
1092	ILE ? ILE->inits() : CPLIE->getInitExprs();
1093	InitListElements = InitExprs.size();
1094
1095	// If this is a multi-dimensional array new, we will initialize multiple
1096	// elements with each init list element.
1097	QualType AllocType = E->getAllocatedType();
1098	if (const ConstantArrayType *CAT = dyn_cast_or_null<ConstantArrayType>(
1099	AllocType->getAsArrayTypeUnsafe())) {
1100	ElementTy = ConvertTypeForMem(T: AllocType);
1101	CurPtr = CurPtr.withElementType(ElemTy: ElementTy);
1102	InitListElements *= getContext().getConstantArrayElementCount(CA: CAT);
1103	}
1104
1105	// Enter a partial-destruction Cleanup if necessary.
1106	if (needsEHCleanup(kind: DtorKind)) {
1107	// In principle we could tell the Cleanup where we are more
1108	// directly, but the control flow can get so varied here that it
1109	// would actually be quite complex. Therefore we go through an
1110	// alloca.
1111	EndOfInit = CreateTempAlloca(BeginPtr.getType(), getPointerAlign(),
1112	"array.init.end");
1113	CleanupDominator = Builder.CreateStore(Val: BeginPtr.getPointer(), Addr: EndOfInit);
1114	pushIrregularPartialArrayCleanup(arrayBegin: BeginPtr.getPointer(), arrayEndPointer: EndOfInit,
1115	elementType: ElementType, elementAlignment: ElementAlign,
1116	destroyer: getDestroyer(destructionKind: DtorKind));
1117	Cleanup = EHStack.stable_begin();
1118	}
1119
1120	CharUnits StartAlign = CurPtr.getAlignment();
1121	unsigned i = 0;
1122	for (const Expr *IE : InitExprs) {
1123	// Tell the cleanup that it needs to destroy up to this
1124	// element. TODO: some of these stores can be trivially
1125	// observed to be unnecessary.
1126	if (EndOfInit.isValid()) {
1127	Builder.CreateStore(Val: CurPtr.getPointer(), Addr: EndOfInit);
1128	}
1129	// FIXME: If the last initializer is an incomplete initializer list for
1130	// an array, and we have an array filler, we can fold together the two
1131	// initialization loops.
1132	StoreAnyExprIntoOneUnit(CGF&: *this, Init: IE, AllocType: IE->getType(), NewPtr: CurPtr,
1133	MayOverlap: AggValueSlot::DoesNotOverlap);
1134	CurPtr = Address(Builder.CreateInBoundsGEP(
1135	Ty: CurPtr.getElementType(), Ptr: CurPtr.getPointer(),
1136	IdxList: Builder.getSize(N: 1), Name: "array.exp.next"),
1137	CurPtr.getElementType(),
1138	StartAlign.alignmentAtOffset(offset: (++i) * ElementSize));
1139	}
1140
1141	// The remaining elements are filled with the array filler expression.
1142	Init = ILE ? ILE->getArrayFiller() : CPLIE->getArrayFiller();
1143
1144	// Extract the initializer for the individual array elements by pulling
1145	// out the array filler from all the nested initializer lists. This avoids
1146	// generating a nested loop for the initialization.
1147	while (Init && Init->getType()->isConstantArrayType()) {
1148	auto *SubILE = dyn_cast<InitListExpr>(Val: Init);
1149	if (!SubILE)
1150	break;
1151	assert(SubILE->getNumInits() == 0 && "explicit inits in array filler?");
1152	Init = SubILE->getArrayFiller();
1153	}
1154
1155	// Switch back to initializing one base element at a time.
1156	CurPtr = CurPtr.withElementType(ElemTy: BeginPtr.getElementType());
1157	}
1158
1159	// If all elements have already been initialized, skip any further
1160	// initialization.
1161	llvm::ConstantInt *ConstNum = dyn_cast<llvm::ConstantInt>(Val: NumElements);
1162	if (ConstNum && ConstNum->getZExtValue() <= InitListElements) {
1163	// If there was a Cleanup, deactivate it.
1164	if (CleanupDominator)
1165	DeactivateCleanupBlock(Cleanup, DominatingIP: CleanupDominator);
1166	return;
1167	}
1168
1169	assert(Init && "have trailing elements to initialize but no initializer");
1170
1171	// If this is a constructor call, try to optimize it out, and failing that
1172	// emit a single loop to initialize all remaining elements.
1173	if (const CXXConstructExpr *CCE = dyn_cast<CXXConstructExpr>(Val: Init)) {
1174	CXXConstructorDecl *Ctor = CCE->getConstructor();
1175	if (Ctor->isTrivial()) {
1176	// If new expression did not specify value-initialization, then there
1177	// is no initialization.
1178	if (!CCE->requiresZeroInitialization() || Ctor->getParent()->isEmpty())
1179	return;
1180
1181	if (TryMemsetInitialization())
1182	return;
1183	}
1184
1185	// Store the new Cleanup position for irregular Cleanups.
1186	//
1187	// FIXME: Share this cleanup with the constructor call emission rather than
1188	// having it create a cleanup of its own.
1189	if (EndOfInit.isValid())
1190	Builder.CreateStore(Val: CurPtr.getPointer(), Addr: EndOfInit);
1191
1192	// Emit a constructor call loop to initialize the remaining elements.
1193	if (InitListElements)
1194	NumElements = Builder.CreateSub(
1195	LHS: NumElements,
1196	RHS: llvm::ConstantInt::get(Ty: NumElements->getType(), V: InitListElements));
1197	EmitCXXAggrConstructorCall(D: Ctor, NumElements, ArrayPtr: CurPtr, E: CCE,
1198	/*NewPointerIsChecked*/true,
1199	ZeroInitialization: CCE->requiresZeroInitialization());
1200	return;
1201	}
1202
1203	// If this is value-initialization, we can usually use memset.
1204	ImplicitValueInitExpr IVIE(ElementType);
1205	if (isa<ImplicitValueInitExpr>(Val: Init)) {
1206	if (TryMemsetInitialization())
1207	return;
1208
1209	// Switch to an ImplicitValueInitExpr for the element type. This handles
1210	// only one case: multidimensional array new of pointers to members. In
1211	// all other cases, we already have an initializer for the array element.
1212	Init = &IVIE;
1213	}
1214
1215	// At this point we should have found an initializer for the individual
1216	// elements of the array.
1217	assert(getContext().hasSameUnqualifiedType(ElementType, Init->getType()) &&
1218	"got wrong type of element to initialize");
1219
1220	// If we have an empty initializer list, we can usually use memset.
1221	if (auto *ILE = dyn_cast<InitListExpr>(Val: Init))
1222	if (ILE->getNumInits() == 0 && TryMemsetInitialization())
1223	return;
1224
1225	// If we have a struct whose every field is value-initialized, we can
1226	// usually use memset.
1227	if (auto *ILE = dyn_cast<InitListExpr>(Val: Init)) {
1228	if (const RecordType *RType = ILE->getType()->getAs<RecordType>()) {
1229	if (RType->getDecl()->isStruct()) {
1230	unsigned NumElements = 0;
1231	if (auto *CXXRD = dyn_cast<CXXRecordDecl>(RType->getDecl()))
1232	NumElements = CXXRD->getNumBases();
1233	for (auto *Field : RType->getDecl()->fields())
1234	if (!Field->isUnnamedBitfield())
1235	++NumElements;
1236	// FIXME: Recurse into nested InitListExprs.
1237	if (ILE->getNumInits() == NumElements)
1238	for (unsigned i = 0, e = ILE->getNumInits(); i != e; ++i)
1239	if (!isa<ImplicitValueInitExpr>(Val: ILE->getInit(Init: i)))
1240	--NumElements;
1241	if (ILE->getNumInits() == NumElements && TryMemsetInitialization())
1242	return;
1243	}
1244	}
1245	}
1246
1247	// Create the loop blocks.
1248	llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
1249	llvm::BasicBlock *LoopBB = createBasicBlock(name: "new.loop");
1250	llvm::BasicBlock *ContBB = createBasicBlock(name: "new.loop.end");
1251
1252	// Find the end of the array, hoisted out of the loop.
1253	llvm::Value *EndPtr =
1254	Builder.CreateInBoundsGEP(Ty: BeginPtr.getElementType(), Ptr: BeginPtr.getPointer(),
1255	IdxList: NumElements, Name: "array.end");
1256
1257	// If the number of elements isn't constant, we have to now check if there is
1258	// anything left to initialize.
1259	if (!ConstNum) {
1260	llvm::Value *IsEmpty =
1261	Builder.CreateICmpEQ(LHS: CurPtr.getPointer(), RHS: EndPtr, Name: "array.isempty");
1262	Builder.CreateCondBr(Cond: IsEmpty, True: ContBB, False: LoopBB);
1263	}
1264
1265	// Enter the loop.
1266	EmitBlock(BB: LoopBB);
1267
1268	// Set up the current-element phi.
1269	llvm::PHINode *CurPtrPhi =
1270	Builder.CreatePHI(Ty: CurPtr.getType(), NumReservedValues: 2, Name: "array.cur");
1271	CurPtrPhi->addIncoming(V: CurPtr.getPointer(), BB: EntryBB);
1272
1273	CurPtr = Address(CurPtrPhi, CurPtr.getElementType(), ElementAlign);
1274
1275	// Store the new Cleanup position for irregular Cleanups.
1276	if (EndOfInit.isValid())
1277	Builder.CreateStore(Val: CurPtr.getPointer(), Addr: EndOfInit);
1278
1279	// Enter a partial-destruction Cleanup if necessary.
1280	if (!CleanupDominator && needsEHCleanup(kind: DtorKind)) {
1281	pushRegularPartialArrayCleanup(arrayBegin: BeginPtr.getPointer(), arrayEnd: CurPtr.getPointer(),
1282	elementType: ElementType, elementAlignment: ElementAlign,
1283	destroyer: getDestroyer(destructionKind: DtorKind));
1284	Cleanup = EHStack.stable_begin();
1285	CleanupDominator = Builder.CreateUnreachable();
1286	}
1287
1288	// Emit the initializer into this element.
1289	StoreAnyExprIntoOneUnit(CGF&: *this, Init, AllocType: Init->getType(), NewPtr: CurPtr,
1290	MayOverlap: AggValueSlot::DoesNotOverlap);
1291
1292	// Leave the Cleanup if we entered one.
1293	if (CleanupDominator) {
1294	DeactivateCleanupBlock(Cleanup, DominatingIP: CleanupDominator);
1295	CleanupDominator->eraseFromParent();
1296	}
1297
1298	// Advance to the next element by adjusting the pointer type as necessary.
1299	llvm::Value *NextPtr =
1300	Builder.CreateConstInBoundsGEP1_32(Ty: ElementTy, Ptr: CurPtr.getPointer(), Idx0: 1,
1301	Name: "array.next");
1302
1303	// Check whether we've gotten to the end of the array and, if so,
1304	// exit the loop.
1305	llvm::Value *IsEnd = Builder.CreateICmpEQ(LHS: NextPtr, RHS: EndPtr, Name: "array.atend");
1306	Builder.CreateCondBr(Cond: IsEnd, True: ContBB, False: LoopBB);
1307	CurPtrPhi->addIncoming(V: NextPtr, BB: Builder.GetInsertBlock());
1308
1309	EmitBlock(BB: ContBB);
1310	}
1311
1312	static void EmitNewInitializer(CodeGenFunction &CGF, const CXXNewExpr *E,
1313	QualType ElementType, llvm::Type *ElementTy,
1314	Address NewPtr, llvm::Value *NumElements,
1315	llvm::Value *AllocSizeWithoutCookie) {
1316	ApplyDebugLocation DL(CGF, E);
1317	if (E->isArray())
1318	CGF.EmitNewArrayInitializer(E, ElementType, ElementTy, BeginPtr: NewPtr, NumElements,
1319	AllocSizeWithoutCookie);
1320	else if (const Expr *Init = E->getInitializer())
1321	StoreAnyExprIntoOneUnit(CGF, Init, AllocType: E->getAllocatedType(), NewPtr,
1322	MayOverlap: AggValueSlot::DoesNotOverlap);
1323	}
1324
1325	/// Emit a call to an operator new or operator delete function, as implicitly
1326	/// created by new-expressions and delete-expressions.
1327	static RValue EmitNewDeleteCall(CodeGenFunction &CGF,
1328	const FunctionDecl *CalleeDecl,
1329	const FunctionProtoType *CalleeType,
1330	const CallArgList &Args) {
1331	llvm::CallBase *CallOrInvoke;
1332	llvm::Constant *CalleePtr = CGF.CGM.GetAddrOfFunction(GD: CalleeDecl);
1333	CGCallee Callee = CGCallee::forDirect(functionPtr: CalleePtr, abstractInfo: GlobalDecl(CalleeDecl));
1334	RValue RV =
1335	CGF.EmitCall(CallInfo: CGF.CGM.getTypes().arrangeFreeFunctionCall(
1336	Args, CalleeType, /*ChainCall=*/false),
1337	Callee, ReturnValue: ReturnValueSlot(), Args, callOrInvoke: &CallOrInvoke);
1338
1339	/// C++1y [expr.new]p10:
1340	/// [In a new-expression,] an implementation is allowed to omit a call
1341	/// to a replaceable global allocation function.
1342	///
1343	/// We model such elidable calls with the 'builtin' attribute.
1344	llvm::Function *Fn = dyn_cast<llvm::Function>(Val: CalleePtr);
1345	if (CalleeDecl->isReplaceableGlobalAllocationFunction() &&
1346	Fn && Fn->hasFnAttribute(llvm::Attribute::NoBuiltin)) {
1347	CallOrInvoke->addFnAttr(llvm::Attribute::Builtin);
1348	}
1349
1350	return RV;
1351	}
1352
1353	RValue CodeGenFunction::EmitBuiltinNewDeleteCall(const FunctionProtoType *Type,
1354	const CallExpr *TheCall,
1355	bool IsDelete) {
1356	CallArgList Args;
1357	EmitCallArgs(Args, Prototype: Type, ArgRange: TheCall->arguments());
1358	// Find the allocation or deallocation function that we're calling.
1359	ASTContext &Ctx = getContext();
1360	DeclarationName Name = Ctx.DeclarationNames
1361	.getCXXOperatorName(Op: IsDelete ? OO_Delete : OO_New);
1362
1363	for (auto *Decl : Ctx.getTranslationUnitDecl()->lookup(Name))
1364	if (auto *FD = dyn_cast<FunctionDecl>(Decl))
1365	if (Ctx.hasSameType(FD->getType(), QualType(Type, 0)))
1366	return EmitNewDeleteCall(*this, FD, Type, Args);
1367	llvm_unreachable("predeclared global operator new/delete is missing");
1368	}
1369
1370	namespace {
1371	/// The parameters to pass to a usual operator delete.
1372	struct UsualDeleteParams {
1373	bool DestroyingDelete = false;
1374	bool Size = false;
1375	bool Alignment = false;
1376	};
1377	}
1378
1379	static UsualDeleteParams getUsualDeleteParams(const FunctionDecl *FD) {
1380	UsualDeleteParams Params;
1381
1382	const FunctionProtoType *FPT = FD->getType()->castAs<FunctionProtoType>();
1383	auto AI = FPT->param_type_begin(), AE = FPT->param_type_end();
1384
1385	// The first argument is always a void*.
1386	++AI;
1387
1388	// The next parameter may be a std::destroying_delete_t.
1389	if (FD->isDestroyingOperatorDelete()) {
1390	Params.DestroyingDelete = true;
1391	assert(AI != AE);
1392	++AI;
1393	}
1394
1395	// Figure out what other parameters we should be implicitly passing.
1396	if (AI != AE && (*AI)->isIntegerType()) {
1397	Params.Size = true;
1398	++AI;
1399	}
1400
1401	if (AI != AE && (*AI)->isAlignValT()) {
1402	Params.Alignment = true;
1403	++AI;
1404	}
1405
1406	assert(AI == AE && "unexpected usual deallocation function parameter");
1407	return Params;
1408	}
1409
1410	namespace {
1411	/// A cleanup to call the given 'operator delete' function upon abnormal
1412	/// exit from a new expression. Templated on a traits type that deals with
1413	/// ensuring that the arguments dominate the cleanup if necessary.
1414	template<typename Traits>
1415	class CallDeleteDuringNew final : public EHScopeStack::Cleanup {
1416	/// Type used to hold llvm::Value*s.
1417	typedef typename Traits::ValueTy ValueTy;
1418	/// Type used to hold RValues.
1419	typedef typename Traits::RValueTy RValueTy;
1420	struct PlacementArg {
1421	RValueTy ArgValue;
1422	QualType ArgType;
1423	};
1424
1425	unsigned NumPlacementArgs : 31;
1426	LLVM_PREFERRED_TYPE(bool)
1427	unsigned PassAlignmentToPlacementDelete : 1;
1428	const FunctionDecl *OperatorDelete;
1429	ValueTy Ptr;
1430	ValueTy AllocSize;
1431	CharUnits AllocAlign;
1432
1433	PlacementArg *getPlacementArgs() {
1434	return reinterpret_cast<PlacementArg *>(this + 1);
1435	}
1436
1437	public:
1438	static size_t getExtraSize(size_t NumPlacementArgs) {
1439	return NumPlacementArgs * sizeof(PlacementArg);
1440	}
1441
1442	CallDeleteDuringNew(size_t NumPlacementArgs,
1443	const FunctionDecl *OperatorDelete, ValueTy Ptr,
1444	ValueTy AllocSize, bool PassAlignmentToPlacementDelete,
1445	CharUnits AllocAlign)
1446	: NumPlacementArgs(NumPlacementArgs),
1447	PassAlignmentToPlacementDelete(PassAlignmentToPlacementDelete),
1448	OperatorDelete(OperatorDelete), Ptr(Ptr), AllocSize(AllocSize),
1449	AllocAlign(AllocAlign) {}
1450
1451	void setPlacementArg(unsigned I, RValueTy Arg, QualType Type) {
1452	assert(I < NumPlacementArgs && "index out of range");
1453	getPlacementArgs()[I] = {Arg, Type};
1454	}
1455
1456	void Emit(CodeGenFunction &CGF, Flags flags) override {
1457	const auto *FPT = OperatorDelete->getType()->castAs<FunctionProtoType>();
1458	CallArgList DeleteArgs;
1459
1460	// The first argument is always a void* (or C* for a destroying operator
1461	// delete for class type C).
1462	DeleteArgs.add(rvalue: Traits::get(CGF, Ptr), type: FPT->getParamType(0));
1463
1464	// Figure out what other parameters we should be implicitly passing.
1465	UsualDeleteParams Params;
1466	if (NumPlacementArgs) {
1467	// A placement deallocation function is implicitly passed an alignment
1468	// if the placement allocation function was, but is never passed a size.
1469	Params.Alignment = PassAlignmentToPlacementDelete;
1470	} else {
1471	// For a non-placement new-expression, 'operator delete' can take a
1472	// size and/or an alignment if it has the right parameters.
1473	Params = getUsualDeleteParams(FD: OperatorDelete);
1474	}
1475
1476	assert(!Params.DestroyingDelete &&
1477	"should not call destroying delete in a new-expression");
1478
1479	// The second argument can be a std::size_t (for non-placement delete).
1480	if (Params.Size)
1481	DeleteArgs.add(rvalue: Traits::get(CGF, AllocSize),
1482	type: CGF.getContext().getSizeType());
1483
1484	// The next (second or third) argument can be a std::align_val_t, which
1485	// is an enum whose underlying type is std::size_t.
1486	// FIXME: Use the right type as the parameter type. Note that in a call
1487	// to operator delete(size_t, ...), we may not have it available.
1488	if (Params.Alignment)
1489	DeleteArgs.add(rvalue: RValue::get(V: llvm::ConstantInt::get(
1490	Ty: CGF.SizeTy, V: AllocAlign.getQuantity())),
1491	type: CGF.getContext().getSizeType());
1492
1493	// Pass the rest of the arguments, which must match exactly.
1494	for (unsigned I = 0; I != NumPlacementArgs; ++I) {
1495	auto Arg = getPlacementArgs()[I];
1496	DeleteArgs.add(rvalue: Traits::get(CGF, Arg.ArgValue), type: Arg.ArgType);
1497	}
1498
1499	// Call 'operator delete'.
1500	EmitNewDeleteCall(CGF, OperatorDelete, FPT, DeleteArgs);
1501	}
1502	};
1503	}
1504
1505	/// Enter a cleanup to call 'operator delete' if the initializer in a
1506	/// new-expression throws.
1507	static void EnterNewDeleteCleanup(CodeGenFunction &CGF,
1508	const CXXNewExpr *E,
1509	Address NewPtr,
1510	llvm::Value *AllocSize,
1511	CharUnits AllocAlign,
1512	const CallArgList &NewArgs) {
1513	unsigned NumNonPlacementArgs = E->passAlignment() ? 2 : 1;
1514
1515	// If we're not inside a conditional branch, then the cleanup will
1516	// dominate and we can do the easier (and more efficient) thing.
1517	if (!CGF.isInConditionalBranch()) {
1518	struct DirectCleanupTraits {
1519	typedef llvm::Value *ValueTy;
1520	typedef RValue RValueTy;
1521	static RValue get(CodeGenFunction &, ValueTy V) { return RValue::get(V); }
1522	static RValue get(CodeGenFunction &, RValueTy V) { return V; }
1523	};
1524
1525	typedef CallDeleteDuringNew<DirectCleanupTraits> DirectCleanup;
1526
1527	DirectCleanup *Cleanup = CGF.EHStack
1528	.pushCleanupWithExtra<DirectCleanup>(Kind: EHCleanup,
1529	N: E->getNumPlacementArgs(),
1530	A: E->getOperatorDelete(),
1531	A: NewPtr.getPointer(),
1532	A: AllocSize,
1533	A: E->passAlignment(),
1534	A: AllocAlign);
1535	for (unsigned I = 0, N = E->getNumPlacementArgs(); I != N; ++I) {
1536	auto &Arg = NewArgs[I + NumNonPlacementArgs];
1537	Cleanup->setPlacementArg(I, Arg.getRValue(CGF), Arg.Ty);
1538	}
1539
1540	return;
1541	}
1542
1543	// Otherwise, we need to save all this stuff.
1544	DominatingValue<RValue>::saved_type SavedNewPtr =
1545	DominatingValue<RValue>::save(CGF, value: RValue::get(V: NewPtr.getPointer()));
1546	DominatingValue<RValue>::saved_type SavedAllocSize =
1547	DominatingValue<RValue>::save(CGF, value: RValue::get(V: AllocSize));
1548
1549	struct ConditionalCleanupTraits {
1550	typedef DominatingValue<RValue>::saved_type ValueTy;
1551	typedef DominatingValue<RValue>::saved_type RValueTy;
1552	static RValue get(CodeGenFunction &CGF, ValueTy V) {
1553	return V.restore(CGF);
1554	}
1555	};
1556	typedef CallDeleteDuringNew<ConditionalCleanupTraits> ConditionalCleanup;
1557
1558	ConditionalCleanup *Cleanup = CGF.EHStack
1559	.pushCleanupWithExtra<ConditionalCleanup>(Kind: EHCleanup,
1560	N: E->getNumPlacementArgs(),
1561	A: E->getOperatorDelete(),
1562	A: SavedNewPtr,
1563	A: SavedAllocSize,
1564	A: E->passAlignment(),
1565	A: AllocAlign);
1566	for (unsigned I = 0, N = E->getNumPlacementArgs(); I != N; ++I) {
1567	auto &Arg = NewArgs[I + NumNonPlacementArgs];
1568	Cleanup->setPlacementArg(
1569	I, DominatingValue<RValue>::save(CGF, value: Arg.getRValue(CGF)), Arg.Ty);
1570	}
1571
1572	CGF.initFullExprCleanup();
1573	}
1574
1575	llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
1576	// The element type being allocated.
1577	QualType allocType = getContext().getBaseElementType(QT: E->getAllocatedType());
1578
1579	// 1. Build a call to the allocation function.
1580	FunctionDecl *allocator = E->getOperatorNew();
1581
1582	// If there is a brace-initializer or C++20 parenthesized initializer, cannot
1583	// allocate fewer elements than inits.
1584	unsigned minElements = 0;
1585	if (E->isArray() && E->hasInitializer()) {
1586	const Expr *Init = E->getInitializer();
1587	const InitListExpr *ILE = dyn_cast<InitListExpr>(Val: Init);
1588	const CXXParenListInitExpr *CPLIE = dyn_cast<CXXParenListInitExpr>(Val: Init);
1589	const Expr *IgnoreParen = Init->IgnoreParenImpCasts();
1590	if ((ILE && ILE->isStringLiteralInit()) ||
1591	isa<StringLiteral>(Val: IgnoreParen) || isa<ObjCEncodeExpr>(Val: IgnoreParen)) {
1592	minElements =
1593	cast<ConstantArrayType>(Init->getType()->getAsArrayTypeUnsafe())
1594	->getSize()
1595	.getZExtValue();
1596	} else if (ILE || CPLIE) {
1597	minElements = ILE ? ILE->getNumInits() : CPLIE->getInitExprs().size();
1598	}
1599	}
1600
1601	llvm::Value *numElements = nullptr;
1602	llvm::Value *allocSizeWithoutCookie = nullptr;
1603	llvm::Value *allocSize =
1604	EmitCXXNewAllocSize(CGF&: *this, e: E, minElements, numElements,
1605	sizeWithoutCookie&: allocSizeWithoutCookie);
1606	CharUnits allocAlign = getContext().getTypeAlignInChars(T: allocType);
1607
1608	// Emit the allocation call. If the allocator is a global placement
1609	// operator, just "inline" it directly.
1610	Address allocation = Address::invalid();
1611	CallArgList allocatorArgs;
1612	if (allocator->isReservedGlobalPlacementOperator()) {
1613	assert(E->getNumPlacementArgs() == 1);
1614	const Expr *arg = *E->placement_arguments().begin();
1615
1616	LValueBaseInfo BaseInfo;
1617	allocation = EmitPointerWithAlignment(Addr: arg, BaseInfo: &BaseInfo);
1618
1619	// The pointer expression will, in many cases, be an opaque void*.
1620	// In these cases, discard the computed alignment and use the
1621	// formal alignment of the allocated type.
1622	if (BaseInfo.getAlignmentSource() != AlignmentSource::Decl)
1623	allocation = allocation.withAlignment(NewAlignment: allocAlign);
1624
1625	// Set up allocatorArgs for the call to operator delete if it's not
1626	// the reserved global operator.
1627	if (E->getOperatorDelete() &&
1628	!E->getOperatorDelete()->isReservedGlobalPlacementOperator()) {
1629	allocatorArgs.add(rvalue: RValue::get(V: allocSize), type: getContext().getSizeType());
1630	allocatorArgs.add(rvalue: RValue::get(V: allocation.getPointer()), type: arg->getType());
1631	}
1632
1633	} else {
1634	const FunctionProtoType *allocatorType =
1635	allocator->getType()->castAs<FunctionProtoType>();
1636	unsigned ParamsToSkip = 0;
1637
1638	// The allocation size is the first argument.
1639	QualType sizeType = getContext().getSizeType();
1640	allocatorArgs.add(rvalue: RValue::get(V: allocSize), type: sizeType);
1641	++ParamsToSkip;
1642
1643	if (allocSize != allocSizeWithoutCookie) {
1644	CharUnits cookieAlign = getSizeAlign(); // FIXME: Ask the ABI.
1645	allocAlign = std::max(a: allocAlign, b: cookieAlign);
1646	}
1647
1648	// The allocation alignment may be passed as the second argument.
1649	if (E->passAlignment()) {
1650	QualType AlignValT = sizeType;
1651	if (allocatorType->getNumParams() > 1) {
1652	AlignValT = allocatorType->getParamType(i: 1);
1653	assert(getContext().hasSameUnqualifiedType(
1654	AlignValT->castAs<EnumType>()->getDecl()->getIntegerType(),
1655	sizeType) &&
1656	"wrong type for alignment parameter");
1657	++ParamsToSkip;
1658	} else {
1659	// Corner case, passing alignment to 'operator new(size_t, ...)'.
1660	assert(allocator->isVariadic() && "can't pass alignment to allocator");
1661	}
1662	allocatorArgs.add(
1663	rvalue: RValue::get(V: llvm::ConstantInt::get(Ty: SizeTy, V: allocAlign.getQuantity())),
1664	type: AlignValT);
1665	}
1666
1667	// FIXME: Why do we not pass a CalleeDecl here?
1668	EmitCallArgs(Args&: allocatorArgs, Prototype: allocatorType, ArgRange: E->placement_arguments(),
1669	/*AC*/AbstractCallee(), /*ParamsToSkip*/ParamsToSkip);
1670
1671	RValue RV =
1672	EmitNewDeleteCall(CGF&: *this, CalleeDecl: allocator, CalleeType: allocatorType, Args: allocatorArgs);
1673
1674	// Set !heapallocsite metadata on the call to operator new.
1675	if (getDebugInfo())
1676	if (auto *newCall = dyn_cast<llvm::CallBase>(RV.getScalarVal()))
1677	getDebugInfo()->addHeapAllocSiteMetadata(CallSite: newCall, AllocatedTy: allocType,
1678	Loc: E->getExprLoc());
1679
1680	// If this was a call to a global replaceable allocation function that does
1681	// not take an alignment argument, the allocator is known to produce
1682	// storage that's suitably aligned for any object that fits, up to a known
1683	// threshold. Otherwise assume it's suitably aligned for the allocated type.
1684	CharUnits allocationAlign = allocAlign;
1685	if (!E->passAlignment() &&
1686	allocator->isReplaceableGlobalAllocationFunction()) {
1687	unsigned AllocatorAlign = llvm::bit_floor(Value: std::min<uint64_t>(
1688	a: Target.getNewAlign(), b: getContext().getTypeSize(T: allocType)));
1689	allocationAlign = std::max(
1690	a: allocationAlign, b: getContext().toCharUnitsFromBits(BitSize: AllocatorAlign));
1691	}
1692
1693	allocation = Address(RV.getScalarVal(), Int8Ty, allocationAlign);
1694	}
1695
1696	// Emit a null check on the allocation result if the allocation
1697	// function is allowed to return null (because it has a non-throwing
1698	// exception spec or is the reserved placement new) and we have an
1699	// interesting initializer will be running sanitizers on the initialization.
1700	bool nullCheck = E->shouldNullCheckAllocation() &&
1701	(!allocType.isPODType(Context: getContext()) || E->hasInitializer() ||
1702	sanitizePerformTypeCheck());
1703
1704	llvm::BasicBlock *nullCheckBB = nullptr;
1705	llvm::BasicBlock *contBB = nullptr;
1706
1707	// The null-check means that the initializer is conditionally
1708	// evaluated.
1709	ConditionalEvaluation conditional(*this);
1710
1711	if (nullCheck) {
1712	conditional.begin(CGF&: *this);
1713
1714	nullCheckBB = Builder.GetInsertBlock();
1715	llvm::BasicBlock *notNullBB = createBasicBlock(name: "new.notnull");
1716	contBB = createBasicBlock(name: "new.cont");
1717
1718	llvm::Value *isNull =
1719	Builder.CreateIsNull(Arg: allocation.getPointer(), Name: "new.isnull");
1720	Builder.CreateCondBr(Cond: isNull, True: contBB, False: notNullBB);
1721	EmitBlock(BB: notNullBB);
1722	}
1723
1724	// If there's an operator delete, enter a cleanup to call it if an
1725	// exception is thrown.
1726	EHScopeStack::stable_iterator operatorDeleteCleanup;
1727	llvm::Instruction *cleanupDominator = nullptr;
1728	if (E->getOperatorDelete() &&
1729	!E->getOperatorDelete()->isReservedGlobalPlacementOperator()) {
1730	EnterNewDeleteCleanup(CGF&: *this, E, NewPtr: allocation, AllocSize: allocSize, AllocAlign: allocAlign,
1731	NewArgs: allocatorArgs);
1732	operatorDeleteCleanup = EHStack.stable_begin();
1733	cleanupDominator = Builder.CreateUnreachable();
1734	}
1735
1736	assert((allocSize == allocSizeWithoutCookie) ==
1737	CalculateCookiePadding(*this, E).isZero());
1738	if (allocSize != allocSizeWithoutCookie) {
1739	assert(E->isArray());
1740	allocation = CGM.getCXXABI().InitializeArrayCookie(CGF&: *this, NewPtr: allocation,
1741	NumElements: numElements,
1742	expr: E, ElementType: allocType);
1743	}
1744
1745	llvm::Type *elementTy = ConvertTypeForMem(T: allocType);
1746	Address result = allocation.withElementType(ElemTy: elementTy);
1747
1748	// Passing pointer through launder.invariant.group to avoid propagation of
1749	// vptrs information which may be included in previous type.
1750	// To not break LTO with different optimizations levels, we do it regardless
1751	// of optimization level.
1752	if (CGM.getCodeGenOpts().StrictVTablePointers &&
1753	allocator->isReservedGlobalPlacementOperator())
1754	result = Builder.CreateLaunderInvariantGroup(Addr: result);
1755
1756	// Emit sanitizer checks for pointer value now, so that in the case of an
1757	// array it was checked only once and not at each constructor call. We may
1758	// have already checked that the pointer is non-null.
1759	// FIXME: If we have an array cookie and a potentially-throwing allocator,
1760	// we'll null check the wrong pointer here.
1761	SanitizerSet SkippedChecks;
1762	SkippedChecks.set(K: SanitizerKind::Null, Value: nullCheck);
1763	EmitTypeCheck(TCK: CodeGenFunction::TCK_ConstructorCall,
1764	Loc: E->getAllocatedTypeSourceInfo()->getTypeLoc().getBeginLoc(),
1765	V: result.getPointer(), Type: allocType, Alignment: result.getAlignment(),
1766	SkippedChecks, ArraySize: numElements);
1767
1768	EmitNewInitializer(CGF&: *this, E, ElementType: allocType, ElementTy: elementTy, NewPtr: result, NumElements: numElements,
1769	AllocSizeWithoutCookie: allocSizeWithoutCookie);
1770	llvm::Value *resultPtr = result.getPointer();
1771	if (E->isArray()) {
1772	// NewPtr is a pointer to the base element type. If we're
1773	// allocating an array of arrays, we'll need to cast back to the
1774	// array pointer type.
1775	llvm::Type *resultType = ConvertTypeForMem(T: E->getType());
1776	if (resultPtr->getType() != resultType)
1777	resultPtr = Builder.CreateBitCast(V: resultPtr, DestTy: resultType);
1778	}
1779
1780	// Deactivate the 'operator delete' cleanup if we finished
1781	// initialization.
1782	if (operatorDeleteCleanup.isValid()) {
1783	DeactivateCleanupBlock(Cleanup: operatorDeleteCleanup, DominatingIP: cleanupDominator);
1784	cleanupDominator->eraseFromParent();
1785	}
1786
1787	if (nullCheck) {
1788	conditional.end(CGF&: *this);
1789
1790	llvm::BasicBlock *notNullBB = Builder.GetInsertBlock();
1791	EmitBlock(BB: contBB);
1792
1793	llvm::PHINode *PHI = Builder.CreatePHI(Ty: resultPtr->getType(), NumReservedValues: 2);
1794	PHI->addIncoming(V: resultPtr, BB: notNullBB);
1795	PHI->addIncoming(V: llvm::Constant::getNullValue(Ty: resultPtr->getType()),
1796	BB: nullCheckBB);
1797
1798	resultPtr = PHI;
1799	}
1800
1801	return resultPtr;
1802	}
1803
1804	void CodeGenFunction::EmitDeleteCall(const FunctionDecl *DeleteFD,
1805	llvm::Value *Ptr, QualType DeleteTy,
1806	llvm::Value *NumElements,
1807	CharUnits CookieSize) {
1808	assert((!NumElements && CookieSize.isZero()) ||
1809	DeleteFD->getOverloadedOperator() == OO_Array_Delete);
1810
1811	const auto *DeleteFTy = DeleteFD->getType()->castAs<FunctionProtoType>();
1812	CallArgList DeleteArgs;
1813
1814	auto Params = getUsualDeleteParams(FD: DeleteFD);
1815	auto ParamTypeIt = DeleteFTy->param_type_begin();
1816
1817	// Pass the pointer itself.
1818	QualType ArgTy = *ParamTypeIt++;
1819	llvm::Value *DeletePtr = Builder.CreateBitCast(V: Ptr, DestTy: ConvertType(T: ArgTy));
1820	DeleteArgs.add(rvalue: RValue::get(V: DeletePtr), type: ArgTy);
1821
1822	// Pass the std::destroying_delete tag if present.
1823	llvm::AllocaInst *DestroyingDeleteTag = nullptr;
1824	if (Params.DestroyingDelete) {
1825	QualType DDTag = *ParamTypeIt++;
1826	llvm::Type *Ty = getTypes().ConvertType(T: DDTag);
1827	CharUnits Align = CGM.getNaturalTypeAlignment(T: DDTag);
1828	DestroyingDeleteTag = CreateTempAlloca(Ty, Name: "destroying.delete.tag");
1829	DestroyingDeleteTag->setAlignment(Align.getAsAlign());
1830	DeleteArgs.add(
1831	rvalue: RValue::getAggregate(addr: Address(DestroyingDeleteTag, Ty, Align)), type: DDTag);
1832	}
1833
1834	// Pass the size if the delete function has a size_t parameter.
1835	if (Params.Size) {
1836	QualType SizeType = *ParamTypeIt++;
1837	CharUnits DeleteTypeSize = getContext().getTypeSizeInChars(T: DeleteTy);
1838	llvm::Value *Size = llvm::ConstantInt::get(Ty: ConvertType(T: SizeType),
1839	V: DeleteTypeSize.getQuantity());
1840
1841	// For array new, multiply by the number of elements.
1842	if (NumElements)
1843	Size = Builder.CreateMul(LHS: Size, RHS: NumElements);
1844
1845	// If there is a cookie, add the cookie size.
1846	if (!CookieSize.isZero())
1847	Size = Builder.CreateAdd(
1848	LHS: Size, RHS: llvm::ConstantInt::get(Ty: SizeTy, V: CookieSize.getQuantity()));
1849
1850	DeleteArgs.add(rvalue: RValue::get(V: Size), type: SizeType);
1851	}
1852
1853	// Pass the alignment if the delete function has an align_val_t parameter.
1854	if (Params.Alignment) {
1855	QualType AlignValType = *ParamTypeIt++;
1856	CharUnits DeleteTypeAlign =
1857	getContext().toCharUnitsFromBits(BitSize: getContext().getTypeAlignIfKnown(
1858	T: DeleteTy, NeedsPreferredAlignment: true /* NeedsPreferredAlignment */));
1859	llvm::Value *Align = llvm::ConstantInt::get(Ty: ConvertType(T: AlignValType),
1860	V: DeleteTypeAlign.getQuantity());
1861	DeleteArgs.add(rvalue: RValue::get(V: Align), type: AlignValType);
1862	}
1863
1864	assert(ParamTypeIt == DeleteFTy->param_type_end() &&
1865	"unknown parameter to usual delete function");
1866
1867	// Emit the call to delete.
1868	EmitNewDeleteCall(*this, DeleteFD, DeleteFTy, DeleteArgs);
1869
1870	// If call argument lowering didn't use the destroying_delete_t alloca,
1871	// remove it again.
1872	if (DestroyingDeleteTag && DestroyingDeleteTag->use_empty())
1873	DestroyingDeleteTag->eraseFromParent();
1874	}
1875
1876	namespace {
1877	/// Calls the given 'operator delete' on a single object.
1878	struct CallObjectDelete final : EHScopeStack::Cleanup {
1879	llvm::Value *Ptr;
1880	const FunctionDecl *OperatorDelete;
1881	QualType ElementType;
1882
1883	CallObjectDelete(llvm::Value *Ptr,
1884	const FunctionDecl *OperatorDelete,
1885	QualType ElementType)
1886	: Ptr(Ptr), OperatorDelete(OperatorDelete), ElementType(ElementType) {}
1887
1888	void Emit(CodeGenFunction &CGF, Flags flags) override {
1889	CGF.EmitDeleteCall(OperatorDelete, Ptr, ElementType);
1890	}
1891	};
1892	}
1893
1894	void
1895	CodeGenFunction::pushCallObjectDeleteCleanup(const FunctionDecl *OperatorDelete,
1896	llvm::Value *CompletePtr,
1897	QualType ElementType) {
1898	EHStack.pushCleanup<CallObjectDelete>(Kind: NormalAndEHCleanup, A: CompletePtr,
1899	A: OperatorDelete, A: ElementType);
1900	}
1901
1902	/// Emit the code for deleting a single object with a destroying operator
1903	/// delete. If the element type has a non-virtual destructor, Ptr has already
1904	/// been converted to the type of the parameter of 'operator delete'. Otherwise
1905	/// Ptr points to an object of the static type.
1906	static void EmitDestroyingObjectDelete(CodeGenFunction &CGF,
1907	const CXXDeleteExpr *DE, Address Ptr,
1908	QualType ElementType) {
1909	auto *Dtor = ElementType->getAsCXXRecordDecl()->getDestructor();
1910	if (Dtor && Dtor->isVirtual())
1911	CGF.CGM.getCXXABI().emitVirtualObjectDelete(CGF, DE, Ptr, ElementType,
1912	Dtor);
1913	else
1914	CGF.EmitDeleteCall(DeleteFD: DE->getOperatorDelete(), Ptr: Ptr.getPointer(), DeleteTy: ElementType);
1915	}
1916
1917	/// Emit the code for deleting a single object.
1918	/// \return \c true if we started emitting UnconditionalDeleteBlock, \c false
1919	/// if not.
1920	static bool EmitObjectDelete(CodeGenFunction &CGF,
1921	const CXXDeleteExpr *DE,
1922	Address Ptr,
1923	QualType ElementType,
1924	llvm::BasicBlock *UnconditionalDeleteBlock) {
1925	// C++11 [expr.delete]p3:
1926	// If the static type of the object to be deleted is different from its
1927	// dynamic type, the static type shall be a base class of the dynamic type
1928	// of the object to be deleted and the static type shall have a virtual
1929	// destructor or the behavior is undefined.
1930	CGF.EmitTypeCheck(TCK: CodeGenFunction::TCK_MemberCall,
1931	Loc: DE->getExprLoc(), V: Ptr.getPointer(),
1932	Type: ElementType);
1933
1934	const FunctionDecl *OperatorDelete = DE->getOperatorDelete();
1935	assert(!OperatorDelete->isDestroyingOperatorDelete());
1936
1937	// Find the destructor for the type, if applicable. If the
1938	// destructor is virtual, we'll just emit the vcall and return.
1939	const CXXDestructorDecl *Dtor = nullptr;
1940	if (const RecordType *RT = ElementType->getAs<RecordType>()) {
1941	CXXRecordDecl *RD = cast<CXXRecordDecl>(Val: RT->getDecl());
1942	if (RD->hasDefinition() && !RD->hasTrivialDestructor()) {
1943	Dtor = RD->getDestructor();
1944
1945	if (Dtor->isVirtual()) {
1946	bool UseVirtualCall = true;
1947	const Expr *Base = DE->getArgument();
1948	if (auto *DevirtualizedDtor =
1949	dyn_cast_or_null<const CXXDestructorDecl>(
1950	Dtor->getDevirtualizedMethod(
1951	Base, CGF.CGM.getLangOpts().AppleKext))) {
1952	UseVirtualCall = false;
1953	const CXXRecordDecl *DevirtualizedClass =
1954	DevirtualizedDtor->getParent();
1955	if (declaresSameEntity(getCXXRecord(E: Base), DevirtualizedClass)) {
1956	// Devirtualized to the class of the base type (the type of the
1957	// whole expression).
1958	Dtor = DevirtualizedDtor;
1959	} else {
1960	// Devirtualized to some other type. Would need to cast the this
1961	// pointer to that type but we don't have support for that yet, so
1962	// do a virtual call. FIXME: handle the case where it is
1963	// devirtualized to the derived type (the type of the inner
1964	// expression) as in EmitCXXMemberOrOperatorMemberCallExpr.
1965	UseVirtualCall = true;
1966	}
1967	}
1968	if (UseVirtualCall) {
1969	CGF.CGM.getCXXABI().emitVirtualObjectDelete(CGF, DE, Ptr, ElementType,
1970	Dtor);
1971	return false;
1972	}
1973	}
1974	}
1975	}
1976
1977	// Make sure that we call delete even if the dtor throws.
1978	// This doesn't have to a conditional cleanup because we're going
1979	// to pop it off in a second.
1980	CGF.EHStack.pushCleanup<CallObjectDelete>(Kind: NormalAndEHCleanup,
1981	A: Ptr.getPointer(),
1982	A: OperatorDelete, A: ElementType);
1983
1984	if (Dtor)
1985	CGF.EmitCXXDestructorCall(D: Dtor, Type: Dtor_Complete,
1986	/*ForVirtualBase=*/false,
1987	/*Delegating=*/false,
1988	This: Ptr, ThisTy: ElementType);
1989	else if (auto Lifetime = ElementType.getObjCLifetime()) {
1990	switch (Lifetime) {
1991	case Qualifiers::OCL_None:
1992	case Qualifiers::OCL_ExplicitNone:
1993	case Qualifiers::OCL_Autoreleasing:
1994	break;
1995
1996	case Qualifiers::OCL_Strong:
1997	CGF.EmitARCDestroyStrong(addr: Ptr, precise: ARCPreciseLifetime);
1998	break;
1999
2000	case Qualifiers::OCL_Weak:
2001	CGF.EmitARCDestroyWeak(addr: Ptr);
2002	break;
2003	}
2004	}
2005
2006	// When optimizing for size, call 'operator delete' unconditionally.
2007	if (CGF.CGM.getCodeGenOpts().OptimizeSize > 1) {
2008	CGF.EmitBlock(BB: UnconditionalDeleteBlock);
2009	CGF.PopCleanupBlock();
2010	return true;
2011	}
2012
2013	CGF.PopCleanupBlock();
2014	return false;
2015	}
2016
2017	namespace {
2018	/// Calls the given 'operator delete' on an array of objects.
2019	struct CallArrayDelete final : EHScopeStack::Cleanup {
2020	llvm::Value *Ptr;
2021	const FunctionDecl *OperatorDelete;
2022	llvm::Value *NumElements;
2023	QualType ElementType;
2024	CharUnits CookieSize;
2025
2026	CallArrayDelete(llvm::Value *Ptr,
2027	const FunctionDecl *OperatorDelete,
2028	llvm::Value *NumElements,
2029	QualType ElementType,
2030	CharUnits CookieSize)
2031	: Ptr(Ptr), OperatorDelete(OperatorDelete), NumElements(NumElements),
2032	ElementType(ElementType), CookieSize(CookieSize) {}
2033
2034	void Emit(CodeGenFunction &CGF, Flags flags) override {
2035	CGF.EmitDeleteCall(OperatorDelete, Ptr, ElementType, NumElements,
2036	CookieSize);
2037	}
2038	};
2039	}
2040
2041	/// Emit the code for deleting an array of objects.
2042	static void EmitArrayDelete(CodeGenFunction &CGF,
2043	const CXXDeleteExpr *E,
2044	Address deletedPtr,
2045	QualType elementType) {
2046	llvm::Value *numElements = nullptr;
2047	llvm::Value *allocatedPtr = nullptr;
2048	CharUnits cookieSize;
2049	CGF.CGM.getCXXABI().ReadArrayCookie(CGF, Ptr: deletedPtr, expr: E, ElementType: elementType,
2050	NumElements&: numElements, AllocPtr&: allocatedPtr, CookieSize&: cookieSize);
2051
2052	assert(allocatedPtr && "ReadArrayCookie didn't set allocated pointer");
2053
2054	// Make sure that we call delete even if one of the dtors throws.
2055	const FunctionDecl *operatorDelete = E->getOperatorDelete();
2056	CGF.EHStack.pushCleanup<CallArrayDelete>(Kind: NormalAndEHCleanup,
2057	A: allocatedPtr, A: operatorDelete,
2058	A: numElements, A: elementType,
2059	A: cookieSize);
2060
2061	// Destroy the elements.
2062	if (QualType::DestructionKind dtorKind = elementType.isDestructedType()) {
2063	assert(numElements && "no element count for a type with a destructor!");
2064
2065	CharUnits elementSize = CGF.getContext().getTypeSizeInChars(T: elementType);
2066	CharUnits elementAlign =
2067	deletedPtr.getAlignment().alignmentOfArrayElement(elementSize);
2068
2069	llvm::Value *arrayBegin = deletedPtr.getPointer();
2070	llvm::Value *arrayEnd = CGF.Builder.CreateInBoundsGEP(
2071	Ty: deletedPtr.getElementType(), Ptr: arrayBegin, IdxList: numElements, Name: "delete.end");
2072
2073	// Note that it is legal to allocate a zero-length array, and we
2074	// can never fold the check away because the length should always
2075	// come from a cookie.
2076	CGF.emitArrayDestroy(begin: arrayBegin, end: arrayEnd, elementType, elementAlign,
2077	destroyer: CGF.getDestroyer(destructionKind: dtorKind),
2078	/*checkZeroLength*/ true,
2079	useEHCleanup: CGF.needsEHCleanup(kind: dtorKind));
2080	}
2081
2082	// Pop the cleanup block.
2083	CGF.PopCleanupBlock();
2084	}
2085
2086	void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) {
2087	const Expr *Arg = E->getArgument();
2088	Address Ptr = EmitPointerWithAlignment(Addr: Arg);
2089
2090	// Null check the pointer.
2091	//
2092	// We could avoid this null check if we can determine that the object
2093	// destruction is trivial and doesn't require an array cookie; we can
2094	// unconditionally perform the operator delete call in that case. For now, we
2095	// assume that deleted pointers are null rarely enough that it's better to
2096	// keep the branch. This might be worth revisiting for a -O0 code size win.
2097	llvm::BasicBlock *DeleteNotNull = createBasicBlock(name: "delete.notnull");
2098	llvm::BasicBlock *DeleteEnd = createBasicBlock(name: "delete.end");
2099
2100	llvm::Value *IsNull = Builder.CreateIsNull(Arg: Ptr.getPointer(), Name: "isnull");
2101
2102	Builder.CreateCondBr(Cond: IsNull, True: DeleteEnd, False: DeleteNotNull);
2103	EmitBlock(BB: DeleteNotNull);
2104	Ptr.setKnownNonNull();
2105
2106	QualType DeleteTy = E->getDestroyedType();
2107
2108	// A destroying operator delete overrides the entire operation of the
2109	// delete expression.
2110	if (E->getOperatorDelete()->isDestroyingOperatorDelete()) {
2111	EmitDestroyingObjectDelete(CGF&: *this, DE: E, Ptr, ElementType: DeleteTy);
2112	EmitBlock(BB: DeleteEnd);
2113	return;
2114	}
2115
2116	// We might be deleting a pointer to array. If so, GEP down to the
2117	// first non-array element.
2118	// (this assumes that A(*)[3][7] is converted to [3 x [7 x %A]]*)
2119	if (DeleteTy->isConstantArrayType()) {
2120	llvm::Value *Zero = Builder.getInt32(C: 0);
2121	SmallVector<llvm::Value*,8> GEP;
2122
2123	GEP.push_back(Elt: Zero); // point at the outermost array
2124
2125	// For each layer of array type we're pointing at:
2126	while (const ConstantArrayType *Arr
2127	= getContext().getAsConstantArrayType(T: DeleteTy)) {
2128	// 1. Unpeel the array type.
2129	DeleteTy = Arr->getElementType();
2130
2131	// 2. GEP to the first element of the array.
2132	GEP.push_back(Elt: Zero);
2133	}
2134
2135	Ptr = Address(Builder.CreateInBoundsGEP(Ty: Ptr.getElementType(),
2136	Ptr: Ptr.getPointer(), IdxList: GEP, Name: "del.first"),
2137	ConvertTypeForMem(T: DeleteTy), Ptr.getAlignment(),
2138	Ptr.isKnownNonNull());
2139	}
2140
2141	assert(ConvertTypeForMem(DeleteTy) == Ptr.getElementType());
2142
2143	if (E->isArrayForm()) {
2144	EmitArrayDelete(CGF&: *this, E, deletedPtr: Ptr, elementType: DeleteTy);
2145	EmitBlock(BB: DeleteEnd);
2146	} else {
2147	if (!EmitObjectDelete(CGF&: *this, DE: E, Ptr, ElementType: DeleteTy, UnconditionalDeleteBlock: DeleteEnd))
2148	EmitBlock(BB: DeleteEnd);
2149	}
2150	}
2151
2152	static bool isGLValueFromPointerDeref(const Expr *E) {
2153	E = E->IgnoreParens();
2154
2155	if (const auto *CE = dyn_cast<CastExpr>(Val: E)) {
2156	if (!CE->getSubExpr()->isGLValue())
2157	return false;
2158	return isGLValueFromPointerDeref(E: CE->getSubExpr());
2159	}
2160
2161	if (const auto *OVE = dyn_cast<OpaqueValueExpr>(Val: E))
2162	return isGLValueFromPointerDeref(E: OVE->getSourceExpr());
2163
2164	if (const auto *BO = dyn_cast<BinaryOperator>(Val: E))
2165	if (BO->getOpcode() == BO_Comma)
2166	return isGLValueFromPointerDeref(E: BO->getRHS());
2167
2168	if (const auto *ACO = dyn_cast<AbstractConditionalOperator>(Val: E))
2169	return isGLValueFromPointerDeref(E: ACO->getTrueExpr()) ||
2170	isGLValueFromPointerDeref(E: ACO->getFalseExpr());
2171
2172	// C++11 [expr.sub]p1:
2173	// The expression E1[E2] is identical (by definition) to *((E1)+(E2))
2174	if (isa<ArraySubscriptExpr>(Val: E))
2175	return true;
2176
2177	if (const auto *UO = dyn_cast<UnaryOperator>(Val: E))
2178	if (UO->getOpcode() == UO_Deref)
2179	return true;
2180
2181	return false;
2182	}
2183
2184	static llvm::Value *EmitTypeidFromVTable(CodeGenFunction &CGF, const Expr *E,
2185	llvm::Type *StdTypeInfoPtrTy) {
2186	// Get the vtable pointer.
2187	Address ThisPtr = CGF.EmitLValue(E).getAddress(CGF);
2188
2189	QualType SrcRecordTy = E->getType();
2190
2191	// C++ [class.cdtor]p4:
2192	// If the operand of typeid refers to the object under construction or
2193	// destruction and the static type of the operand is neither the constructor
2194	// or destructor’s class nor one of its bases, the behavior is undefined.
2195	CGF.EmitTypeCheck(TCK: CodeGenFunction::TCK_DynamicOperation, Loc: E->getExprLoc(),
2196	V: ThisPtr.getPointer(), Type: SrcRecordTy);
2197
2198	// C++ [expr.typeid]p2:
2199	// If the glvalue expression is obtained by applying the unary * operator to
2200	// a pointer and the pointer is a null pointer value, the typeid expression
2201	// throws the std::bad_typeid exception.
2202	//
2203	// However, this paragraph's intent is not clear. We choose a very generous
2204	// interpretation which implores us to consider comma operators, conditional
2205	// operators, parentheses and other such constructs.
2206	if (CGF.CGM.getCXXABI().shouldTypeidBeNullChecked(
2207	IsDeref: isGLValueFromPointerDeref(E), SrcRecordTy)) {
2208	llvm::BasicBlock *BadTypeidBlock =
2209	CGF.createBasicBlock(name: "typeid.bad_typeid");
2210	llvm::BasicBlock *EndBlock = CGF.createBasicBlock(name: "typeid.end");
2211
2212	llvm::Value *IsNull = CGF.Builder.CreateIsNull(Arg: ThisPtr.getPointer());
2213	CGF.Builder.CreateCondBr(Cond: IsNull, True: BadTypeidBlock, False: EndBlock);
2214
2215	CGF.EmitBlock(BB: BadTypeidBlock);
2216	CGF.CGM.getCXXABI().EmitBadTypeidCall(CGF);
2217	CGF.EmitBlock(BB: EndBlock);
2218	}
2219
2220	return CGF.CGM.getCXXABI().EmitTypeid(CGF, SrcRecordTy, ThisPtr,
2221	StdTypeInfoPtrTy);
2222	}
2223
2224	llvm::Value *CodeGenFunction::EmitCXXTypeidExpr(const CXXTypeidExpr *E) {
2225	llvm::Type *PtrTy = llvm::PointerType::getUnqual(C&: getLLVMContext());
2226	LangAS GlobAS = CGM.GetGlobalVarAddressSpace(D: nullptr);
2227
2228	auto MaybeASCast = [=](auto &&TypeInfo) {
2229	if (GlobAS == LangAS::Default)
2230	return TypeInfo;
2231	return getTargetHooks().performAddrSpaceCast(CGM,TypeInfo, GlobAS,
2232	LangAS::Default, PtrTy);
2233	};
2234
2235	if (E->isTypeOperand()) {
2236	llvm::Constant *TypeInfo =
2237	CGM.GetAddrOfRTTIDescriptor(Ty: E->getTypeOperand(Context&: getContext()));
2238	return MaybeASCast(TypeInfo);
2239	}
2240
2241	// C++ [expr.typeid]p2:
2242	// When typeid is applied to a glvalue expression whose type is a
2243	// polymorphic class type, the result refers to a std::type_info object
2244	// representing the type of the most derived object (that is, the dynamic
2245	// type) to which the glvalue refers.
2246	// If the operand is already most derived object, no need to look up vtable.
2247	if (E->isPotentiallyEvaluated() && !E->isMostDerived(Context&: getContext()))
2248	return EmitTypeidFromVTable(CGF&: *this, E: E->getExprOperand(), StdTypeInfoPtrTy: PtrTy);
2249
2250	QualType OperandTy = E->getExprOperand()->getType();
2251	return MaybeASCast(CGM.GetAddrOfRTTIDescriptor(Ty: OperandTy));
2252	}
2253
2254	static llvm::Value *EmitDynamicCastToNull(CodeGenFunction &CGF,
2255	QualType DestTy) {
2256	llvm::Type *DestLTy = CGF.ConvertType(T: DestTy);
2257	if (DestTy->isPointerType())
2258	return llvm::Constant::getNullValue(Ty: DestLTy);
2259
2260	/// C++ [expr.dynamic.cast]p9:
2261	/// A failed cast to reference type throws std::bad_cast
2262	if (!CGF.CGM.getCXXABI().EmitBadCastCall(CGF))
2263	return nullptr;
2264
2265	CGF.Builder.ClearInsertionPoint();
2266	return llvm::PoisonValue::get(T: DestLTy);
2267	}
2268
2269	llvm::Value *CodeGenFunction::EmitDynamicCast(Address ThisAddr,
2270	const CXXDynamicCastExpr *DCE) {
2271	CGM.EmitExplicitCastExprType(DCE, this);
2272	QualType DestTy = DCE->getTypeAsWritten();
2273
2274	QualType SrcTy = DCE->getSubExpr()->getType();
2275
2276	// C++ [expr.dynamic.cast]p7:
2277	// If T is "pointer to cv void," then the result is a pointer to the most
2278	// derived object pointed to by v.
2279	bool IsDynamicCastToVoid = DestTy->isVoidPointerType();
2280	QualType SrcRecordTy;
2281	QualType DestRecordTy;
2282	if (IsDynamicCastToVoid) {
2283	SrcRecordTy = SrcTy->getPointeeType();
2284	// No DestRecordTy.
2285	} else if (const PointerType *DestPTy = DestTy->getAs<PointerType>()) {
2286	SrcRecordTy = SrcTy->castAs<PointerType>()->getPointeeType();
2287	DestRecordTy = DestPTy->getPointeeType();
2288	} else {
2289	SrcRecordTy = SrcTy;
2290	DestRecordTy = DestTy->castAs<ReferenceType>()->getPointeeType();
2291	}
2292
2293	// C++ [class.cdtor]p5:
2294	// If the operand of the dynamic_cast refers to the object under
2295	// construction or destruction and the static type of the operand is not a
2296	// pointer to or object of the constructor or destructor’s own class or one
2297	// of its bases, the dynamic_cast results in undefined behavior.
2298	EmitTypeCheck(TCK: TCK_DynamicOperation, Loc: DCE->getExprLoc(), V: ThisAddr.getPointer(),
2299	Type: SrcRecordTy);
2300
2301	if (DCE->isAlwaysNull()) {
2302	if (llvm::Value *T = EmitDynamicCastToNull(CGF&: *this, DestTy)) {
2303	// Expression emission is expected to retain a valid insertion point.
2304	if (!Builder.GetInsertBlock())
2305	EmitBlock(BB: createBasicBlock(name: "dynamic_cast.unreachable"));
2306	return T;
2307	}
2308	}
2309
2310	assert(SrcRecordTy->isRecordType() && "source type must be a record type!");
2311
2312	// If the destination is effectively final, the cast succeeds if and only
2313	// if the dynamic type of the pointer is exactly the destination type.
2314	bool IsExact = !IsDynamicCastToVoid &&
2315	CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2316	DestRecordTy->getAsCXXRecordDecl()->isEffectivelyFinal() &&
2317	CGM.getCXXABI().shouldEmitExactDynamicCast(DestRecordTy);
2318
2319	// C++ [expr.dynamic.cast]p4:
2320	// If the value of v is a null pointer value in the pointer case, the result
2321	// is the null pointer value of type T.
2322	bool ShouldNullCheckSrcValue =
2323	IsExact || CGM.getCXXABI().shouldDynamicCastCallBeNullChecked(
2324	SrcIsPtr: SrcTy->isPointerType(), SrcRecordTy);
2325
2326	llvm::BasicBlock *CastNull = nullptr;
2327	llvm::BasicBlock *CastNotNull = nullptr;
2328	llvm::BasicBlock *CastEnd = createBasicBlock(name: "dynamic_cast.end");
2329
2330	if (ShouldNullCheckSrcValue) {
2331	CastNull = createBasicBlock(name: "dynamic_cast.null");
2332	CastNotNull = createBasicBlock(name: "dynamic_cast.notnull");
2333
2334	llvm::Value *IsNull = Builder.CreateIsNull(Arg: ThisAddr.getPointer());
2335	Builder.CreateCondBr(Cond: IsNull, True: CastNull, False: CastNotNull);
2336	EmitBlock(BB: CastNotNull);
2337	}
2338
2339	llvm::Value *Value;
2340	if (IsDynamicCastToVoid) {
2341	Value = CGM.getCXXABI().emitDynamicCastToVoid(CGF&: *this, Value: ThisAddr, SrcRecordTy);
2342	} else if (IsExact) {
2343	// If the destination type is effectively final, this pointer points to the
2344	// right type if and only if its vptr has the right value.
2345	Value = CGM.getCXXABI().emitExactDynamicCast(
2346	CGF&: *this, Value: ThisAddr, SrcRecordTy, DestTy, DestRecordTy, CastSuccess: CastEnd, CastFail: CastNull);
2347	} else {
2348	assert(DestRecordTy->isRecordType() &&
2349	"destination type must be a record type!");
2350	Value = CGM.getCXXABI().emitDynamicCastCall(CGF&: *this, Value: ThisAddr, SrcRecordTy,
2351	DestTy, DestRecordTy, CastEnd);
2352	}
2353	CastNotNull = Builder.GetInsertBlock();
2354
2355	llvm::Value *NullValue = nullptr;
2356	if (ShouldNullCheckSrcValue) {
2357	EmitBranch(Block: CastEnd);
2358
2359	EmitBlock(BB: CastNull);
2360	NullValue = EmitDynamicCastToNull(CGF&: *this, DestTy);
2361	CastNull = Builder.GetInsertBlock();
2362
2363	EmitBranch(Block: CastEnd);
2364	}
2365
2366	EmitBlock(BB: CastEnd);
2367
2368	if (CastNull) {
2369	llvm::PHINode *PHI = Builder.CreatePHI(Ty: Value->getType(), NumReservedValues: 2);
2370	PHI->addIncoming(V: Value, BB: CastNotNull);
2371	PHI->addIncoming(V: NullValue, BB: CastNull);
2372
2373	Value = PHI;
2374	}
2375
2376	return Value;
2377	}
2378