1	//===--- MicrosoftCXXABI.cpp - Emit LLVM Code from ASTs for a Module ------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This provides C++ code generation targeting the Microsoft Visual C++ ABI.
10	// The class in this file generates structures that follow the Microsoft
11	// Visual C++ ABI, which is actually not very well documented at all outside
12	// of Microsoft.
13	//
14	//===----------------------------------------------------------------------===//
15
16	#include "ABIInfo.h"
17	#include "CGCXXABI.h"
18	#include "CGCleanup.h"
19	#include "CGVTables.h"
20	#include "CodeGenModule.h"
21	#include "CodeGenTypes.h"
22	#include "TargetInfo.h"
23	#include "clang/AST/Attr.h"
24	#include "clang/AST/CXXInheritance.h"
25	#include "clang/AST/Decl.h"
26	#include "clang/AST/DeclCXX.h"
27	#include "clang/AST/StmtCXX.h"
28	#include "clang/AST/VTableBuilder.h"
29	#include "clang/CodeGen/ConstantInitBuilder.h"
30	#include "llvm/ADT/StringExtras.h"
31	#include "llvm/ADT/StringSet.h"
32	#include "llvm/IR/Intrinsics.h"
33
34	using namespace clang;
35	using namespace CodeGen;
36
37	namespace {
38
39	/// Holds all the vbtable globals for a given class.
40	struct VBTableGlobals {
41	const VPtrInfoVector *VBTables;
42	SmallVector<llvm::GlobalVariable *, 2> Globals;
43	};
44
45	class MicrosoftCXXABI : public CGCXXABI {
46	public:
47	MicrosoftCXXABI(CodeGenModule &CGM)
48	: CGCXXABI(CGM), BaseClassDescriptorType(nullptr),
49	ClassHierarchyDescriptorType(nullptr),
50	CompleteObjectLocatorType(nullptr), CatchableTypeType(nullptr),
51	ThrowInfoType(nullptr) {
52	assert(!(CGM.getLangOpts().isExplicitDefaultVisibilityExportMapping() ||
53	CGM.getLangOpts().isAllDefaultVisibilityExportMapping()) &&
54	"visibility export mapping option unimplemented in this ABI");
55	}
56
57	bool HasThisReturn(GlobalDecl GD) const override;
58	bool hasMostDerivedReturn(GlobalDecl GD) const override;
59
60	bool classifyReturnType(CGFunctionInfo &FI) const override;
61
62	RecordArgABI getRecordArgABI(const CXXRecordDecl *RD) const override;
63
64	bool isSRetParameterAfterThis() const override { return true; }
65
66	bool isThisCompleteObject(GlobalDecl GD) const override {
67	// The Microsoft ABI doesn't use separate complete-object vs.
68	// base-object variants of constructors, but it does of destructors.
69	if (isa<CXXDestructorDecl>(Val: GD.getDecl())) {
70	switch (GD.getDtorType()) {
71	case Dtor_Complete:
72	case Dtor_Deleting:
73	return true;
74
75	case Dtor_Base:
76	return false;
77
78	case Dtor_Comdat: llvm_unreachable("emitting dtor comdat as function?");
79	}
80	llvm_unreachable("bad dtor kind");
81	}
82
83	// No other kinds.
84	return false;
85	}
86
87	size_t getSrcArgforCopyCtor(const CXXConstructorDecl *CD,
88	FunctionArgList &Args) const override {
89	assert(Args.size() >= 2 &&
90	"expected the arglist to have at least two args!");
91	// The 'most_derived' parameter goes second if the ctor is variadic and
92	// has v-bases.
93	if (CD->getParent()->getNumVBases() > 0 &&
94	CD->getType()->castAs<FunctionProtoType>()->isVariadic())
95	return 2;
96	return 1;
97	}
98
99	std::vector<CharUnits> getVBPtrOffsets(const CXXRecordDecl *RD) override {
100	std::vector<CharUnits> VBPtrOffsets;
101	const ASTContext &Context = getContext();
102	const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
103
104	const VBTableGlobals &VBGlobals = enumerateVBTables(RD);
105	for (const std::unique_ptr<VPtrInfo> &VBT : *VBGlobals.VBTables) {
106	const ASTRecordLayout &SubobjectLayout =
107	Context.getASTRecordLayout(VBT->IntroducingObject);
108	CharUnits Offs = VBT->NonVirtualOffset;
109	Offs += SubobjectLayout.getVBPtrOffset();
110	if (VBT->getVBaseWithVPtr())
111	Offs += Layout.getVBaseClassOffset(VBase: VBT->getVBaseWithVPtr());
112	VBPtrOffsets.push_back(x: Offs);
113	}
114	llvm::array_pod_sort(Start: VBPtrOffsets.begin(), End: VBPtrOffsets.end());
115	return VBPtrOffsets;
116	}
117
118	StringRef GetPureVirtualCallName() override { return "_purecall"; }
119	StringRef GetDeletedVirtualCallName() override { return "_purecall"; }
120
121	void emitVirtualObjectDelete(CodeGenFunction &CGF, const CXXDeleteExpr *DE,
122	Address Ptr, QualType ElementType,
123	const CXXDestructorDecl *Dtor) override;
124
125	void emitRethrow(CodeGenFunction &CGF, bool isNoReturn) override;
126	void emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) override;
127
128	void emitBeginCatch(CodeGenFunction &CGF, const CXXCatchStmt *C) override;
129
130	llvm::GlobalVariable *getMSCompleteObjectLocator(const CXXRecordDecl *RD,
131	const VPtrInfo &Info);
132
133	llvm::Constant *getAddrOfRTTIDescriptor(QualType Ty) override;
134	CatchTypeInfo
135	getAddrOfCXXCatchHandlerType(QualType Ty, QualType CatchHandlerType) override;
136
137	/// MSVC needs an extra flag to indicate a catchall.
138	CatchTypeInfo getCatchAllTypeInfo() override {
139	// For -EHa catch(...) must handle HW exception
140	// Adjective = HT_IsStdDotDot (0x40), only catch C++ exceptions
141	if (getContext().getLangOpts().EHAsynch)
142	return CatchTypeInfo{.RTTI: nullptr, .Flags: 0};
143	else
144	return CatchTypeInfo{.RTTI: nullptr, .Flags: 0x40};
145	}
146
147	bool shouldTypeidBeNullChecked(bool IsDeref, QualType SrcRecordTy) override;
148	void EmitBadTypeidCall(CodeGenFunction &CGF) override;
149	llvm::Value *EmitTypeid(CodeGenFunction &CGF, QualType SrcRecordTy,
150	Address ThisPtr,
151	llvm::Type *StdTypeInfoPtrTy) override;
152
153	bool shouldDynamicCastCallBeNullChecked(bool SrcIsPtr,
154	QualType SrcRecordTy) override;
155
156	bool shouldEmitExactDynamicCast(QualType DestRecordTy) override {
157	// TODO: Add support for exact dynamic_casts.
158	return false;
159	}
160	llvm::Value *emitExactDynamicCast(CodeGenFunction &CGF, Address Value,
161	QualType SrcRecordTy, QualType DestTy,
162	QualType DestRecordTy,
163	llvm::BasicBlock *CastSuccess,
164	llvm::BasicBlock *CastFail) override {
165	llvm_unreachable("unsupported");
166	}
167
168	llvm::Value *emitDynamicCastCall(CodeGenFunction &CGF, Address Value,
169	QualType SrcRecordTy, QualType DestTy,
170	QualType DestRecordTy,
171	llvm::BasicBlock *CastEnd) override;
172
173	llvm::Value *emitDynamicCastToVoid(CodeGenFunction &CGF, Address Value,
174	QualType SrcRecordTy) override;
175
176	bool EmitBadCastCall(CodeGenFunction &CGF) override;
177	bool canSpeculativelyEmitVTable(const CXXRecordDecl *RD) const override {
178	return false;
179	}
180
181	llvm::Value *
182	GetVirtualBaseClassOffset(CodeGenFunction &CGF, Address This,
183	const CXXRecordDecl *ClassDecl,
184	const CXXRecordDecl *BaseClassDecl) override;
185
186	llvm::BasicBlock *
187	EmitCtorCompleteObjectHandler(CodeGenFunction &CGF,
188	const CXXRecordDecl *RD) override;
189
190	llvm::BasicBlock *
191	EmitDtorCompleteObjectHandler(CodeGenFunction &CGF);
192
193	void initializeHiddenVirtualInheritanceMembers(CodeGenFunction &CGF,
194	const CXXRecordDecl *RD) override;
195
196	void EmitCXXConstructors(const CXXConstructorDecl *D) override;
197
198	// Background on MSVC destructors
199	// ==============================
200	//
201	// Both Itanium and MSVC ABIs have destructor variants. The variant names
202	// roughly correspond in the following way:
203	// Itanium Microsoft
204	// Base -> no name, just ~Class
205	// Complete -> vbase destructor
206	// Deleting -> scalar deleting destructor
207	// vector deleting destructor
208	//
209	// The base and complete destructors are the same as in Itanium, although the
210	// complete destructor does not accept a VTT parameter when there are virtual
211	// bases. A separate mechanism involving vtordisps is used to ensure that
212	// virtual methods of destroyed subobjects are not called.
213	//
214	// The deleting destructors accept an i32 bitfield as a second parameter. Bit
215	// 1 indicates if the memory should be deleted. Bit 2 indicates if the this
216	// pointer points to an array. The scalar deleting destructor assumes that
217	// bit 2 is zero, and therefore does not contain a loop.
218	//
219	// For virtual destructors, only one entry is reserved in the vftable, and it
220	// always points to the vector deleting destructor. The vector deleting
221	// destructor is the most general, so it can be used to destroy objects in
222	// place, delete single heap objects, or delete arrays.
223	//
224	// A TU defining a non-inline destructor is only guaranteed to emit a base
225	// destructor, and all of the other variants are emitted on an as-needed basis
226	// in COMDATs. Because a non-base destructor can be emitted in a TU that
227	// lacks a definition for the destructor, non-base destructors must always
228	// delegate to or alias the base destructor.
229
230	AddedStructorArgCounts
231	buildStructorSignature(GlobalDecl GD,
232	SmallVectorImpl<CanQualType> &ArgTys) override;
233
234	/// Non-base dtors should be emitted as delegating thunks in this ABI.
235	bool useThunkForDtorVariant(const CXXDestructorDecl *Dtor,
236	CXXDtorType DT) const override {
237	return DT != Dtor_Base;
238	}
239
240	void setCXXDestructorDLLStorage(llvm::GlobalValue *GV,
241	const CXXDestructorDecl *Dtor,
242	CXXDtorType DT) const override;
243
244	llvm::GlobalValue::LinkageTypes
245	getCXXDestructorLinkage(GVALinkage Linkage, const CXXDestructorDecl *Dtor,
246	CXXDtorType DT) const override;
247
248	void EmitCXXDestructors(const CXXDestructorDecl *D) override;
249
250	const CXXRecordDecl *getThisArgumentTypeForMethod(GlobalDecl GD) override {
251	auto *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
252
253	if (MD->isVirtual()) {
254	GlobalDecl LookupGD = GD;
255	if (const auto *DD = dyn_cast<CXXDestructorDecl>(Val: MD)) {
256	// Complete dtors take a pointer to the complete object,
257	// thus don't need adjustment.
258	if (GD.getDtorType() == Dtor_Complete)
259	return MD->getParent();
260
261	// There's only Dtor_Deleting in vftable but it shares the this
262	// adjustment with the base one, so look up the deleting one instead.
263	LookupGD = GlobalDecl(DD, Dtor_Deleting);
264	}
265	MethodVFTableLocation ML =
266	CGM.getMicrosoftVTableContext().getMethodVFTableLocation(GD: LookupGD);
267
268	// The vbases might be ordered differently in the final overrider object
269	// and the complete object, so the "this" argument may sometimes point to
270	// memory that has no particular type (e.g. past the complete object).
271	// In this case, we just use a generic pointer type.
272	// FIXME: might want to have a more precise type in the non-virtual
273	// multiple inheritance case.
274	if (ML.VBase || !ML.VFPtrOffset.isZero())
275	return nullptr;
276	}
277	return MD->getParent();
278	}
279
280	Address
281	adjustThisArgumentForVirtualFunctionCall(CodeGenFunction &CGF, GlobalDecl GD,
282	Address This,
283	bool VirtualCall) override;
284
285	void addImplicitStructorParams(CodeGenFunction &CGF, QualType &ResTy,
286	FunctionArgList &Params) override;
287
288	void EmitInstanceFunctionProlog(CodeGenFunction &CGF) override;
289
290	AddedStructorArgs getImplicitConstructorArgs(CodeGenFunction &CGF,
291	const CXXConstructorDecl *D,
292	CXXCtorType Type,
293	bool ForVirtualBase,
294	bool Delegating) override;
295
296	llvm::Value *getCXXDestructorImplicitParam(CodeGenFunction &CGF,
297	const CXXDestructorDecl *DD,
298	CXXDtorType Type,
299	bool ForVirtualBase,
300	bool Delegating) override;
301
302	void EmitDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *DD,
303	CXXDtorType Type, bool ForVirtualBase,
304	bool Delegating, Address This,
305	QualType ThisTy) override;
306
307	void emitVTableTypeMetadata(const VPtrInfo &Info, const CXXRecordDecl *RD,
308	llvm::GlobalVariable *VTable);
309
310	void emitVTableDefinitions(CodeGenVTables &CGVT,
311	const CXXRecordDecl *RD) override;
312
313	bool isVirtualOffsetNeededForVTableField(CodeGenFunction &CGF,
314	CodeGenFunction::VPtr Vptr) override;
315
316	/// Don't initialize vptrs if dynamic class
317	/// is marked with the 'novtable' attribute.
318	bool doStructorsInitializeVPtrs(const CXXRecordDecl *VTableClass) override {
319	return !VTableClass->hasAttr<MSNoVTableAttr>();
320	}
321
322	llvm::Constant *
323	getVTableAddressPoint(BaseSubobject Base,
324	const CXXRecordDecl *VTableClass) override;
325
326	llvm::Value *getVTableAddressPointInStructor(
327	CodeGenFunction &CGF, const CXXRecordDecl *VTableClass,
328	BaseSubobject Base, const CXXRecordDecl *NearestVBase) override;
329
330	llvm::Constant *
331	getVTableAddressPointForConstExpr(BaseSubobject Base,
332	const CXXRecordDecl *VTableClass) override;
333
334	llvm::GlobalVariable *getAddrOfVTable(const CXXRecordDecl *RD,
335	CharUnits VPtrOffset) override;
336
337	CGCallee getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD,
338	Address This, llvm::Type *Ty,
339	SourceLocation Loc) override;
340
341	llvm::Value *EmitVirtualDestructorCall(CodeGenFunction &CGF,
342	const CXXDestructorDecl *Dtor,
343	CXXDtorType DtorType, Address This,
344	DeleteOrMemberCallExpr E) override;
345
346	void adjustCallArgsForDestructorThunk(CodeGenFunction &CGF, GlobalDecl GD,
347	CallArgList &CallArgs) override {
348	assert(GD.getDtorType() == Dtor_Deleting &&
349	"Only deleting destructor thunks are available in this ABI");
350	CallArgs.add(rvalue: RValue::get(V: getStructorImplicitParamValue(CGF)),
351	type: getContext().IntTy);
352	}
353
354	void emitVirtualInheritanceTables(const CXXRecordDecl *RD) override;
355
356	llvm::GlobalVariable *
357	getAddrOfVBTable(const VPtrInfo &VBT, const CXXRecordDecl *RD,
358	llvm::GlobalVariable::LinkageTypes Linkage);
359
360	llvm::GlobalVariable *
361	getAddrOfVirtualDisplacementMap(const CXXRecordDecl *SrcRD,
362	const CXXRecordDecl *DstRD) {
363	SmallString<256> OutName;
364	llvm::raw_svector_ostream Out(OutName);
365	getMangleContext().mangleCXXVirtualDisplacementMap(SrcRD, DstRD, Out);
366	StringRef MangledName = OutName.str();
367
368	if (auto *VDispMap = CGM.getModule().getNamedGlobal(Name: MangledName))
369	return VDispMap;
370
371	MicrosoftVTableContext &VTContext = CGM.getMicrosoftVTableContext();
372	unsigned NumEntries = 1 + SrcRD->getNumVBases();
373	SmallVector<llvm::Constant *, 4> Map(NumEntries,
374	llvm::UndefValue::get(T: CGM.IntTy));
375	Map[0] = llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0);
376	bool AnyDifferent = false;
377	for (const auto &I : SrcRD->vbases()) {
378	const CXXRecordDecl *VBase = I.getType()->getAsCXXRecordDecl();
379	if (!DstRD->isVirtuallyDerivedFrom(Base: VBase))
380	continue;
381
382	unsigned SrcVBIndex = VTContext.getVBTableIndex(Derived: SrcRD, VBase);
383	unsigned DstVBIndex = VTContext.getVBTableIndex(Derived: DstRD, VBase);
384	Map[SrcVBIndex] = llvm::ConstantInt::get(Ty: CGM.IntTy, V: DstVBIndex * 4);
385	AnyDifferent |= SrcVBIndex != DstVBIndex;
386	}
387	// This map would be useless, don't use it.
388	if (!AnyDifferent)
389	return nullptr;
390
391	llvm::ArrayType *VDispMapTy = llvm::ArrayType::get(ElementType: CGM.IntTy, NumElements: Map.size());
392	llvm::Constant *Init = llvm::ConstantArray::get(T: VDispMapTy, V: Map);
393	llvm::GlobalValue::LinkageTypes Linkage =
394	SrcRD->isExternallyVisible() && DstRD->isExternallyVisible()
395	? llvm::GlobalValue::LinkOnceODRLinkage
396	: llvm::GlobalValue::InternalLinkage;
397	auto *VDispMap = new llvm::GlobalVariable(
398	CGM.getModule(), VDispMapTy, /*isConstant=*/true, Linkage,
399	/*Initializer=*/Init, MangledName);
400	return VDispMap;
401	}
402
403	void emitVBTableDefinition(const VPtrInfo &VBT, const CXXRecordDecl *RD,
404	llvm::GlobalVariable *GV) const;
405
406	void setThunkLinkage(llvm::Function *Thunk, bool ForVTable,
407	GlobalDecl GD, bool ReturnAdjustment) override {
408	GVALinkage Linkage =
409	getContext().GetGVALinkageForFunction(FD: cast<FunctionDecl>(Val: GD.getDecl()));
410
411	if (Linkage == GVA_Internal)
412	Thunk->setLinkage(llvm::GlobalValue::InternalLinkage);
413	else if (ReturnAdjustment)
414	Thunk->setLinkage(llvm::GlobalValue::WeakODRLinkage);
415	else
416	Thunk->setLinkage(llvm::GlobalValue::LinkOnceODRLinkage);
417	}
418
419	bool exportThunk() override { return false; }
420
421	llvm::Value *performThisAdjustment(CodeGenFunction &CGF, Address This,
422	const ThisAdjustment &TA) override;
423
424	llvm::Value *performReturnAdjustment(CodeGenFunction &CGF, Address Ret,
425	const ReturnAdjustment &RA) override;
426
427	void EmitThreadLocalInitFuncs(
428	CodeGenModule &CGM, ArrayRef<const VarDecl *> CXXThreadLocals,
429	ArrayRef<llvm::Function *> CXXThreadLocalInits,
430	ArrayRef<const VarDecl *> CXXThreadLocalInitVars) override;
431
432	bool usesThreadWrapperFunction(const VarDecl *VD) const override {
433	return getContext().getLangOpts().isCompatibleWithMSVC(
434	MajorVersion: LangOptions::MSVC2019_5) &&
435	(!isEmittedWithConstantInitializer(VD) || mayNeedDestruction(VD));
436	}
437	LValue EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, const VarDecl *VD,
438	QualType LValType) override;
439
440	void EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D,
441	llvm::GlobalVariable *DeclPtr,
442	bool PerformInit) override;
443	void registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D,
444	llvm::FunctionCallee Dtor,
445	llvm::Constant *Addr) override;
446
447	// ==== Notes on array cookies =========
448	//
449	// MSVC seems to only use cookies when the class has a destructor; a
450	// two-argument usual array deallocation function isn't sufficient.
451	//
452	// For example, this code prints "100" and "1":
453	// struct A {
454	// char x;
455	// void *operator new[](size_t sz) {
456	// printf("%u\n", sz);
457	// return malloc(sz);
458	// }
459	// void operator delete[](void *p, size_t sz) {
460	// printf("%u\n", sz);
461	// free(p);
462	// }
463	// };
464	// int main() {
465	// A *p = new A[100];
466	// delete[] p;
467	// }
468	// Whereas it prints "104" and "104" if you give A a destructor.
469
470	bool requiresArrayCookie(const CXXDeleteExpr *expr,
471	QualType elementType) override;
472	bool requiresArrayCookie(const CXXNewExpr *expr) override;
473	CharUnits getArrayCookieSizeImpl(QualType type) override;
474	Address InitializeArrayCookie(CodeGenFunction &CGF,
475	Address NewPtr,
476	llvm::Value *NumElements,
477	const CXXNewExpr *expr,
478	QualType ElementType) override;
479	llvm::Value *readArrayCookieImpl(CodeGenFunction &CGF,
480	Address allocPtr,
481	CharUnits cookieSize) override;
482
483	friend struct MSRTTIBuilder;
484
485	bool isImageRelative() const {
486	return CGM.getTarget().getPointerWidth(AddrSpace: LangAS::Default) == 64;
487	}
488
489	// 5 routines for constructing the llvm types for MS RTTI structs.
490	llvm::StructType *getTypeDescriptorType(StringRef TypeInfoString) {
491	llvm::SmallString<32> TDTypeName("rtti.TypeDescriptor");
492	TDTypeName += llvm::utostr(X: TypeInfoString.size());
493	llvm::StructType *&TypeDescriptorType =
494	TypeDescriptorTypeMap[TypeInfoString.size()];
495	if (TypeDescriptorType)
496	return TypeDescriptorType;
497	llvm::Type *FieldTypes[] = {
498	CGM.Int8PtrPtrTy,
499	CGM.Int8PtrTy,
500	llvm::ArrayType::get(ElementType: CGM.Int8Ty, NumElements: TypeInfoString.size() + 1)};
501	TypeDescriptorType =
502	llvm::StructType::create(Context&: CGM.getLLVMContext(), Elements: FieldTypes, Name: TDTypeName);
503	return TypeDescriptorType;
504	}
505
506	llvm::Type *getImageRelativeType(llvm::Type *PtrType) {
507	if (!isImageRelative())
508	return PtrType;
509	return CGM.IntTy;
510	}
511
512	llvm::StructType *getBaseClassDescriptorType() {
513	if (BaseClassDescriptorType)
514	return BaseClassDescriptorType;
515	llvm::Type *FieldTypes[] = {
516	getImageRelativeType(PtrType: CGM.Int8PtrTy),
517	CGM.IntTy,
518	CGM.IntTy,
519	CGM.IntTy,
520	CGM.IntTy,
521	CGM.IntTy,
522	getImageRelativeType(PtrType: getClassHierarchyDescriptorType()->getPointerTo()),
523	};
524	BaseClassDescriptorType = llvm::StructType::create(
525	Context&: CGM.getLLVMContext(), Elements: FieldTypes, Name: "rtti.BaseClassDescriptor");
526	return BaseClassDescriptorType;
527	}
528
529	llvm::StructType *getClassHierarchyDescriptorType() {
530	if (ClassHierarchyDescriptorType)
531	return ClassHierarchyDescriptorType;
532	// Forward-declare RTTIClassHierarchyDescriptor to break a cycle.
533	ClassHierarchyDescriptorType = llvm::StructType::create(
534	Context&: CGM.getLLVMContext(), Name: "rtti.ClassHierarchyDescriptor");
535	llvm::Type *FieldTypes[] = {
536	CGM.IntTy,
537	CGM.IntTy,
538	CGM.IntTy,
539	getImageRelativeType(
540	PtrType: getBaseClassDescriptorType()->getPointerTo()->getPointerTo()),
541	};
542	ClassHierarchyDescriptorType->setBody(Elements: FieldTypes);
543	return ClassHierarchyDescriptorType;
544	}
545
546	llvm::StructType *getCompleteObjectLocatorType() {
547	if (CompleteObjectLocatorType)
548	return CompleteObjectLocatorType;
549	CompleteObjectLocatorType = llvm::StructType::create(
550	Context&: CGM.getLLVMContext(), Name: "rtti.CompleteObjectLocator");
551	llvm::Type *FieldTypes[] = {
552	CGM.IntTy,
553	CGM.IntTy,
554	CGM.IntTy,
555	getImageRelativeType(PtrType: CGM.Int8PtrTy),
556	getImageRelativeType(PtrType: getClassHierarchyDescriptorType()->getPointerTo()),
557	getImageRelativeType(PtrType: CompleteObjectLocatorType),
558	};
559	llvm::ArrayRef<llvm::Type *> FieldTypesRef(FieldTypes);
560	if (!isImageRelative())
561	FieldTypesRef = FieldTypesRef.drop_back();
562	CompleteObjectLocatorType->setBody(Elements: FieldTypesRef);
563	return CompleteObjectLocatorType;
564	}
565
566	llvm::GlobalVariable *getImageBase() {
567	StringRef Name = "__ImageBase";
568	if (llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(Name))
569	return GV;
570
571	auto *GV = new llvm::GlobalVariable(CGM.getModule(), CGM.Int8Ty,
572	/*isConstant=*/true,
573	llvm::GlobalValue::ExternalLinkage,
574	/*Initializer=*/nullptr, Name);
575	CGM.setDSOLocal(GV);
576	return GV;
577	}
578
579	llvm::Constant *getImageRelativeConstant(llvm::Constant *PtrVal) {
580	if (!isImageRelative())
581	return PtrVal;
582
583	if (PtrVal->isNullValue())
584	return llvm::Constant::getNullValue(Ty: CGM.IntTy);
585
586	llvm::Constant *ImageBaseAsInt =
587	llvm::ConstantExpr::getPtrToInt(C: getImageBase(), Ty: CGM.IntPtrTy);
588	llvm::Constant *PtrValAsInt =
589	llvm::ConstantExpr::getPtrToInt(C: PtrVal, Ty: CGM.IntPtrTy);
590	llvm::Constant *Diff =
591	llvm::ConstantExpr::getSub(C1: PtrValAsInt, C2: ImageBaseAsInt,
592	/*HasNUW=*/true, /*HasNSW=*/true);
593	return llvm::ConstantExpr::getTrunc(C: Diff, Ty: CGM.IntTy);
594	}
595
596	private:
597	MicrosoftMangleContext &getMangleContext() {
598	return cast<MicrosoftMangleContext>(Val&: CodeGen::CGCXXABI::getMangleContext());
599	}
600
601	llvm::Constant *getZeroInt() {
602	return llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0);
603	}
604
605	llvm::Constant *getAllOnesInt() {
606	return llvm::Constant::getAllOnesValue(Ty: CGM.IntTy);
607	}
608
609	CharUnits getVirtualFunctionPrologueThisAdjustment(GlobalDecl GD) override;
610
611	void
612	GetNullMemberPointerFields(const MemberPointerType *MPT,
613	llvm::SmallVectorImpl<llvm::Constant *> &fields);
614
615	/// Shared code for virtual base adjustment. Returns the offset from
616	/// the vbptr to the virtual base. Optionally returns the address of the
617	/// vbptr itself.
618	llvm::Value *GetVBaseOffsetFromVBPtr(CodeGenFunction &CGF,
619	Address Base,
620	llvm::Value *VBPtrOffset,
621	llvm::Value *VBTableOffset,
622	llvm::Value **VBPtr = nullptr);
623
624	llvm::Value *GetVBaseOffsetFromVBPtr(CodeGenFunction &CGF,
625	Address Base,
626	int32_t VBPtrOffset,
627	int32_t VBTableOffset,
628	llvm::Value **VBPtr = nullptr) {
629	assert(VBTableOffset % 4 == 0 && "should be byte offset into table of i32s");
630	llvm::Value *VBPOffset = llvm::ConstantInt::get(Ty: CGM.IntTy, V: VBPtrOffset),
631	*VBTOffset = llvm::ConstantInt::get(Ty: CGM.IntTy, V: VBTableOffset);
632	return GetVBaseOffsetFromVBPtr(CGF, Base, VBPtrOffset: VBPOffset, VBTableOffset: VBTOffset, VBPtr);
633	}
634
635	std::tuple<Address, llvm::Value *, const CXXRecordDecl *>
636	performBaseAdjustment(CodeGenFunction &CGF, Address Value,
637	QualType SrcRecordTy);
638
639	/// Performs a full virtual base adjustment. Used to dereference
640	/// pointers to members of virtual bases.
641	llvm::Value *AdjustVirtualBase(CodeGenFunction &CGF, const Expr *E,
642	const CXXRecordDecl *RD, Address Base,
643	llvm::Value *VirtualBaseAdjustmentOffset,
644	llvm::Value *VBPtrOffset /* optional */);
645
646	/// Emits a full member pointer with the fields common to data and
647	/// function member pointers.
648	llvm::Constant *EmitFullMemberPointer(llvm::Constant *FirstField,
649	bool IsMemberFunction,
650	const CXXRecordDecl *RD,
651	CharUnits NonVirtualBaseAdjustment,
652	unsigned VBTableIndex);
653
654	bool MemberPointerConstantIsNull(const MemberPointerType *MPT,
655	llvm::Constant *MP);
656
657	/// - Initialize all vbptrs of 'this' with RD as the complete type.
658	void EmitVBPtrStores(CodeGenFunction &CGF, const CXXRecordDecl *RD);
659
660	/// Caching wrapper around VBTableBuilder::enumerateVBTables().
661	const VBTableGlobals &enumerateVBTables(const CXXRecordDecl *RD);
662
663	/// Generate a thunk for calling a virtual member function MD.
664	llvm::Function *EmitVirtualMemPtrThunk(const CXXMethodDecl *MD,
665	const MethodVFTableLocation &ML);
666
667	llvm::Constant *EmitMemberDataPointer(const CXXRecordDecl *RD,
668	CharUnits offset);
669
670	public:
671	llvm::Type *ConvertMemberPointerType(const MemberPointerType *MPT) override;
672
673	bool isZeroInitializable(const MemberPointerType *MPT) override;
674
675	bool isMemberPointerConvertible(const MemberPointerType *MPT) const override {
676	const CXXRecordDecl *RD = MPT->getMostRecentCXXRecordDecl();
677	return RD->hasAttr<MSInheritanceAttr>();
678	}
679
680	llvm::Constant *EmitNullMemberPointer(const MemberPointerType *MPT) override;
681
682	llvm::Constant *EmitMemberDataPointer(const MemberPointerType *MPT,
683	CharUnits offset) override;
684	llvm::Constant *EmitMemberFunctionPointer(const CXXMethodDecl *MD) override;
685	llvm::Constant *EmitMemberPointer(const APValue &MP, QualType MPT) override;
686
687	llvm::Value *EmitMemberPointerComparison(CodeGenFunction &CGF,
688	llvm::Value *L,
689	llvm::Value *R,
690	const MemberPointerType *MPT,
691	bool Inequality) override;
692
693	llvm::Value *EmitMemberPointerIsNotNull(CodeGenFunction &CGF,
694	llvm::Value *MemPtr,
695	const MemberPointerType *MPT) override;
696
697	llvm::Value *
698	EmitMemberDataPointerAddress(CodeGenFunction &CGF, const Expr *E,
699	Address Base, llvm::Value *MemPtr,
700	const MemberPointerType *MPT) override;
701
702	llvm::Value *EmitNonNullMemberPointerConversion(
703	const MemberPointerType *SrcTy, const MemberPointerType *DstTy,
704	CastKind CK, CastExpr::path_const_iterator PathBegin,
705	CastExpr::path_const_iterator PathEnd, llvm::Value *Src,
706	CGBuilderTy &Builder);
707
708	llvm::Value *EmitMemberPointerConversion(CodeGenFunction &CGF,
709	const CastExpr *E,
710	llvm::Value *Src) override;
711
712	llvm::Constant *EmitMemberPointerConversion(const CastExpr *E,
713	llvm::Constant *Src) override;
714
715	llvm::Constant *EmitMemberPointerConversion(
716	const MemberPointerType *SrcTy, const MemberPointerType *DstTy,
717	CastKind CK, CastExpr::path_const_iterator PathBegin,
718	CastExpr::path_const_iterator PathEnd, llvm::Constant *Src);
719
720	CGCallee
721	EmitLoadOfMemberFunctionPointer(CodeGenFunction &CGF, const Expr *E,
722	Address This, llvm::Value *&ThisPtrForCall,
723	llvm::Value *MemPtr,
724	const MemberPointerType *MPT) override;
725
726	void emitCXXStructor(GlobalDecl GD) override;
727
728	llvm::StructType *getCatchableTypeType() {
729	if (CatchableTypeType)
730	return CatchableTypeType;
731	llvm::Type *FieldTypes[] = {
732	CGM.IntTy, // Flags
733	getImageRelativeType(PtrType: CGM.Int8PtrTy), // TypeDescriptor
734	CGM.IntTy, // NonVirtualAdjustment
735	CGM.IntTy, // OffsetToVBPtr
736	CGM.IntTy, // VBTableIndex
737	CGM.IntTy, // Size
738	getImageRelativeType(PtrType: CGM.Int8PtrTy) // CopyCtor
739	};
740	CatchableTypeType = llvm::StructType::create(
741	Context&: CGM.getLLVMContext(), Elements: FieldTypes, Name: "eh.CatchableType");
742	return CatchableTypeType;
743	}
744
745	llvm::StructType *getCatchableTypeArrayType(uint32_t NumEntries) {
746	llvm::StructType *&CatchableTypeArrayType =
747	CatchableTypeArrayTypeMap[NumEntries];
748	if (CatchableTypeArrayType)
749	return CatchableTypeArrayType;
750
751	llvm::SmallString<23> CTATypeName("eh.CatchableTypeArray.");
752	CTATypeName += llvm::utostr(X: NumEntries);
753	llvm::Type *CTType =
754	getImageRelativeType(PtrType: getCatchableTypeType()->getPointerTo());
755	llvm::Type *FieldTypes[] = {
756	CGM.IntTy, // NumEntries
757	llvm::ArrayType::get(ElementType: CTType, NumElements: NumEntries) // CatchableTypes
758	};
759	CatchableTypeArrayType =
760	llvm::StructType::create(Context&: CGM.getLLVMContext(), Elements: FieldTypes, Name: CTATypeName);
761	return CatchableTypeArrayType;
762	}
763
764	llvm::StructType *getThrowInfoType() {
765	if (ThrowInfoType)
766	return ThrowInfoType;
767	llvm::Type *FieldTypes[] = {
768	CGM.IntTy, // Flags
769	getImageRelativeType(PtrType: CGM.Int8PtrTy), // CleanupFn
770	getImageRelativeType(PtrType: CGM.Int8PtrTy), // ForwardCompat
771	getImageRelativeType(PtrType: CGM.Int8PtrTy) // CatchableTypeArray
772	};
773	ThrowInfoType = llvm::StructType::create(Context&: CGM.getLLVMContext(), Elements: FieldTypes,
774	Name: "eh.ThrowInfo");
775	return ThrowInfoType;
776	}
777
778	llvm::FunctionCallee getThrowFn() {
779	// _CxxThrowException is passed an exception object and a ThrowInfo object
780	// which describes the exception.
781	llvm::Type *Args[] = {CGM.Int8PtrTy, getThrowInfoType()->getPointerTo()};
782	llvm::FunctionType *FTy =
783	llvm::FunctionType::get(Result: CGM.VoidTy, Params: Args, /*isVarArg=*/false);
784	llvm::FunctionCallee Throw =
785	CGM.CreateRuntimeFunction(Ty: FTy, Name: "_CxxThrowException");
786	// _CxxThrowException is stdcall on 32-bit x86 platforms.
787	if (CGM.getTarget().getTriple().getArch() == llvm::Triple::x86) {
788	if (auto *Fn = dyn_cast<llvm::Function>(Val: Throw.getCallee()))
789	Fn->setCallingConv(llvm::CallingConv::X86_StdCall);
790	}
791	return Throw;
792	}
793
794	llvm::Function *getAddrOfCXXCtorClosure(const CXXConstructorDecl *CD,
795	CXXCtorType CT);
796
797	llvm::Constant *getCatchableType(QualType T,
798	uint32_t NVOffset = 0,
799	int32_t VBPtrOffset = -1,
800	uint32_t VBIndex = 0);
801
802	llvm::GlobalVariable *getCatchableTypeArray(QualType T);
803
804	llvm::GlobalVariable *getThrowInfo(QualType T) override;
805
806	std::pair<llvm::Value *, const CXXRecordDecl *>
807	LoadVTablePtr(CodeGenFunction &CGF, Address This,
808	const CXXRecordDecl *RD) override;
809
810	bool
811	isPermittedToBeHomogeneousAggregate(const CXXRecordDecl *RD) const override;
812
813	private:
814	typedef std::pair<const CXXRecordDecl *, CharUnits> VFTableIdTy;
815	typedef llvm::DenseMap<VFTableIdTy, llvm::GlobalVariable *> VTablesMapTy;
816	typedef llvm::DenseMap<VFTableIdTy, llvm::GlobalValue *> VFTablesMapTy;
817	/// All the vftables that have been referenced.
818	VFTablesMapTy VFTablesMap;
819	VTablesMapTy VTablesMap;
820
821	/// This set holds the record decls we've deferred vtable emission for.
822	llvm::SmallPtrSet<const CXXRecordDecl *, 4> DeferredVFTables;
823
824
825	/// All the vbtables which have been referenced.
826	llvm::DenseMap<const CXXRecordDecl *, VBTableGlobals> VBTablesMap;
827
828	/// Info on the global variable used to guard initialization of static locals.
829	/// The BitIndex field is only used for externally invisible declarations.
830	struct GuardInfo {
831	GuardInfo() = default;
832	llvm::GlobalVariable *Guard = nullptr;
833	unsigned BitIndex = 0;
834	};
835
836	/// Map from DeclContext to the current guard variable. We assume that the
837	/// AST is visited in source code order.
838	llvm::DenseMap<const DeclContext *, GuardInfo> GuardVariableMap;
839	llvm::DenseMap<const DeclContext *, GuardInfo> ThreadLocalGuardVariableMap;
840	llvm::DenseMap<const DeclContext *, unsigned> ThreadSafeGuardNumMap;
841
842	llvm::DenseMap<size_t, llvm::StructType *> TypeDescriptorTypeMap;
843	llvm::StructType *BaseClassDescriptorType;
844	llvm::StructType *ClassHierarchyDescriptorType;
845	llvm::StructType *CompleteObjectLocatorType;
846
847	llvm::DenseMap<QualType, llvm::GlobalVariable *> CatchableTypeArrays;
848
849	llvm::StructType *CatchableTypeType;
850	llvm::DenseMap<uint32_t, llvm::StructType *> CatchableTypeArrayTypeMap;
851	llvm::StructType *ThrowInfoType;
852	};
853
854	}
855
856	CGCXXABI::RecordArgABI
857	MicrosoftCXXABI::getRecordArgABI(const CXXRecordDecl *RD) const {
858	// Use the default C calling convention rules for things that can be passed in
859	// registers, i.e. non-trivially copyable records or records marked with
860	// [[trivial_abi]].
861	if (RD->canPassInRegisters())
862	return RAA_Default;
863
864	switch (CGM.getTarget().getTriple().getArch()) {
865	default:
866	// FIXME: Implement for other architectures.
867	return RAA_Indirect;
868
869	case llvm::Triple::thumb:
870	// Pass things indirectly for now because it is simple.
871	// FIXME: This is incompatible with MSVC for arguments with a dtor and no
872	// copy ctor.
873	return RAA_Indirect;
874
875	case llvm::Triple::x86: {
876	// If the argument has *required* alignment greater than four bytes, pass
877	// it indirectly. Prior to MSVC version 19.14, passing overaligned
878	// arguments was not supported and resulted in a compiler error. In 19.14
879	// and later versions, such arguments are now passed indirectly.
880	TypeInfo Info = getContext().getTypeInfo(RD->getTypeForDecl());
881	if (Info.isAlignRequired() && Info.Align > 4)
882	return RAA_Indirect;
883
884	// If C++ prohibits us from making a copy, construct the arguments directly
885	// into argument memory.
886	return RAA_DirectInMemory;
887	}
888
889	case llvm::Triple::x86_64:
890	case llvm::Triple::aarch64:
891	return RAA_Indirect;
892	}
893
894	llvm_unreachable("invalid enum");
895	}
896
897	void MicrosoftCXXABI::emitVirtualObjectDelete(CodeGenFunction &CGF,
898	const CXXDeleteExpr *DE,
899	Address Ptr,
900	QualType ElementType,
901	const CXXDestructorDecl *Dtor) {
902	// FIXME: Provide a source location here even though there's no
903	// CXXMemberCallExpr for dtor call.
904	bool UseGlobalDelete = DE->isGlobalDelete();
905	CXXDtorType DtorType = UseGlobalDelete ? Dtor_Complete : Dtor_Deleting;
906	llvm::Value *MDThis = EmitVirtualDestructorCall(CGF, Dtor, DtorType, This: Ptr, E: DE);
907	if (UseGlobalDelete)
908	CGF.EmitDeleteCall(DeleteFD: DE->getOperatorDelete(), Ptr: MDThis, DeleteTy: ElementType);
909	}
910
911	void MicrosoftCXXABI::emitRethrow(CodeGenFunction &CGF, bool isNoReturn) {
912	llvm::Value *Args[] = {
913	llvm::ConstantPointerNull::get(T: CGM.Int8PtrTy),
914	llvm::ConstantPointerNull::get(T: getThrowInfoType()->getPointerTo())};
915	llvm::FunctionCallee Fn = getThrowFn();
916	if (isNoReturn)
917	CGF.EmitNoreturnRuntimeCallOrInvoke(callee: Fn, args: Args);
918	else
919	CGF.EmitRuntimeCallOrInvoke(callee: Fn, args: Args);
920	}
921
922	void MicrosoftCXXABI::emitBeginCatch(CodeGenFunction &CGF,
923	const CXXCatchStmt *S) {
924	// In the MS ABI, the runtime handles the copy, and the catch handler is
925	// responsible for destruction.
926	VarDecl *CatchParam = S->getExceptionDecl();
927	llvm::BasicBlock *CatchPadBB = CGF.Builder.GetInsertBlock();
928	llvm::CatchPadInst *CPI =
929	cast<llvm::CatchPadInst>(Val: CatchPadBB->getFirstNonPHI());
930	CGF.CurrentFuncletPad = CPI;
931
932	// If this is a catch-all or the catch parameter is unnamed, we don't need to
933	// emit an alloca to the object.
934	if (!CatchParam || !CatchParam->getDeclName()) {
935	CGF.EHStack.pushCleanup<CatchRetScope>(Kind: NormalCleanup, A: CPI);
936	return;
937	}
938
939	CodeGenFunction::AutoVarEmission var = CGF.EmitAutoVarAlloca(var: *CatchParam);
940	CPI->setArgOperand(i: 2, v: var.getObjectAddress(CGF).getPointer());
941	CGF.EHStack.pushCleanup<CatchRetScope>(Kind: NormalCleanup, A: CPI);
942	CGF.EmitAutoVarCleanups(emission: var);
943	}
944
945	/// We need to perform a generic polymorphic operation (like a typeid
946	/// or a cast), which requires an object with a vfptr. Adjust the
947	/// address to point to an object with a vfptr.
948	std::tuple<Address, llvm::Value *, const CXXRecordDecl *>
949	MicrosoftCXXABI::performBaseAdjustment(CodeGenFunction &CGF, Address Value,
950	QualType SrcRecordTy) {
951	Value = Value.withElementType(ElemTy: CGF.Int8Ty);
952	const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl();
953	const ASTContext &Context = getContext();
954
955	// If the class itself has a vfptr, great. This check implicitly
956	// covers non-virtual base subobjects: a class with its own virtual
957	// functions would be a candidate to be a primary base.
958	if (Context.getASTRecordLayout(SrcDecl).hasExtendableVFPtr())
959	return std::make_tuple(args&: Value, args: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 0),
960	args&: SrcDecl);
961
962	// Okay, one of the vbases must have a vfptr, or else this isn't
963	// actually a polymorphic class.
964	const CXXRecordDecl *PolymorphicBase = nullptr;
965	for (auto &Base : SrcDecl->vbases()) {
966	const CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl();
967	if (Context.getASTRecordLayout(BaseDecl).hasExtendableVFPtr()) {
968	PolymorphicBase = BaseDecl;
969	break;
970	}
971	}
972	assert(PolymorphicBase && "polymorphic class has no apparent vfptr?");
973
974	llvm::Value *Offset =
975	GetVirtualBaseClassOffset(CGF, This: Value, ClassDecl: SrcDecl, BaseClassDecl: PolymorphicBase);
976	llvm::Value *Ptr = CGF.Builder.CreateInBoundsGEP(
977	Ty: Value.getElementType(), Ptr: Value.getPointer(), IdxList: Offset);
978	CharUnits VBaseAlign =
979	CGF.CGM.getVBaseAlignment(DerivedAlign: Value.getAlignment(), Derived: SrcDecl, VBase: PolymorphicBase);
980	return std::make_tuple(args: Address(Ptr, CGF.Int8Ty, VBaseAlign), args&: Offset,
981	args&: PolymorphicBase);
982	}
983
984	bool MicrosoftCXXABI::shouldTypeidBeNullChecked(bool IsDeref,
985	QualType SrcRecordTy) {
986	const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl();
987	return IsDeref &&
988	!getContext().getASTRecordLayout(SrcDecl).hasExtendableVFPtr();
989	}
990
991	static llvm::CallBase *emitRTtypeidCall(CodeGenFunction &CGF,
992	llvm::Value *Argument) {
993	llvm::Type *ArgTypes[] = {CGF.Int8PtrTy};
994	llvm::FunctionType *FTy =
995	llvm::FunctionType::get(Result: CGF.Int8PtrTy, Params: ArgTypes, isVarArg: false);
996	llvm::Value *Args[] = {Argument};
997	llvm::FunctionCallee Fn = CGF.CGM.CreateRuntimeFunction(Ty: FTy, Name: "__RTtypeid");
998	return CGF.EmitRuntimeCallOrInvoke(callee: Fn, args: Args);
999	}
1000
1001	void MicrosoftCXXABI::EmitBadTypeidCall(CodeGenFunction &CGF) {
1002	llvm::CallBase *Call =
1003	emitRTtypeidCall(CGF, Argument: llvm::Constant::getNullValue(Ty: CGM.VoidPtrTy));
1004	Call->setDoesNotReturn();
1005	CGF.Builder.CreateUnreachable();
1006	}
1007
1008	llvm::Value *MicrosoftCXXABI::EmitTypeid(CodeGenFunction &CGF,
1009	QualType SrcRecordTy,
1010	Address ThisPtr,
1011	llvm::Type *StdTypeInfoPtrTy) {
1012	std::tie(args&: ThisPtr, args: std::ignore, args: std::ignore) =
1013	performBaseAdjustment(CGF, Value: ThisPtr, SrcRecordTy);
1014	llvm::CallBase *Typeid = emitRTtypeidCall(CGF, Argument: ThisPtr.getPointer());
1015	return CGF.Builder.CreateBitCast(V: Typeid, DestTy: StdTypeInfoPtrTy);
1016	}
1017
1018	bool MicrosoftCXXABI::shouldDynamicCastCallBeNullChecked(bool SrcIsPtr,
1019	QualType SrcRecordTy) {
1020	const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl();
1021	return SrcIsPtr &&
1022	!getContext().getASTRecordLayout(SrcDecl).hasExtendableVFPtr();
1023	}
1024
1025	llvm::Value *MicrosoftCXXABI::emitDynamicCastCall(
1026	CodeGenFunction &CGF, Address This, QualType SrcRecordTy, QualType DestTy,
1027	QualType DestRecordTy, llvm::BasicBlock *CastEnd) {
1028	llvm::Value *SrcRTTI =
1029	CGF.CGM.GetAddrOfRTTIDescriptor(Ty: SrcRecordTy.getUnqualifiedType());
1030	llvm::Value *DestRTTI =
1031	CGF.CGM.GetAddrOfRTTIDescriptor(Ty: DestRecordTy.getUnqualifiedType());
1032
1033	llvm::Value *Offset;
1034	std::tie(args&: This, args&: Offset, args: std::ignore) =
1035	performBaseAdjustment(CGF, Value: This, SrcRecordTy);
1036	llvm::Value *ThisPtr = This.getPointer();
1037	Offset = CGF.Builder.CreateTrunc(V: Offset, DestTy: CGF.Int32Ty);
1038
1039	// PVOID __RTDynamicCast(
1040	// PVOID inptr,
1041	// LONG VfDelta,
1042	// PVOID SrcType,
1043	// PVOID TargetType,
1044	// BOOL isReference)
1045	llvm::Type *ArgTypes[] = {CGF.Int8PtrTy, CGF.Int32Ty, CGF.Int8PtrTy,
1046	CGF.Int8PtrTy, CGF.Int32Ty};
1047	llvm::FunctionCallee Function = CGF.CGM.CreateRuntimeFunction(
1048	Ty: llvm::FunctionType::get(Result: CGF.Int8PtrTy, Params: ArgTypes, isVarArg: false),
1049	Name: "__RTDynamicCast");
1050	llvm::Value *Args[] = {
1051	ThisPtr, Offset, SrcRTTI, DestRTTI,
1052	llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: DestTy->isReferenceType())};
1053	return CGF.EmitRuntimeCallOrInvoke(callee: Function, args: Args);
1054	}
1055
1056	llvm::Value *MicrosoftCXXABI::emitDynamicCastToVoid(CodeGenFunction &CGF,
1057	Address Value,
1058	QualType SrcRecordTy) {
1059	std::tie(args&: Value, args: std::ignore, args: std::ignore) =
1060	performBaseAdjustment(CGF, Value, SrcRecordTy);
1061
1062	// PVOID __RTCastToVoid(
1063	// PVOID inptr)
1064	llvm::Type *ArgTypes[] = {CGF.Int8PtrTy};
1065	llvm::FunctionCallee Function = CGF.CGM.CreateRuntimeFunction(
1066	Ty: llvm::FunctionType::get(Result: CGF.Int8PtrTy, Params: ArgTypes, isVarArg: false),
1067	Name: "__RTCastToVoid");
1068	llvm::Value *Args[] = {Value.getPointer()};
1069	return CGF.EmitRuntimeCall(callee: Function, args: Args);
1070	}
1071
1072	bool MicrosoftCXXABI::EmitBadCastCall(CodeGenFunction &CGF) {
1073	return false;
1074	}
1075
1076	llvm::Value *MicrosoftCXXABI::GetVirtualBaseClassOffset(
1077	CodeGenFunction &CGF, Address This, const CXXRecordDecl *ClassDecl,
1078	const CXXRecordDecl *BaseClassDecl) {
1079	const ASTContext &Context = getContext();
1080	int64_t VBPtrChars =
1081	Context.getASTRecordLayout(ClassDecl).getVBPtrOffset().getQuantity();
1082	llvm::Value *VBPtrOffset = llvm::ConstantInt::get(Ty: CGM.PtrDiffTy, V: VBPtrChars);
1083	CharUnits IntSize = Context.getTypeSizeInChars(Context.IntTy);
1084	CharUnits VBTableChars =
1085	IntSize *
1086	CGM.getMicrosoftVTableContext().getVBTableIndex(Derived: ClassDecl, VBase: BaseClassDecl);
1087	llvm::Value *VBTableOffset =
1088	llvm::ConstantInt::get(Ty: CGM.IntTy, V: VBTableChars.getQuantity());
1089
1090	llvm::Value *VBPtrToNewBase =
1091	GetVBaseOffsetFromVBPtr(CGF, Base: This, VBPtrOffset, VBTableOffset);
1092	VBPtrToNewBase =
1093	CGF.Builder.CreateSExtOrBitCast(V: VBPtrToNewBase, DestTy: CGM.PtrDiffTy);
1094	return CGF.Builder.CreateNSWAdd(LHS: VBPtrOffset, RHS: VBPtrToNewBase);
1095	}
1096
1097	bool MicrosoftCXXABI::HasThisReturn(GlobalDecl GD) const {
1098	return isa<CXXConstructorDecl>(Val: GD.getDecl());
1099	}
1100
1101	static bool isDeletingDtor(GlobalDecl GD) {
1102	return isa<CXXDestructorDecl>(Val: GD.getDecl()) &&
1103	GD.getDtorType() == Dtor_Deleting;
1104	}
1105
1106	bool MicrosoftCXXABI::hasMostDerivedReturn(GlobalDecl GD) const {
1107	return isDeletingDtor(GD);
1108	}
1109
1110	static bool isTrivialForMSVC(const CXXRecordDecl *RD, QualType Ty,
1111	CodeGenModule &CGM) {
1112	// On AArch64, HVAs that can be passed in registers can also be returned
1113	// in registers. (Note this is using the MSVC definition of an HVA; see
1114	// isPermittedToBeHomogeneousAggregate().)
1115	const Type *Base = nullptr;
1116	uint64_t NumElts = 0;
1117	if (CGM.getTarget().getTriple().isAArch64() &&
1118	CGM.getTypes().getABIInfo().isHomogeneousAggregate(Ty, Base, Members&: NumElts) &&
1119	isa<VectorType>(Val: Base)) {
1120	return true;
1121	}
1122
1123	// We use the C++14 definition of an aggregate, so we also
1124	// check for:
1125	// No private or protected non static data members.
1126	// No base classes
1127	// No virtual functions
1128	// Additionally, we need to ensure that there is a trivial copy assignment
1129	// operator, a trivial destructor and no user-provided constructors.
1130	if (RD->hasProtectedFields() || RD->hasPrivateFields())
1131	return false;
1132	if (RD->getNumBases() > 0)
1133	return false;
1134	if (RD->isPolymorphic())
1135	return false;
1136	if (RD->hasNonTrivialCopyAssignment())
1137	return false;
1138	for (const CXXConstructorDecl *Ctor : RD->ctors())
1139	if (Ctor->isUserProvided())
1140	return false;
1141	if (RD->hasNonTrivialDestructor())
1142	return false;
1143	return true;
1144	}
1145
1146	bool MicrosoftCXXABI::classifyReturnType(CGFunctionInfo &FI) const {
1147	const CXXRecordDecl *RD = FI.getReturnType()->getAsCXXRecordDecl();
1148	if (!RD)
1149	return false;
1150
1151	bool isTrivialForABI = RD->canPassInRegisters() &&
1152	isTrivialForMSVC(RD, Ty: FI.getReturnType(), CGM);
1153
1154	// MSVC always returns structs indirectly from C++ instance methods.
1155	bool isIndirectReturn = !isTrivialForABI || FI.isInstanceMethod();
1156
1157	if (isIndirectReturn) {
1158	CharUnits Align = CGM.getContext().getTypeAlignInChars(T: FI.getReturnType());
1159	FI.getReturnInfo() = ABIArgInfo::getIndirect(Alignment: Align, /*ByVal=*/false);
1160
1161	// MSVC always passes `this` before the `sret` parameter.
1162	FI.getReturnInfo().setSRetAfterThis(FI.isInstanceMethod());
1163
1164	// On AArch64, use the `inreg` attribute if the object is considered to not
1165	// be trivially copyable, or if this is an instance method struct return.
1166	FI.getReturnInfo().setInReg(CGM.getTarget().getTriple().isAArch64());
1167
1168	return true;
1169	}
1170
1171	// Otherwise, use the C ABI rules.
1172	return false;
1173	}
1174
1175	llvm::BasicBlock *
1176	MicrosoftCXXABI::EmitCtorCompleteObjectHandler(CodeGenFunction &CGF,
1177	const CXXRecordDecl *RD) {
1178	llvm::Value *IsMostDerivedClass = getStructorImplicitParamValue(CGF);
1179	assert(IsMostDerivedClass &&
1180	"ctor for a class with virtual bases must have an implicit parameter");
1181	llvm::Value *IsCompleteObject =
1182	CGF.Builder.CreateIsNotNull(Arg: IsMostDerivedClass, Name: "is_complete_object");
1183
1184	llvm::BasicBlock *CallVbaseCtorsBB = CGF.createBasicBlock(name: "ctor.init_vbases");
1185	llvm::BasicBlock *SkipVbaseCtorsBB = CGF.createBasicBlock(name: "ctor.skip_vbases");
1186	CGF.Builder.CreateCondBr(Cond: IsCompleteObject,
1187	True: CallVbaseCtorsBB, False: SkipVbaseCtorsBB);
1188
1189	CGF.EmitBlock(BB: CallVbaseCtorsBB);
1190
1191	// Fill in the vbtable pointers here.
1192	EmitVBPtrStores(CGF, RD);
1193
1194	// CGF will put the base ctor calls in this basic block for us later.
1195
1196	return SkipVbaseCtorsBB;
1197	}
1198
1199	llvm::BasicBlock *
1200	MicrosoftCXXABI::EmitDtorCompleteObjectHandler(CodeGenFunction &CGF) {
1201	llvm::Value *IsMostDerivedClass = getStructorImplicitParamValue(CGF);
1202	assert(IsMostDerivedClass &&
1203	"ctor for a class with virtual bases must have an implicit parameter");
1204	llvm::Value *IsCompleteObject =
1205	CGF.Builder.CreateIsNotNull(Arg: IsMostDerivedClass, Name: "is_complete_object");
1206
1207	llvm::BasicBlock *CallVbaseDtorsBB = CGF.createBasicBlock(name: "Dtor.dtor_vbases");
1208	llvm::BasicBlock *SkipVbaseDtorsBB = CGF.createBasicBlock(name: "Dtor.skip_vbases");
1209	CGF.Builder.CreateCondBr(Cond: IsCompleteObject,
1210	True: CallVbaseDtorsBB, False: SkipVbaseDtorsBB);
1211
1212	CGF.EmitBlock(BB: CallVbaseDtorsBB);
1213	// CGF will put the base dtor calls in this basic block for us later.
1214
1215	return SkipVbaseDtorsBB;
1216	}
1217
1218	void MicrosoftCXXABI::initializeHiddenVirtualInheritanceMembers(
1219	CodeGenFunction &CGF, const CXXRecordDecl *RD) {
1220	// In most cases, an override for a vbase virtual method can adjust
1221	// the "this" parameter by applying a constant offset.
1222	// However, this is not enough while a constructor or a destructor of some
1223	// class X is being executed if all the following conditions are met:
1224	// - X has virtual bases, (1)
1225	// - X overrides a virtual method M of a vbase Y, (2)
1226	// - X itself is a vbase of the most derived class.
1227	//
1228	// If (1) and (2) are true, the vtorDisp for vbase Y is a hidden member of X
1229	// which holds the extra amount of "this" adjustment we must do when we use
1230	// the X vftables (i.e. during X ctor or dtor).
1231	// Outside the ctors and dtors, the values of vtorDisps are zero.
1232
1233	const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
1234	typedef ASTRecordLayout::VBaseOffsetsMapTy VBOffsets;
1235	const VBOffsets &VBaseMap = Layout.getVBaseOffsetsMap();
1236	CGBuilderTy &Builder = CGF.Builder;
1237
1238	llvm::Value *Int8This = nullptr; // Initialize lazily.
1239
1240	for (const CXXBaseSpecifier &S : RD->vbases()) {
1241	const CXXRecordDecl *VBase = S.getType()->getAsCXXRecordDecl();
1242	auto I = VBaseMap.find(Val: VBase);
1243	assert(I != VBaseMap.end());
1244	if (!I->second.hasVtorDisp())
1245	continue;
1246
1247	llvm::Value *VBaseOffset =
1248	GetVirtualBaseClassOffset(CGF, This: getThisAddress(CGF), ClassDecl: RD, BaseClassDecl: VBase);
1249	uint64_t ConstantVBaseOffset = I->second.VBaseOffset.getQuantity();
1250
1251	// vtorDisp_for_vbase = vbptr[vbase_idx] - offsetof(RD, vbase).
1252	llvm::Value *VtorDispValue = Builder.CreateSub(
1253	LHS: VBaseOffset, RHS: llvm::ConstantInt::get(Ty: CGM.PtrDiffTy, V: ConstantVBaseOffset),
1254	Name: "vtordisp.value");
1255	VtorDispValue = Builder.CreateTruncOrBitCast(V: VtorDispValue, DestTy: CGF.Int32Ty);
1256
1257	if (!Int8This)
1258	Int8This = getThisValue(CGF);
1259
1260	llvm::Value *VtorDispPtr =
1261	Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: Int8This, IdxList: VBaseOffset);
1262	// vtorDisp is always the 32-bits before the vbase in the class layout.
1263	VtorDispPtr = Builder.CreateConstGEP1_32(Ty: CGF.Int8Ty, Ptr: VtorDispPtr, Idx0: -4);
1264
1265	Builder.CreateAlignedStore(Val: VtorDispValue, Addr: VtorDispPtr,
1266	Align: CharUnits::fromQuantity(Quantity: 4));
1267	}
1268	}
1269
1270	static bool hasDefaultCXXMethodCC(ASTContext &Context,
1271	const CXXMethodDecl *MD) {
1272	CallingConv ExpectedCallingConv = Context.getDefaultCallingConvention(
1273	/*IsVariadic=*/false, /*IsCXXMethod=*/true);
1274	CallingConv ActualCallingConv =
1275	MD->getType()->castAs<FunctionProtoType>()->getCallConv();
1276	return ExpectedCallingConv == ActualCallingConv;
1277	}
1278
1279	void MicrosoftCXXABI::EmitCXXConstructors(const CXXConstructorDecl *D) {
1280	// There's only one constructor type in this ABI.
1281	CGM.EmitGlobal(D: GlobalDecl(D, Ctor_Complete));
1282
1283	// Exported default constructors either have a simple call-site where they use
1284	// the typical calling convention and have a single 'this' pointer for an
1285	// argument -or- they get a wrapper function which appropriately thunks to the
1286	// real default constructor. This thunk is the default constructor closure.
1287	if (D->hasAttr<DLLExportAttr>() && D->isDefaultConstructor() &&
1288	D->isDefined()) {
1289	if (!hasDefaultCXXMethodCC(getContext(), D) || D->getNumParams() != 0) {
1290	llvm::Function *Fn = getAddrOfCXXCtorClosure(CD: D, CT: Ctor_DefaultClosure);
1291	Fn->setLinkage(llvm::GlobalValue::WeakODRLinkage);
1292	CGM.setGVProperties(Fn, D);
1293	}
1294	}
1295	}
1296
1297	void MicrosoftCXXABI::EmitVBPtrStores(CodeGenFunction &CGF,
1298	const CXXRecordDecl *RD) {
1299	Address This = getThisAddress(CGF);
1300	This = This.withElementType(ElemTy: CGM.Int8Ty);
1301	const ASTContext &Context = getContext();
1302	const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
1303
1304	const VBTableGlobals &VBGlobals = enumerateVBTables(RD);
1305	for (unsigned I = 0, E = VBGlobals.VBTables->size(); I != E; ++I) {
1306	const std::unique_ptr<VPtrInfo> &VBT = (*VBGlobals.VBTables)[I];
1307	llvm::GlobalVariable *GV = VBGlobals.Globals[I];
1308	const ASTRecordLayout &SubobjectLayout =
1309	Context.getASTRecordLayout(VBT->IntroducingObject);
1310	CharUnits Offs = VBT->NonVirtualOffset;
1311	Offs += SubobjectLayout.getVBPtrOffset();
1312	if (VBT->getVBaseWithVPtr())
1313	Offs += Layout.getVBaseClassOffset(VBase: VBT->getVBaseWithVPtr());
1314	Address VBPtr = CGF.Builder.CreateConstInBoundsByteGEP(Addr: This, Offset: Offs);
1315	llvm::Value *GVPtr =
1316	CGF.Builder.CreateConstInBoundsGEP2_32(Ty: GV->getValueType(), Ptr: GV, Idx0: 0, Idx1: 0);
1317	VBPtr = VBPtr.withElementType(ElemTy: GVPtr->getType());
1318	CGF.Builder.CreateStore(Val: GVPtr, Addr: VBPtr);
1319	}
1320	}
1321
1322	CGCXXABI::AddedStructorArgCounts
1323	MicrosoftCXXABI::buildStructorSignature(GlobalDecl GD,
1324	SmallVectorImpl<CanQualType> &ArgTys) {
1325	AddedStructorArgCounts Added;
1326	// TODO: 'for base' flag
1327	if (isa<CXXDestructorDecl>(Val: GD.getDecl()) &&
1328	GD.getDtorType() == Dtor_Deleting) {
1329	// The scalar deleting destructor takes an implicit int parameter.
1330	ArgTys.push_back(Elt: getContext().IntTy);
1331	++Added.Suffix;
1332	}
1333	auto *CD = dyn_cast<CXXConstructorDecl>(Val: GD.getDecl());
1334	if (!CD)
1335	return Added;
1336
1337	// All parameters are already in place except is_most_derived, which goes
1338	// after 'this' if it's variadic and last if it's not.
1339
1340	const CXXRecordDecl *Class = CD->getParent();
1341	const FunctionProtoType *FPT = CD->getType()->castAs<FunctionProtoType>();
1342	if (Class->getNumVBases()) {
1343	if (FPT->isVariadic()) {
1344	ArgTys.insert(ArgTys.begin() + 1, getContext().IntTy);
1345	++Added.Prefix;
1346	} else {
1347	ArgTys.push_back(Elt: getContext().IntTy);
1348	++Added.Suffix;
1349	}
1350	}
1351
1352	return Added;
1353	}
1354
1355	void MicrosoftCXXABI::setCXXDestructorDLLStorage(llvm::GlobalValue *GV,
1356	const CXXDestructorDecl *Dtor,
1357	CXXDtorType DT) const {
1358	// Deleting destructor variants are never imported or exported. Give them the
1359	// default storage class.
1360	if (DT == Dtor_Deleting) {
1361	GV->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
1362	} else {
1363	const NamedDecl *ND = Dtor;
1364	CGM.setDLLImportDLLExport(GV, D: ND);
1365	}
1366	}
1367
1368	llvm::GlobalValue::LinkageTypes MicrosoftCXXABI::getCXXDestructorLinkage(
1369	GVALinkage Linkage, const CXXDestructorDecl *Dtor, CXXDtorType DT) const {
1370	// Internal things are always internal, regardless of attributes. After this,
1371	// we know the thunk is externally visible.
1372	if (Linkage == GVA_Internal)
1373	return llvm::GlobalValue::InternalLinkage;
1374
1375	switch (DT) {
1376	case Dtor_Base:
1377	// The base destructor most closely tracks the user-declared constructor, so
1378	// we delegate back to the normal declarator case.
1379	return CGM.getLLVMLinkageForDeclarator(Dtor, Linkage);
1380	case Dtor_Complete:
1381	// The complete destructor is like an inline function, but it may be
1382	// imported and therefore must be exported as well. This requires changing
1383	// the linkage if a DLL attribute is present.
1384	if (Dtor->hasAttr<DLLExportAttr>())
1385	return llvm::GlobalValue::WeakODRLinkage;
1386	if (Dtor->hasAttr<DLLImportAttr>())
1387	return llvm::GlobalValue::AvailableExternallyLinkage;
1388	return llvm::GlobalValue::LinkOnceODRLinkage;
1389	case Dtor_Deleting:
1390	// Deleting destructors are like inline functions. They have vague linkage
1391	// and are emitted everywhere they are used. They are internal if the class
1392	// is internal.
1393	return llvm::GlobalValue::LinkOnceODRLinkage;
1394	case Dtor_Comdat:
1395	llvm_unreachable("MS C++ ABI does not support comdat dtors");
1396	}
1397	llvm_unreachable("invalid dtor type");
1398	}
1399
1400	void MicrosoftCXXABI::EmitCXXDestructors(const CXXDestructorDecl *D) {
1401	// The TU defining a dtor is only guaranteed to emit a base destructor. All
1402	// other destructor variants are delegating thunks.
1403	CGM.EmitGlobal(D: GlobalDecl(D, Dtor_Base));
1404
1405	// If the class is dllexported, emit the complete (vbase) destructor wherever
1406	// the base dtor is emitted.
1407	// FIXME: To match MSVC, this should only be done when the class is exported
1408	// with -fdllexport-inlines enabled.
1409	if (D->getParent()->getNumVBases() > 0 && D->hasAttr<DLLExportAttr>())
1410	CGM.EmitGlobal(D: GlobalDecl(D, Dtor_Complete));
1411	}
1412
1413	CharUnits
1414	MicrosoftCXXABI::getVirtualFunctionPrologueThisAdjustment(GlobalDecl GD) {
1415	const CXXMethodDecl *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
1416
1417	if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(Val: MD)) {
1418	// Complete destructors take a pointer to the complete object as a
1419	// parameter, thus don't need this adjustment.
1420	if (GD.getDtorType() == Dtor_Complete)
1421	return CharUnits();
1422
1423	// There's no Dtor_Base in vftable but it shares the this adjustment with
1424	// the deleting one, so look it up instead.
1425	GD = GlobalDecl(DD, Dtor_Deleting);
1426	}
1427
1428	MethodVFTableLocation ML =
1429	CGM.getMicrosoftVTableContext().getMethodVFTableLocation(GD);
1430	CharUnits Adjustment = ML.VFPtrOffset;
1431
1432	// Normal virtual instance methods need to adjust from the vfptr that first
1433	// defined the virtual method to the virtual base subobject, but destructors
1434	// do not. The vector deleting destructor thunk applies this adjustment for
1435	// us if necessary.
1436	if (isa<CXXDestructorDecl>(Val: MD))
1437	Adjustment = CharUnits::Zero();
1438
1439	if (ML.VBase) {
1440	const ASTRecordLayout &DerivedLayout =
1441	getContext().getASTRecordLayout(MD->getParent());
1442	Adjustment += DerivedLayout.getVBaseClassOffset(VBase: ML.VBase);
1443	}
1444
1445	return Adjustment;
1446	}
1447
1448	Address MicrosoftCXXABI::adjustThisArgumentForVirtualFunctionCall(
1449	CodeGenFunction &CGF, GlobalDecl GD, Address This,
1450	bool VirtualCall) {
1451	if (!VirtualCall) {
1452	// If the call of a virtual function is not virtual, we just have to
1453	// compensate for the adjustment the virtual function does in its prologue.
1454	CharUnits Adjustment = getVirtualFunctionPrologueThisAdjustment(GD);
1455	if (Adjustment.isZero())
1456	return This;
1457
1458	This = This.withElementType(ElemTy: CGF.Int8Ty);
1459	assert(Adjustment.isPositive());
1460	return CGF.Builder.CreateConstByteGEP(Addr: This, Offset: Adjustment);
1461	}
1462
1463	const CXXMethodDecl *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
1464
1465	GlobalDecl LookupGD = GD;
1466	if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(Val: MD)) {
1467	// Complete dtors take a pointer to the complete object,
1468	// thus don't need adjustment.
1469	if (GD.getDtorType() == Dtor_Complete)
1470	return This;
1471
1472	// There's only Dtor_Deleting in vftable but it shares the this adjustment
1473	// with the base one, so look up the deleting one instead.
1474	LookupGD = GlobalDecl(DD, Dtor_Deleting);
1475	}
1476	MethodVFTableLocation ML =
1477	CGM.getMicrosoftVTableContext().getMethodVFTableLocation(GD: LookupGD);
1478
1479	CharUnits StaticOffset = ML.VFPtrOffset;
1480
1481	// Base destructors expect 'this' to point to the beginning of the base
1482	// subobject, not the first vfptr that happens to contain the virtual dtor.
1483	// However, we still need to apply the virtual base adjustment.
1484	if (isa<CXXDestructorDecl>(Val: MD) && GD.getDtorType() == Dtor_Base)
1485	StaticOffset = CharUnits::Zero();
1486
1487	Address Result = This;
1488	if (ML.VBase) {
1489	Result = Result.withElementType(ElemTy: CGF.Int8Ty);
1490
1491	const CXXRecordDecl *Derived = MD->getParent();
1492	const CXXRecordDecl *VBase = ML.VBase;
1493	llvm::Value *VBaseOffset =
1494	GetVirtualBaseClassOffset(CGF, This: Result, ClassDecl: Derived, BaseClassDecl: VBase);
1495	llvm::Value *VBasePtr = CGF.Builder.CreateInBoundsGEP(
1496	Ty: Result.getElementType(), Ptr: Result.getPointer(), IdxList: VBaseOffset);
1497	CharUnits VBaseAlign =
1498	CGF.CGM.getVBaseAlignment(DerivedAlign: Result.getAlignment(), Derived, VBase);
1499	Result = Address(VBasePtr, CGF.Int8Ty, VBaseAlign);
1500	}
1501	if (!StaticOffset.isZero()) {
1502	assert(StaticOffset.isPositive());
1503	Result = Result.withElementType(ElemTy: CGF.Int8Ty);
1504	if (ML.VBase) {
1505	// Non-virtual adjustment might result in a pointer outside the allocated
1506	// object, e.g. if the final overrider class is laid out after the virtual
1507	// base that declares a method in the most derived class.
1508	// FIXME: Update the code that emits this adjustment in thunks prologues.
1509	Result = CGF.Builder.CreateConstByteGEP(Addr: Result, Offset: StaticOffset);
1510	} else {
1511	Result = CGF.Builder.CreateConstInBoundsByteGEP(Addr: Result, Offset: StaticOffset);
1512	}
1513	}
1514	return Result;
1515	}
1516
1517	void MicrosoftCXXABI::addImplicitStructorParams(CodeGenFunction &CGF,
1518	QualType &ResTy,
1519	FunctionArgList &Params) {
1520	ASTContext &Context = getContext();
1521	const CXXMethodDecl *MD = cast<CXXMethodDecl>(Val: CGF.CurGD.getDecl());
1522	assert(isa<CXXConstructorDecl>(MD) || isa<CXXDestructorDecl>(MD));
1523	if (isa<CXXConstructorDecl>(Val: MD) && MD->getParent()->getNumVBases()) {
1524	auto *IsMostDerived = ImplicitParamDecl::Create(
1525	Context, /*DC=*/nullptr, CGF.CurGD.getDecl()->getLocation(),
1526	&Context.Idents.get(Name: "is_most_derived"), Context.IntTy,
1527	ImplicitParamKind::Other);
1528	// The 'most_derived' parameter goes second if the ctor is variadic and last
1529	// if it's not. Dtors can't be variadic.
1530	const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
1531	if (FPT->isVariadic())
1532	Params.insert(Params.begin() + 1, IsMostDerived);
1533	else
1534	Params.push_back(Elt: IsMostDerived);
1535	getStructorImplicitParamDecl(CGF) = IsMostDerived;
1536	} else if (isDeletingDtor(GD: CGF.CurGD)) {
1537	auto *ShouldDelete = ImplicitParamDecl::Create(
1538	Context, /*DC=*/nullptr, CGF.CurGD.getDecl()->getLocation(),
1539	&Context.Idents.get(Name: "should_call_delete"), Context.IntTy,
1540	ImplicitParamKind::Other);
1541	Params.push_back(Elt: ShouldDelete);
1542	getStructorImplicitParamDecl(CGF) = ShouldDelete;
1543	}
1544	}
1545
1546	void MicrosoftCXXABI::EmitInstanceFunctionProlog(CodeGenFunction &CGF) {
1547	// Naked functions have no prolog.
1548	if (CGF.CurFuncDecl && CGF.CurFuncDecl->hasAttr<NakedAttr>())
1549	return;
1550
1551	// Overridden virtual methods of non-primary bases need to adjust the incoming
1552	// 'this' pointer in the prologue. In this hierarchy, C::b will subtract
1553	// sizeof(void*) to adjust from B* to C*:
1554	// struct A { virtual void a(); };
1555	// struct B { virtual void b(); };
1556	// struct C : A, B { virtual void b(); };
1557	//
1558	// Leave the value stored in the 'this' alloca unadjusted, so that the
1559	// debugger sees the unadjusted value. Microsoft debuggers require this, and
1560	// will apply the ThisAdjustment in the method type information.
1561	// FIXME: Do something better for DWARF debuggers, which won't expect this,
1562	// without making our codegen depend on debug info settings.
1563	llvm::Value *This = loadIncomingCXXThis(CGF);
1564	const CXXMethodDecl *MD = cast<CXXMethodDecl>(Val: CGF.CurGD.getDecl());
1565	if (!CGF.CurFuncIsThunk && MD->isVirtual()) {
1566	CharUnits Adjustment = getVirtualFunctionPrologueThisAdjustment(GD: CGF.CurGD);
1567	if (!Adjustment.isZero()) {
1568	assert(Adjustment.isPositive());
1569	This = CGF.Builder.CreateConstInBoundsGEP1_32(Ty: CGF.Int8Ty, Ptr: This,
1570	Idx0: -Adjustment.getQuantity());
1571	}
1572	}
1573	setCXXABIThisValue(CGF, ThisPtr: This);
1574
1575	// If this is a function that the ABI specifies returns 'this', initialize
1576	// the return slot to 'this' at the start of the function.
1577	//
1578	// Unlike the setting of return types, this is done within the ABI
1579	// implementation instead of by clients of CGCXXABI because:
1580	// 1) getThisValue is currently protected
1581	// 2) in theory, an ABI could implement 'this' returns some other way;
1582	// HasThisReturn only specifies a contract, not the implementation
1583	if (HasThisReturn(GD: CGF.CurGD) || hasMostDerivedReturn(GD: CGF.CurGD))
1584	CGF.Builder.CreateStore(Val: getThisValue(CGF), Addr: CGF.ReturnValue);
1585
1586	if (isa<CXXConstructorDecl>(Val: MD) && MD->getParent()->getNumVBases()) {
1587	assert(getStructorImplicitParamDecl(CGF) &&
1588	"no implicit parameter for a constructor with virtual bases?");
1589	getStructorImplicitParamValue(CGF)
1590	= CGF.Builder.CreateLoad(
1591	Addr: CGF.GetAddrOfLocalVar(getStructorImplicitParamDecl(CGF)),
1592	Name: "is_most_derived");
1593	}
1594
1595	if (isDeletingDtor(GD: CGF.CurGD)) {
1596	assert(getStructorImplicitParamDecl(CGF) &&
1597	"no implicit parameter for a deleting destructor?");
1598	getStructorImplicitParamValue(CGF)
1599	= CGF.Builder.CreateLoad(
1600	Addr: CGF.GetAddrOfLocalVar(getStructorImplicitParamDecl(CGF)),
1601	Name: "should_call_delete");
1602	}
1603	}
1604
1605	CGCXXABI::AddedStructorArgs MicrosoftCXXABI::getImplicitConstructorArgs(
1606	CodeGenFunction &CGF, const CXXConstructorDecl *D, CXXCtorType Type,
1607	bool ForVirtualBase, bool Delegating) {
1608	assert(Type == Ctor_Complete || Type == Ctor_Base);
1609
1610	// Check if we need a 'most_derived' parameter.
1611	if (!D->getParent()->getNumVBases())
1612	return AddedStructorArgs{};
1613
1614	// Add the 'most_derived' argument second if we are variadic or last if not.
1615	const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
1616	llvm::Value *MostDerivedArg;
1617	if (Delegating) {
1618	MostDerivedArg = getStructorImplicitParamValue(CGF);
1619	} else {
1620	MostDerivedArg = llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: Type == Ctor_Complete);
1621	}
1622	if (FPT->isVariadic()) {
1623	return AddedStructorArgs::prefix(Args: {{MostDerivedArg, getContext().IntTy}});
1624	}
1625	return AddedStructorArgs::suffix(Args: {{MostDerivedArg, getContext().IntTy}});
1626	}
1627
1628	llvm::Value *MicrosoftCXXABI::getCXXDestructorImplicitParam(
1629	CodeGenFunction &CGF, const CXXDestructorDecl *DD, CXXDtorType Type,
1630	bool ForVirtualBase, bool Delegating) {
1631	return nullptr;
1632	}
1633
1634	void MicrosoftCXXABI::EmitDestructorCall(CodeGenFunction &CGF,
1635	const CXXDestructorDecl *DD,
1636	CXXDtorType Type, bool ForVirtualBase,
1637	bool Delegating, Address This,
1638	QualType ThisTy) {
1639	// Use the base destructor variant in place of the complete destructor variant
1640	// if the class has no virtual bases. This effectively implements some of the
1641	// -mconstructor-aliases optimization, but as part of the MS C++ ABI.
1642	if (Type == Dtor_Complete && DD->getParent()->getNumVBases() == 0)
1643	Type = Dtor_Base;
1644
1645	GlobalDecl GD(DD, Type);
1646	CGCallee Callee = CGCallee::forDirect(functionPtr: CGM.getAddrOfCXXStructor(GD), abstractInfo: GD);
1647
1648	if (DD->isVirtual()) {
1649	assert(Type != CXXDtorType::Dtor_Deleting &&
1650	"The deleting destructor should only be called via a virtual call");
1651	This = adjustThisArgumentForVirtualFunctionCall(CGF, GD: GlobalDecl(DD, Type),
1652	This, VirtualCall: false);
1653	}
1654
1655	llvm::BasicBlock *BaseDtorEndBB = nullptr;
1656	if (ForVirtualBase && isa<CXXConstructorDecl>(Val: CGF.CurCodeDecl)) {
1657	BaseDtorEndBB = EmitDtorCompleteObjectHandler(CGF);
1658	}
1659
1660	llvm::Value *Implicit =
1661	getCXXDestructorImplicitParam(CGF, DD, Type, ForVirtualBase,
1662	Delegating); // = nullptr
1663	CGF.EmitCXXDestructorCall(Dtor: GD, Callee, This: This.getPointer(), ThisTy,
1664	/*ImplicitParam=*/Implicit,
1665	/*ImplicitParamTy=*/QualType(), E: nullptr);
1666	if (BaseDtorEndBB) {
1667	// Complete object handler should continue to be the remaining
1668	CGF.Builder.CreateBr(Dest: BaseDtorEndBB);
1669	CGF.EmitBlock(BB: BaseDtorEndBB);
1670	}
1671	}
1672
1673	void MicrosoftCXXABI::emitVTableTypeMetadata(const VPtrInfo &Info,
1674	const CXXRecordDecl *RD,
1675	llvm::GlobalVariable *VTable) {
1676	// Emit type metadata on vtables with LTO or IR instrumentation.
1677	// In IR instrumentation, the type metadata could be used to find out vtable
1678	// definitions (for type profiling) among all global variables.
1679	if (!CGM.getCodeGenOpts().LTOUnit &&
1680	!CGM.getCodeGenOpts().hasProfileIRInstr())
1681	return;
1682
1683	// TODO: Should VirtualFunctionElimination also be supported here?
1684	// See similar handling in CodeGenModule::EmitVTableTypeMetadata.
1685	if (CGM.getCodeGenOpts().WholeProgramVTables) {
1686	llvm::DenseSet<const CXXRecordDecl *> Visited;
1687	llvm::GlobalObject::VCallVisibility TypeVis =
1688	CGM.GetVCallVisibilityLevel(RD, Visited);
1689	if (TypeVis != llvm::GlobalObject::VCallVisibilityPublic)
1690	VTable->setVCallVisibilityMetadata(TypeVis);
1691	}
1692
1693	// The location of the first virtual function pointer in the virtual table,
1694	// aka the "address point" on Itanium. This is at offset 0 if RTTI is
1695	// disabled, or sizeof(void*) if RTTI is enabled.
1696	CharUnits AddressPoint =
1697	getContext().getLangOpts().RTTIData
1698	? getContext().toCharUnitsFromBits(
1699	BitSize: getContext().getTargetInfo().getPointerWidth(AddrSpace: LangAS::Default))
1700	: CharUnits::Zero();
1701
1702	if (Info.PathToIntroducingObject.empty()) {
1703	CGM.AddVTableTypeMetadata(VTable, Offset: AddressPoint, RD);
1704	return;
1705	}
1706
1707	// Add a bitset entry for the least derived base belonging to this vftable.
1708	CGM.AddVTableTypeMetadata(VTable, Offset: AddressPoint,
1709	RD: Info.PathToIntroducingObject.back());
1710
1711	// Add a bitset entry for each derived class that is laid out at the same
1712	// offset as the least derived base.
1713	for (unsigned I = Info.PathToIntroducingObject.size() - 1; I != 0; --I) {
1714	const CXXRecordDecl *DerivedRD = Info.PathToIntroducingObject[I - 1];
1715	const CXXRecordDecl *BaseRD = Info.PathToIntroducingObject[I];
1716
1717	const ASTRecordLayout &Layout =
1718	getContext().getASTRecordLayout(DerivedRD);
1719	CharUnits Offset;
1720	auto VBI = Layout.getVBaseOffsetsMap().find(Val: BaseRD);
1721	if (VBI == Layout.getVBaseOffsetsMap().end())
1722	Offset = Layout.getBaseClassOffset(Base: BaseRD);
1723	else
1724	Offset = VBI->second.VBaseOffset;
1725	if (!Offset.isZero())
1726	return;
1727	CGM.AddVTableTypeMetadata(VTable, Offset: AddressPoint, RD: DerivedRD);
1728	}
1729
1730	// Finally do the same for the most derived class.
1731	if (Info.FullOffsetInMDC.isZero())
1732	CGM.AddVTableTypeMetadata(VTable, Offset: AddressPoint, RD);
1733	}
1734
1735	void MicrosoftCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT,
1736	const CXXRecordDecl *RD) {
1737	MicrosoftVTableContext &VFTContext = CGM.getMicrosoftVTableContext();
1738	const VPtrInfoVector &VFPtrs = VFTContext.getVFPtrOffsets(RD);
1739
1740	for (const std::unique_ptr<VPtrInfo>& Info : VFPtrs) {
1741	llvm::GlobalVariable *VTable = getAddrOfVTable(RD, VPtrOffset: Info->FullOffsetInMDC);
1742	if (VTable->hasInitializer())
1743	continue;
1744
1745	const VTableLayout &VTLayout =
1746	VFTContext.getVFTableLayout(RD, VFPtrOffset: Info->FullOffsetInMDC);
1747
1748	llvm::Constant *RTTI = nullptr;
1749	if (any_of(Range: VTLayout.vtable_components(),
1750	P: [](const VTableComponent &VTC) { return VTC.isRTTIKind(); }))
1751	RTTI = getMSCompleteObjectLocator(RD, Info: *Info);
1752
1753	ConstantInitBuilder builder(CGM);
1754	auto components = builder.beginStruct();
1755	CGVT.createVTableInitializer(builder&: components, layout: VTLayout, rtti: RTTI,
1756	vtableHasLocalLinkage: VTable->hasLocalLinkage());
1757	components.finishAndSetAsInitializer(global: VTable);
1758
1759	emitVTableTypeMetadata(Info: *Info, RD, VTable);
1760	}
1761	}
1762
1763	bool MicrosoftCXXABI::isVirtualOffsetNeededForVTableField(
1764	CodeGenFunction &CGF, CodeGenFunction::VPtr Vptr) {
1765	return Vptr.NearestVBase != nullptr;
1766	}
1767
1768	llvm::Value *MicrosoftCXXABI::getVTableAddressPointInStructor(
1769	CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, BaseSubobject Base,
1770	const CXXRecordDecl *NearestVBase) {
1771	llvm::Constant *VTableAddressPoint = getVTableAddressPoint(Base, VTableClass);
1772	if (!VTableAddressPoint) {
1773	assert(Base.getBase()->getNumVBases() &&
1774	!getContext().getASTRecordLayout(Base.getBase()).hasOwnVFPtr());
1775	}
1776	return VTableAddressPoint;
1777	}
1778
1779	static void mangleVFTableName(MicrosoftMangleContext &MangleContext,
1780	const CXXRecordDecl *RD, const VPtrInfo &VFPtr,
1781	SmallString<256> &Name) {
1782	llvm::raw_svector_ostream Out(Name);
1783	MangleContext.mangleCXXVFTable(Derived: RD, BasePath: VFPtr.MangledPath, Out);
1784	}
1785
1786	llvm::Constant *
1787	MicrosoftCXXABI::getVTableAddressPoint(BaseSubobject Base,
1788	const CXXRecordDecl *VTableClass) {
1789	(void)getAddrOfVTable(RD: VTableClass, VPtrOffset: Base.getBaseOffset());
1790	VFTableIdTy ID(VTableClass, Base.getBaseOffset());
1791	return VFTablesMap[ID];
1792	}
1793
1794	llvm::Constant *MicrosoftCXXABI::getVTableAddressPointForConstExpr(
1795	BaseSubobject Base, const CXXRecordDecl *VTableClass) {
1796	llvm::Constant *VFTable = getVTableAddressPoint(Base, VTableClass);
1797	assert(VFTable && "Couldn't find a vftable for the given base?");
1798	return VFTable;
1799	}
1800
1801	llvm::GlobalVariable *MicrosoftCXXABI::getAddrOfVTable(const CXXRecordDecl *RD,
1802	CharUnits VPtrOffset) {
1803	// getAddrOfVTable may return 0 if asked to get an address of a vtable which
1804	// shouldn't be used in the given record type. We want to cache this result in
1805	// VFTablesMap, thus a simple zero check is not sufficient.
1806
1807	VFTableIdTy ID(RD, VPtrOffset);
1808	VTablesMapTy::iterator I;
1809	bool Inserted;
1810	std::tie(args&: I, args&: Inserted) = VTablesMap.insert(KV: std::make_pair(x&: ID, y: nullptr));
1811	if (!Inserted)
1812	return I->second;
1813
1814	llvm::GlobalVariable *&VTable = I->second;
1815
1816	MicrosoftVTableContext &VTContext = CGM.getMicrosoftVTableContext();
1817	const VPtrInfoVector &VFPtrs = VTContext.getVFPtrOffsets(RD);
1818
1819	if (DeferredVFTables.insert(Ptr: RD).second) {
1820	// We haven't processed this record type before.
1821	// Queue up this vtable for possible deferred emission.
1822	CGM.addDeferredVTable(RD);
1823
1824	#ifndef NDEBUG
1825	// Create all the vftables at once in order to make sure each vftable has
1826	// a unique mangled name.
1827	llvm::StringSet<> ObservedMangledNames;
1828	for (size_t J = 0, F = VFPtrs.size(); J != F; ++J) {
1829	SmallString<256> Name;
1830	mangleVFTableName(MangleContext&: getMangleContext(), RD, VFPtr: *VFPtrs[J], Name);
1831	if (!ObservedMangledNames.insert(key: Name.str()).second)
1832	llvm_unreachable("Already saw this mangling before?");
1833	}
1834	#endif
1835	}
1836
1837	const std::unique_ptr<VPtrInfo> *VFPtrI =
1838	llvm::find_if(Range: VFPtrs, P: [&](const std::unique_ptr<VPtrInfo> &VPI) {
1839	return VPI->FullOffsetInMDC == VPtrOffset;
1840	});
1841	if (VFPtrI == VFPtrs.end()) {
1842	VFTablesMap[ID] = nullptr;
1843	return nullptr;
1844	}
1845	const std::unique_ptr<VPtrInfo> &VFPtr = *VFPtrI;
1846
1847	SmallString<256> VFTableName;
1848	mangleVFTableName(MangleContext&: getMangleContext(), RD, VFPtr: *VFPtr, Name&: VFTableName);
1849
1850	// Classes marked __declspec(dllimport) need vftables generated on the
1851	// import-side in order to support features like constexpr. No other
1852	// translation unit relies on the emission of the local vftable, translation
1853	// units are expected to generate them as needed.
1854	//
1855	// Because of this unique behavior, we maintain this logic here instead of
1856	// getVTableLinkage.
1857	llvm::GlobalValue::LinkageTypes VFTableLinkage =
1858	RD->hasAttr<DLLImportAttr>() ? llvm::GlobalValue::LinkOnceODRLinkage
1859	: CGM.getVTableLinkage(RD);
1860	bool VFTableComesFromAnotherTU =
1861	llvm::GlobalValue::isAvailableExternallyLinkage(Linkage: VFTableLinkage) ||
1862	llvm::GlobalValue::isExternalLinkage(Linkage: VFTableLinkage);
1863	bool VTableAliasIsRequred =
1864	!VFTableComesFromAnotherTU && getContext().getLangOpts().RTTIData;
1865
1866	if (llvm::GlobalValue *VFTable =
1867	CGM.getModule().getNamedGlobal(Name: VFTableName)) {
1868	VFTablesMap[ID] = VFTable;
1869	VTable = VTableAliasIsRequred
1870	? cast<llvm::GlobalVariable>(
1871	Val: cast<llvm::GlobalAlias>(Val: VFTable)->getAliaseeObject())
1872	: cast<llvm::GlobalVariable>(Val: VFTable);
1873	return VTable;
1874	}
1875
1876	const VTableLayout &VTLayout =
1877	VTContext.getVFTableLayout(RD, VFPtrOffset: VFPtr->FullOffsetInMDC);
1878	llvm::GlobalValue::LinkageTypes VTableLinkage =
1879	VTableAliasIsRequred ? llvm::GlobalValue::PrivateLinkage : VFTableLinkage;
1880
1881	StringRef VTableName = VTableAliasIsRequred ? StringRef() : VFTableName.str();
1882
1883	llvm::Type *VTableType = CGM.getVTables().getVTableType(layout: VTLayout);
1884
1885	// Create a backing variable for the contents of VTable. The VTable may
1886	// or may not include space for a pointer to RTTI data.
1887	llvm::GlobalValue *VFTable;
1888	VTable = new llvm::GlobalVariable(CGM.getModule(), VTableType,
1889	/*isConstant=*/true, VTableLinkage,
1890	/*Initializer=*/nullptr, VTableName);
1891	VTable->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
1892
1893	llvm::Comdat *C = nullptr;
1894	if (!VFTableComesFromAnotherTU &&
1895	llvm::GlobalValue::isWeakForLinker(Linkage: VFTableLinkage))
1896	C = CGM.getModule().getOrInsertComdat(Name: VFTableName.str());
1897
1898	// Only insert a pointer into the VFTable for RTTI data if we are not
1899	// importing it. We never reference the RTTI data directly so there is no
1900	// need to make room for it.
1901	if (VTableAliasIsRequred) {
1902	llvm::Value *GEPIndices[] = {llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 0),
1903	llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 0),
1904	llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 1)};
1905	// Create a GEP which points just after the first entry in the VFTable,
1906	// this should be the location of the first virtual method.
1907	llvm::Constant *VTableGEP = llvm::ConstantExpr::getInBoundsGetElementPtr(
1908	Ty: VTable->getValueType(), C: VTable, IdxList: GEPIndices);
1909	if (llvm::GlobalValue::isWeakForLinker(Linkage: VFTableLinkage)) {
1910	VFTableLinkage = llvm::GlobalValue::ExternalLinkage;
1911	if (C)
1912	C->setSelectionKind(llvm::Comdat::Largest);
1913	}
1914	VFTable = llvm::GlobalAlias::create(Ty: CGM.Int8PtrTy,
1915	/*AddressSpace=*/0, Linkage: VFTableLinkage,
1916	Name: VFTableName.str(), Aliasee: VTableGEP,
1917	Parent: &CGM.getModule());
1918	VFTable->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
1919	} else {
1920	// We don't need a GlobalAlias to be a symbol for the VTable if we won't
1921	// be referencing any RTTI data.
1922	// The GlobalVariable will end up being an appropriate definition of the
1923	// VFTable.
1924	VFTable = VTable;
1925	}
1926	if (C)
1927	VTable->setComdat(C);
1928
1929	if (RD->hasAttr<DLLExportAttr>())
1930	VFTable->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass);
1931
1932	VFTablesMap[ID] = VFTable;
1933	return VTable;
1934	}
1935
1936	CGCallee MicrosoftCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF,
1937	GlobalDecl GD,
1938	Address This,
1939	llvm::Type *Ty,
1940	SourceLocation Loc) {
1941	CGBuilderTy &Builder = CGF.Builder;
1942
1943	Ty = Ty->getPointerTo();
1944	Address VPtr =
1945	adjustThisArgumentForVirtualFunctionCall(CGF, GD, This, VirtualCall: true);
1946
1947	auto *MethodDecl = cast<CXXMethodDecl>(Val: GD.getDecl());
1948	llvm::Value *VTable = CGF.GetVTablePtr(This: VPtr, VTableTy: Ty->getPointerTo(),
1949	VTableClass: MethodDecl->getParent());
1950
1951	MicrosoftVTableContext &VFTContext = CGM.getMicrosoftVTableContext();
1952	MethodVFTableLocation ML = VFTContext.getMethodVFTableLocation(GD);
1953
1954	// Compute the identity of the most derived class whose virtual table is
1955	// located at the MethodVFTableLocation ML.
1956	auto getObjectWithVPtr = [&] {
1957	return llvm::find_if(Range: VFTContext.getVFPtrOffsets(
1958	RD: ML.VBase ? ML.VBase : MethodDecl->getParent()),
1959	P: [&](const std::unique_ptr<VPtrInfo> &Info) {
1960	return Info->FullOffsetInMDC == ML.VFPtrOffset;
1961	})
1962	->get()
1963	->ObjectWithVPtr;
1964	};
1965
1966	llvm::Value *VFunc;
1967	if (CGF.ShouldEmitVTableTypeCheckedLoad(RD: MethodDecl->getParent())) {
1968	VFunc = CGF.EmitVTableTypeCheckedLoad(
1969	RD: getObjectWithVPtr(), VTable, VTableTy: Ty,
1970	VTableByteOffset: ML.Index *
1971	CGM.getContext().getTargetInfo().getPointerWidth(AddrSpace: LangAS::Default) /
1972	8);
1973	} else {
1974	if (CGM.getCodeGenOpts().PrepareForLTO)
1975	CGF.EmitTypeMetadataCodeForVCall(RD: getObjectWithVPtr(), VTable, Loc);
1976
1977	llvm::Value *VFuncPtr =
1978	Builder.CreateConstInBoundsGEP1_64(Ty, Ptr: VTable, Idx0: ML.Index, Name: "vfn");
1979	VFunc = Builder.CreateAlignedLoad(Ty, VFuncPtr, CGF.getPointerAlign());
1980	}
1981
1982	CGCallee Callee(GD, VFunc);
1983	return Callee;
1984	}
1985
1986	llvm::Value *MicrosoftCXXABI::EmitVirtualDestructorCall(
1987	CodeGenFunction &CGF, const CXXDestructorDecl *Dtor, CXXDtorType DtorType,
1988	Address This, DeleteOrMemberCallExpr E) {
1989	auto *CE = E.dyn_cast<const CXXMemberCallExpr *>();
1990	auto *D = E.dyn_cast<const CXXDeleteExpr *>();
1991	assert((CE != nullptr) ^ (D != nullptr));
1992	assert(CE == nullptr || CE->arg_begin() == CE->arg_end());
1993	assert(DtorType == Dtor_Deleting || DtorType == Dtor_Complete);
1994
1995	// We have only one destructor in the vftable but can get both behaviors
1996	// by passing an implicit int parameter.
1997	GlobalDecl GD(Dtor, Dtor_Deleting);
1998	const CGFunctionInfo *FInfo =
1999	&CGM.getTypes().arrangeCXXStructorDeclaration(GD);
2000	llvm::FunctionType *Ty = CGF.CGM.getTypes().GetFunctionType(Info: *FInfo);
2001	CGCallee Callee = CGCallee::forVirtual(CE, GD, This, Ty);
2002
2003	ASTContext &Context = getContext();
2004	llvm::Value *ImplicitParam = llvm::ConstantInt::get(
2005	Ty: llvm::IntegerType::getInt32Ty(C&: CGF.getLLVMContext()),
2006	V: DtorType == Dtor_Deleting);
2007
2008	QualType ThisTy;
2009	if (CE) {
2010	ThisTy = CE->getObjectType();
2011	} else {
2012	ThisTy = D->getDestroyedType();
2013	}
2014
2015	This = adjustThisArgumentForVirtualFunctionCall(CGF, GD, This, VirtualCall: true);
2016	RValue RV = CGF.EmitCXXDestructorCall(GD, Callee, This.getPointer(), ThisTy,
2017	ImplicitParam, Context.IntTy, CE);
2018	return RV.getScalarVal();
2019	}
2020
2021	const VBTableGlobals &
2022	MicrosoftCXXABI::enumerateVBTables(const CXXRecordDecl *RD) {
2023	// At this layer, we can key the cache off of a single class, which is much
2024	// easier than caching each vbtable individually.
2025	llvm::DenseMap<const CXXRecordDecl*, VBTableGlobals>::iterator Entry;
2026	bool Added;
2027	std::tie(args&: Entry, args&: Added) =
2028	VBTablesMap.insert(KV: std::make_pair(x&: RD, y: VBTableGlobals()));
2029	VBTableGlobals &VBGlobals = Entry->second;
2030	if (!Added)
2031	return VBGlobals;
2032
2033	MicrosoftVTableContext &Context = CGM.getMicrosoftVTableContext();
2034	VBGlobals.VBTables = &Context.enumerateVBTables(RD);
2035
2036	// Cache the globals for all vbtables so we don't have to recompute the
2037	// mangled names.
2038	llvm::GlobalVariable::LinkageTypes Linkage = CGM.getVTableLinkage(RD);
2039	for (VPtrInfoVector::const_iterator I = VBGlobals.VBTables->begin(),
2040	E = VBGlobals.VBTables->end();
2041	I != E; ++I) {
2042	VBGlobals.Globals.push_back(Elt: getAddrOfVBTable(VBT: **I, RD, Linkage));
2043	}
2044
2045	return VBGlobals;
2046	}
2047
2048	llvm::Function *
2049	MicrosoftCXXABI::EmitVirtualMemPtrThunk(const CXXMethodDecl *MD,
2050	const MethodVFTableLocation &ML) {
2051	assert(!isa<CXXConstructorDecl>(MD) && !isa<CXXDestructorDecl>(MD) &&
2052	"can't form pointers to ctors or virtual dtors");
2053
2054	// Calculate the mangled name.
2055	SmallString<256> ThunkName;
2056	llvm::raw_svector_ostream Out(ThunkName);
2057	getMangleContext().mangleVirtualMemPtrThunk(MD, ML, Out);
2058
2059	// If the thunk has been generated previously, just return it.
2060	if (llvm::GlobalValue *GV = CGM.getModule().getNamedValue(Name: ThunkName))
2061	return cast<llvm::Function>(Val: GV);
2062
2063	// Create the llvm::Function.
2064	const CGFunctionInfo &FnInfo =
2065	CGM.getTypes().arrangeUnprototypedMustTailThunk(MD);
2066	llvm::FunctionType *ThunkTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
2067	llvm::Function *ThunkFn =
2068	llvm::Function::Create(Ty: ThunkTy, Linkage: llvm::Function::ExternalLinkage,
2069	N: ThunkName.str(), M: &CGM.getModule());
2070	assert(ThunkFn->getName() == ThunkName && "name was uniqued!");
2071
2072	ThunkFn->setLinkage(MD->isExternallyVisible()
2073	? llvm::GlobalValue::LinkOnceODRLinkage
2074	: llvm::GlobalValue::InternalLinkage);
2075	if (MD->isExternallyVisible())
2076	ThunkFn->setComdat(CGM.getModule().getOrInsertComdat(Name: ThunkFn->getName()));
2077
2078	CGM.SetLLVMFunctionAttributes(MD, FnInfo, ThunkFn, /*IsThunk=*/false);
2079	CGM.SetLLVMFunctionAttributesForDefinition(MD, ThunkFn);
2080
2081	// Add the "thunk" attribute so that LLVM knows that the return type is
2082	// meaningless. These thunks can be used to call functions with differing
2083	// return types, and the caller is required to cast the prototype
2084	// appropriately to extract the correct value.
2085	ThunkFn->addFnAttr(Kind: "thunk");
2086
2087	// These thunks can be compared, so they are not unnamed.
2088	ThunkFn->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::None);
2089
2090	// Start codegen.
2091	CodeGenFunction CGF(CGM);
2092	CGF.CurGD = GlobalDecl(MD);
2093	CGF.CurFuncIsThunk = true;
2094
2095	// Build FunctionArgs, but only include the implicit 'this' parameter
2096	// declaration.
2097	FunctionArgList FunctionArgs;
2098	buildThisParam(CGF, Params&: FunctionArgs);
2099
2100	// Start defining the function.
2101	CGF.StartFunction(GD: GlobalDecl(), RetTy: FnInfo.getReturnType(), Fn: ThunkFn, FnInfo,
2102	Args: FunctionArgs, Loc: MD->getLocation(), StartLoc: SourceLocation());
2103
2104	ApplyDebugLocation AL(CGF, MD->getLocation());
2105	setCXXABIThisValue(CGF, ThisPtr: loadIncomingCXXThis(CGF));
2106
2107	// Load the vfptr and then callee from the vftable. The callee should have
2108	// adjusted 'this' so that the vfptr is at offset zero.
2109	llvm::Type *ThunkPtrTy = ThunkTy->getPointerTo();
2110	llvm::Value *VTable = CGF.GetVTablePtr(
2111	This: getThisAddress(CGF), VTableTy: ThunkPtrTy->getPointerTo(), VTableClass: MD->getParent());
2112
2113	llvm::Value *VFuncPtr = CGF.Builder.CreateConstInBoundsGEP1_64(
2114	Ty: ThunkPtrTy, Ptr: VTable, Idx0: ML.Index, Name: "vfn");
2115	llvm::Value *Callee =
2116	CGF.Builder.CreateAlignedLoad(ThunkPtrTy, VFuncPtr, CGF.getPointerAlign());
2117
2118	CGF.EmitMustTailThunk(MD, getThisValue(CGF), {ThunkTy, Callee});
2119
2120	return ThunkFn;
2121	}
2122
2123	void MicrosoftCXXABI::emitVirtualInheritanceTables(const CXXRecordDecl *RD) {
2124	const VBTableGlobals &VBGlobals = enumerateVBTables(RD);
2125	for (unsigned I = 0, E = VBGlobals.VBTables->size(); I != E; ++I) {
2126	const std::unique_ptr<VPtrInfo>& VBT = (*VBGlobals.VBTables)[I];
2127	llvm::GlobalVariable *GV = VBGlobals.Globals[I];
2128	if (GV->isDeclaration())
2129	emitVBTableDefinition(VBT: *VBT, RD, GV);
2130	}
2131	}
2132
2133	llvm::GlobalVariable *
2134	MicrosoftCXXABI::getAddrOfVBTable(const VPtrInfo &VBT, const CXXRecordDecl *RD,
2135	llvm::GlobalVariable::LinkageTypes Linkage) {
2136	SmallString<256> OutName;
2137	llvm::raw_svector_ostream Out(OutName);
2138	getMangleContext().mangleCXXVBTable(Derived: RD, BasePath: VBT.MangledPath, Out);
2139	StringRef Name = OutName.str();
2140
2141	llvm::ArrayType *VBTableType =
2142	llvm::ArrayType::get(ElementType: CGM.IntTy, NumElements: 1 + VBT.ObjectWithVPtr->getNumVBases());
2143
2144	assert(!CGM.getModule().getNamedGlobal(Name) &&
2145	"vbtable with this name already exists: mangling bug?");
2146	CharUnits Alignment =
2147	CGM.getContext().getTypeAlignInChars(CGM.getContext().IntTy);
2148	llvm::GlobalVariable *GV = CGM.CreateOrReplaceCXXRuntimeVariable(
2149	Name, Ty: VBTableType, Linkage, Alignment: Alignment.getAsAlign());
2150	GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
2151
2152	if (RD->hasAttr<DLLImportAttr>())
2153	GV->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass);
2154	else if (RD->hasAttr<DLLExportAttr>())
2155	GV->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass);
2156
2157	if (!GV->hasExternalLinkage())
2158	emitVBTableDefinition(VBT, RD, GV);
2159
2160	return GV;
2161	}
2162
2163	void MicrosoftCXXABI::emitVBTableDefinition(const VPtrInfo &VBT,
2164	const CXXRecordDecl *RD,
2165	llvm::GlobalVariable *GV) const {
2166	const CXXRecordDecl *ObjectWithVPtr = VBT.ObjectWithVPtr;
2167
2168	assert(RD->getNumVBases() && ObjectWithVPtr->getNumVBases() &&
2169	"should only emit vbtables for classes with vbtables");
2170
2171	const ASTRecordLayout &BaseLayout =
2172	getContext().getASTRecordLayout(VBT.IntroducingObject);
2173	const ASTRecordLayout &DerivedLayout = getContext().getASTRecordLayout(RD);
2174
2175	SmallVector<llvm::Constant *, 4> Offsets(1 + ObjectWithVPtr->getNumVBases(),
2176	nullptr);
2177
2178	// The offset from ObjectWithVPtr's vbptr to itself always leads.
2179	CharUnits VBPtrOffset = BaseLayout.getVBPtrOffset();
2180	Offsets[0] = llvm::ConstantInt::get(Ty: CGM.IntTy, V: -VBPtrOffset.getQuantity());
2181
2182	MicrosoftVTableContext &Context = CGM.getMicrosoftVTableContext();
2183	for (const auto &I : ObjectWithVPtr->vbases()) {
2184	const CXXRecordDecl *VBase = I.getType()->getAsCXXRecordDecl();
2185	CharUnits Offset = DerivedLayout.getVBaseClassOffset(VBase);
2186	assert(!Offset.isNegative());
2187
2188	// Make it relative to the subobject vbptr.
2189	CharUnits CompleteVBPtrOffset = VBT.NonVirtualOffset + VBPtrOffset;
2190	if (VBT.getVBaseWithVPtr())
2191	CompleteVBPtrOffset +=
2192	DerivedLayout.getVBaseClassOffset(VBase: VBT.getVBaseWithVPtr());
2193	Offset -= CompleteVBPtrOffset;
2194
2195	unsigned VBIndex = Context.getVBTableIndex(Derived: ObjectWithVPtr, VBase);
2196	assert(Offsets[VBIndex] == nullptr && "The same vbindex seen twice?");
2197	Offsets[VBIndex] = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Offset.getQuantity());
2198	}
2199
2200	assert(Offsets.size() ==
2201	cast<llvm::ArrayType>(GV->getValueType())->getNumElements());
2202	llvm::ArrayType *VBTableType =
2203	llvm::ArrayType::get(ElementType: CGM.IntTy, NumElements: Offsets.size());
2204	llvm::Constant *Init = llvm::ConstantArray::get(T: VBTableType, V: Offsets);
2205	GV->setInitializer(Init);
2206
2207	if (RD->hasAttr<DLLImportAttr>())
2208	GV->setLinkage(llvm::GlobalVariable::AvailableExternallyLinkage);
2209	}
2210
2211	llvm::Value *MicrosoftCXXABI::performThisAdjustment(CodeGenFunction &CGF,
2212	Address This,
2213	const ThisAdjustment &TA) {
2214	if (TA.isEmpty())
2215	return This.getPointer();
2216
2217	This = This.withElementType(ElemTy: CGF.Int8Ty);
2218
2219	llvm::Value *V;
2220	if (TA.Virtual.isEmpty()) {
2221	V = This.getPointer();
2222	} else {
2223	assert(TA.Virtual.Microsoft.VtordispOffset < 0);
2224	// Adjust the this argument based on the vtordisp value.
2225	Address VtorDispPtr =
2226	CGF.Builder.CreateConstInBoundsByteGEP(Addr: This,
2227	Offset: CharUnits::fromQuantity(Quantity: TA.Virtual.Microsoft.VtordispOffset));
2228	VtorDispPtr = VtorDispPtr.withElementType(ElemTy: CGF.Int32Ty);
2229	llvm::Value *VtorDisp = CGF.Builder.CreateLoad(Addr: VtorDispPtr, Name: "vtordisp");
2230	V = CGF.Builder.CreateGEP(Ty: This.getElementType(), Ptr: This.getPointer(),
2231	IdxList: CGF.Builder.CreateNeg(V: VtorDisp));
2232
2233	// Unfortunately, having applied the vtordisp means that we no
2234	// longer really have a known alignment for the vbptr step.
2235	// We'll assume the vbptr is pointer-aligned.
2236
2237	if (TA.Virtual.Microsoft.VBPtrOffset) {
2238	// If the final overrider is defined in a virtual base other than the one
2239	// that holds the vfptr, we have to use a vtordispex thunk which looks up
2240	// the vbtable of the derived class.
2241	assert(TA.Virtual.Microsoft.VBPtrOffset > 0);
2242	assert(TA.Virtual.Microsoft.VBOffsetOffset >= 0);
2243	llvm::Value *VBPtr;
2244	llvm::Value *VBaseOffset = GetVBaseOffsetFromVBPtr(
2245	CGF, Base: Address(V, CGF.Int8Ty, CGF.getPointerAlign()),
2246	VBPtrOffset: -TA.Virtual.Microsoft.VBPtrOffset,
2247	VBTableOffset: TA.Virtual.Microsoft.VBOffsetOffset, VBPtr: &VBPtr);
2248	V = CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: VBPtr, IdxList: VBaseOffset);
2249	}
2250	}
2251
2252	if (TA.NonVirtual) {
2253	// Non-virtual adjustment might result in a pointer outside the allocated
2254	// object, e.g. if the final overrider class is laid out after the virtual
2255	// base that declares a method in the most derived class.
2256	V = CGF.Builder.CreateConstGEP1_32(Ty: CGF.Int8Ty, Ptr: V, Idx0: TA.NonVirtual);
2257	}
2258
2259	// Don't need to bitcast back, the call CodeGen will handle this.
2260	return V;
2261	}
2262
2263	llvm::Value *
2264	MicrosoftCXXABI::performReturnAdjustment(CodeGenFunction &CGF, Address Ret,
2265	const ReturnAdjustment &RA) {
2266	if (RA.isEmpty())
2267	return Ret.getPointer();
2268
2269	Ret = Ret.withElementType(ElemTy: CGF.Int8Ty);
2270
2271	llvm::Value *V = Ret.getPointer();
2272	if (RA.Virtual.Microsoft.VBIndex) {
2273	assert(RA.Virtual.Microsoft.VBIndex > 0);
2274	int32_t IntSize = CGF.getIntSize().getQuantity();
2275	llvm::Value *VBPtr;
2276	llvm::Value *VBaseOffset =
2277	GetVBaseOffsetFromVBPtr(CGF, Base: Ret, VBPtrOffset: RA.Virtual.Microsoft.VBPtrOffset,
2278	VBTableOffset: IntSize * RA.Virtual.Microsoft.VBIndex, VBPtr: &VBPtr);
2279	V = CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: VBPtr, IdxList: VBaseOffset);
2280	}
2281
2282	if (RA.NonVirtual)
2283	V = CGF.Builder.CreateConstInBoundsGEP1_32(Ty: CGF.Int8Ty, Ptr: V, Idx0: RA.NonVirtual);
2284
2285	return V;
2286	}
2287
2288	bool MicrosoftCXXABI::requiresArrayCookie(const CXXDeleteExpr *expr,
2289	QualType elementType) {
2290	// Microsoft seems to completely ignore the possibility of a
2291	// two-argument usual deallocation function.
2292	return elementType.isDestructedType();
2293	}
2294
2295	bool MicrosoftCXXABI::requiresArrayCookie(const CXXNewExpr *expr) {
2296	// Microsoft seems to completely ignore the possibility of a
2297	// two-argument usual deallocation function.
2298	return expr->getAllocatedType().isDestructedType();
2299	}
2300
2301	CharUnits MicrosoftCXXABI::getArrayCookieSizeImpl(QualType type) {
2302	// The array cookie is always a size_t; we then pad that out to the
2303	// alignment of the element type.
2304	ASTContext &Ctx = getContext();
2305	return std::max(a: Ctx.getTypeSizeInChars(T: Ctx.getSizeType()),
2306	b: Ctx.getTypeAlignInChars(T: type));
2307	}
2308
2309	llvm::Value *MicrosoftCXXABI::readArrayCookieImpl(CodeGenFunction &CGF,
2310	Address allocPtr,
2311	CharUnits cookieSize) {
2312	Address numElementsPtr = allocPtr.withElementType(ElemTy: CGF.SizeTy);
2313	return CGF.Builder.CreateLoad(Addr: numElementsPtr);
2314	}
2315
2316	Address MicrosoftCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
2317	Address newPtr,
2318	llvm::Value *numElements,
2319	const CXXNewExpr *expr,
2320	QualType elementType) {
2321	assert(requiresArrayCookie(expr));
2322
2323	// The size of the cookie.
2324	CharUnits cookieSize = getArrayCookieSizeImpl(type: elementType);
2325
2326	// Compute an offset to the cookie.
2327	Address cookiePtr = newPtr;
2328
2329	// Write the number of elements into the appropriate slot.
2330	Address numElementsPtr = cookiePtr.withElementType(ElemTy: CGF.SizeTy);
2331	CGF.Builder.CreateStore(Val: numElements, Addr: numElementsPtr);
2332
2333	// Finally, compute a pointer to the actual data buffer by skipping
2334	// over the cookie completely.
2335	return CGF.Builder.CreateConstInBoundsByteGEP(Addr: newPtr, Offset: cookieSize);
2336	}
2337
2338	static void emitGlobalDtorWithTLRegDtor(CodeGenFunction &CGF, const VarDecl &VD,
2339	llvm::FunctionCallee Dtor,
2340	llvm::Constant *Addr) {
2341	// Create a function which calls the destructor.
2342	llvm::Constant *DtorStub = CGF.createAtExitStub(VD, Dtor, Addr);
2343
2344	// extern "C" int __tlregdtor(void (*f)(void));
2345	llvm::FunctionType *TLRegDtorTy = llvm::FunctionType::get(
2346	Result: CGF.IntTy, Params: DtorStub->getType(), /*isVarArg=*/false);
2347
2348	llvm::FunctionCallee TLRegDtor = CGF.CGM.CreateRuntimeFunction(
2349	Ty: TLRegDtorTy, Name: "__tlregdtor", ExtraAttrs: llvm::AttributeList(), /*Local=*/true);
2350	if (llvm::Function *TLRegDtorFn =
2351	dyn_cast<llvm::Function>(Val: TLRegDtor.getCallee()))
2352	TLRegDtorFn->setDoesNotThrow();
2353
2354	CGF.EmitNounwindRuntimeCall(callee: TLRegDtor, args: DtorStub);
2355	}
2356
2357	void MicrosoftCXXABI::registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D,
2358	llvm::FunctionCallee Dtor,
2359	llvm::Constant *Addr) {
2360	if (D.isNoDestroy(CGM.getContext()))
2361	return;
2362
2363	if (D.getTLSKind())
2364	return emitGlobalDtorWithTLRegDtor(CGF, VD: D, Dtor, Addr);
2365
2366	// HLSL doesn't support atexit.
2367	if (CGM.getLangOpts().HLSL)
2368	return CGM.AddCXXDtorEntry(DtorFn: Dtor, Object: Addr);
2369
2370	// The default behavior is to use atexit.
2371	CGF.registerGlobalDtorWithAtExit(D, fn: Dtor, addr: Addr);
2372	}
2373
2374	void MicrosoftCXXABI::EmitThreadLocalInitFuncs(
2375	CodeGenModule &CGM, ArrayRef<const VarDecl *> CXXThreadLocals,
2376	ArrayRef<llvm::Function *> CXXThreadLocalInits,
2377	ArrayRef<const VarDecl *> CXXThreadLocalInitVars) {
2378	if (CXXThreadLocalInits.empty())
2379	return;
2380
2381	CGM.AppendLinkerOptions(Opts: CGM.getTarget().getTriple().getArch() ==
2382	llvm::Triple::x86
2383	? "/include:___dyn_tls_init@12"
2384	: "/include:__dyn_tls_init");
2385
2386	// This will create a GV in the .CRT$XDU section. It will point to our
2387	// initialization function. The CRT will call all of these function
2388	// pointers at start-up time and, eventually, at thread-creation time.
2389	auto AddToXDU = [&CGM](llvm::Function *InitFunc) {
2390	llvm::GlobalVariable *InitFuncPtr = new llvm::GlobalVariable(
2391	CGM.getModule(), InitFunc->getType(), /*isConstant=*/true,
2392	llvm::GlobalVariable::InternalLinkage, InitFunc,
2393	Twine(InitFunc->getName(), "$initializer$"));
2394	InitFuncPtr->setSection(".CRT$XDU");
2395	// This variable has discardable linkage, we have to add it to @llvm.used to
2396	// ensure it won't get discarded.
2397	CGM.addUsedGlobal(GV: InitFuncPtr);
2398	return InitFuncPtr;
2399	};
2400
2401	std::vector<llvm::Function *> NonComdatInits;
2402	for (size_t I = 0, E = CXXThreadLocalInitVars.size(); I != E; ++I) {
2403	llvm::GlobalVariable *GV = cast<llvm::GlobalVariable>(
2404	Val: CGM.GetGlobalValue(Ref: CGM.getMangledName(GD: CXXThreadLocalInitVars[I])));
2405	llvm::Function *F = CXXThreadLocalInits[I];
2406
2407	// If the GV is already in a comdat group, then we have to join it.
2408	if (llvm::Comdat *C = GV->getComdat())
2409	AddToXDU(F)->setComdat(C);
2410	else
2411	NonComdatInits.push_back(x: F);
2412	}
2413
2414	if (!NonComdatInits.empty()) {
2415	llvm::FunctionType *FTy =
2416	llvm::FunctionType::get(Result: CGM.VoidTy, /*isVarArg=*/false);
2417	llvm::Function *InitFunc = CGM.CreateGlobalInitOrCleanUpFunction(
2418	ty: FTy, name: "__tls_init", FI: CGM.getTypes().arrangeNullaryFunction(),
2419	Loc: SourceLocation(), /*TLS=*/true);
2420	CodeGenFunction(CGM).GenerateCXXGlobalInitFunc(Fn: InitFunc, CXXThreadLocals: NonComdatInits);
2421
2422	AddToXDU(InitFunc);
2423	}
2424	}
2425
2426	static llvm::GlobalValue *getTlsGuardVar(CodeGenModule &CGM) {
2427	// __tls_guard comes from the MSVC runtime and reflects
2428	// whether TLS has been initialized for a particular thread.
2429	// It is set from within __dyn_tls_init by the runtime.
2430	// Every library and executable has its own variable.
2431	llvm::Type *VTy = llvm::Type::getInt8Ty(C&: CGM.getLLVMContext());
2432	llvm::Constant *TlsGuardConstant =
2433	CGM.CreateRuntimeVariable(Ty: VTy, Name: "__tls_guard");
2434	llvm::GlobalValue *TlsGuard = cast<llvm::GlobalValue>(Val: TlsGuardConstant);
2435
2436	TlsGuard->setThreadLocal(true);
2437
2438	return TlsGuard;
2439	}
2440
2441	static llvm::FunctionCallee getDynTlsOnDemandInitFn(CodeGenModule &CGM) {
2442	// __dyn_tls_on_demand_init comes from the MSVC runtime and triggers
2443	// dynamic TLS initialization by calling __dyn_tls_init internally.
2444	llvm::FunctionType *FTy =
2445	llvm::FunctionType::get(Result: llvm::Type::getVoidTy(C&: CGM.getLLVMContext()), Params: {},
2446	/*isVarArg=*/false);
2447	return CGM.CreateRuntimeFunction(
2448	FTy, "__dyn_tls_on_demand_init",
2449	llvm::AttributeList::get(CGM.getLLVMContext(),
2450	llvm::AttributeList::FunctionIndex,
2451	llvm::Attribute::NoUnwind),
2452	/*Local=*/true);
2453	}
2454
2455	static void emitTlsGuardCheck(CodeGenFunction &CGF, llvm::GlobalValue *TlsGuard,
2456	llvm::BasicBlock *DynInitBB,
2457	llvm::BasicBlock *ContinueBB) {
2458	llvm::LoadInst *TlsGuardValue =
2459	CGF.Builder.CreateLoad(Addr: Address(TlsGuard, CGF.Int8Ty, CharUnits::One()));
2460	llvm::Value *CmpResult =
2461	CGF.Builder.CreateICmpEQ(LHS: TlsGuardValue, RHS: CGF.Builder.getInt8(C: 0));
2462	CGF.Builder.CreateCondBr(Cond: CmpResult, True: DynInitBB, False: ContinueBB);
2463	}
2464
2465	static void emitDynamicTlsInitializationCall(CodeGenFunction &CGF,
2466	llvm::GlobalValue *TlsGuard,
2467	llvm::BasicBlock *ContinueBB) {
2468	llvm::FunctionCallee Initializer = getDynTlsOnDemandInitFn(CGM&: CGF.CGM);
2469	llvm::Function *InitializerFunction =
2470	cast<llvm::Function>(Val: Initializer.getCallee());
2471	llvm::CallInst *CallVal = CGF.Builder.CreateCall(Callee: InitializerFunction);
2472	CallVal->setCallingConv(InitializerFunction->getCallingConv());
2473
2474	CGF.Builder.CreateBr(Dest: ContinueBB);
2475	}
2476
2477	static void emitDynamicTlsInitialization(CodeGenFunction &CGF) {
2478	llvm::BasicBlock *DynInitBB =
2479	CGF.createBasicBlock(name: "dyntls.dyn_init", parent: CGF.CurFn);
2480	llvm::BasicBlock *ContinueBB =
2481	CGF.createBasicBlock(name: "dyntls.continue", parent: CGF.CurFn);
2482
2483	llvm::GlobalValue *TlsGuard = getTlsGuardVar(CGM&: CGF.CGM);
2484
2485	emitTlsGuardCheck(CGF, TlsGuard, DynInitBB, ContinueBB);
2486	CGF.Builder.SetInsertPoint(DynInitBB);
2487	emitDynamicTlsInitializationCall(CGF, TlsGuard, ContinueBB);
2488	CGF.Builder.SetInsertPoint(ContinueBB);
2489	}
2490
2491	LValue MicrosoftCXXABI::EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF,
2492	const VarDecl *VD,
2493	QualType LValType) {
2494	// Dynamic TLS initialization works by checking the state of a
2495	// guard variable (__tls_guard) to see whether TLS initialization
2496	// for a thread has happend yet.
2497	// If not, the initialization is triggered on-demand
2498	// by calling __dyn_tls_on_demand_init.
2499	emitDynamicTlsInitialization(CGF);
2500
2501	// Emit the variable just like any regular global variable.
2502
2503	llvm::Value *V = CGF.CGM.GetAddrOfGlobalVar(D: VD);
2504	llvm::Type *RealVarTy = CGF.getTypes().ConvertTypeForMem(T: VD->getType());
2505
2506	CharUnits Alignment = CGF.getContext().getDeclAlign(VD);
2507	Address Addr(V, RealVarTy, Alignment);
2508
2509	LValue LV = VD->getType()->isReferenceType()
2510	? CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(),
2511	AlignmentSource::Decl)
2512	: CGF.MakeAddrLValue(Addr, T: LValType, Source: AlignmentSource::Decl);
2513	return LV;
2514	}
2515
2516	static ConstantAddress getInitThreadEpochPtr(CodeGenModule &CGM) {
2517	StringRef VarName("_Init_thread_epoch");
2518	CharUnits Align = CGM.getIntAlign();
2519	if (auto *GV = CGM.getModule().getNamedGlobal(Name: VarName))
2520	return ConstantAddress(GV, GV->getValueType(), Align);
2521	auto *GV = new llvm::GlobalVariable(
2522	CGM.getModule(), CGM.IntTy,
2523	/*isConstant=*/false, llvm::GlobalVariable::ExternalLinkage,
2524	/*Initializer=*/nullptr, VarName,
2525	/*InsertBefore=*/nullptr, llvm::GlobalVariable::GeneralDynamicTLSModel);
2526	GV->setAlignment(Align.getAsAlign());
2527	return ConstantAddress(GV, GV->getValueType(), Align);
2528	}
2529
2530	static llvm::FunctionCallee getInitThreadHeaderFn(CodeGenModule &CGM) {
2531	llvm::FunctionType *FTy =
2532	llvm::FunctionType::get(Result: llvm::Type::getVoidTy(C&: CGM.getLLVMContext()),
2533	Params: CGM.IntTy->getPointerTo(), /*isVarArg=*/false);
2534	return CGM.CreateRuntimeFunction(
2535	FTy, "_Init_thread_header",
2536	llvm::AttributeList::get(CGM.getLLVMContext(),
2537	llvm::AttributeList::FunctionIndex,
2538	llvm::Attribute::NoUnwind),
2539	/*Local=*/true);
2540	}
2541
2542	static llvm::FunctionCallee getInitThreadFooterFn(CodeGenModule &CGM) {
2543	llvm::FunctionType *FTy =
2544	llvm::FunctionType::get(Result: llvm::Type::getVoidTy(C&: CGM.getLLVMContext()),
2545	Params: CGM.IntTy->getPointerTo(), /*isVarArg=*/false);
2546	return CGM.CreateRuntimeFunction(
2547	FTy, "_Init_thread_footer",
2548	llvm::AttributeList::get(CGM.getLLVMContext(),
2549	llvm::AttributeList::FunctionIndex,
2550	llvm::Attribute::NoUnwind),
2551	/*Local=*/true);
2552	}
2553
2554	static llvm::FunctionCallee getInitThreadAbortFn(CodeGenModule &CGM) {
2555	llvm::FunctionType *FTy =
2556	llvm::FunctionType::get(Result: llvm::Type::getVoidTy(C&: CGM.getLLVMContext()),
2557	Params: CGM.IntTy->getPointerTo(), /*isVarArg=*/false);
2558	return CGM.CreateRuntimeFunction(
2559	FTy, "_Init_thread_abort",
2560	llvm::AttributeList::get(CGM.getLLVMContext(),
2561	llvm::AttributeList::FunctionIndex,
2562	llvm::Attribute::NoUnwind),
2563	/*Local=*/true);
2564	}
2565
2566	namespace {
2567	struct ResetGuardBit final : EHScopeStack::Cleanup {
2568	Address Guard;
2569	unsigned GuardNum;
2570	ResetGuardBit(Address Guard, unsigned GuardNum)
2571	: Guard(Guard), GuardNum(GuardNum) {}
2572
2573	void Emit(CodeGenFunction &CGF, Flags flags) override {
2574	// Reset the bit in the mask so that the static variable may be
2575	// reinitialized.
2576	CGBuilderTy &Builder = CGF.Builder;
2577	llvm::LoadInst *LI = Builder.CreateLoad(Addr: Guard);
2578	llvm::ConstantInt *Mask =
2579	llvm::ConstantInt::get(Ty: CGF.IntTy, V: ~(1ULL << GuardNum));
2580	Builder.CreateStore(Val: Builder.CreateAnd(LHS: LI, RHS: Mask), Addr: Guard);
2581	}
2582	};
2583
2584	struct CallInitThreadAbort final : EHScopeStack::Cleanup {
2585	llvm::Value *Guard;
2586	CallInitThreadAbort(Address Guard) : Guard(Guard.getPointer()) {}
2587
2588	void Emit(CodeGenFunction &CGF, Flags flags) override {
2589	// Calling _Init_thread_abort will reset the guard's state.
2590	CGF.EmitNounwindRuntimeCall(callee: getInitThreadAbortFn(CGM&: CGF.CGM), args: Guard);
2591	}
2592	};
2593	}
2594
2595	void MicrosoftCXXABI::EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D,
2596	llvm::GlobalVariable *GV,
2597	bool PerformInit) {
2598	// MSVC only uses guards for static locals.
2599	if (!D.isStaticLocal()) {
2600	assert(GV->hasWeakLinkage() || GV->hasLinkOnceLinkage());
2601	// GlobalOpt is allowed to discard the initializer, so use linkonce_odr.
2602	llvm::Function *F = CGF.CurFn;
2603	F->setLinkage(llvm::GlobalValue::LinkOnceODRLinkage);
2604	F->setComdat(CGM.getModule().getOrInsertComdat(Name: F->getName()));
2605	CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit);
2606	return;
2607	}
2608
2609	bool ThreadlocalStatic = D.getTLSKind();
2610	bool ThreadsafeStatic = getContext().getLangOpts().ThreadsafeStatics;
2611
2612	// Thread-safe static variables which aren't thread-specific have a
2613	// per-variable guard.
2614	bool HasPerVariableGuard = ThreadsafeStatic && !ThreadlocalStatic;
2615
2616	CGBuilderTy &Builder = CGF.Builder;
2617	llvm::IntegerType *GuardTy = CGF.Int32Ty;
2618	llvm::ConstantInt *Zero = llvm::ConstantInt::get(Ty: GuardTy, V: 0);
2619	CharUnits GuardAlign = CharUnits::fromQuantity(Quantity: 4);
2620
2621	// Get the guard variable for this function if we have one already.
2622	GuardInfo *GI = nullptr;
2623	if (ThreadlocalStatic)
2624	GI = &ThreadLocalGuardVariableMap[D.getDeclContext()];
2625	else if (!ThreadsafeStatic)
2626	GI = &GuardVariableMap[D.getDeclContext()];
2627
2628	llvm::GlobalVariable *GuardVar = GI ? GI->Guard : nullptr;
2629	unsigned GuardNum;
2630	if (D.isExternallyVisible()) {
2631	// Externally visible variables have to be numbered in Sema to properly
2632	// handle unreachable VarDecls.
2633	GuardNum = getContext().getStaticLocalNumber(VD: &D);
2634	assert(GuardNum > 0);
2635	GuardNum--;
2636	} else if (HasPerVariableGuard) {
2637	GuardNum = ThreadSafeGuardNumMap[D.getDeclContext()]++;
2638	} else {
2639	// Non-externally visible variables are numbered here in CodeGen.
2640	GuardNum = GI->BitIndex++;
2641	}
2642
2643	if (!HasPerVariableGuard && GuardNum >= 32) {
2644	if (D.isExternallyVisible())
2645	ErrorUnsupportedABI(CGF, S: "more than 32 guarded initializations");
2646	GuardNum %= 32;
2647	GuardVar = nullptr;
2648	}
2649
2650	if (!GuardVar) {
2651	// Mangle the name for the guard.
2652	SmallString<256> GuardName;
2653	{
2654	llvm::raw_svector_ostream Out(GuardName);
2655	if (HasPerVariableGuard)
2656	getMangleContext().mangleThreadSafeStaticGuardVariable(VD: &D, GuardNum,
2657	Out);
2658	else
2659	getMangleContext().mangleStaticGuardVariable(D: &D, Out);
2660	}
2661
2662	// Create the guard variable with a zero-initializer. Just absorb linkage,
2663	// visibility and dll storage class from the guarded variable.
2664	GuardVar =
2665	new llvm::GlobalVariable(CGM.getModule(), GuardTy, /*isConstant=*/false,
2666	GV->getLinkage(), Zero, GuardName.str());
2667	GuardVar->setVisibility(GV->getVisibility());
2668	GuardVar->setDLLStorageClass(GV->getDLLStorageClass());
2669	GuardVar->setAlignment(GuardAlign.getAsAlign());
2670	if (GuardVar->isWeakForLinker())
2671	GuardVar->setComdat(
2672	CGM.getModule().getOrInsertComdat(Name: GuardVar->getName()));
2673	if (D.getTLSKind())
2674	CGM.setTLSMode(GV: GuardVar, D);
2675	if (GI && !HasPerVariableGuard)
2676	GI->Guard = GuardVar;
2677	}
2678
2679	ConstantAddress GuardAddr(GuardVar, GuardTy, GuardAlign);
2680
2681	assert(GuardVar->getLinkage() == GV->getLinkage() &&
2682	"static local from the same function had different linkage");
2683
2684	if (!HasPerVariableGuard) {
2685	// Pseudo code for the test:
2686	// if (!(GuardVar & MyGuardBit)) {
2687	// GuardVar |= MyGuardBit;
2688	// ... initialize the object ...;
2689	// }
2690
2691	// Test our bit from the guard variable.
2692	llvm::ConstantInt *Bit = llvm::ConstantInt::get(Ty: GuardTy, V: 1ULL << GuardNum);
2693	llvm::LoadInst *LI = Builder.CreateLoad(Addr: GuardAddr);
2694	llvm::Value *NeedsInit =
2695	Builder.CreateICmpEQ(LHS: Builder.CreateAnd(LHS: LI, RHS: Bit), RHS: Zero);
2696	llvm::BasicBlock *InitBlock = CGF.createBasicBlock(name: "init");
2697	llvm::BasicBlock *EndBlock = CGF.createBasicBlock(name: "init.end");
2698	CGF.EmitCXXGuardedInitBranch(NeedsInit, InitBlock, NoInitBlock: EndBlock,
2699	Kind: CodeGenFunction::GuardKind::VariableGuard, D: &D);
2700
2701	// Set our bit in the guard variable and emit the initializer and add a global
2702	// destructor if appropriate.
2703	CGF.EmitBlock(BB: InitBlock);
2704	Builder.CreateStore(Val: Builder.CreateOr(LHS: LI, RHS: Bit), Addr: GuardAddr);
2705	CGF.EHStack.pushCleanup<ResetGuardBit>(Kind: EHCleanup, A: GuardAddr, A: GuardNum);
2706	CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit);
2707	CGF.PopCleanupBlock();
2708	Builder.CreateBr(Dest: EndBlock);
2709
2710	// Continue.
2711	CGF.EmitBlock(BB: EndBlock);
2712	} else {
2713	// Pseudo code for the test:
2714	// if (TSS > _Init_thread_epoch) {
2715	// _Init_thread_header(&TSS);
2716	// if (TSS == -1) {
2717	// ... initialize the object ...;
2718	// _Init_thread_footer(&TSS);
2719	// }
2720	// }
2721	//
2722	// The algorithm is almost identical to what can be found in the appendix
2723	// found in N2325.
2724
2725	// This BasicBLock determines whether or not we have any work to do.
2726	llvm::LoadInst *FirstGuardLoad = Builder.CreateLoad(Addr: GuardAddr);
2727	FirstGuardLoad->setOrdering(llvm::AtomicOrdering::Unordered);
2728	llvm::LoadInst *InitThreadEpoch =
2729	Builder.CreateLoad(Addr: getInitThreadEpochPtr(CGM));
2730	llvm::Value *IsUninitialized =
2731	Builder.CreateICmpSGT(LHS: FirstGuardLoad, RHS: InitThreadEpoch);
2732	llvm::BasicBlock *AttemptInitBlock = CGF.createBasicBlock(name: "init.attempt");
2733	llvm::BasicBlock *EndBlock = CGF.createBasicBlock(name: "init.end");
2734	CGF.EmitCXXGuardedInitBranch(NeedsInit: IsUninitialized, InitBlock: AttemptInitBlock, NoInitBlock: EndBlock,
2735	Kind: CodeGenFunction::GuardKind::VariableGuard, D: &D);
2736
2737	// This BasicBlock attempts to determine whether or not this thread is
2738	// responsible for doing the initialization.
2739	CGF.EmitBlock(BB: AttemptInitBlock);
2740	CGF.EmitNounwindRuntimeCall(callee: getInitThreadHeaderFn(CGM),
2741	args: GuardAddr.getPointer());
2742	llvm::LoadInst *SecondGuardLoad = Builder.CreateLoad(Addr: GuardAddr);
2743	SecondGuardLoad->setOrdering(llvm::AtomicOrdering::Unordered);
2744	llvm::Value *ShouldDoInit =
2745	Builder.CreateICmpEQ(LHS: SecondGuardLoad, RHS: getAllOnesInt());
2746	llvm::BasicBlock *InitBlock = CGF.createBasicBlock(name: "init");
2747	Builder.CreateCondBr(Cond: ShouldDoInit, True: InitBlock, False: EndBlock);
2748
2749	// Ok, we ended up getting selected as the initializing thread.
2750	CGF.EmitBlock(BB: InitBlock);
2751	CGF.EHStack.pushCleanup<CallInitThreadAbort>(Kind: EHCleanup, A: GuardAddr);
2752	CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit);
2753	CGF.PopCleanupBlock();
2754	CGF.EmitNounwindRuntimeCall(callee: getInitThreadFooterFn(CGM),
2755	args: GuardAddr.getPointer());
2756	Builder.CreateBr(Dest: EndBlock);
2757
2758	CGF.EmitBlock(BB: EndBlock);
2759	}
2760	}
2761
2762	bool MicrosoftCXXABI::isZeroInitializable(const MemberPointerType *MPT) {
2763	// Null-ness for function memptrs only depends on the first field, which is
2764	// the function pointer. The rest don't matter, so we can zero initialize.
2765	if (MPT->isMemberFunctionPointer())
2766	return true;
2767
2768	// The virtual base adjustment field is always -1 for null, so if we have one
2769	// we can't zero initialize. The field offset is sometimes also -1 if 0 is a
2770	// valid field offset.
2771	const CXXRecordDecl *RD = MPT->getMostRecentCXXRecordDecl();
2772	MSInheritanceModel Inheritance = RD->getMSInheritanceModel();
2773	return (!inheritanceModelHasVBTableOffsetField(Inheritance) &&
2774	RD->nullFieldOffsetIsZero());
2775	}
2776
2777	llvm::Type *
2778	MicrosoftCXXABI::ConvertMemberPointerType(const MemberPointerType *MPT) {
2779	const CXXRecordDecl *RD = MPT->getMostRecentCXXRecordDecl();
2780	MSInheritanceModel Inheritance = RD->getMSInheritanceModel();
2781	llvm::SmallVector<llvm::Type *, 4> fields;
2782	if (MPT->isMemberFunctionPointer())
2783	fields.push_back(Elt: CGM.VoidPtrTy); // FunctionPointerOrVirtualThunk
2784	else
2785	fields.push_back(Elt: CGM.IntTy); // FieldOffset
2786
2787	if (inheritanceModelHasNVOffsetField(IsMemberFunction: MPT->isMemberFunctionPointer(),
2788	Inheritance))
2789	fields.push_back(Elt: CGM.IntTy);
2790	if (inheritanceModelHasVBPtrOffsetField(Inheritance))
2791	fields.push_back(Elt: CGM.IntTy);
2792	if (inheritanceModelHasVBTableOffsetField(Inheritance))
2793	fields.push_back(Elt: CGM.IntTy); // VirtualBaseAdjustmentOffset
2794
2795	if (fields.size() == 1)
2796	return fields[0];
2797	return llvm::StructType::get(Context&: CGM.getLLVMContext(), Elements: fields);
2798	}
2799
2800	void MicrosoftCXXABI::
2801	GetNullMemberPointerFields(const MemberPointerType *MPT,
2802	llvm::SmallVectorImpl<llvm::Constant *> &fields) {
2803	assert(fields.empty());
2804	const CXXRecordDecl *RD = MPT->getMostRecentCXXRecordDecl();
2805	MSInheritanceModel Inheritance = RD->getMSInheritanceModel();
2806	if (MPT->isMemberFunctionPointer()) {
2807	// FunctionPointerOrVirtualThunk
2808	fields.push_back(Elt: llvm::Constant::getNullValue(Ty: CGM.VoidPtrTy));
2809	} else {
2810	if (RD->nullFieldOffsetIsZero())
2811	fields.push_back(Elt: getZeroInt()); // FieldOffset
2812	else
2813	fields.push_back(Elt: getAllOnesInt()); // FieldOffset
2814	}
2815
2816	if (inheritanceModelHasNVOffsetField(IsMemberFunction: MPT->isMemberFunctionPointer(),
2817	Inheritance))
2818	fields.push_back(Elt: getZeroInt());
2819	if (inheritanceModelHasVBPtrOffsetField(Inheritance))
2820	fields.push_back(Elt: getZeroInt());
2821	if (inheritanceModelHasVBTableOffsetField(Inheritance))
2822	fields.push_back(Elt: getAllOnesInt());
2823	}
2824
2825	llvm::Constant *
2826	MicrosoftCXXABI::EmitNullMemberPointer(const MemberPointerType *MPT) {
2827	llvm::SmallVector<llvm::Constant *, 4> fields;
2828	GetNullMemberPointerFields(MPT, fields);
2829	if (fields.size() == 1)
2830	return fields[0];
2831	llvm::Constant *Res = llvm::ConstantStruct::getAnon(V: fields);
2832	assert(Res->getType() == ConvertMemberPointerType(MPT));
2833	return Res;
2834	}
2835
2836	llvm::Constant *
2837	MicrosoftCXXABI::EmitFullMemberPointer(llvm::Constant *FirstField,
2838	bool IsMemberFunction,
2839	const CXXRecordDecl *RD,
2840	CharUnits NonVirtualBaseAdjustment,
2841	unsigned VBTableIndex) {
2842	MSInheritanceModel Inheritance = RD->getMSInheritanceModel();
2843
2844	// Single inheritance class member pointer are represented as scalars instead
2845	// of aggregates.
2846	if (inheritanceModelHasOnlyOneField(IsMemberFunction, Inheritance))
2847	return FirstField;
2848
2849	llvm::SmallVector<llvm::Constant *, 4> fields;
2850	fields.push_back(Elt: FirstField);
2851
2852	if (inheritanceModelHasNVOffsetField(IsMemberFunction, Inheritance))
2853	fields.push_back(Elt: llvm::ConstantInt::get(
2854	Ty: CGM.IntTy, V: NonVirtualBaseAdjustment.getQuantity()));
2855
2856	if (inheritanceModelHasVBPtrOffsetField(Inheritance)) {
2857	CharUnits Offs = CharUnits::Zero();
2858	if (VBTableIndex)
2859	Offs = getContext().getASTRecordLayout(RD).getVBPtrOffset();
2860	fields.push_back(Elt: llvm::ConstantInt::get(Ty: CGM.IntTy, V: Offs.getQuantity()));
2861	}
2862
2863	// The rest of the fields are adjusted by conversions to a more derived class.
2864	if (inheritanceModelHasVBTableOffsetField(Inheritance))
2865	fields.push_back(Elt: llvm::ConstantInt::get(Ty: CGM.IntTy, V: VBTableIndex));
2866
2867	return llvm::ConstantStruct::getAnon(V: fields);
2868	}
2869
2870	llvm::Constant *
2871	MicrosoftCXXABI::EmitMemberDataPointer(const MemberPointerType *MPT,
2872	CharUnits offset) {
2873	return EmitMemberDataPointer(RD: MPT->getMostRecentCXXRecordDecl(), offset);
2874	}
2875
2876	llvm::Constant *MicrosoftCXXABI::EmitMemberDataPointer(const CXXRecordDecl *RD,
2877	CharUnits offset) {
2878	if (RD->getMSInheritanceModel() ==
2879	MSInheritanceModel::Virtual)
2880	offset -= getContext().getOffsetOfBaseWithVBPtr(RD);
2881	llvm::Constant *FirstField =
2882	llvm::ConstantInt::get(Ty: CGM.IntTy, V: offset.getQuantity());
2883	return EmitFullMemberPointer(FirstField, /*IsMemberFunction=*/false, RD,
2884	NonVirtualBaseAdjustment: CharUnits::Zero(), /*VBTableIndex=*/0);
2885	}
2886
2887	llvm::Constant *MicrosoftCXXABI::EmitMemberPointer(const APValue &MP,
2888	QualType MPType) {
2889	const MemberPointerType *DstTy = MPType->castAs<MemberPointerType>();
2890	const ValueDecl *MPD = MP.getMemberPointerDecl();
2891	if (!MPD)
2892	return EmitNullMemberPointer(MPT: DstTy);
2893
2894	ASTContext &Ctx = getContext();
2895	ArrayRef<const CXXRecordDecl *> MemberPointerPath = MP.getMemberPointerPath();
2896
2897	llvm::Constant *C;
2898	if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: MPD)) {
2899	C = EmitMemberFunctionPointer(MD);
2900	} else {
2901	// For a pointer to data member, start off with the offset of the field in
2902	// the class in which it was declared, and convert from there if necessary.
2903	// For indirect field decls, get the outermost anonymous field and use the
2904	// parent class.
2905	CharUnits FieldOffset = Ctx.toCharUnitsFromBits(BitSize: Ctx.getFieldOffset(FD: MPD));
2906	const FieldDecl *FD = dyn_cast<FieldDecl>(Val: MPD);
2907	if (!FD)
2908	FD = cast<FieldDecl>(Val: *cast<IndirectFieldDecl>(Val: MPD)->chain_begin());
2909	const CXXRecordDecl *RD = cast<CXXRecordDecl>(Val: FD->getParent());
2910	RD = RD->getMostRecentNonInjectedDecl();
2911	C = EmitMemberDataPointer(RD, offset: FieldOffset);
2912	}
2913
2914	if (!MemberPointerPath.empty()) {
2915	const CXXRecordDecl *SrcRD = cast<CXXRecordDecl>(MPD->getDeclContext());
2916	const Type *SrcRecTy = Ctx.getTypeDeclType(SrcRD).getTypePtr();
2917	const MemberPointerType *SrcTy =
2918	Ctx.getMemberPointerType(T: DstTy->getPointeeType(), Cls: SrcRecTy)
2919	->castAs<MemberPointerType>();
2920
2921	bool DerivedMember = MP.isMemberPointerToDerivedMember();
2922	SmallVector<const CXXBaseSpecifier *, 4> DerivedToBasePath;
2923	const CXXRecordDecl *PrevRD = SrcRD;
2924	for (const CXXRecordDecl *PathElem : MemberPointerPath) {
2925	const CXXRecordDecl *Base = nullptr;
2926	const CXXRecordDecl *Derived = nullptr;
2927	if (DerivedMember) {
2928	Base = PathElem;
2929	Derived = PrevRD;
2930	} else {
2931	Base = PrevRD;
2932	Derived = PathElem;
2933	}
2934	for (const CXXBaseSpecifier &BS : Derived->bases())
2935	if (BS.getType()->getAsCXXRecordDecl()->getCanonicalDecl() ==
2936	Base->getCanonicalDecl())
2937	DerivedToBasePath.push_back(Elt: &BS);
2938	PrevRD = PathElem;
2939	}
2940	assert(DerivedToBasePath.size() == MemberPointerPath.size());
2941
2942	CastKind CK = DerivedMember ? CK_DerivedToBaseMemberPointer
2943	: CK_BaseToDerivedMemberPointer;
2944	C = EmitMemberPointerConversion(SrcTy, DstTy, CK, PathBegin: DerivedToBasePath.begin(),
2945	PathEnd: DerivedToBasePath.end(), Src: C);
2946	}
2947	return C;
2948	}
2949
2950	llvm::Constant *
2951	MicrosoftCXXABI::EmitMemberFunctionPointer(const CXXMethodDecl *MD) {
2952	assert(MD->isInstance() && "Member function must not be static!");
2953
2954	CharUnits NonVirtualBaseAdjustment = CharUnits::Zero();
2955	const CXXRecordDecl *RD = MD->getParent()->getMostRecentNonInjectedDecl();
2956	CodeGenTypes &Types = CGM.getTypes();
2957
2958	unsigned VBTableIndex = 0;
2959	llvm::Constant *FirstField;
2960	const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
2961	if (!MD->isVirtual()) {
2962	llvm::Type *Ty;
2963	// Check whether the function has a computable LLVM signature.
2964	if (Types.isFuncTypeConvertible(FPT)) {
2965	// The function has a computable LLVM signature; use the correct type.
2966	Ty = Types.GetFunctionType(Info: Types.arrangeCXXMethodDeclaration(MD));
2967	} else {
2968	// Use an arbitrary non-function type to tell GetAddrOfFunction that the
2969	// function type is incomplete.
2970	Ty = CGM.PtrDiffTy;
2971	}
2972	FirstField = CGM.GetAddrOfFunction(MD, Ty);
2973	} else {
2974	auto &VTableContext = CGM.getMicrosoftVTableContext();
2975	MethodVFTableLocation ML = VTableContext.getMethodVFTableLocation(MD);
2976	FirstField = EmitVirtualMemPtrThunk(MD, ML);
2977	// Include the vfptr adjustment if the method is in a non-primary vftable.
2978	NonVirtualBaseAdjustment += ML.VFPtrOffset;
2979	if (ML.VBase)
2980	VBTableIndex = VTableContext.getVBTableIndex(Derived: RD, VBase: ML.VBase) * 4;
2981	}
2982
2983	if (VBTableIndex == 0 &&
2984	RD->getMSInheritanceModel() ==
2985	MSInheritanceModel::Virtual)
2986	NonVirtualBaseAdjustment -= getContext().getOffsetOfBaseWithVBPtr(RD);
2987
2988	// The rest of the fields are common with data member pointers.
2989	return EmitFullMemberPointer(FirstField, /*IsMemberFunction=*/true, RD,
2990	NonVirtualBaseAdjustment, VBTableIndex);
2991	}
2992
2993	/// Member pointers are the same if they're either bitwise identical *or* both
2994	/// null. Null-ness for function members is determined by the first field,
2995	/// while for data member pointers we must compare all fields.
2996	llvm::Value *
2997	MicrosoftCXXABI::EmitMemberPointerComparison(CodeGenFunction &CGF,
2998	llvm::Value *L,
2999	llvm::Value *R,
3000	const MemberPointerType *MPT,
3001	bool Inequality) {
3002	CGBuilderTy &Builder = CGF.Builder;
3003
3004	// Handle != comparisons by switching the sense of all boolean operations.
3005	llvm::ICmpInst::Predicate Eq;
3006	llvm::Instruction::BinaryOps And, Or;
3007	if (Inequality) {
3008	Eq = llvm::ICmpInst::ICMP_NE;
3009	And = llvm::Instruction::Or;
3010	Or = llvm::Instruction::And;
3011	} else {
3012	Eq = llvm::ICmpInst::ICMP_EQ;
3013	And = llvm::Instruction::And;
3014	Or = llvm::Instruction::Or;
3015	}
3016
3017	// If this is a single field member pointer (single inheritance), this is a
3018	// single icmp.
3019	const CXXRecordDecl *RD = MPT->getMostRecentCXXRecordDecl();
3020	MSInheritanceModel Inheritance = RD->getMSInheritanceModel();
3021	if (inheritanceModelHasOnlyOneField(IsMemberFunction: MPT->isMemberFunctionPointer(),
3022	Inheritance))
3023	return Builder.CreateICmp(P: Eq, LHS: L, RHS: R);
3024
3025	// Compare the first field.
3026	llvm::Value *L0 = Builder.CreateExtractValue(Agg: L, Idxs: 0, Name: "lhs.0");
3027	llvm::Value *R0 = Builder.CreateExtractValue(Agg: R, Idxs: 0, Name: "rhs.0");
3028	llvm::Value *Cmp0 = Builder.CreateICmp(P: Eq, LHS: L0, RHS: R0, Name: "memptr.cmp.first");
3029
3030	// Compare everything other than the first field.
3031	llvm::Value *Res = nullptr;
3032	llvm::StructType *LType = cast<llvm::StructType>(Val: L->getType());
3033	for (unsigned I = 1, E = LType->getNumElements(); I != E; ++I) {
3034	llvm::Value *LF = Builder.CreateExtractValue(Agg: L, Idxs: I);
3035	llvm::Value *RF = Builder.CreateExtractValue(Agg: R, Idxs: I);
3036	llvm::Value *Cmp = Builder.CreateICmp(P: Eq, LHS: LF, RHS: RF, Name: "memptr.cmp.rest");
3037	if (Res)
3038	Res = Builder.CreateBinOp(Opc: And, LHS: Res, RHS: Cmp);
3039	else
3040	Res = Cmp;
3041	}
3042
3043	// Check if the first field is 0 if this is a function pointer.
3044	if (MPT->isMemberFunctionPointer()) {
3045	// (l1 == r1 && ...) || l0 == 0
3046	llvm::Value *Zero = llvm::Constant::getNullValue(Ty: L0->getType());
3047	llvm::Value *IsZero = Builder.CreateICmp(P: Eq, LHS: L0, RHS: Zero, Name: "memptr.cmp.iszero");
3048	Res = Builder.CreateBinOp(Opc: Or, LHS: Res, RHS: IsZero);
3049	}
3050
3051	// Combine the comparison of the first field, which must always be true for
3052	// this comparison to succeeed.
3053	return Builder.CreateBinOp(Opc: And, LHS: Res, RHS: Cmp0, Name: "memptr.cmp");
3054	}
3055
3056	llvm::Value *
3057	MicrosoftCXXABI::EmitMemberPointerIsNotNull(CodeGenFunction &CGF,
3058	llvm::Value *MemPtr,
3059	const MemberPointerType *MPT) {
3060	CGBuilderTy &Builder = CGF.Builder;
3061	llvm::SmallVector<llvm::Constant *, 4> fields;
3062	// We only need one field for member functions.
3063	if (MPT->isMemberFunctionPointer())
3064	fields.push_back(Elt: llvm::Constant::getNullValue(Ty: CGM.VoidPtrTy));
3065	else
3066	GetNullMemberPointerFields(MPT, fields);
3067	assert(!fields.empty());
3068	llvm::Value *FirstField = MemPtr;
3069	if (MemPtr->getType()->isStructTy())
3070	FirstField = Builder.CreateExtractValue(Agg: MemPtr, Idxs: 0);
3071	llvm::Value *Res = Builder.CreateICmpNE(LHS: FirstField, RHS: fields[0], Name: "memptr.cmp0");
3072
3073	// For function member pointers, we only need to test the function pointer
3074	// field. The other fields if any can be garbage.
3075	if (MPT->isMemberFunctionPointer())
3076	return Res;
3077
3078	// Otherwise, emit a series of compares and combine the results.
3079	for (int I = 1, E = fields.size(); I < E; ++I) {
3080	llvm::Value *Field = Builder.CreateExtractValue(Agg: MemPtr, Idxs: I);
3081	llvm::Value *Next = Builder.CreateICmpNE(LHS: Field, RHS: fields[I], Name: "memptr.cmp");
3082	Res = Builder.CreateOr(LHS: Res, RHS: Next, Name: "memptr.tobool");
3083	}
3084	return Res;
3085	}
3086
3087	bool MicrosoftCXXABI::MemberPointerConstantIsNull(const MemberPointerType *MPT,
3088	llvm::Constant *Val) {
3089	// Function pointers are null if the pointer in the first field is null.
3090	if (MPT->isMemberFunctionPointer()) {
3091	llvm::Constant *FirstField = Val->getType()->isStructTy() ?
3092	Val->getAggregateElement(Elt: 0U) : Val;
3093	return FirstField->isNullValue();
3094	}
3095
3096	// If it's not a function pointer and it's zero initializable, we can easily
3097	// check zero.
3098	if (isZeroInitializable(MPT) && Val->isNullValue())
3099	return true;
3100
3101	// Otherwise, break down all the fields for comparison. Hopefully these
3102	// little Constants are reused, while a big null struct might not be.
3103	llvm::SmallVector<llvm::Constant *, 4> Fields;
3104	GetNullMemberPointerFields(MPT, fields&: Fields);
3105	if (Fields.size() == 1) {
3106	assert(Val->getType()->isIntegerTy());
3107	return Val == Fields[0];
3108	}
3109
3110	unsigned I, E;
3111	for (I = 0, E = Fields.size(); I != E; ++I) {
3112	if (Val->getAggregateElement(Elt: I) != Fields[I])
3113	break;
3114	}
3115	return I == E;
3116	}
3117
3118	llvm::Value *
3119	MicrosoftCXXABI::GetVBaseOffsetFromVBPtr(CodeGenFunction &CGF,
3120	Address This,
3121	llvm::Value *VBPtrOffset,
3122	llvm::Value *VBTableOffset,
3123	llvm::Value **VBPtrOut) {
3124	CGBuilderTy &Builder = CGF.Builder;
3125	// Load the vbtable pointer from the vbptr in the instance.
3126	llvm::Value *VBPtr = Builder.CreateInBoundsGEP(Ty: CGM.Int8Ty, Ptr: This.getPointer(),
3127	IdxList: VBPtrOffset, Name: "vbptr");
3128	if (VBPtrOut)
3129	*VBPtrOut = VBPtr;
3130
3131	CharUnits VBPtrAlign;
3132	if (auto CI = dyn_cast<llvm::ConstantInt>(Val: VBPtrOffset)) {
3133	VBPtrAlign = This.getAlignment().alignmentAtOffset(
3134	offset: CharUnits::fromQuantity(Quantity: CI->getSExtValue()));
3135	} else {
3136	VBPtrAlign = CGF.getPointerAlign();
3137	}
3138
3139	llvm::Value *VBTable = Builder.CreateAlignedLoad(
3140	Ty: CGM.Int32Ty->getPointerTo(AddrSpace: 0), Addr: VBPtr, Align: VBPtrAlign, Name: "vbtable");
3141
3142	// Translate from byte offset to table index. It improves analyzability.
3143	llvm::Value *VBTableIndex = Builder.CreateAShr(
3144	LHS: VBTableOffset, RHS: llvm::ConstantInt::get(Ty: VBTableOffset->getType(), V: 2),
3145	Name: "vbtindex", /*isExact=*/true);
3146
3147	// Load an i32 offset from the vb-table.
3148	llvm::Value *VBaseOffs =
3149	Builder.CreateInBoundsGEP(Ty: CGM.Int32Ty, Ptr: VBTable, IdxList: VBTableIndex);
3150	return Builder.CreateAlignedLoad(Ty: CGM.Int32Ty, Addr: VBaseOffs,
3151	Align: CharUnits::fromQuantity(Quantity: 4), Name: "vbase_offs");
3152	}
3153
3154	// Returns an adjusted base cast to i8*, since we do more address arithmetic on
3155	// it.
3156	llvm::Value *MicrosoftCXXABI::AdjustVirtualBase(
3157	CodeGenFunction &CGF, const Expr *E, const CXXRecordDecl *RD,
3158	Address Base, llvm::Value *VBTableOffset, llvm::Value *VBPtrOffset) {
3159	CGBuilderTy &Builder = CGF.Builder;
3160	Base = Base.withElementType(ElemTy: CGM.Int8Ty);
3161	llvm::BasicBlock *OriginalBB = nullptr;
3162	llvm::BasicBlock *SkipAdjustBB = nullptr;
3163	llvm::BasicBlock *VBaseAdjustBB = nullptr;
3164
3165	// In the unspecified inheritance model, there might not be a vbtable at all,
3166	// in which case we need to skip the virtual base lookup. If there is a
3167	// vbtable, the first entry is a no-op entry that gives back the original
3168	// base, so look for a virtual base adjustment offset of zero.
3169	if (VBPtrOffset) {
3170	OriginalBB = Builder.GetInsertBlock();
3171	VBaseAdjustBB = CGF.createBasicBlock(name: "memptr.vadjust");
3172	SkipAdjustBB = CGF.createBasicBlock(name: "memptr.skip_vadjust");
3173	llvm::Value *IsVirtual =
3174	Builder.CreateICmpNE(LHS: VBTableOffset, RHS: getZeroInt(),
3175	Name: "memptr.is_vbase");
3176	Builder.CreateCondBr(Cond: IsVirtual, True: VBaseAdjustBB, False: SkipAdjustBB);
3177	CGF.EmitBlock(BB: VBaseAdjustBB);
3178	}
3179
3180	// If we weren't given a dynamic vbptr offset, RD should be complete and we'll
3181	// know the vbptr offset.
3182	if (!VBPtrOffset) {
3183	CharUnits offs = CharUnits::Zero();
3184	if (!RD->hasDefinition()) {
3185	DiagnosticsEngine &Diags = CGF.CGM.getDiags();
3186	unsigned DiagID = Diags.getCustomDiagID(
3187	L: DiagnosticsEngine::Error,
3188	FormatString: "member pointer representation requires a "
3189	"complete class type for %0 to perform this expression");
3190	Diags.Report(Loc: E->getExprLoc(), DiagID) << RD << E->getSourceRange();
3191	} else if (RD->getNumVBases())
3192	offs = getContext().getASTRecordLayout(RD).getVBPtrOffset();
3193	VBPtrOffset = llvm::ConstantInt::get(Ty: CGM.IntTy, V: offs.getQuantity());
3194	}
3195	llvm::Value *VBPtr = nullptr;
3196	llvm::Value *VBaseOffs =
3197	GetVBaseOffsetFromVBPtr(CGF, This: Base, VBPtrOffset, VBTableOffset, VBPtrOut: &VBPtr);
3198	llvm::Value *AdjustedBase =
3199	Builder.CreateInBoundsGEP(Ty: CGM.Int8Ty, Ptr: VBPtr, IdxList: VBaseOffs);
3200
3201	// Merge control flow with the case where we didn't have to adjust.
3202	if (VBaseAdjustBB) {
3203	Builder.CreateBr(Dest: SkipAdjustBB);
3204	CGF.EmitBlock(BB: SkipAdjustBB);
3205	llvm::PHINode *Phi = Builder.CreatePHI(Ty: CGM.Int8PtrTy, NumReservedValues: 2, Name: "memptr.base");
3206	Phi->addIncoming(V: Base.getPointer(), BB: OriginalBB);
3207	Phi->addIncoming(V: AdjustedBase, BB: VBaseAdjustBB);
3208	return Phi;
3209	}
3210	return AdjustedBase;
3211	}
3212
3213	llvm::Value *MicrosoftCXXABI::EmitMemberDataPointerAddress(
3214	CodeGenFunction &CGF, const Expr *E, Address Base, llvm::Value *MemPtr,
3215	const MemberPointerType *MPT) {
3216	assert(MPT->isMemberDataPointer());
3217	CGBuilderTy &Builder = CGF.Builder;
3218	const CXXRecordDecl *RD = MPT->getMostRecentCXXRecordDecl();
3219	MSInheritanceModel Inheritance = RD->getMSInheritanceModel();
3220
3221	// Extract the fields we need, regardless of model. We'll apply them if we
3222	// have them.
3223	llvm::Value *FieldOffset = MemPtr;
3224	llvm::Value *VirtualBaseAdjustmentOffset = nullptr;
3225	llvm::Value *VBPtrOffset = nullptr;
3226	if (MemPtr->getType()->isStructTy()) {
3227	// We need to extract values.
3228	unsigned I = 0;
3229	FieldOffset = Builder.CreateExtractValue(Agg: MemPtr, Idxs: I++);
3230	if (inheritanceModelHasVBPtrOffsetField(Inheritance))
3231	VBPtrOffset = Builder.CreateExtractValue(Agg: MemPtr, Idxs: I++);
3232	if (inheritanceModelHasVBTableOffsetField(Inheritance))
3233	VirtualBaseAdjustmentOffset = Builder.CreateExtractValue(Agg: MemPtr, Idxs: I++);
3234	}
3235
3236	llvm::Value *Addr;
3237	if (VirtualBaseAdjustmentOffset) {
3238	Addr = AdjustVirtualBase(CGF, E, RD, Base, VBTableOffset: VirtualBaseAdjustmentOffset,
3239	VBPtrOffset);
3240	} else {
3241	Addr = Base.getPointer();
3242	}
3243
3244	// Apply the offset, which we assume is non-null.
3245	return Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: Addr, IdxList: FieldOffset,
3246	Name: "memptr.offset");
3247	}
3248
3249	llvm::Value *
3250	MicrosoftCXXABI::EmitMemberPointerConversion(CodeGenFunction &CGF,
3251	const CastExpr *E,
3252	llvm::Value *Src) {
3253	assert(E->getCastKind() == CK_DerivedToBaseMemberPointer ||
3254	E->getCastKind() == CK_BaseToDerivedMemberPointer ||
3255	E->getCastKind() == CK_ReinterpretMemberPointer);
3256
3257	// Use constant emission if we can.
3258	if (isa<llvm::Constant>(Val: Src))
3259	return EmitMemberPointerConversion(E, Src: cast<llvm::Constant>(Val: Src));
3260
3261	// We may be adding or dropping fields from the member pointer, so we need
3262	// both types and the inheritance models of both records.
3263	const MemberPointerType *SrcTy =
3264	E->getSubExpr()->getType()->castAs<MemberPointerType>();
3265	const MemberPointerType *DstTy = E->getType()->castAs<MemberPointerType>();
3266	bool IsFunc = SrcTy->isMemberFunctionPointer();
3267
3268	// If the classes use the same null representation, reinterpret_cast is a nop.
3269	bool IsReinterpret = E->getCastKind() == CK_ReinterpretMemberPointer;
3270	if (IsReinterpret && IsFunc)
3271	return Src;
3272
3273	CXXRecordDecl *SrcRD = SrcTy->getMostRecentCXXRecordDecl();
3274	CXXRecordDecl *DstRD = DstTy->getMostRecentCXXRecordDecl();
3275	if (IsReinterpret &&
3276	SrcRD->nullFieldOffsetIsZero() == DstRD->nullFieldOffsetIsZero())
3277	return Src;
3278
3279	CGBuilderTy &Builder = CGF.Builder;
3280
3281	// Branch past the conversion if Src is null.
3282	llvm::Value *IsNotNull = EmitMemberPointerIsNotNull(CGF, MemPtr: Src, MPT: SrcTy);
3283	llvm::Constant *DstNull = EmitNullMemberPointer(MPT: DstTy);
3284
3285	// C++ 5.2.10p9: The null member pointer value is converted to the null member
3286	// pointer value of the destination type.
3287	if (IsReinterpret) {
3288	// For reinterpret casts, sema ensures that src and dst are both functions
3289	// or data and have the same size, which means the LLVM types should match.
3290	assert(Src->getType() == DstNull->getType());
3291	return Builder.CreateSelect(C: IsNotNull, True: Src, False: DstNull);
3292	}
3293
3294	llvm::BasicBlock *OriginalBB = Builder.GetInsertBlock();
3295	llvm::BasicBlock *ConvertBB = CGF.createBasicBlock(name: "memptr.convert");
3296	llvm::BasicBlock *ContinueBB = CGF.createBasicBlock(name: "memptr.converted");
3297	Builder.CreateCondBr(Cond: IsNotNull, True: ConvertBB, False: ContinueBB);
3298	CGF.EmitBlock(BB: ConvertBB);
3299
3300	llvm::Value *Dst = EmitNonNullMemberPointerConversion(
3301	SrcTy, DstTy, CK: E->getCastKind(), PathBegin: E->path_begin(), PathEnd: E->path_end(), Src,
3302	Builder);
3303
3304	Builder.CreateBr(Dest: ContinueBB);
3305
3306	// In the continuation, choose between DstNull and Dst.
3307	CGF.EmitBlock(BB: ContinueBB);
3308	llvm::PHINode *Phi = Builder.CreatePHI(Ty: DstNull->getType(), NumReservedValues: 2, Name: "memptr.converted");
3309	Phi->addIncoming(V: DstNull, BB: OriginalBB);
3310	Phi->addIncoming(V: Dst, BB: ConvertBB);
3311	return Phi;
3312	}
3313
3314	llvm::Value *MicrosoftCXXABI::EmitNonNullMemberPointerConversion(
3315	const MemberPointerType *SrcTy, const MemberPointerType *DstTy, CastKind CK,
3316	CastExpr::path_const_iterator PathBegin,
3317	CastExpr::path_const_iterator PathEnd, llvm::Value *Src,
3318	CGBuilderTy &Builder) {
3319	const CXXRecordDecl *SrcRD = SrcTy->getMostRecentCXXRecordDecl();
3320	const CXXRecordDecl *DstRD = DstTy->getMostRecentCXXRecordDecl();
3321	MSInheritanceModel SrcInheritance = SrcRD->getMSInheritanceModel();
3322	MSInheritanceModel DstInheritance = DstRD->getMSInheritanceModel();
3323	bool IsFunc = SrcTy->isMemberFunctionPointer();
3324	bool IsConstant = isa<llvm::Constant>(Val: Src);
3325
3326	// Decompose src.
3327	llvm::Value *FirstField = Src;
3328	llvm::Value *NonVirtualBaseAdjustment = getZeroInt();
3329	llvm::Value *VirtualBaseAdjustmentOffset = getZeroInt();
3330	llvm::Value *VBPtrOffset = getZeroInt();
3331	if (!inheritanceModelHasOnlyOneField(IsMemberFunction: IsFunc, Inheritance: SrcInheritance)) {
3332	// We need to extract values.
3333	unsigned I = 0;
3334	FirstField = Builder.CreateExtractValue(Agg: Src, Idxs: I++);
3335	if (inheritanceModelHasNVOffsetField(IsMemberFunction: IsFunc, Inheritance: SrcInheritance))
3336	NonVirtualBaseAdjustment = Builder.CreateExtractValue(Agg: Src, Idxs: I++);
3337	if (inheritanceModelHasVBPtrOffsetField(Inheritance: SrcInheritance))
3338	VBPtrOffset = Builder.CreateExtractValue(Agg: Src, Idxs: I++);
3339	if (inheritanceModelHasVBTableOffsetField(Inheritance: SrcInheritance))
3340	VirtualBaseAdjustmentOffset = Builder.CreateExtractValue(Agg: Src, Idxs: I++);
3341	}
3342
3343	bool IsDerivedToBase = (CK == CK_DerivedToBaseMemberPointer);
3344	const MemberPointerType *DerivedTy = IsDerivedToBase ? SrcTy : DstTy;
3345	const CXXRecordDecl *DerivedClass = DerivedTy->getMostRecentCXXRecordDecl();
3346
3347	// For data pointers, we adjust the field offset directly. For functions, we
3348	// have a separate field.
3349	llvm::Value *&NVAdjustField = IsFunc ? NonVirtualBaseAdjustment : FirstField;
3350
3351	// The virtual inheritance model has a quirk: the virtual base table is always
3352	// referenced when dereferencing a member pointer even if the member pointer
3353	// is non-virtual. This is accounted for by adjusting the non-virtual offset
3354	// to point backwards to the top of the MDC from the first VBase. Undo this
3355	// adjustment to normalize the member pointer.
3356	llvm::Value *SrcVBIndexEqZero =
3357	Builder.CreateICmpEQ(LHS: VirtualBaseAdjustmentOffset, RHS: getZeroInt());
3358	if (SrcInheritance == MSInheritanceModel::Virtual) {
3359	if (int64_t SrcOffsetToFirstVBase =
3360	getContext().getOffsetOfBaseWithVBPtr(RD: SrcRD).getQuantity()) {
3361	llvm::Value *UndoSrcAdjustment = Builder.CreateSelect(
3362	C: SrcVBIndexEqZero,
3363	True: llvm::ConstantInt::get(Ty: CGM.IntTy, V: SrcOffsetToFirstVBase),
3364	False: getZeroInt());
3365	NVAdjustField = Builder.CreateNSWAdd(LHS: NVAdjustField, RHS: UndoSrcAdjustment);
3366	}
3367	}
3368
3369	// A non-zero vbindex implies that we are dealing with a source member in a
3370	// floating virtual base in addition to some non-virtual offset. If the
3371	// vbindex is zero, we are dealing with a source that exists in a non-virtual,
3372	// fixed, base. The difference between these two cases is that the vbindex +
3373	// nvoffset *always* point to the member regardless of what context they are
3374	// evaluated in so long as the vbindex is adjusted. A member inside a fixed
3375	// base requires explicit nv adjustment.
3376	llvm::Constant *BaseClassOffset = llvm::ConstantInt::get(
3377	Ty: CGM.IntTy,
3378	V: CGM.computeNonVirtualBaseClassOffset(DerivedClass, Start: PathBegin, End: PathEnd)
3379	.getQuantity());
3380
3381	llvm::Value *NVDisp;
3382	if (IsDerivedToBase)
3383	NVDisp = Builder.CreateNSWSub(LHS: NVAdjustField, RHS: BaseClassOffset, Name: "adj");
3384	else
3385	NVDisp = Builder.CreateNSWAdd(LHS: NVAdjustField, RHS: BaseClassOffset, Name: "adj");
3386
3387	NVAdjustField = Builder.CreateSelect(C: SrcVBIndexEqZero, True: NVDisp, False: getZeroInt());
3388
3389	// Update the vbindex to an appropriate value in the destination because
3390	// SrcRD's vbtable might not be a strict prefix of the one in DstRD.
3391	llvm::Value *DstVBIndexEqZero = SrcVBIndexEqZero;
3392	if (inheritanceModelHasVBTableOffsetField(Inheritance: DstInheritance) &&
3393	inheritanceModelHasVBTableOffsetField(Inheritance: SrcInheritance)) {
3394	if (llvm::GlobalVariable *VDispMap =
3395	getAddrOfVirtualDisplacementMap(SrcRD, DstRD)) {
3396	llvm::Value *VBIndex = Builder.CreateExactUDiv(
3397	LHS: VirtualBaseAdjustmentOffset, RHS: llvm::ConstantInt::get(Ty: CGM.IntTy, V: 4));
3398	if (IsConstant) {
3399	llvm::Constant *Mapping = VDispMap->getInitializer();
3400	VirtualBaseAdjustmentOffset =
3401	Mapping->getAggregateElement(Elt: cast<llvm::Constant>(Val: VBIndex));
3402	} else {
3403	llvm::Value *Idxs[] = {getZeroInt(), VBIndex};
3404	VirtualBaseAdjustmentOffset = Builder.CreateAlignedLoad(
3405	Ty: CGM.IntTy, Addr: Builder.CreateInBoundsGEP(Ty: VDispMap->getValueType(),
3406	Ptr: VDispMap, IdxList: Idxs),
3407	Align: CharUnits::fromQuantity(Quantity: 4));
3408	}
3409
3410	DstVBIndexEqZero =
3411	Builder.CreateICmpEQ(LHS: VirtualBaseAdjustmentOffset, RHS: getZeroInt());
3412	}
3413	}
3414
3415	// Set the VBPtrOffset to zero if the vbindex is zero. Otherwise, initialize
3416	// it to the offset of the vbptr.
3417	if (inheritanceModelHasVBPtrOffsetField(Inheritance: DstInheritance)) {
3418	llvm::Value *DstVBPtrOffset = llvm::ConstantInt::get(
3419	Ty: CGM.IntTy,
3420	V: getContext().getASTRecordLayout(DstRD).getVBPtrOffset().getQuantity());
3421	VBPtrOffset =
3422	Builder.CreateSelect(C: DstVBIndexEqZero, True: getZeroInt(), False: DstVBPtrOffset);
3423	}
3424
3425	// Likewise, apply a similar adjustment so that dereferencing the member
3426	// pointer correctly accounts for the distance between the start of the first
3427	// virtual base and the top of the MDC.
3428	if (DstInheritance == MSInheritanceModel::Virtual) {
3429	if (int64_t DstOffsetToFirstVBase =
3430	getContext().getOffsetOfBaseWithVBPtr(RD: DstRD).getQuantity()) {
3431	llvm::Value *DoDstAdjustment = Builder.CreateSelect(
3432	C: DstVBIndexEqZero,
3433	True: llvm::ConstantInt::get(Ty: CGM.IntTy, V: DstOffsetToFirstVBase),
3434	False: getZeroInt());
3435	NVAdjustField = Builder.CreateNSWSub(LHS: NVAdjustField, RHS: DoDstAdjustment);
3436	}
3437	}
3438
3439	// Recompose dst from the null struct and the adjusted fields from src.
3440	llvm::Value *Dst;
3441	if (inheritanceModelHasOnlyOneField(IsMemberFunction: IsFunc, Inheritance: DstInheritance)) {
3442	Dst = FirstField;
3443	} else {
3444	Dst = llvm::UndefValue::get(T: ConvertMemberPointerType(MPT: DstTy));
3445	unsigned Idx = 0;
3446	Dst = Builder.CreateInsertValue(Agg: Dst, Val: FirstField, Idxs: Idx++);
3447	if (inheritanceModelHasNVOffsetField(IsMemberFunction: IsFunc, Inheritance: DstInheritance))
3448	Dst = Builder.CreateInsertValue(Agg: Dst, Val: NonVirtualBaseAdjustment, Idxs: Idx++);
3449	if (inheritanceModelHasVBPtrOffsetField(Inheritance: DstInheritance))
3450	Dst = Builder.CreateInsertValue(Agg: Dst, Val: VBPtrOffset, Idxs: Idx++);
3451	if (inheritanceModelHasVBTableOffsetField(Inheritance: DstInheritance))
3452	Dst = Builder.CreateInsertValue(Agg: Dst, Val: VirtualBaseAdjustmentOffset, Idxs: Idx++);
3453	}
3454	return Dst;
3455	}
3456
3457	llvm::Constant *
3458	MicrosoftCXXABI::EmitMemberPointerConversion(const CastExpr *E,
3459	llvm::Constant *Src) {
3460	const MemberPointerType *SrcTy =
3461	E->getSubExpr()->getType()->castAs<MemberPointerType>();
3462	const MemberPointerType *DstTy = E->getType()->castAs<MemberPointerType>();
3463
3464	CastKind CK = E->getCastKind();
3465
3466	return EmitMemberPointerConversion(SrcTy, DstTy, CK, PathBegin: E->path_begin(),
3467	PathEnd: E->path_end(), Src);
3468	}
3469
3470	llvm::Constant *MicrosoftCXXABI::EmitMemberPointerConversion(
3471	const MemberPointerType *SrcTy, const MemberPointerType *DstTy, CastKind CK,
3472	CastExpr::path_const_iterator PathBegin,
3473	CastExpr::path_const_iterator PathEnd, llvm::Constant *Src) {
3474	assert(CK == CK_DerivedToBaseMemberPointer ||
3475	CK == CK_BaseToDerivedMemberPointer ||
3476	CK == CK_ReinterpretMemberPointer);
3477	// If src is null, emit a new null for dst. We can't return src because dst
3478	// might have a new representation.
3479	if (MemberPointerConstantIsNull(MPT: SrcTy, Val: Src))
3480	return EmitNullMemberPointer(MPT: DstTy);
3481
3482	// We don't need to do anything for reinterpret_casts of non-null member
3483	// pointers. We should only get here when the two type representations have
3484	// the same size.
3485	if (CK == CK_ReinterpretMemberPointer)
3486	return Src;
3487
3488	CGBuilderTy Builder(CGM, CGM.getLLVMContext());
3489	auto *Dst = cast<llvm::Constant>(Val: EmitNonNullMemberPointerConversion(
3490	SrcTy, DstTy, CK, PathBegin, PathEnd, Src, Builder));
3491
3492	return Dst;
3493	}
3494
3495	CGCallee MicrosoftCXXABI::EmitLoadOfMemberFunctionPointer(
3496	CodeGenFunction &CGF, const Expr *E, Address This,
3497	llvm::Value *&ThisPtrForCall, llvm::Value *MemPtr,
3498	const MemberPointerType *MPT) {
3499	assert(MPT->isMemberFunctionPointer());
3500	const FunctionProtoType *FPT =
3501	MPT->getPointeeType()->castAs<FunctionProtoType>();
3502	const CXXRecordDecl *RD = MPT->getMostRecentCXXRecordDecl();
3503	CGBuilderTy &Builder = CGF.Builder;
3504
3505	MSInheritanceModel Inheritance = RD->getMSInheritanceModel();
3506
3507	// Extract the fields we need, regardless of model. We'll apply them if we
3508	// have them.
3509	llvm::Value *FunctionPointer = MemPtr;
3510	llvm::Value *NonVirtualBaseAdjustment = nullptr;
3511	llvm::Value *VirtualBaseAdjustmentOffset = nullptr;
3512	llvm::Value *VBPtrOffset = nullptr;
3513	if (MemPtr->getType()->isStructTy()) {
3514	// We need to extract values.
3515	unsigned I = 0;
3516	FunctionPointer = Builder.CreateExtractValue(Agg: MemPtr, Idxs: I++);
3517	if (inheritanceModelHasNVOffsetField(IsMemberFunction: MPT, Inheritance))
3518	NonVirtualBaseAdjustment = Builder.CreateExtractValue(Agg: MemPtr, Idxs: I++);
3519	if (inheritanceModelHasVBPtrOffsetField(Inheritance))
3520	VBPtrOffset = Builder.CreateExtractValue(Agg: MemPtr, Idxs: I++);
3521	if (inheritanceModelHasVBTableOffsetField(Inheritance))
3522	VirtualBaseAdjustmentOffset = Builder.CreateExtractValue(Agg: MemPtr, Idxs: I++);
3523	}
3524
3525	if (VirtualBaseAdjustmentOffset) {
3526	ThisPtrForCall = AdjustVirtualBase(CGF, E, RD, Base: This,
3527	VBTableOffset: VirtualBaseAdjustmentOffset, VBPtrOffset);
3528	} else {
3529	ThisPtrForCall = This.getPointer();
3530	}
3531
3532	if (NonVirtualBaseAdjustment)
3533	ThisPtrForCall = Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: ThisPtrForCall,
3534	IdxList: NonVirtualBaseAdjustment);
3535
3536	CGCallee Callee(FPT, FunctionPointer);
3537	return Callee;
3538	}
3539
3540	CGCXXABI *clang::CodeGen::CreateMicrosoftCXXABI(CodeGenModule &CGM) {
3541	return new MicrosoftCXXABI(CGM);
3542	}
3543
3544	// MS RTTI Overview:
3545	// The run time type information emitted by cl.exe contains 5 distinct types of
3546	// structures. Many of them reference each other.
3547	//
3548	// TypeInfo: Static classes that are returned by typeid.
3549	//
3550	// CompleteObjectLocator: Referenced by vftables. They contain information
3551	// required for dynamic casting, including OffsetFromTop. They also contain
3552	// a reference to the TypeInfo for the type and a reference to the
3553	// CompleteHierarchyDescriptor for the type.
3554	//
3555	// ClassHierarchyDescriptor: Contains information about a class hierarchy.
3556	// Used during dynamic_cast to walk a class hierarchy. References a base
3557	// class array and the size of said array.
3558	//
3559	// BaseClassArray: Contains a list of classes in a hierarchy. BaseClassArray is
3560	// somewhat of a misnomer because the most derived class is also in the list
3561	// as well as multiple copies of virtual bases (if they occur multiple times
3562	// in the hierarchy.) The BaseClassArray contains one BaseClassDescriptor for
3563	// every path in the hierarchy, in pre-order depth first order. Note, we do
3564	// not declare a specific llvm type for BaseClassArray, it's merely an array
3565	// of BaseClassDescriptor pointers.
3566	//
3567	// BaseClassDescriptor: Contains information about a class in a class hierarchy.
3568	// BaseClassDescriptor is also somewhat of a misnomer for the same reason that
3569	// BaseClassArray is. It contains information about a class within a
3570	// hierarchy such as: is this base is ambiguous and what is its offset in the
3571	// vbtable. The names of the BaseClassDescriptors have all of their fields
3572	// mangled into them so they can be aggressively deduplicated by the linker.
3573
3574	static llvm::GlobalVariable *getTypeInfoVTable(CodeGenModule &CGM) {
3575	StringRef MangledName("??_7type_info@@6B@");
3576	if (auto VTable = CGM.getModule().getNamedGlobal(Name: MangledName))
3577	return VTable;
3578	return new llvm::GlobalVariable(CGM.getModule(), CGM.Int8PtrTy,
3579	/*isConstant=*/true,
3580	llvm::GlobalVariable::ExternalLinkage,
3581	/*Initializer=*/nullptr, MangledName);
3582	}
3583
3584	namespace {
3585
3586	/// A Helper struct that stores information about a class in a class
3587	/// hierarchy. The information stored in these structs struct is used during
3588	/// the generation of ClassHierarchyDescriptors and BaseClassDescriptors.
3589	// During RTTI creation, MSRTTIClasses are stored in a contiguous array with
3590	// implicit depth first pre-order tree connectivity. getFirstChild and
3591	// getNextSibling allow us to walk the tree efficiently.
3592	struct MSRTTIClass {
3593	enum {
3594	IsPrivateOnPath = 1 | 8,
3595	IsAmbiguous = 2,
3596	IsPrivate = 4,
3597	IsVirtual = 16,
3598	HasHierarchyDescriptor = 64
3599	};
3600	MSRTTIClass(const CXXRecordDecl *RD) : RD(RD) {}
3601	uint32_t initialize(const MSRTTIClass *Parent,
3602	const CXXBaseSpecifier *Specifier);
3603
3604	MSRTTIClass *getFirstChild() { return this + 1; }
3605	static MSRTTIClass *getNextChild(MSRTTIClass *Child) {
3606	return Child + 1 + Child->NumBases;
3607	}
3608
3609	const CXXRecordDecl *RD, *VirtualRoot;
3610	uint32_t Flags, NumBases, OffsetInVBase;
3611	};
3612
3613	/// Recursively initialize the base class array.
3614	uint32_t MSRTTIClass::initialize(const MSRTTIClass *Parent,
3615	const CXXBaseSpecifier *Specifier) {
3616	Flags = HasHierarchyDescriptor;
3617	if (!Parent) {
3618	VirtualRoot = nullptr;
3619	OffsetInVBase = 0;
3620	} else {
3621	if (Specifier->getAccessSpecifier() != AS_public)
3622	Flags |= IsPrivate | IsPrivateOnPath;
3623	if (Specifier->isVirtual()) {
3624	Flags |= IsVirtual;
3625	VirtualRoot = RD;
3626	OffsetInVBase = 0;
3627	} else {
3628	if (Parent->Flags & IsPrivateOnPath)
3629	Flags |= IsPrivateOnPath;
3630	VirtualRoot = Parent->VirtualRoot;
3631	OffsetInVBase = Parent->OffsetInVBase + RD->getASTContext()
3632	.getASTRecordLayout(Parent->RD).getBaseClassOffset(RD).getQuantity();
3633	}
3634	}
3635	NumBases = 0;
3636	MSRTTIClass *Child = getFirstChild();
3637	for (const CXXBaseSpecifier &Base : RD->bases()) {
3638	NumBases += Child->initialize(Parent: this, Specifier: &Base) + 1;
3639	Child = getNextChild(Child);
3640	}
3641	return NumBases;
3642	}
3643
3644	static llvm::GlobalValue::LinkageTypes getLinkageForRTTI(QualType Ty) {
3645	switch (Ty->getLinkage()) {
3646	case Linkage::Invalid:
3647	llvm_unreachable("Linkage hasn't been computed!");
3648
3649	case Linkage::None:
3650	case Linkage::Internal:
3651	case Linkage::UniqueExternal:
3652	return llvm::GlobalValue::InternalLinkage;
3653
3654	case Linkage::VisibleNone:
3655	case Linkage::Module:
3656	case Linkage::External:
3657	return llvm::GlobalValue::LinkOnceODRLinkage;
3658	}
3659	llvm_unreachable("Invalid linkage!");
3660	}
3661
3662	/// An ephemeral helper class for building MS RTTI types. It caches some
3663	/// calls to the module and information about the most derived class in a
3664	/// hierarchy.
3665	struct MSRTTIBuilder {
3666	enum {
3667	HasBranchingHierarchy = 1,
3668	HasVirtualBranchingHierarchy = 2,
3669	HasAmbiguousBases = 4
3670	};
3671
3672	MSRTTIBuilder(MicrosoftCXXABI &ABI, const CXXRecordDecl *RD)
3673	: CGM(ABI.CGM), Context(CGM.getContext()),
3674	VMContext(CGM.getLLVMContext()), Module(CGM.getModule()), RD(RD),
3675	Linkage(getLinkageForRTTI(Ty: CGM.getContext().getTagDeclType(RD))),
3676	ABI(ABI) {}
3677
3678	llvm::GlobalVariable *getBaseClassDescriptor(const MSRTTIClass &Classes);
3679	llvm::GlobalVariable *
3680	getBaseClassArray(SmallVectorImpl<MSRTTIClass> &Classes);
3681	llvm::GlobalVariable *getClassHierarchyDescriptor();
3682	llvm::GlobalVariable *getCompleteObjectLocator(const VPtrInfo &Info);
3683
3684	CodeGenModule &CGM;
3685	ASTContext &Context;
3686	llvm::LLVMContext &VMContext;
3687	llvm::Module &Module;
3688	const CXXRecordDecl *RD;
3689	llvm::GlobalVariable::LinkageTypes Linkage;
3690	MicrosoftCXXABI &ABI;
3691	};
3692
3693	} // namespace
3694
3695	/// Recursively serializes a class hierarchy in pre-order depth first
3696	/// order.
3697	static void serializeClassHierarchy(SmallVectorImpl<MSRTTIClass> &Classes,
3698	const CXXRecordDecl *RD) {
3699	Classes.push_back(Elt: MSRTTIClass(RD));
3700	for (const CXXBaseSpecifier &Base : RD->bases())
3701	serializeClassHierarchy(Classes, RD: Base.getType()->getAsCXXRecordDecl());
3702	}
3703
3704	/// Find ambiguity among base classes.
3705	static void
3706	detectAmbiguousBases(SmallVectorImpl<MSRTTIClass> &Classes) {
3707	llvm::SmallPtrSet<const CXXRecordDecl *, 8> VirtualBases;
3708	llvm::SmallPtrSet<const CXXRecordDecl *, 8> UniqueBases;
3709	llvm::SmallPtrSet<const CXXRecordDecl *, 8> AmbiguousBases;
3710	for (MSRTTIClass *Class = &Classes.front(); Class <= &Classes.back();) {
3711	if ((Class->Flags & MSRTTIClass::IsVirtual) &&
3712	!VirtualBases.insert(Ptr: Class->RD).second) {
3713	Class = MSRTTIClass::getNextChild(Child: Class);
3714	continue;
3715	}
3716	if (!UniqueBases.insert(Ptr: Class->RD).second)
3717	AmbiguousBases.insert(Ptr: Class->RD);
3718	Class++;
3719	}
3720	if (AmbiguousBases.empty())
3721	return;
3722	for (MSRTTIClass &Class : Classes)
3723	if (AmbiguousBases.count(Ptr: Class.RD))
3724	Class.Flags |= MSRTTIClass::IsAmbiguous;
3725	}
3726
3727	llvm::GlobalVariable *MSRTTIBuilder::getClassHierarchyDescriptor() {
3728	SmallString<256> MangledName;
3729	{
3730	llvm::raw_svector_ostream Out(MangledName);
3731	ABI.getMangleContext().mangleCXXRTTIClassHierarchyDescriptor(Derived: RD, Out);
3732	}
3733
3734	// Check to see if we've already declared this ClassHierarchyDescriptor.
3735	if (auto CHD = Module.getNamedGlobal(Name: MangledName))
3736	return CHD;
3737
3738	// Serialize the class hierarchy and initialize the CHD Fields.
3739	SmallVector<MSRTTIClass, 8> Classes;
3740	serializeClassHierarchy(Classes, RD);
3741	Classes.front().initialize(/*Parent=*/nullptr, /*Specifier=*/nullptr);
3742	detectAmbiguousBases(Classes);
3743	int Flags = 0;
3744	for (const MSRTTIClass &Class : Classes) {
3745	if (Class.RD->getNumBases() > 1)
3746	Flags |= HasBranchingHierarchy;
3747	// Note: cl.exe does not calculate "HasAmbiguousBases" correctly. We
3748	// believe the field isn't actually used.
3749	if (Class.Flags & MSRTTIClass::IsAmbiguous)
3750	Flags |= HasAmbiguousBases;
3751	}
3752	if ((Flags & HasBranchingHierarchy) && RD->getNumVBases() != 0)
3753	Flags |= HasVirtualBranchingHierarchy;
3754	// These gep indices are used to get the address of the first element of the
3755	// base class array.
3756	llvm::Value *GEPIndices[] = {llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0),
3757	llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0)};
3758
3759	// Forward-declare the class hierarchy descriptor
3760	auto Type = ABI.getClassHierarchyDescriptorType();
3761	auto CHD = new llvm::GlobalVariable(Module, Type, /*isConstant=*/true, Linkage,
3762	/*Initializer=*/nullptr,
3763	MangledName);
3764	if (CHD->isWeakForLinker())
3765	CHD->setComdat(CGM.getModule().getOrInsertComdat(Name: CHD->getName()));
3766
3767	auto *Bases = getBaseClassArray(Classes);
3768
3769	// Initialize the base class ClassHierarchyDescriptor.
3770	llvm::Constant *Fields[] = {
3771	llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0), // reserved by the runtime
3772	llvm::ConstantInt::get(Ty: CGM.IntTy, V: Flags),
3773	llvm::ConstantInt::get(Ty: CGM.IntTy, V: Classes.size()),
3774	ABI.getImageRelativeConstant(PtrVal: llvm::ConstantExpr::getInBoundsGetElementPtr(
3775	Ty: Bases->getValueType(), C: Bases,
3776	IdxList: llvm::ArrayRef<llvm::Value *>(GEPIndices))),
3777	};
3778	CHD->setInitializer(llvm::ConstantStruct::get(T: Type, V: Fields));
3779	return CHD;
3780	}
3781
3782	llvm::GlobalVariable *
3783	MSRTTIBuilder::getBaseClassArray(SmallVectorImpl<MSRTTIClass> &Classes) {
3784	SmallString<256> MangledName;
3785	{
3786	llvm::raw_svector_ostream Out(MangledName);
3787	ABI.getMangleContext().mangleCXXRTTIBaseClassArray(Derived: RD, Out);
3788	}
3789
3790	// Forward-declare the base class array.
3791	// cl.exe pads the base class array with 1 (in 32 bit mode) or 4 (in 64 bit
3792	// mode) bytes of padding. We provide a pointer sized amount of padding by
3793	// adding +1 to Classes.size(). The sections have pointer alignment and are
3794	// marked pick-any so it shouldn't matter.
3795	llvm::Type *PtrType = ABI.getImageRelativeType(
3796	PtrType: ABI.getBaseClassDescriptorType()->getPointerTo());
3797	auto *ArrType = llvm::ArrayType::get(ElementType: PtrType, NumElements: Classes.size() + 1);
3798	auto *BCA =
3799	new llvm::GlobalVariable(Module, ArrType,
3800	/*isConstant=*/true, Linkage,
3801	/*Initializer=*/nullptr, MangledName);
3802	if (BCA->isWeakForLinker())
3803	BCA->setComdat(CGM.getModule().getOrInsertComdat(Name: BCA->getName()));
3804
3805	// Initialize the BaseClassArray.
3806	SmallVector<llvm::Constant *, 8> BaseClassArrayData;
3807	for (MSRTTIClass &Class : Classes)
3808	BaseClassArrayData.push_back(
3809	Elt: ABI.getImageRelativeConstant(PtrVal: getBaseClassDescriptor(Classes: Class)));
3810	BaseClassArrayData.push_back(Elt: llvm::Constant::getNullValue(Ty: PtrType));
3811	BCA->setInitializer(llvm::ConstantArray::get(T: ArrType, V: BaseClassArrayData));
3812	return BCA;
3813	}
3814
3815	llvm::GlobalVariable *
3816	MSRTTIBuilder::getBaseClassDescriptor(const MSRTTIClass &Class) {
3817	// Compute the fields for the BaseClassDescriptor. They are computed up front
3818	// because they are mangled into the name of the object.
3819	uint32_t OffsetInVBTable = 0;
3820	int32_t VBPtrOffset = -1;
3821	if (Class.VirtualRoot) {
3822	auto &VTableContext = CGM.getMicrosoftVTableContext();
3823	OffsetInVBTable = VTableContext.getVBTableIndex(Derived: RD, VBase: Class.VirtualRoot) * 4;
3824	VBPtrOffset = Context.getASTRecordLayout(RD).getVBPtrOffset().getQuantity();
3825	}
3826
3827	SmallString<256> MangledName;
3828	{
3829	llvm::raw_svector_ostream Out(MangledName);
3830	ABI.getMangleContext().mangleCXXRTTIBaseClassDescriptor(
3831	Derived: Class.RD, NVOffset: Class.OffsetInVBase, VBPtrOffset, VBTableOffset: OffsetInVBTable,
3832	Flags: Class.Flags, Out);
3833	}
3834
3835	// Check to see if we've already declared this object.
3836	if (auto BCD = Module.getNamedGlobal(Name: MangledName))
3837	return BCD;
3838
3839	// Forward-declare the base class descriptor.
3840	auto Type = ABI.getBaseClassDescriptorType();
3841	auto BCD =
3842	new llvm::GlobalVariable(Module, Type, /*isConstant=*/true, Linkage,
3843	/*Initializer=*/nullptr, MangledName);
3844	if (BCD->isWeakForLinker())
3845	BCD->setComdat(CGM.getModule().getOrInsertComdat(Name: BCD->getName()));
3846
3847	// Initialize the BaseClassDescriptor.
3848	llvm::Constant *Fields[] = {
3849	ABI.getImageRelativeConstant(
3850	PtrVal: ABI.getAddrOfRTTIDescriptor(Ty: Context.getTypeDeclType(Class.RD))),
3851	llvm::ConstantInt::get(Ty: CGM.IntTy, V: Class.NumBases),
3852	llvm::ConstantInt::get(Ty: CGM.IntTy, V: Class.OffsetInVBase),
3853	llvm::ConstantInt::get(Ty: CGM.IntTy, V: VBPtrOffset),
3854	llvm::ConstantInt::get(Ty: CGM.IntTy, V: OffsetInVBTable),
3855	llvm::ConstantInt::get(Ty: CGM.IntTy, V: Class.Flags),
3856	ABI.getImageRelativeConstant(
3857	PtrVal: MSRTTIBuilder(ABI, Class.RD).getClassHierarchyDescriptor()),
3858	};
3859	BCD->setInitializer(llvm::ConstantStruct::get(T: Type, V: Fields));
3860	return BCD;
3861	}
3862
3863	llvm::GlobalVariable *
3864	MSRTTIBuilder::getCompleteObjectLocator(const VPtrInfo &Info) {
3865	SmallString<256> MangledName;
3866	{
3867	llvm::raw_svector_ostream Out(MangledName);
3868	ABI.getMangleContext().mangleCXXRTTICompleteObjectLocator(Derived: RD, BasePath: Info.MangledPath, Out);
3869	}
3870
3871	// Check to see if we've already computed this complete object locator.
3872	if (auto COL = Module.getNamedGlobal(Name: MangledName))
3873	return COL;
3874
3875	// Compute the fields of the complete object locator.
3876	int OffsetToTop = Info.FullOffsetInMDC.getQuantity();
3877	int VFPtrOffset = 0;
3878	// The offset includes the vtordisp if one exists.
3879	if (const CXXRecordDecl *VBase = Info.getVBaseWithVPtr())
3880	if (Context.getASTRecordLayout(RD)
3881	.getVBaseOffsetsMap()
3882	.find(Val: VBase)
3883	->second.hasVtorDisp())
3884	VFPtrOffset = Info.NonVirtualOffset.getQuantity() + 4;
3885
3886	// Forward-declare the complete object locator.
3887	llvm::StructType *Type = ABI.getCompleteObjectLocatorType();
3888	auto COL = new llvm::GlobalVariable(Module, Type, /*isConstant=*/true, Linkage,
3889	/*Initializer=*/nullptr, MangledName);
3890
3891	// Initialize the CompleteObjectLocator.
3892	llvm::Constant *Fields[] = {
3893	llvm::ConstantInt::get(Ty: CGM.IntTy, V: ABI.isImageRelative()),
3894	llvm::ConstantInt::get(Ty: CGM.IntTy, V: OffsetToTop),
3895	llvm::ConstantInt::get(Ty: CGM.IntTy, V: VFPtrOffset),
3896	ABI.getImageRelativeConstant(
3897	PtrVal: CGM.GetAddrOfRTTIDescriptor(Ty: Context.getTypeDeclType(RD))),
3898	ABI.getImageRelativeConstant(PtrVal: getClassHierarchyDescriptor()),
3899	ABI.getImageRelativeConstant(PtrVal: COL),
3900	};
3901	llvm::ArrayRef<llvm::Constant *> FieldsRef(Fields);
3902	if (!ABI.isImageRelative())
3903	FieldsRef = FieldsRef.drop_back();
3904	COL->setInitializer(llvm::ConstantStruct::get(T: Type, V: FieldsRef));
3905	if (COL->isWeakForLinker())
3906	COL->setComdat(CGM.getModule().getOrInsertComdat(Name: COL->getName()));
3907	return COL;
3908	}
3909
3910	static QualType decomposeTypeForEH(ASTContext &Context, QualType T,
3911	bool &IsConst, bool &IsVolatile,
3912	bool &IsUnaligned) {
3913	T = Context.getExceptionObjectType(T);
3914
3915	// C++14 [except.handle]p3:
3916	// A handler is a match for an exception object of type E if [...]
3917	// - the handler is of type cv T or const T& where T is a pointer type and
3918	// E is a pointer type that can be converted to T by [...]
3919	// - a qualification conversion
3920	IsConst = false;
3921	IsVolatile = false;
3922	IsUnaligned = false;
3923	QualType PointeeType = T->getPointeeType();
3924	if (!PointeeType.isNull()) {
3925	IsConst = PointeeType.isConstQualified();
3926	IsVolatile = PointeeType.isVolatileQualified();
3927	IsUnaligned = PointeeType.getQualifiers().hasUnaligned();
3928	}
3929
3930	// Member pointer types like "const int A::*" are represented by having RTTI
3931	// for "int A::*" and separately storing the const qualifier.
3932	if (const auto *MPTy = T->getAs<MemberPointerType>())
3933	T = Context.getMemberPointerType(T: PointeeType.getUnqualifiedType(),
3934	Cls: MPTy->getClass());
3935
3936	// Pointer types like "const int * const *" are represented by having RTTI
3937	// for "const int **" and separately storing the const qualifier.
3938	if (T->isPointerType())
3939	T = Context.getPointerType(T: PointeeType.getUnqualifiedType());
3940
3941	return T;
3942	}
3943
3944	CatchTypeInfo
3945	MicrosoftCXXABI::getAddrOfCXXCatchHandlerType(QualType Type,
3946	QualType CatchHandlerType) {
3947	// TypeDescriptors for exceptions never have qualified pointer types,
3948	// qualifiers are stored separately in order to support qualification
3949	// conversions.
3950	bool IsConst, IsVolatile, IsUnaligned;
3951	Type =
3952	decomposeTypeForEH(Context&: getContext(), T: Type, IsConst, IsVolatile, IsUnaligned);
3953
3954	bool IsReference = CatchHandlerType->isReferenceType();
3955
3956	uint32_t Flags = 0;
3957	if (IsConst)
3958	Flags |= 1;
3959	if (IsVolatile)
3960	Flags |= 2;
3961	if (IsUnaligned)
3962	Flags |= 4;
3963	if (IsReference)
3964	Flags |= 8;
3965
3966	return CatchTypeInfo{.RTTI: getAddrOfRTTIDescriptor(Ty: Type)->stripPointerCasts(),
3967	.Flags: Flags};
3968	}
3969
3970	/// Gets a TypeDescriptor. Returns a llvm::Constant * rather than a
3971	/// llvm::GlobalVariable * because different type descriptors have different
3972	/// types, and need to be abstracted. They are abstracting by casting the
3973	/// address to an Int8PtrTy.
3974	llvm::Constant *MicrosoftCXXABI::getAddrOfRTTIDescriptor(QualType Type) {
3975	SmallString<256> MangledName;
3976	{
3977	llvm::raw_svector_ostream Out(MangledName);
3978	getMangleContext().mangleCXXRTTI(T: Type, Out);
3979	}
3980
3981	// Check to see if we've already declared this TypeDescriptor.
3982	if (llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(Name: MangledName))
3983	return GV;
3984
3985	// Note for the future: If we would ever like to do deferred emission of
3986	// RTTI, check if emitting vtables opportunistically need any adjustment.
3987
3988	// Compute the fields for the TypeDescriptor.
3989	SmallString<256> TypeInfoString;
3990	{
3991	llvm::raw_svector_ostream Out(TypeInfoString);
3992	getMangleContext().mangleCXXRTTIName(T: Type, Out);
3993	}
3994
3995	// Declare and initialize the TypeDescriptor.
3996	llvm::Constant *Fields[] = {
3997	getTypeInfoVTable(CGM), // VFPtr
3998	llvm::ConstantPointerNull::get(T: CGM.Int8PtrTy), // Runtime data
3999	llvm::ConstantDataArray::getString(Context&: CGM.getLLVMContext(), Initializer: TypeInfoString)};
4000	llvm::StructType *TypeDescriptorType =
4001	getTypeDescriptorType(TypeInfoString);
4002	auto *Var = new llvm::GlobalVariable(
4003	CGM.getModule(), TypeDescriptorType, /*isConstant=*/false,
4004	getLinkageForRTTI(Ty: Type),
4005	llvm::ConstantStruct::get(T: TypeDescriptorType, V: Fields),
4006	MangledName);
4007	if (Var->isWeakForLinker())
4008	Var->setComdat(CGM.getModule().getOrInsertComdat(Name: Var->getName()));
4009	return Var;
4010	}
4011
4012	/// Gets or a creates a Microsoft CompleteObjectLocator.
4013	llvm::GlobalVariable *
4014	MicrosoftCXXABI::getMSCompleteObjectLocator(const CXXRecordDecl *RD,
4015	const VPtrInfo &Info) {
4016	return MSRTTIBuilder(*this, RD).getCompleteObjectLocator(Info);
4017	}
4018
4019	void MicrosoftCXXABI::emitCXXStructor(GlobalDecl GD) {
4020	if (auto *ctor = dyn_cast<CXXConstructorDecl>(Val: GD.getDecl())) {
4021	// There are no constructor variants, always emit the complete destructor.
4022	llvm::Function *Fn =
4023	CGM.codegenCXXStructor(GD: GD.getWithCtorType(Type: Ctor_Complete));
4024	CGM.maybeSetTrivialComdat(*ctor, *Fn);
4025	return;
4026	}
4027
4028	auto *dtor = cast<CXXDestructorDecl>(Val: GD.getDecl());
4029
4030	// Emit the base destructor if the base and complete (vbase) destructors are
4031	// equivalent. This effectively implements -mconstructor-aliases as part of
4032	// the ABI.
4033	if (GD.getDtorType() == Dtor_Complete &&
4034	dtor->getParent()->getNumVBases() == 0)
4035	GD = GD.getWithDtorType(Type: Dtor_Base);
4036
4037	// The base destructor is equivalent to the base destructor of its
4038	// base class if there is exactly one non-virtual base class with a
4039	// non-trivial destructor, there are no fields with a non-trivial
4040	// destructor, and the body of the destructor is trivial.
4041	if (GD.getDtorType() == Dtor_Base && !CGM.TryEmitBaseDestructorAsAlias(D: dtor))
4042	return;
4043
4044	llvm::Function *Fn = CGM.codegenCXXStructor(GD);
4045	if (Fn->isWeakForLinker())
4046	Fn->setComdat(CGM.getModule().getOrInsertComdat(Name: Fn->getName()));
4047	}
4048
4049	llvm::Function *
4050	MicrosoftCXXABI::getAddrOfCXXCtorClosure(const CXXConstructorDecl *CD,
4051	CXXCtorType CT) {
4052	assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure);
4053
4054	// Calculate the mangled name.
4055	SmallString<256> ThunkName;
4056	llvm::raw_svector_ostream Out(ThunkName);
4057	getMangleContext().mangleName(GD: GlobalDecl(CD, CT), Out);
4058
4059	// If the thunk has been generated previously, just return it.
4060	if (llvm::GlobalValue *GV = CGM.getModule().getNamedValue(Name: ThunkName))
4061	return cast<llvm::Function>(Val: GV);
4062
4063	// Create the llvm::Function.
4064	const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeMSCtorClosure(CD, CT);
4065	llvm::FunctionType *ThunkTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
4066	const CXXRecordDecl *RD = CD->getParent();
4067	QualType RecordTy = getContext().getRecordType(RD);
4068	llvm::Function *ThunkFn = llvm::Function::Create(
4069	Ty: ThunkTy, Linkage: getLinkageForRTTI(Ty: RecordTy), N: ThunkName.str(), M: &CGM.getModule());
4070	ThunkFn->setCallingConv(static_cast<llvm::CallingConv::ID>(
4071	FnInfo.getEffectiveCallingConvention()));
4072	if (ThunkFn->isWeakForLinker())
4073	ThunkFn->setComdat(CGM.getModule().getOrInsertComdat(Name: ThunkFn->getName()));
4074	bool IsCopy = CT == Ctor_CopyingClosure;
4075
4076	// Start codegen.
4077	CodeGenFunction CGF(CGM);
4078	CGF.CurGD = GlobalDecl(CD, Ctor_Complete);
4079
4080	// Build FunctionArgs.
4081	FunctionArgList FunctionArgs;
4082
4083	// A constructor always starts with a 'this' pointer as its first argument.
4084	buildThisParam(CGF, Params&: FunctionArgs);
4085
4086	// Following the 'this' pointer is a reference to the source object that we
4087	// are copying from.
4088	ImplicitParamDecl SrcParam(
4089	getContext(), /*DC=*/nullptr, SourceLocation(),
4090	&getContext().Idents.get(Name: "src"),
4091	getContext().getLValueReferenceType(T: RecordTy,
4092	/*SpelledAsLValue=*/true),
4093	ImplicitParamKind::Other);
4094	if (IsCopy)
4095	FunctionArgs.push_back(&SrcParam);
4096
4097	// Constructors for classes which utilize virtual bases have an additional
4098	// parameter which indicates whether or not it is being delegated to by a more
4099	// derived constructor.
4100	ImplicitParamDecl IsMostDerived(getContext(), /*DC=*/nullptr,
4101	SourceLocation(),
4102	&getContext().Idents.get(Name: "is_most_derived"),
4103	getContext().IntTy, ImplicitParamKind::Other);
4104	// Only add the parameter to the list if the class has virtual bases.
4105	if (RD->getNumVBases() > 0)
4106	FunctionArgs.push_back(&IsMostDerived);
4107
4108	// Start defining the function.
4109	auto NL = ApplyDebugLocation::CreateEmpty(CGF);
4110	CGF.StartFunction(GD: GlobalDecl(), RetTy: FnInfo.getReturnType(), Fn: ThunkFn, FnInfo,
4111	Args: FunctionArgs, Loc: CD->getLocation(), StartLoc: SourceLocation());
4112	// Create a scope with an artificial location for the body of this function.
4113	auto AL = ApplyDebugLocation::CreateArtificial(CGF);
4114	setCXXABIThisValue(CGF, ThisPtr: loadIncomingCXXThis(CGF));
4115	llvm::Value *This = getThisValue(CGF);
4116
4117	llvm::Value *SrcVal =
4118	IsCopy ? CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(&SrcParam), Name: "src")
4119	: nullptr;
4120
4121	CallArgList Args;
4122
4123	// Push the this ptr.
4124	Args.add(rvalue: RValue::get(V: This), type: CD->getThisType());
4125
4126	// Push the src ptr.
4127	if (SrcVal)
4128	Args.add(rvalue: RValue::get(V: SrcVal), type: SrcParam.getType());
4129
4130	// Add the rest of the default arguments.
4131	SmallVector<const Stmt *, 4> ArgVec;
4132	ArrayRef<ParmVarDecl *> params = CD->parameters().drop_front(IsCopy ? 1 : 0);
4133	for (const ParmVarDecl *PD : params) {
4134	assert(PD->hasDefaultArg() && "ctor closure lacks default args");
4135	ArgVec.push_back(PD->getDefaultArg());
4136	}
4137
4138	CodeGenFunction::RunCleanupsScope Cleanups(CGF);
4139
4140	const auto *FPT = CD->getType()->castAs<FunctionProtoType>();
4141	CGF.EmitCallArgs(Args, Prototype: FPT, ArgRange: llvm::ArrayRef(ArgVec), AC: CD, ParamsToSkip: IsCopy ? 1 : 0);
4142
4143	// Insert any ABI-specific implicit constructor arguments.
4144	AddedStructorArgCounts ExtraArgs =
4145	addImplicitConstructorArgs(CGF, D: CD, Type: Ctor_Complete,
4146	/*ForVirtualBase=*/false,
4147	/*Delegating=*/false, Args);
4148	// Call the destructor with our arguments.
4149	llvm::Constant *CalleePtr =
4150	CGM.getAddrOfCXXStructor(GD: GlobalDecl(CD, Ctor_Complete));
4151	CGCallee Callee =
4152	CGCallee::forDirect(functionPtr: CalleePtr, abstractInfo: GlobalDecl(CD, Ctor_Complete));
4153	const CGFunctionInfo &CalleeInfo = CGM.getTypes().arrangeCXXConstructorCall(
4154	Args, D: CD, CtorKind: Ctor_Complete, ExtraPrefixArgs: ExtraArgs.Prefix, ExtraSuffixArgs: ExtraArgs.Suffix);
4155	CGF.EmitCall(CallInfo: CalleeInfo, Callee, ReturnValue: ReturnValueSlot(), Args);
4156
4157	Cleanups.ForceCleanup();
4158
4159	// Emit the ret instruction, remove any temporary instructions created for the
4160	// aid of CodeGen.
4161	CGF.FinishFunction(EndLoc: SourceLocation());
4162
4163	return ThunkFn;
4164	}
4165
4166	llvm::Constant *MicrosoftCXXABI::getCatchableType(QualType T,
4167	uint32_t NVOffset,
4168	int32_t VBPtrOffset,
4169	uint32_t VBIndex) {
4170	assert(!T->isReferenceType());
4171
4172	CXXRecordDecl *RD = T->getAsCXXRecordDecl();
4173	const CXXConstructorDecl *CD =
4174	RD ? CGM.getContext().getCopyConstructorForExceptionObject(RD) : nullptr;
4175	CXXCtorType CT = Ctor_Complete;
4176	if (CD)
4177	if (!hasDefaultCXXMethodCC(getContext(), CD) || CD->getNumParams() != 1)
4178	CT = Ctor_CopyingClosure;
4179
4180	uint32_t Size = getContext().getTypeSizeInChars(T).getQuantity();
4181	SmallString<256> MangledName;
4182	{
4183	llvm::raw_svector_ostream Out(MangledName);
4184	getMangleContext().mangleCXXCatchableType(T, CD, CT, Size, NVOffset,
4185	VBPtrOffset, VBIndex, Out);
4186	}
4187	if (llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(Name: MangledName))
4188	return getImageRelativeConstant(PtrVal: GV);
4189
4190	// The TypeDescriptor is used by the runtime to determine if a catch handler
4191	// is appropriate for the exception object.
4192	llvm::Constant *TD = getImageRelativeConstant(PtrVal: getAddrOfRTTIDescriptor(Type: T));
4193
4194	// The runtime is responsible for calling the copy constructor if the
4195	// exception is caught by value.
4196	llvm::Constant *CopyCtor;
4197	if (CD) {
4198	if (CT == Ctor_CopyingClosure)
4199	CopyCtor = getAddrOfCXXCtorClosure(CD, CT: Ctor_CopyingClosure);
4200	else
4201	CopyCtor = CGM.getAddrOfCXXStructor(GD: GlobalDecl(CD, Ctor_Complete));
4202	} else {
4203	CopyCtor = llvm::Constant::getNullValue(Ty: CGM.Int8PtrTy);
4204	}
4205	CopyCtor = getImageRelativeConstant(PtrVal: CopyCtor);
4206
4207	bool IsScalar = !RD;
4208	bool HasVirtualBases = false;
4209	bool IsStdBadAlloc = false; // std::bad_alloc is special for some reason.
4210	QualType PointeeType = T;
4211	if (T->isPointerType())
4212	PointeeType = T->getPointeeType();
4213	if (const CXXRecordDecl *RD = PointeeType->getAsCXXRecordDecl()) {
4214	HasVirtualBases = RD->getNumVBases() > 0;
4215	if (IdentifierInfo *II = RD->getIdentifier())
4216	IsStdBadAlloc = II->isStr(Str: "bad_alloc") && RD->isInStdNamespace();
4217	}
4218
4219	// Encode the relevant CatchableType properties into the Flags bitfield.
4220	// FIXME: Figure out how bits 2 or 8 can get set.
4221	uint32_t Flags = 0;
4222	if (IsScalar)
4223	Flags |= 1;
4224	if (HasVirtualBases)
4225	Flags |= 4;
4226	if (IsStdBadAlloc)
4227	Flags |= 16;
4228
4229	llvm::Constant *Fields[] = {
4230	llvm::ConstantInt::get(Ty: CGM.IntTy, V: Flags), // Flags
4231	TD, // TypeDescriptor
4232	llvm::ConstantInt::get(Ty: CGM.IntTy, V: NVOffset), // NonVirtualAdjustment
4233	llvm::ConstantInt::get(Ty: CGM.IntTy, V: VBPtrOffset), // OffsetToVBPtr
4234	llvm::ConstantInt::get(Ty: CGM.IntTy, V: VBIndex), // VBTableIndex
4235	llvm::ConstantInt::get(Ty: CGM.IntTy, V: Size), // Size
4236	CopyCtor // CopyCtor
4237	};
4238	llvm::StructType *CTType = getCatchableTypeType();
4239	auto *GV = new llvm::GlobalVariable(
4240	CGM.getModule(), CTType, /*isConstant=*/true, getLinkageForRTTI(Ty: T),
4241	llvm::ConstantStruct::get(T: CTType, V: Fields), MangledName);
4242	GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4243	GV->setSection(".xdata");
4244	if (GV->isWeakForLinker())
4245	GV->setComdat(CGM.getModule().getOrInsertComdat(Name: GV->getName()));
4246	return getImageRelativeConstant(PtrVal: GV);
4247	}
4248
4249	llvm::GlobalVariable *MicrosoftCXXABI::getCatchableTypeArray(QualType T) {
4250	assert(!T->isReferenceType());
4251
4252	// See if we've already generated a CatchableTypeArray for this type before.
4253	llvm::GlobalVariable *&CTA = CatchableTypeArrays[T];
4254	if (CTA)
4255	return CTA;
4256
4257	// Ensure that we don't have duplicate entries in our CatchableTypeArray by
4258	// using a SmallSetVector. Duplicates may arise due to virtual bases
4259	// occurring more than once in the hierarchy.
4260	llvm::SmallSetVector<llvm::Constant *, 2> CatchableTypes;
4261
4262	// C++14 [except.handle]p3:
4263	// A handler is a match for an exception object of type E if [...]
4264	// - the handler is of type cv T or cv T& and T is an unambiguous public
4265	// base class of E, or
4266	// - the handler is of type cv T or const T& where T is a pointer type and
4267	// E is a pointer type that can be converted to T by [...]
4268	// - a standard pointer conversion (4.10) not involving conversions to
4269	// pointers to private or protected or ambiguous classes
4270	const CXXRecordDecl *MostDerivedClass = nullptr;
4271	bool IsPointer = T->isPointerType();
4272	if (IsPointer)
4273	MostDerivedClass = T->getPointeeType()->getAsCXXRecordDecl();
4274	else
4275	MostDerivedClass = T->getAsCXXRecordDecl();
4276
4277	// Collect all the unambiguous public bases of the MostDerivedClass.
4278	if (MostDerivedClass) {
4279	const ASTContext &Context = getContext();
4280	const ASTRecordLayout &MostDerivedLayout =
4281	Context.getASTRecordLayout(MostDerivedClass);
4282	MicrosoftVTableContext &VTableContext = CGM.getMicrosoftVTableContext();
4283	SmallVector<MSRTTIClass, 8> Classes;
4284	serializeClassHierarchy(Classes, RD: MostDerivedClass);
4285	Classes.front().initialize(/*Parent=*/nullptr, /*Specifier=*/nullptr);
4286	detectAmbiguousBases(Classes);
4287	for (const MSRTTIClass &Class : Classes) {
4288	// Skip any ambiguous or private bases.
4289	if (Class.Flags &
4290	(MSRTTIClass::IsPrivateOnPath | MSRTTIClass::IsAmbiguous))
4291	continue;
4292	// Write down how to convert from a derived pointer to a base pointer.
4293	uint32_t OffsetInVBTable = 0;
4294	int32_t VBPtrOffset = -1;
4295	if (Class.VirtualRoot) {
4296	OffsetInVBTable =
4297	VTableContext.getVBTableIndex(Derived: MostDerivedClass, VBase: Class.VirtualRoot)*4;
4298	VBPtrOffset = MostDerivedLayout.getVBPtrOffset().getQuantity();
4299	}
4300
4301	// Turn our record back into a pointer if the exception object is a
4302	// pointer.
4303	QualType RTTITy = QualType(Class.RD->getTypeForDecl(), 0);
4304	if (IsPointer)
4305	RTTITy = Context.getPointerType(T: RTTITy);
4306	CatchableTypes.insert(X: getCatchableType(T: RTTITy, NVOffset: Class.OffsetInVBase,
4307	VBPtrOffset, VBIndex: OffsetInVBTable));
4308	}
4309	}
4310
4311	// C++14 [except.handle]p3:
4312	// A handler is a match for an exception object of type E if
4313	// - The handler is of type cv T or cv T& and E and T are the same type
4314	// (ignoring the top-level cv-qualifiers)
4315	CatchableTypes.insert(X: getCatchableType(T));
4316
4317	// C++14 [except.handle]p3:
4318	// A handler is a match for an exception object of type E if
4319	// - the handler is of type cv T or const T& where T is a pointer type and
4320	// E is a pointer type that can be converted to T by [...]
4321	// - a standard pointer conversion (4.10) not involving conversions to
4322	// pointers to private or protected or ambiguous classes
4323	//
4324	// C++14 [conv.ptr]p2:
4325	// A prvalue of type "pointer to cv T," where T is an object type, can be
4326	// converted to a prvalue of type "pointer to cv void".
4327	if (IsPointer && T->getPointeeType()->isObjectType())
4328	CatchableTypes.insert(getCatchableType(T: getContext().VoidPtrTy));
4329
4330	// C++14 [except.handle]p3:
4331	// A handler is a match for an exception object of type E if [...]
4332	// - the handler is of type cv T or const T& where T is a pointer or
4333	// pointer to member type and E is std::nullptr_t.
4334	//
4335	// We cannot possibly list all possible pointer types here, making this
4336	// implementation incompatible with the standard. However, MSVC includes an
4337	// entry for pointer-to-void in this case. Let's do the same.
4338	if (T->isNullPtrType())
4339	CatchableTypes.insert(getCatchableType(T: getContext().VoidPtrTy));
4340
4341	uint32_t NumEntries = CatchableTypes.size();
4342	llvm::Type *CTType =
4343	getImageRelativeType(PtrType: getCatchableTypeType()->getPointerTo());
4344	llvm::ArrayType *AT = llvm::ArrayType::get(ElementType: CTType, NumElements: NumEntries);
4345	llvm::StructType *CTAType = getCatchableTypeArrayType(NumEntries);
4346	llvm::Constant *Fields[] = {
4347	llvm::ConstantInt::get(Ty: CGM.IntTy, V: NumEntries), // NumEntries
4348	llvm::ConstantArray::get(
4349	T: AT, V: llvm::ArrayRef(CatchableTypes.begin(),
4350	CatchableTypes.end())) // CatchableTypes
4351	};
4352	SmallString<256> MangledName;
4353	{
4354	llvm::raw_svector_ostream Out(MangledName);
4355	getMangleContext().mangleCXXCatchableTypeArray(T, NumEntries, Out);
4356	}
4357	CTA = new llvm::GlobalVariable(
4358	CGM.getModule(), CTAType, /*isConstant=*/true, getLinkageForRTTI(Ty: T),
4359	llvm::ConstantStruct::get(T: CTAType, V: Fields), MangledName);
4360	CTA->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4361	CTA->setSection(".xdata");
4362	if (CTA->isWeakForLinker())
4363	CTA->setComdat(CGM.getModule().getOrInsertComdat(Name: CTA->getName()));
4364	return CTA;
4365	}
4366
4367	llvm::GlobalVariable *MicrosoftCXXABI::getThrowInfo(QualType T) {
4368	bool IsConst, IsVolatile, IsUnaligned;
4369	T = decomposeTypeForEH(Context&: getContext(), T, IsConst, IsVolatile, IsUnaligned);
4370
4371	// The CatchableTypeArray enumerates the various (CV-unqualified) types that
4372	// the exception object may be caught as.
4373	llvm::GlobalVariable *CTA = getCatchableTypeArray(T);
4374	// The first field in a CatchableTypeArray is the number of CatchableTypes.
4375	// This is used as a component of the mangled name which means that we need to
4376	// know what it is in order to see if we have previously generated the
4377	// ThrowInfo.
4378	uint32_t NumEntries =
4379	cast<llvm::ConstantInt>(Val: CTA->getInitializer()->getAggregateElement(Elt: 0U))
4380	->getLimitedValue();
4381
4382	SmallString<256> MangledName;
4383	{
4384	llvm::raw_svector_ostream Out(MangledName);
4385	getMangleContext().mangleCXXThrowInfo(T, IsConst, IsVolatile, IsUnaligned,
4386	NumEntries, Out);
4387	}
4388
4389	// Reuse a previously generated ThrowInfo if we have generated an appropriate
4390	// one before.
4391	if (llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(Name: MangledName))
4392	return GV;
4393
4394	// The RTTI TypeDescriptor uses an unqualified type but catch clauses must
4395	// be at least as CV qualified. Encode this requirement into the Flags
4396	// bitfield.
4397	uint32_t Flags = 0;
4398	if (IsConst)
4399	Flags |= 1;
4400	if (IsVolatile)
4401	Flags |= 2;
4402	if (IsUnaligned)
4403	Flags |= 4;
4404
4405	// The cleanup-function (a destructor) must be called when the exception
4406	// object's lifetime ends.
4407	llvm::Constant *CleanupFn = llvm::Constant::getNullValue(Ty: CGM.Int8PtrTy);
4408	if (const CXXRecordDecl *RD = T->getAsCXXRecordDecl())
4409	if (CXXDestructorDecl *DtorD = RD->getDestructor())
4410	if (!DtorD->isTrivial())
4411	CleanupFn = CGM.getAddrOfCXXStructor(GD: GlobalDecl(DtorD, Dtor_Complete));
4412	// This is unused as far as we can tell, initialize it to null.
4413	llvm::Constant *ForwardCompat =
4414	getImageRelativeConstant(PtrVal: llvm::Constant::getNullValue(Ty: CGM.Int8PtrTy));
4415	llvm::Constant *PointerToCatchableTypes = getImageRelativeConstant(PtrVal: CTA);
4416	llvm::StructType *TIType = getThrowInfoType();
4417	llvm::Constant *Fields[] = {
4418	llvm::ConstantInt::get(Ty: CGM.IntTy, V: Flags), // Flags
4419	getImageRelativeConstant(PtrVal: CleanupFn), // CleanupFn
4420	ForwardCompat, // ForwardCompat
4421	PointerToCatchableTypes // CatchableTypeArray
4422	};
4423	auto *GV = new llvm::GlobalVariable(
4424	CGM.getModule(), TIType, /*isConstant=*/true, getLinkageForRTTI(Ty: T),
4425	llvm::ConstantStruct::get(T: TIType, V: Fields), MangledName.str());
4426	GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4427	GV->setSection(".xdata");
4428	if (GV->isWeakForLinker())
4429	GV->setComdat(CGM.getModule().getOrInsertComdat(Name: GV->getName()));
4430	return GV;
4431	}
4432
4433	void MicrosoftCXXABI::emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) {
4434	const Expr *SubExpr = E->getSubExpr();
4435	assert(SubExpr && "SubExpr cannot be null");
4436	QualType ThrowType = SubExpr->getType();
4437	// The exception object lives on the stack and it's address is passed to the
4438	// runtime function.
4439	Address AI = CGF.CreateMemTemp(T: ThrowType);
4440	CGF.EmitAnyExprToMem(E: SubExpr, Location: AI, Quals: ThrowType.getQualifiers(),
4441	/*IsInit=*/IsInitializer: true);
4442
4443	// The so-called ThrowInfo is used to describe how the exception object may be
4444	// caught.
4445	llvm::GlobalVariable *TI = getThrowInfo(T: ThrowType);
4446
4447	// Call into the runtime to throw the exception.
4448	llvm::Value *Args[] = {
4449	AI.getPointer(),
4450	TI
4451	};
4452	CGF.EmitNoreturnRuntimeCallOrInvoke(callee: getThrowFn(), args: Args);
4453	}
4454
4455	std::pair<llvm::Value *, const CXXRecordDecl *>
4456	MicrosoftCXXABI::LoadVTablePtr(CodeGenFunction &CGF, Address This,
4457	const CXXRecordDecl *RD) {
4458	std::tie(args&: This, args: std::ignore, args&: RD) =
4459	performBaseAdjustment(CGF, Value: This, SrcRecordTy: QualType(RD->getTypeForDecl(), 0));
4460	return {CGF.GetVTablePtr(This, VTableTy: CGM.Int8PtrTy, VTableClass: RD), RD};
4461	}
4462
4463	bool MicrosoftCXXABI::isPermittedToBeHomogeneousAggregate(
4464	const CXXRecordDecl *RD) const {
4465	// All aggregates are permitted to be HFA on non-ARM platforms, which mostly
4466	// affects vectorcall on x64/x86.
4467	if (!CGM.getTarget().getTriple().isAArch64())
4468	return true;
4469	// MSVC Windows on Arm64 has its own rules for determining if a type is HFA
4470	// that are inconsistent with the AAPCS64 ABI. The following are our best
4471	// determination of those rules so far, based on observation of MSVC's
4472	// behavior.
4473	if (RD->isEmpty())
4474	return false;
4475	if (RD->isPolymorphic())
4476	return false;
4477	if (RD->hasNonTrivialCopyAssignment())
4478	return false;
4479	if (RD->hasNonTrivialDestructor())
4480	return false;
4481	if (RD->hasNonTrivialDefaultConstructor())
4482	return false;
4483	// These two are somewhat redundant given the caller
4484	// (ABIInfo::isHomogeneousAggregate) checks the bases and fields, but that
4485	// caller doesn't consider empty bases/fields to be non-homogenous, but it
4486	// looks like Microsoft's AArch64 ABI does care about these empty types &
4487	// anything containing/derived from one is non-homogeneous.
4488	// Instead we could add another CXXABI entry point to query this property and
4489	// have ABIInfo::isHomogeneousAggregate use that property.
4490	// I don't think any other of the features listed above could be true of a
4491	// base/field while not true of the outer struct. For example, if you have a
4492	// base/field that has an non-trivial copy assignment/dtor/default ctor, then
4493	// the outer struct's corresponding operation must be non-trivial.
4494	for (const CXXBaseSpecifier &B : RD->bases()) {
4495	if (const CXXRecordDecl *FRD = B.getType()->getAsCXXRecordDecl()) {
4496	if (!isPermittedToBeHomogeneousAggregate(RD: FRD))
4497	return false;
4498	}
4499	}
4500	// empty fields seem to be caught by the ABIInfo::isHomogeneousAggregate
4501	// checking for padding - but maybe there are ways to end up with an empty
4502	// field without padding? Not that I know of, so don't check fields here &
4503	// rely on the padding check.
4504	return true;
4505	}
4506