1	//===- AddressSanitizer.cpp - memory error detector -----------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of AddressSanitizer, an address basic correctness
10	// checker.
11	// Details of the algorithm:
12	// https://github.com/google/sanitizers/wiki/AddressSanitizerAlgorithm
13	//
14	// FIXME: This sanitizer does not yet handle scalable vectors
15	//
16	//===----------------------------------------------------------------------===//
17
18	#include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
19	#include "llvm/ADT/ArrayRef.h"
20	#include "llvm/ADT/DenseMap.h"
21	#include "llvm/ADT/DepthFirstIterator.h"
22	#include "llvm/ADT/SmallPtrSet.h"
23	#include "llvm/ADT/SmallVector.h"
24	#include "llvm/ADT/Statistic.h"
25	#include "llvm/ADT/StringExtras.h"
26	#include "llvm/ADT/StringRef.h"
27	#include "llvm/ADT/Twine.h"
28	#include "llvm/Analysis/GlobalsModRef.h"
29	#include "llvm/Analysis/MemoryBuiltins.h"
30	#include "llvm/Analysis/StackSafetyAnalysis.h"
31	#include "llvm/Analysis/TargetLibraryInfo.h"
32	#include "llvm/Analysis/ValueTracking.h"
33	#include "llvm/BinaryFormat/MachO.h"
34	#include "llvm/Demangle/Demangle.h"
35	#include "llvm/IR/Argument.h"
36	#include "llvm/IR/Attributes.h"
37	#include "llvm/IR/BasicBlock.h"
38	#include "llvm/IR/Comdat.h"
39	#include "llvm/IR/Constant.h"
40	#include "llvm/IR/Constants.h"
41	#include "llvm/IR/DIBuilder.h"
42	#include "llvm/IR/DataLayout.h"
43	#include "llvm/IR/DebugInfoMetadata.h"
44	#include "llvm/IR/DebugLoc.h"
45	#include "llvm/IR/DerivedTypes.h"
46	#include "llvm/IR/Function.h"
47	#include "llvm/IR/GlobalAlias.h"
48	#include "llvm/IR/GlobalValue.h"
49	#include "llvm/IR/GlobalVariable.h"
50	#include "llvm/IR/IRBuilder.h"
51	#include "llvm/IR/InlineAsm.h"
52	#include "llvm/IR/InstVisitor.h"
53	#include "llvm/IR/InstrTypes.h"
54	#include "llvm/IR/Instruction.h"
55	#include "llvm/IR/Instructions.h"
56	#include "llvm/IR/IntrinsicInst.h"
57	#include "llvm/IR/Intrinsics.h"
58	#include "llvm/IR/LLVMContext.h"
59	#include "llvm/IR/MDBuilder.h"
60	#include "llvm/IR/Metadata.h"
61	#include "llvm/IR/Module.h"
62	#include "llvm/IR/Type.h"
63	#include "llvm/IR/Use.h"
64	#include "llvm/IR/Value.h"
65	#include "llvm/MC/MCSectionMachO.h"
66	#include "llvm/Support/Casting.h"
67	#include "llvm/Support/CommandLine.h"
68	#include "llvm/Support/Debug.h"
69	#include "llvm/Support/ErrorHandling.h"
70	#include "llvm/Support/MathExtras.h"
71	#include "llvm/Support/raw_ostream.h"
72	#include "llvm/TargetParser/Triple.h"
73	#include "llvm/Transforms/Instrumentation.h"
74	#include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
75	#include "llvm/Transforms/Instrumentation/AddressSanitizerOptions.h"
76	#include "llvm/Transforms/Utils/ASanStackFrameLayout.h"
77	#include "llvm/Transforms/Utils/BasicBlockUtils.h"
78	#include "llvm/Transforms/Utils/Local.h"
79	#include "llvm/Transforms/Utils/ModuleUtils.h"
80	#include "llvm/Transforms/Utils/PromoteMemToReg.h"
81	#include <algorithm>
82	#include <cassert>
83	#include <cstddef>
84	#include <cstdint>
85	#include <iomanip>
86	#include <limits>
87	#include <sstream>
88	#include <string>
89	#include <tuple>
90
91	using namespace llvm;
92
93	#define DEBUG_TYPE "asan"
94
95	static const uint64_t kDefaultShadowScale = 3;
96	static const uint64_t kDefaultShadowOffset32 = 1ULL << 29;
97	static const uint64_t kDefaultShadowOffset64 = 1ULL << 44;
98	static const uint64_t kDynamicShadowSentinel =
99	std::numeric_limits<uint64_t>::max();
100	static const uint64_t kSmallX86_64ShadowOffsetBase = 0x7FFFFFFF; // < 2G.
101	static const uint64_t kSmallX86_64ShadowOffsetAlignMask = ~0xFFFULL;
102	static const uint64_t kLinuxKasan_ShadowOffset64 = 0xdffffc0000000000;
103	static const uint64_t kPPC64_ShadowOffset64 = 1ULL << 44;
104	static const uint64_t kSystemZ_ShadowOffset64 = 1ULL << 52;
105	static const uint64_t kMIPS_ShadowOffsetN32 = 1ULL << 29;
106	static const uint64_t kMIPS32_ShadowOffset32 = 0x0aaa0000;
107	static const uint64_t kMIPS64_ShadowOffset64 = 1ULL << 37;
108	static const uint64_t kAArch64_ShadowOffset64 = 1ULL << 36;
109	static const uint64_t kLoongArch64_ShadowOffset64 = 1ULL << 46;
110	static const uint64_t kRISCV64_ShadowOffset64 = 0xd55550000;
111	static const uint64_t kFreeBSD_ShadowOffset32 = 1ULL << 30;
112	static const uint64_t kFreeBSD_ShadowOffset64 = 1ULL << 46;
113	static const uint64_t kFreeBSDAArch64_ShadowOffset64 = 1ULL << 47;
114	static const uint64_t kFreeBSDKasan_ShadowOffset64 = 0xdffff7c000000000;
115	static const uint64_t kNetBSD_ShadowOffset32 = 1ULL << 30;
116	static const uint64_t kNetBSD_ShadowOffset64 = 1ULL << 46;
117	static const uint64_t kNetBSDKasan_ShadowOffset64 = 0xdfff900000000000;
118	static const uint64_t kPS_ShadowOffset64 = 1ULL << 40;
119	static const uint64_t kWindowsShadowOffset32 = 3ULL << 28;
120	static const uint64_t kEmscriptenShadowOffset = 0;
121
122	// The shadow memory space is dynamically allocated.
123	static const uint64_t kWindowsShadowOffset64 = kDynamicShadowSentinel;
124
125	static const size_t kMinStackMallocSize = 1 << 6; // 64B
126	static const size_t kMaxStackMallocSize = 1 << 16; // 64K
127	static const uintptr_t kCurrentStackFrameMagic = 0x41B58AB3;
128	static const uintptr_t kRetiredStackFrameMagic = 0x45E0360E;
129
130	const char kAsanModuleCtorName[] = "asan.module_ctor";
131	const char kAsanModuleDtorName[] = "asan.module_dtor";
132	static const uint64_t kAsanCtorAndDtorPriority = 1;
133	// On Emscripten, the system needs more than one priorities for constructors.
134	static const uint64_t kAsanEmscriptenCtorAndDtorPriority = 50;
135	const char kAsanReportErrorTemplate[] = "__asan_report_";
136	const char kAsanRegisterGlobalsName[] = "__asan_register_globals";
137	const char kAsanUnregisterGlobalsName[] = "__asan_unregister_globals";
138	const char kAsanRegisterImageGlobalsName[] = "__asan_register_image_globals";
139	const char kAsanUnregisterImageGlobalsName[] =
140	"__asan_unregister_image_globals";
141	const char kAsanRegisterElfGlobalsName[] = "__asan_register_elf_globals";
142	const char kAsanUnregisterElfGlobalsName[] = "__asan_unregister_elf_globals";
143	const char kAsanPoisonGlobalsName[] = "__asan_before_dynamic_init";
144	const char kAsanUnpoisonGlobalsName[] = "__asan_after_dynamic_init";
145	const char kAsanInitName[] = "__asan_init";
146	const char kAsanVersionCheckNamePrefix[] = "__asan_version_mismatch_check_v";
147	const char kAsanPtrCmp[] = "__sanitizer_ptr_cmp";
148	const char kAsanPtrSub[] = "__sanitizer_ptr_sub";
149	const char kAsanHandleNoReturnName[] = "__asan_handle_no_return";
150	static const int kMaxAsanStackMallocSizeClass = 10;
151	const char kAsanStackMallocNameTemplate[] = "__asan_stack_malloc_";
152	const char kAsanStackMallocAlwaysNameTemplate[] =
153	"__asan_stack_malloc_always_";
154	const char kAsanStackFreeNameTemplate[] = "__asan_stack_free_";
155	const char kAsanGenPrefix[] = "___asan_gen_";
156	const char kODRGenPrefix[] = "__odr_asan_gen_";
157	const char kSanCovGenPrefix[] = "__sancov_gen_";
158	const char kAsanSetShadowPrefix[] = "__asan_set_shadow_";
159	const char kAsanPoisonStackMemoryName[] = "__asan_poison_stack_memory";
160	const char kAsanUnpoisonStackMemoryName[] = "__asan_unpoison_stack_memory";
161
162	// ASan version script has __asan_* wildcard. Triple underscore prevents a
163	// linker (gold) warning about attempting to export a local symbol.
164	const char kAsanGlobalsRegisteredFlagName[] = "___asan_globals_registered";
165
166	const char kAsanOptionDetectUseAfterReturn[] =
167	"__asan_option_detect_stack_use_after_return";
168
169	const char kAsanShadowMemoryDynamicAddress[] =
170	"__asan_shadow_memory_dynamic_address";
171
172	const char kAsanAllocaPoison[] = "__asan_alloca_poison";
173	const char kAsanAllocasUnpoison[] = "__asan_allocas_unpoison";
174
175	const char kAMDGPUAddressSharedName[] = "llvm.amdgcn.is.shared";
176	const char kAMDGPUAddressPrivateName[] = "llvm.amdgcn.is.private";
177	const char kAMDGPUBallotName[] = "llvm.amdgcn.ballot.i64";
178	const char kAMDGPUUnreachableName[] = "llvm.amdgcn.unreachable";
179
180	// Accesses sizes are powers of two: 1, 2, 4, 8, 16.
181	static const size_t kNumberOfAccessSizes = 5;
182
183	static const uint64_t kAllocaRzSize = 32;
184
185	// ASanAccessInfo implementation constants.
186	constexpr size_t kCompileKernelShift = 0;
187	constexpr size_t kCompileKernelMask = 0x1;
188	constexpr size_t kAccessSizeIndexShift = 1;
189	constexpr size_t kAccessSizeIndexMask = 0xf;
190	constexpr size_t kIsWriteShift = 5;
191	constexpr size_t kIsWriteMask = 0x1;
192
193	// Command-line flags.
194
195	static cl::opt<bool> ClEnableKasan(
196	"asan-kernel", cl::desc("Enable KernelAddressSanitizer instrumentation"),
197	cl::Hidden, cl::init(Val: false));
198
199	static cl::opt<bool> ClRecover(
200	"asan-recover",
201	cl::desc("Enable recovery mode (continue-after-error)."),
202	cl::Hidden, cl::init(Val: false));
203
204	static cl::opt<bool> ClInsertVersionCheck(
205	"asan-guard-against-version-mismatch",
206	cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden,
207	cl::init(Val: true));
208
209	// This flag may need to be replaced with -f[no-]asan-reads.
210	static cl::opt<bool> ClInstrumentReads("asan-instrument-reads",
211	cl::desc("instrument read instructions"),
212	cl::Hidden, cl::init(Val: true));
213
214	static cl::opt<bool> ClInstrumentWrites(
215	"asan-instrument-writes", cl::desc("instrument write instructions"),
216	cl::Hidden, cl::init(Val: true));
217
218	static cl::opt<bool>
219	ClUseStackSafety("asan-use-stack-safety", cl::Hidden, cl::init(Val: true),
220	cl::Hidden, cl::desc("Use Stack Safety analysis results"),
221	cl::Optional);
222
223	static cl::opt<bool> ClInstrumentAtomics(
224	"asan-instrument-atomics",
225	cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
226	cl::init(Val: true));
227
228	static cl::opt<bool>
229	ClInstrumentByval("asan-instrument-byval",
230	cl::desc("instrument byval call arguments"), cl::Hidden,
231	cl::init(Val: true));
232
233	static cl::opt<bool> ClAlwaysSlowPath(
234	"asan-always-slow-path",
235	cl::desc("use instrumentation with slow path for all accesses"), cl::Hidden,
236	cl::init(Val: false));
237
238	static cl::opt<bool> ClForceDynamicShadow(
239	"asan-force-dynamic-shadow",
240	cl::desc("Load shadow address into a local variable for each function"),
241	cl::Hidden, cl::init(Val: false));
242
243	static cl::opt<bool>
244	ClWithIfunc("asan-with-ifunc",
245	cl::desc("Access dynamic shadow through an ifunc global on "
246	"platforms that support this"),
247	cl::Hidden, cl::init(Val: true));
248
249	static cl::opt<bool> ClWithIfuncSuppressRemat(
250	"asan-with-ifunc-suppress-remat",
251	cl::desc("Suppress rematerialization of dynamic shadow address by passing "
252	"it through inline asm in prologue."),
253	cl::Hidden, cl::init(Val: true));
254
255	// This flag limits the number of instructions to be instrumented
256	// in any given BB. Normally, this should be set to unlimited (INT_MAX),
257	// but due to http://llvm.org/bugs/show_bug.cgi?id=12652 we temporary
258	// set it to 10000.
259	static cl::opt<int> ClMaxInsnsToInstrumentPerBB(
260	"asan-max-ins-per-bb", cl::init(Val: 10000),
261	cl::desc("maximal number of instructions to instrument in any given BB"),
262	cl::Hidden);
263
264	// This flag may need to be replaced with -f[no]asan-stack.
265	static cl::opt<bool> ClStack("asan-stack", cl::desc("Handle stack memory"),
266	cl::Hidden, cl::init(Val: true));
267	static cl::opt<uint32_t> ClMaxInlinePoisoningSize(
268	"asan-max-inline-poisoning-size",
269	cl::desc(
270	"Inline shadow poisoning for blocks up to the given size in bytes."),
271	cl::Hidden, cl::init(Val: 64));
272
273	static cl::opt<AsanDetectStackUseAfterReturnMode> ClUseAfterReturn(
274	"asan-use-after-return",
275	cl::desc("Sets the mode of detection for stack-use-after-return."),
276	cl::values(
277	clEnumValN(AsanDetectStackUseAfterReturnMode::Never, "never",
278	"Never detect stack use after return."),
279	clEnumValN(
280	AsanDetectStackUseAfterReturnMode::Runtime, "runtime",
281	"Detect stack use after return if "
282	"binary flag 'ASAN_OPTIONS=detect_stack_use_after_return' is set."),
283	clEnumValN(AsanDetectStackUseAfterReturnMode::Always, "always",
284	"Always detect stack use after return.")),
285	cl::Hidden, cl::init(Val: AsanDetectStackUseAfterReturnMode::Runtime));
286
287	static cl::opt<bool> ClRedzoneByvalArgs("asan-redzone-byval-args",
288	cl::desc("Create redzones for byval "
289	"arguments (extra copy "
290	"required)"), cl::Hidden,
291	cl::init(Val: true));
292
293	static cl::opt<bool> ClUseAfterScope("asan-use-after-scope",
294	cl::desc("Check stack-use-after-scope"),
295	cl::Hidden, cl::init(Val: false));
296
297	// This flag may need to be replaced with -f[no]asan-globals.
298	static cl::opt<bool> ClGlobals("asan-globals",
299	cl::desc("Handle global objects"), cl::Hidden,
300	cl::init(Val: true));
301
302	static cl::opt<bool> ClInitializers("asan-initialization-order",
303	cl::desc("Handle C++ initializer order"),
304	cl::Hidden, cl::init(Val: true));
305
306	static cl::opt<bool> ClInvalidPointerPairs(
307	"asan-detect-invalid-pointer-pair",
308	cl::desc("Instrument <, <=, >, >=, - with pointer operands"), cl::Hidden,
309	cl::init(Val: false));
310
311	static cl::opt<bool> ClInvalidPointerCmp(
312	"asan-detect-invalid-pointer-cmp",
313	cl::desc("Instrument <, <=, >, >= with pointer operands"), cl::Hidden,
314	cl::init(Val: false));
315
316	static cl::opt<bool> ClInvalidPointerSub(
317	"asan-detect-invalid-pointer-sub",
318	cl::desc("Instrument - operations with pointer operands"), cl::Hidden,
319	cl::init(Val: false));
320
321	static cl::opt<unsigned> ClRealignStack(
322	"asan-realign-stack",
323	cl::desc("Realign stack to the value of this flag (power of two)"),
324	cl::Hidden, cl::init(Val: 32));
325
326	static cl::opt<int> ClInstrumentationWithCallsThreshold(
327	"asan-instrumentation-with-call-threshold",
328	cl::desc("If the function being instrumented contains more than "
329	"this number of memory accesses, use callbacks instead of "
330	"inline checks (-1 means never use callbacks)."),
331	cl::Hidden, cl::init(Val: 7000));
332
333	static cl::opt<std::string> ClMemoryAccessCallbackPrefix(
334	"asan-memory-access-callback-prefix",
335	cl::desc("Prefix for memory access callbacks"), cl::Hidden,
336	cl::init(Val: "__asan_"));
337
338	static cl::opt<bool> ClKasanMemIntrinCallbackPrefix(
339	"asan-kernel-mem-intrinsic-prefix",
340	cl::desc("Use prefix for memory intrinsics in KASAN mode"), cl::Hidden,
341	cl::init(Val: false));
342
343	static cl::opt<bool>
344	ClInstrumentDynamicAllocas("asan-instrument-dynamic-allocas",
345	cl::desc("instrument dynamic allocas"),
346	cl::Hidden, cl::init(Val: true));
347
348	static cl::opt<bool> ClSkipPromotableAllocas(
349	"asan-skip-promotable-allocas",
350	cl::desc("Do not instrument promotable allocas"), cl::Hidden,
351	cl::init(Val: true));
352
353	static cl::opt<AsanCtorKind> ClConstructorKind(
354	"asan-constructor-kind",
355	cl::desc("Sets the ASan constructor kind"),
356	cl::values(clEnumValN(AsanCtorKind::None, "none", "No constructors"),
357	clEnumValN(AsanCtorKind::Global, "global",
358	"Use global constructors")),
359	cl::init(Val: AsanCtorKind::Global), cl::Hidden);
360	// These flags allow to change the shadow mapping.
361	// The shadow mapping looks like
362	// Shadow = (Mem >> scale) + offset
363
364	static cl::opt<int> ClMappingScale("asan-mapping-scale",
365	cl::desc("scale of asan shadow mapping"),
366	cl::Hidden, cl::init(Val: 0));
367
368	static cl::opt<uint64_t>
369	ClMappingOffset("asan-mapping-offset",
370	cl::desc("offset of asan shadow mapping [EXPERIMENTAL]"),
371	cl::Hidden, cl::init(Val: 0));
372
373	// Optimization flags. Not user visible, used mostly for testing
374	// and benchmarking the tool.
375
376	static cl::opt<bool> ClOpt("asan-opt", cl::desc("Optimize instrumentation"),
377	cl::Hidden, cl::init(Val: true));
378
379	static cl::opt<bool> ClOptimizeCallbacks("asan-optimize-callbacks",
380	cl::desc("Optimize callbacks"),
381	cl::Hidden, cl::init(Val: false));
382
383	static cl::opt<bool> ClOptSameTemp(
384	"asan-opt-same-temp", cl::desc("Instrument the same temp just once"),
385	cl::Hidden, cl::init(Val: true));
386
387	static cl::opt<bool> ClOptGlobals("asan-opt-globals",
388	cl::desc("Don't instrument scalar globals"),
389	cl::Hidden, cl::init(Val: true));
390
391	static cl::opt<bool> ClOptStack(
392	"asan-opt-stack", cl::desc("Don't instrument scalar stack variables"),
393	cl::Hidden, cl::init(Val: false));
394
395	static cl::opt<bool> ClDynamicAllocaStack(
396	"asan-stack-dynamic-alloca",
397	cl::desc("Use dynamic alloca to represent stack variables"), cl::Hidden,
398	cl::init(Val: true));
399
400	static cl::opt<uint32_t> ClForceExperiment(
401	"asan-force-experiment",
402	cl::desc("Force optimization experiment (for testing)"), cl::Hidden,
403	cl::init(Val: 0));
404
405	static cl::opt<bool>
406	ClUsePrivateAlias("asan-use-private-alias",
407	cl::desc("Use private aliases for global variables"),
408	cl::Hidden, cl::init(Val: true));
409
410	static cl::opt<bool>
411	ClUseOdrIndicator("asan-use-odr-indicator",
412	cl::desc("Use odr indicators to improve ODR reporting"),
413	cl::Hidden, cl::init(Val: true));
414
415	static cl::opt<bool>
416	ClUseGlobalsGC("asan-globals-live-support",
417	cl::desc("Use linker features to support dead "
418	"code stripping of globals"),
419	cl::Hidden, cl::init(Val: true));
420
421	// This is on by default even though there is a bug in gold:
422	// https://sourceware.org/bugzilla/show_bug.cgi?id=19002
423	static cl::opt<bool>
424	ClWithComdat("asan-with-comdat",
425	cl::desc("Place ASan constructors in comdat sections"),
426	cl::Hidden, cl::init(Val: true));
427
428	static cl::opt<AsanDtorKind> ClOverrideDestructorKind(
429	"asan-destructor-kind",
430	cl::desc("Sets the ASan destructor kind. The default is to use the value "
431	"provided to the pass constructor"),
432	cl::values(clEnumValN(AsanDtorKind::None, "none", "No destructors"),
433	clEnumValN(AsanDtorKind::Global, "global",
434	"Use global destructors")),
435	cl::init(Val: AsanDtorKind::Invalid), cl::Hidden);
436
437	// Debug flags.
438
439	static cl::opt<int> ClDebug("asan-debug", cl::desc("debug"), cl::Hidden,
440	cl::init(Val: 0));
441
442	static cl::opt<int> ClDebugStack("asan-debug-stack", cl::desc("debug stack"),
443	cl::Hidden, cl::init(Val: 0));
444
445	static cl::opt<std::string> ClDebugFunc("asan-debug-func", cl::Hidden,
446	cl::desc("Debug func"));
447
448	static cl::opt<int> ClDebugMin("asan-debug-min", cl::desc("Debug min inst"),
449	cl::Hidden, cl::init(Val: -1));
450
451	static cl::opt<int> ClDebugMax("asan-debug-max", cl::desc("Debug max inst"),
452	cl::Hidden, cl::init(Val: -1));
453
454	STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
455	STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
456	STATISTIC(NumOptimizedAccessesToGlobalVar,
457	"Number of optimized accesses to global vars");
458	STATISTIC(NumOptimizedAccessesToStackVar,
459	"Number of optimized accesses to stack vars");
460
461	namespace {
462
463	/// This struct defines the shadow mapping using the rule:
464	/// shadow = (mem >> Scale) ADD-or-OR Offset.
465	/// If InGlobal is true, then
466	/// extern char __asan_shadow[];
467	/// shadow = (mem >> Scale) + &__asan_shadow
468	struct ShadowMapping {
469	int Scale;
470	uint64_t Offset;
471	bool OrShadowOffset;
472	bool InGlobal;
473	};
474
475	} // end anonymous namespace
476
477	static ShadowMapping getShadowMapping(const Triple &TargetTriple, int LongSize,
478	bool IsKasan) {
479	bool IsAndroid = TargetTriple.isAndroid();
480	bool IsIOS = TargetTriple.isiOS() || TargetTriple.isWatchOS() ||
481	TargetTriple.isDriverKit();
482	bool IsMacOS = TargetTriple.isMacOSX();
483	bool IsFreeBSD = TargetTriple.isOSFreeBSD();
484	bool IsNetBSD = TargetTriple.isOSNetBSD();
485	bool IsPS = TargetTriple.isPS();
486	bool IsLinux = TargetTriple.isOSLinux();
487	bool IsPPC64 = TargetTriple.getArch() == Triple::ppc64 ||
488	TargetTriple.getArch() == Triple::ppc64le;
489	bool IsSystemZ = TargetTriple.getArch() == Triple::systemz;
490	bool IsX86_64 = TargetTriple.getArch() == Triple::x86_64;
491	bool IsMIPSN32ABI = TargetTriple.getEnvironment() == Triple::GNUABIN32;
492	bool IsMIPS32 = TargetTriple.isMIPS32();
493	bool IsMIPS64 = TargetTriple.isMIPS64();
494	bool IsArmOrThumb = TargetTriple.isARM() || TargetTriple.isThumb();
495	bool IsAArch64 = TargetTriple.getArch() == Triple::aarch64 ||
496	TargetTriple.getArch() == Triple::aarch64_be;
497	bool IsLoongArch64 = TargetTriple.isLoongArch64();
498	bool IsRISCV64 = TargetTriple.getArch() == Triple::riscv64;
499	bool IsWindows = TargetTriple.isOSWindows();
500	bool IsFuchsia = TargetTriple.isOSFuchsia();
501	bool IsEmscripten = TargetTriple.isOSEmscripten();
502	bool IsAMDGPU = TargetTriple.isAMDGPU();
503
504	ShadowMapping Mapping;
505
506	Mapping.Scale = kDefaultShadowScale;
507	if (ClMappingScale.getNumOccurrences() > 0) {
508	Mapping.Scale = ClMappingScale;
509	}
510
511	if (LongSize == 32) {
512	if (IsAndroid)
513	Mapping.Offset = kDynamicShadowSentinel;
514	else if (IsMIPSN32ABI)
515	Mapping.Offset = kMIPS_ShadowOffsetN32;
516	else if (IsMIPS32)
517	Mapping.Offset = kMIPS32_ShadowOffset32;
518	else if (IsFreeBSD)
519	Mapping.Offset = kFreeBSD_ShadowOffset32;
520	else if (IsNetBSD)
521	Mapping.Offset = kNetBSD_ShadowOffset32;
522	else if (IsIOS)
523	Mapping.Offset = kDynamicShadowSentinel;
524	else if (IsWindows)
525	Mapping.Offset = kWindowsShadowOffset32;
526	else if (IsEmscripten)
527	Mapping.Offset = kEmscriptenShadowOffset;
528	else
529	Mapping.Offset = kDefaultShadowOffset32;
530	} else { // LongSize == 64
531	// Fuchsia is always PIE, which means that the beginning of the address
532	// space is always available.
533	if (IsFuchsia)
534	Mapping.Offset = 0;
535	else if (IsPPC64)
536	Mapping.Offset = kPPC64_ShadowOffset64;
537	else if (IsSystemZ)
538	Mapping.Offset = kSystemZ_ShadowOffset64;
539	else if (IsFreeBSD && IsAArch64)
540	Mapping.Offset = kFreeBSDAArch64_ShadowOffset64;
541	else if (IsFreeBSD && !IsMIPS64) {
542	if (IsKasan)
543	Mapping.Offset = kFreeBSDKasan_ShadowOffset64;
544	else
545	Mapping.Offset = kFreeBSD_ShadowOffset64;
546	} else if (IsNetBSD) {
547	if (IsKasan)
548	Mapping.Offset = kNetBSDKasan_ShadowOffset64;
549	else
550	Mapping.Offset = kNetBSD_ShadowOffset64;
551	} else if (IsPS)
552	Mapping.Offset = kPS_ShadowOffset64;
553	else if (IsLinux && IsX86_64) {
554	if (IsKasan)
555	Mapping.Offset = kLinuxKasan_ShadowOffset64;
556	else
557	Mapping.Offset = (kSmallX86_64ShadowOffsetBase &
558	(kSmallX86_64ShadowOffsetAlignMask << Mapping.Scale));
559	} else if (IsWindows && IsX86_64) {
560	Mapping.Offset = kWindowsShadowOffset64;
561	} else if (IsMIPS64)
562	Mapping.Offset = kMIPS64_ShadowOffset64;
563	else if (IsIOS)
564	Mapping.Offset = kDynamicShadowSentinel;
565	else if (IsMacOS && IsAArch64)
566	Mapping.Offset = kDynamicShadowSentinel;
567	else if (IsAArch64)
568	Mapping.Offset = kAArch64_ShadowOffset64;
569	else if (IsLoongArch64)
570	Mapping.Offset = kLoongArch64_ShadowOffset64;
571	else if (IsRISCV64)
572	Mapping.Offset = kRISCV64_ShadowOffset64;
573	else if (IsAMDGPU)
574	Mapping.Offset = (kSmallX86_64ShadowOffsetBase &
575	(kSmallX86_64ShadowOffsetAlignMask << Mapping.Scale));
576	else
577	Mapping.Offset = kDefaultShadowOffset64;
578	}
579
580	if (ClForceDynamicShadow) {
581	Mapping.Offset = kDynamicShadowSentinel;
582	}
583
584	if (ClMappingOffset.getNumOccurrences() > 0) {
585	Mapping.Offset = ClMappingOffset;
586	}
587
588	// OR-ing shadow offset if more efficient (at least on x86) if the offset
589	// is a power of two, but on ppc64 and loongarch64 we have to use add since
590	// the shadow offset is not necessarily 1/8-th of the address space. On
591	// SystemZ, we could OR the constant in a single instruction, but it's more
592	// efficient to load it once and use indexed addressing.
593	Mapping.OrShadowOffset = !IsAArch64 && !IsPPC64 && !IsSystemZ && !IsPS &&
594	!IsRISCV64 && !IsLoongArch64 &&
595	!(Mapping.Offset & (Mapping.Offset - 1)) &&
596	Mapping.Offset != kDynamicShadowSentinel;
597	bool IsAndroidWithIfuncSupport =
598	IsAndroid && !TargetTriple.isAndroidVersionLT(Major: 21);
599	Mapping.InGlobal = ClWithIfunc && IsAndroidWithIfuncSupport && IsArmOrThumb;
600
601	return Mapping;
602	}
603
604	namespace llvm {
605	void getAddressSanitizerParams(const Triple &TargetTriple, int LongSize,
606	bool IsKasan, uint64_t *ShadowBase,
607	int *MappingScale, bool *OrShadowOffset) {
608	auto Mapping = getShadowMapping(TargetTriple, LongSize, IsKasan);
609	*ShadowBase = Mapping.Offset;
610	*MappingScale = Mapping.Scale;
611	*OrShadowOffset = Mapping.OrShadowOffset;
612	}
613
614	ASanAccessInfo::ASanAccessInfo(int32_t Packed)
615	: Packed(Packed),
616	AccessSizeIndex((Packed >> kAccessSizeIndexShift) & kAccessSizeIndexMask),
617	IsWrite((Packed >> kIsWriteShift) & kIsWriteMask),
618	CompileKernel((Packed >> kCompileKernelShift) & kCompileKernelMask) {}
619
620	ASanAccessInfo::ASanAccessInfo(bool IsWrite, bool CompileKernel,
621	uint8_t AccessSizeIndex)
622	: Packed((IsWrite << kIsWriteShift) +
623	(CompileKernel << kCompileKernelShift) +
624	(AccessSizeIndex << kAccessSizeIndexShift)),
625	AccessSizeIndex(AccessSizeIndex), IsWrite(IsWrite),
626	CompileKernel(CompileKernel) {}
627
628	} // namespace llvm
629
630	static uint64_t getRedzoneSizeForScale(int MappingScale) {
631	// Redzone used for stack and globals is at least 32 bytes.
632	// For scales 6 and 7, the redzone has to be 64 and 128 bytes respectively.
633	return std::max(a: 32U, b: 1U << MappingScale);
634	}
635
636	static uint64_t GetCtorAndDtorPriority(Triple &TargetTriple) {
637	if (TargetTriple.isOSEmscripten()) {
638	return kAsanEmscriptenCtorAndDtorPriority;
639	} else {
640	return kAsanCtorAndDtorPriority;
641	}
642	}
643
644	namespace {
645
646	/// AddressSanitizer: instrument the code in module to find memory bugs.
647	struct AddressSanitizer {
648	AddressSanitizer(Module &M, const StackSafetyGlobalInfo *SSGI,
649	int InstrumentationWithCallsThreshold,
650	uint32_t MaxInlinePoisoningSize, bool CompileKernel = false,
651	bool Recover = false, bool UseAfterScope = false,
652	AsanDetectStackUseAfterReturnMode UseAfterReturn =
653	AsanDetectStackUseAfterReturnMode::Runtime)
654	: CompileKernel(ClEnableKasan.getNumOccurrences() > 0 ? ClEnableKasan
655	: CompileKernel),
656	Recover(ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover),
657	UseAfterScope(UseAfterScope || ClUseAfterScope),
658	UseAfterReturn(ClUseAfterReturn.getNumOccurrences() ? ClUseAfterReturn
659	: UseAfterReturn),
660	SSGI(SSGI),
661	InstrumentationWithCallsThreshold(
662	ClInstrumentationWithCallsThreshold.getNumOccurrences() > 0
663	? ClInstrumentationWithCallsThreshold
664	: InstrumentationWithCallsThreshold),
665	MaxInlinePoisoningSize(ClMaxInlinePoisoningSize.getNumOccurrences() > 0
666	? ClMaxInlinePoisoningSize
667	: MaxInlinePoisoningSize) {
668	C = &(M.getContext());
669	DL = &M.getDataLayout();
670	LongSize = M.getDataLayout().getPointerSizeInBits();
671	IntptrTy = Type::getIntNTy(C&: *C, N: LongSize);
672	PtrTy = PointerType::getUnqual(C&: *C);
673	Int32Ty = Type::getInt32Ty(C&: *C);
674	TargetTriple = Triple(M.getTargetTriple());
675
676	Mapping = getShadowMapping(TargetTriple, LongSize, IsKasan: this->CompileKernel);
677
678	assert(this->UseAfterReturn != AsanDetectStackUseAfterReturnMode::Invalid);
679	}
680
681	TypeSize getAllocaSizeInBytes(const AllocaInst &AI) const {
682	return *AI.getAllocationSize(DL: AI.getModule()->getDataLayout());
683	}
684
685	/// Check if we want (and can) handle this alloca.
686	bool isInterestingAlloca(const AllocaInst &AI);
687
688	bool ignoreAccess(Instruction *Inst, Value *Ptr);
689	void getInterestingMemoryOperands(
690	Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting);
691
692	void instrumentMop(ObjectSizeOffsetVisitor &ObjSizeVis,
693	InterestingMemoryOperand &O, bool UseCalls,
694	const DataLayout &DL);
695	void instrumentPointerComparisonOrSubtraction(Instruction *I);
696	void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore,
697	Value *Addr, MaybeAlign Alignment,
698	uint32_t TypeStoreSize, bool IsWrite,
699	Value *SizeArgument, bool UseCalls, uint32_t Exp);
700	Instruction *instrumentAMDGPUAddress(Instruction *OrigIns,
701	Instruction *InsertBefore, Value *Addr,
702	uint32_t TypeStoreSize, bool IsWrite,
703	Value *SizeArgument);
704	Instruction *genAMDGPUReportBlock(IRBuilder<> &IRB, Value *Cond,
705	bool Recover);
706	void instrumentUnusualSizeOrAlignment(Instruction *I,
707	Instruction *InsertBefore, Value *Addr,
708	TypeSize TypeStoreSize, bool IsWrite,
709	Value *SizeArgument, bool UseCalls,
710	uint32_t Exp);
711	void instrumentMaskedLoadOrStore(AddressSanitizer *Pass, const DataLayout &DL,
712	Type *IntptrTy, Value *Mask, Value *EVL,
713	Value *Stride, Instruction *I, Value *Addr,
714	MaybeAlign Alignment, unsigned Granularity,
715	Type *OpType, bool IsWrite,
716	Value *SizeArgument, bool UseCalls,
717	uint32_t Exp);
718	Value *createSlowPathCmp(IRBuilder<> &IRB, Value *AddrLong,
719	Value *ShadowValue, uint32_t TypeStoreSize);
720	Instruction *generateCrashCode(Instruction *InsertBefore, Value *Addr,
721	bool IsWrite, size_t AccessSizeIndex,
722	Value *SizeArgument, uint32_t Exp);
723	void instrumentMemIntrinsic(MemIntrinsic *MI);
724	Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
725	bool suppressInstrumentationSiteForDebug(int &Instrumented);
726	bool instrumentFunction(Function &F, const TargetLibraryInfo *TLI);
727	bool maybeInsertAsanInitAtFunctionEntry(Function &F);
728	bool maybeInsertDynamicShadowAtFunctionEntry(Function &F);
729	void markEscapedLocalAllocas(Function &F);
730
731	private:
732	friend struct FunctionStackPoisoner;
733
734	void initializeCallbacks(Module &M, const TargetLibraryInfo *TLI);
735
736	bool LooksLikeCodeInBug11395(Instruction *I);
737	bool GlobalIsLinkerInitialized(GlobalVariable *G);
738	bool isSafeAccess(ObjectSizeOffsetVisitor &ObjSizeVis, Value *Addr,
739	TypeSize TypeStoreSize) const;
740
741	/// Helper to cleanup per-function state.
742	struct FunctionStateRAII {
743	AddressSanitizer *Pass;
744
745	FunctionStateRAII(AddressSanitizer *Pass) : Pass(Pass) {
746	assert(Pass->ProcessedAllocas.empty() &&
747	"last pass forgot to clear cache");
748	assert(!Pass->LocalDynamicShadow);
749	}
750
751	~FunctionStateRAII() {
752	Pass->LocalDynamicShadow = nullptr;
753	Pass->ProcessedAllocas.clear();
754	}
755	};
756
757	LLVMContext *C;
758	const DataLayout *DL;
759	Triple TargetTriple;
760	int LongSize;
761	bool CompileKernel;
762	bool Recover;
763	bool UseAfterScope;
764	AsanDetectStackUseAfterReturnMode UseAfterReturn;
765	Type *IntptrTy;
766	Type *Int32Ty;
767	PointerType *PtrTy;
768	ShadowMapping Mapping;
769	FunctionCallee AsanHandleNoReturnFunc;
770	FunctionCallee AsanPtrCmpFunction, AsanPtrSubFunction;
771	Constant *AsanShadowGlobal;
772
773	// These arrays is indexed by AccessIsWrite, Experiment and log2(AccessSize).
774	FunctionCallee AsanErrorCallback[2][2][kNumberOfAccessSizes];
775	FunctionCallee AsanMemoryAccessCallback[2][2][kNumberOfAccessSizes];
776
777	// These arrays is indexed by AccessIsWrite and Experiment.
778	FunctionCallee AsanErrorCallbackSized[2][2];
779	FunctionCallee AsanMemoryAccessCallbackSized[2][2];
780
781	FunctionCallee AsanMemmove, AsanMemcpy, AsanMemset;
782	Value *LocalDynamicShadow = nullptr;
783	const StackSafetyGlobalInfo *SSGI;
784	DenseMap<const AllocaInst *, bool> ProcessedAllocas;
785
786	FunctionCallee AMDGPUAddressShared;
787	FunctionCallee AMDGPUAddressPrivate;
788	int InstrumentationWithCallsThreshold;
789	uint32_t MaxInlinePoisoningSize;
790	};
791
792	class ModuleAddressSanitizer {
793	public:
794	ModuleAddressSanitizer(Module &M, bool InsertVersionCheck,
795	bool CompileKernel = false, bool Recover = false,
796	bool UseGlobalsGC = true, bool UseOdrIndicator = true,
797	AsanDtorKind DestructorKind = AsanDtorKind::Global,
798	AsanCtorKind ConstructorKind = AsanCtorKind::Global)
799	: CompileKernel(ClEnableKasan.getNumOccurrences() > 0 ? ClEnableKasan
800	: CompileKernel),
801	InsertVersionCheck(ClInsertVersionCheck.getNumOccurrences() > 0
802	? ClInsertVersionCheck
803	: InsertVersionCheck),
804	Recover(ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover),
805	UseGlobalsGC(UseGlobalsGC && ClUseGlobalsGC && !this->CompileKernel),
806	// Enable aliases as they should have no downside with ODR indicators.
807	UsePrivateAlias(ClUsePrivateAlias.getNumOccurrences() > 0
808	? ClUsePrivateAlias
809	: UseOdrIndicator),
810	UseOdrIndicator(ClUseOdrIndicator.getNumOccurrences() > 0
811	? ClUseOdrIndicator
812	: UseOdrIndicator),
813	// Not a typo: ClWithComdat is almost completely pointless without
814	// ClUseGlobalsGC (because then it only works on modules without
815	// globals, which are rare); it is a prerequisite for ClUseGlobalsGC;
816	// and both suffer from gold PR19002 for which UseGlobalsGC constructor
817	// argument is designed as workaround. Therefore, disable both
818	// ClWithComdat and ClUseGlobalsGC unless the frontend says it's ok to
819	// do globals-gc.
820	UseCtorComdat(UseGlobalsGC && ClWithComdat && !this->CompileKernel),
821	DestructorKind(DestructorKind),
822	ConstructorKind(ClConstructorKind.getNumOccurrences() > 0
823	? ClConstructorKind
824	: ConstructorKind) {
825	C = &(M.getContext());
826	int LongSize = M.getDataLayout().getPointerSizeInBits();
827	IntptrTy = Type::getIntNTy(C&: *C, N: LongSize);
828	PtrTy = PointerType::getUnqual(C&: *C);
829	TargetTriple = Triple(M.getTargetTriple());
830	Mapping = getShadowMapping(TargetTriple, LongSize, IsKasan: this->CompileKernel);
831
832	if (ClOverrideDestructorKind != AsanDtorKind::Invalid)
833	this->DestructorKind = ClOverrideDestructorKind;
834	assert(this->DestructorKind != AsanDtorKind::Invalid);
835	}
836
837	bool instrumentModule(Module &);
838
839	private:
840	void initializeCallbacks(Module &M);
841
842	void instrumentGlobals(IRBuilder<> &IRB, Module &M, bool *CtorComdat);
843	void InstrumentGlobalsCOFF(IRBuilder<> &IRB, Module &M,
844	ArrayRef<GlobalVariable *> ExtendedGlobals,
845	ArrayRef<Constant *> MetadataInitializers);
846	void instrumentGlobalsELF(IRBuilder<> &IRB, Module &M,
847	ArrayRef<GlobalVariable *> ExtendedGlobals,
848	ArrayRef<Constant *> MetadataInitializers,
849	const std::string &UniqueModuleId);
850	void InstrumentGlobalsMachO(IRBuilder<> &IRB, Module &M,
851	ArrayRef<GlobalVariable *> ExtendedGlobals,
852	ArrayRef<Constant *> MetadataInitializers);
853	void
854	InstrumentGlobalsWithMetadataArray(IRBuilder<> &IRB, Module &M,
855	ArrayRef<GlobalVariable *> ExtendedGlobals,
856	ArrayRef<Constant *> MetadataInitializers);
857
858	GlobalVariable *CreateMetadataGlobal(Module &M, Constant *Initializer,
859	StringRef OriginalName);
860	void SetComdatForGlobalMetadata(GlobalVariable *G, GlobalVariable *Metadata,
861	StringRef InternalSuffix);
862	Instruction *CreateAsanModuleDtor(Module &M);
863
864	const GlobalVariable *getExcludedAliasedGlobal(const GlobalAlias &GA) const;
865	bool shouldInstrumentGlobal(GlobalVariable *G) const;
866	bool ShouldUseMachOGlobalsSection() const;
867	StringRef getGlobalMetadataSection() const;
868	void poisonOneInitializer(Function &GlobalInit, GlobalValue *ModuleName);
869	void createInitializerPoisonCalls(Module &M, GlobalValue *ModuleName);
870	uint64_t getMinRedzoneSizeForGlobal() const {
871	return getRedzoneSizeForScale(MappingScale: Mapping.Scale);
872	}
873	uint64_t getRedzoneSizeForGlobal(uint64_t SizeInBytes) const;
874	int GetAsanVersion(const Module &M) const;
875
876	bool CompileKernel;
877	bool InsertVersionCheck;
878	bool Recover;
879	bool UseGlobalsGC;
880	bool UsePrivateAlias;
881	bool UseOdrIndicator;
882	bool UseCtorComdat;
883	AsanDtorKind DestructorKind;
884	AsanCtorKind ConstructorKind;
885	Type *IntptrTy;
886	PointerType *PtrTy;
887	LLVMContext *C;
888	Triple TargetTriple;
889	ShadowMapping Mapping;
890	FunctionCallee AsanPoisonGlobals;
891	FunctionCallee AsanUnpoisonGlobals;
892	FunctionCallee AsanRegisterGlobals;
893	FunctionCallee AsanUnregisterGlobals;
894	FunctionCallee AsanRegisterImageGlobals;
895	FunctionCallee AsanUnregisterImageGlobals;
896	FunctionCallee AsanRegisterElfGlobals;
897	FunctionCallee AsanUnregisterElfGlobals;
898
899	Function *AsanCtorFunction = nullptr;
900	Function *AsanDtorFunction = nullptr;
901	};
902
903	// Stack poisoning does not play well with exception handling.
904	// When an exception is thrown, we essentially bypass the code
905	// that unpoisones the stack. This is why the run-time library has
906	// to intercept __cxa_throw (as well as longjmp, etc) and unpoison the entire
907	// stack in the interceptor. This however does not work inside the
908	// actual function which catches the exception. Most likely because the
909	// compiler hoists the load of the shadow value somewhere too high.
910	// This causes asan to report a non-existing bug on 453.povray.
911	// It sounds like an LLVM bug.
912	struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
913	Function &F;
914	AddressSanitizer &ASan;
915	DIBuilder DIB;
916	LLVMContext *C;
917	Type *IntptrTy;
918	Type *IntptrPtrTy;
919	ShadowMapping Mapping;
920
921	SmallVector<AllocaInst *, 16> AllocaVec;
922	SmallVector<AllocaInst *, 16> StaticAllocasToMoveUp;
923	SmallVector<Instruction *, 8> RetVec;
924
925	FunctionCallee AsanStackMallocFunc[kMaxAsanStackMallocSizeClass + 1],
926	AsanStackFreeFunc[kMaxAsanStackMallocSizeClass + 1];
927	FunctionCallee AsanSetShadowFunc[0x100] = {};
928	FunctionCallee AsanPoisonStackMemoryFunc, AsanUnpoisonStackMemoryFunc;
929	FunctionCallee AsanAllocaPoisonFunc, AsanAllocasUnpoisonFunc;
930
931	// Stores a place and arguments of poisoning/unpoisoning call for alloca.
932	struct AllocaPoisonCall {
933	IntrinsicInst *InsBefore;
934	AllocaInst *AI;
935	uint64_t Size;
936	bool DoPoison;
937	};
938	SmallVector<AllocaPoisonCall, 8> DynamicAllocaPoisonCallVec;
939	SmallVector<AllocaPoisonCall, 8> StaticAllocaPoisonCallVec;
940	bool HasUntracedLifetimeIntrinsic = false;
941
942	SmallVector<AllocaInst *, 1> DynamicAllocaVec;
943	SmallVector<IntrinsicInst *, 1> StackRestoreVec;
944	AllocaInst *DynamicAllocaLayout = nullptr;
945	IntrinsicInst *LocalEscapeCall = nullptr;
946
947	bool HasInlineAsm = false;
948	bool HasReturnsTwiceCall = false;
949	bool PoisonStack;
950
951	FunctionStackPoisoner(Function &F, AddressSanitizer &ASan)
952	: F(F), ASan(ASan), DIB(*F.getParent(), /*AllowUnresolved*/ false),
953	C(ASan.C), IntptrTy(ASan.IntptrTy),
954	IntptrPtrTy(PointerType::get(ElementType: IntptrTy, AddressSpace: 0)), Mapping(ASan.Mapping),
955	PoisonStack(ClStack &&
956	!Triple(F.getParent()->getTargetTriple()).isAMDGPU()) {}
957
958	bool runOnFunction() {
959	if (!PoisonStack)
960	return false;
961
962	if (ClRedzoneByvalArgs)
963	copyArgsPassedByValToAllocas();
964
965	// Collect alloca, ret, lifetime instructions etc.
966	for (BasicBlock *BB : depth_first(G: &F.getEntryBlock())) visit(BB&: *BB);
967
968	if (AllocaVec.empty() && DynamicAllocaVec.empty()) return false;
969
970	initializeCallbacks(M&: *F.getParent());
971
972	if (HasUntracedLifetimeIntrinsic) {
973	// If there are lifetime intrinsics which couldn't be traced back to an
974	// alloca, we may not know exactly when a variable enters scope, and
975	// therefore should "fail safe" by not poisoning them.
976	StaticAllocaPoisonCallVec.clear();
977	DynamicAllocaPoisonCallVec.clear();
978	}
979
980	processDynamicAllocas();
981	processStaticAllocas();
982
983	if (ClDebugStack) {
984	LLVM_DEBUG(dbgs() << F);
985	}
986	return true;
987	}
988
989	// Arguments marked with the "byval" attribute are implicitly copied without
990	// using an alloca instruction. To produce redzones for those arguments, we
991	// copy them a second time into memory allocated with an alloca instruction.
992	void copyArgsPassedByValToAllocas();
993
994	// Finds all Alloca instructions and puts
995	// poisoned red zones around all of them.
996	// Then unpoison everything back before the function returns.
997	void processStaticAllocas();
998	void processDynamicAllocas();
999
1000	void createDynamicAllocasInitStorage();
1001
1002	// ----------------------- Visitors.
1003	/// Collect all Ret instructions, or the musttail call instruction if it
1004	/// precedes the return instruction.
1005	void visitReturnInst(ReturnInst &RI) {
1006	if (CallInst *CI = RI.getParent()->getTerminatingMustTailCall())
1007	RetVec.push_back(Elt: CI);
1008	else
1009	RetVec.push_back(Elt: &RI);
1010	}
1011
1012	/// Collect all Resume instructions.
1013	void visitResumeInst(ResumeInst &RI) { RetVec.push_back(Elt: &RI); }
1014
1015	/// Collect all CatchReturnInst instructions.
1016	void visitCleanupReturnInst(CleanupReturnInst &CRI) { RetVec.push_back(Elt: &CRI); }
1017
1018	void unpoisonDynamicAllocasBeforeInst(Instruction *InstBefore,
1019	Value *SavedStack) {
1020	IRBuilder<> IRB(InstBefore);
1021	Value *DynamicAreaPtr = IRB.CreatePtrToInt(V: SavedStack, DestTy: IntptrTy);
1022	// When we insert _asan_allocas_unpoison before @llvm.stackrestore, we
1023	// need to adjust extracted SP to compute the address of the most recent
1024	// alloca. We have a special @llvm.get.dynamic.area.offset intrinsic for
1025	// this purpose.
1026	if (!isa<ReturnInst>(Val: InstBefore)) {
1027	Function *DynamicAreaOffsetFunc = Intrinsic::getDeclaration(
1028	M: InstBefore->getModule(), Intrinsic::id: get_dynamic_area_offset,
1029	Tys: {IntptrTy});
1030
1031	Value *DynamicAreaOffset = IRB.CreateCall(Callee: DynamicAreaOffsetFunc, Args: {});
1032
1033	DynamicAreaPtr = IRB.CreateAdd(LHS: IRB.CreatePtrToInt(V: SavedStack, DestTy: IntptrTy),
1034	RHS: DynamicAreaOffset);
1035	}
1036
1037	IRB.CreateCall(
1038	Callee: AsanAllocasUnpoisonFunc,
1039	Args: {IRB.CreateLoad(Ty: IntptrTy, Ptr: DynamicAllocaLayout), DynamicAreaPtr});
1040	}
1041
1042	// Unpoison dynamic allocas redzones.
1043	void unpoisonDynamicAllocas() {
1044	for (Instruction *Ret : RetVec)
1045	unpoisonDynamicAllocasBeforeInst(InstBefore: Ret, SavedStack: DynamicAllocaLayout);
1046
1047	for (Instruction *StackRestoreInst : StackRestoreVec)
1048	unpoisonDynamicAllocasBeforeInst(InstBefore: StackRestoreInst,
1049	SavedStack: StackRestoreInst->getOperand(i: 0));
1050	}
1051
1052	// Deploy and poison redzones around dynamic alloca call. To do this, we
1053	// should replace this call with another one with changed parameters and
1054	// replace all its uses with new address, so
1055	// addr = alloca type, old_size, align
1056	// is replaced by
1057	// new_size = (old_size + additional_size) * sizeof(type)
1058	// tmp = alloca i8, new_size, max(align, 32)
1059	// addr = tmp + 32 (first 32 bytes are for the left redzone).
1060	// Additional_size is added to make new memory allocation contain not only
1061	// requested memory, but also left, partial and right redzones.
1062	void handleDynamicAllocaCall(AllocaInst *AI);
1063
1064	/// Collect Alloca instructions we want (and can) handle.
1065	void visitAllocaInst(AllocaInst &AI) {
1066	// FIXME: Handle scalable vectors instead of ignoring them.
1067	if (!ASan.isInterestingAlloca(AI) ||
1068	isa<ScalableVectorType>(Val: AI.getAllocatedType())) {
1069	if (AI.isStaticAlloca()) {
1070	// Skip over allocas that are present *before* the first instrumented
1071	// alloca, we don't want to move those around.
1072	if (AllocaVec.empty())
1073	return;
1074
1075	StaticAllocasToMoveUp.push_back(Elt: &AI);
1076	}
1077	return;
1078	}
1079
1080	if (!AI.isStaticAlloca())
1081	DynamicAllocaVec.push_back(Elt: &AI);
1082	else
1083	AllocaVec.push_back(Elt: &AI);
1084	}
1085
1086	/// Collect lifetime intrinsic calls to check for use-after-scope
1087	/// errors.
1088	void visitIntrinsicInst(IntrinsicInst &II) {
1089	Intrinsic::ID ID = II.getIntrinsicID();
1090	if (ID == Intrinsic::stackrestore) StackRestoreVec.push_back(Elt: &II);
1091	if (ID == Intrinsic::localescape) LocalEscapeCall = &II;
1092	if (!ASan.UseAfterScope)
1093	return;
1094	if (!II.isLifetimeStartOrEnd())
1095	return;
1096	// Found lifetime intrinsic, add ASan instrumentation if necessary.
1097	auto *Size = cast<ConstantInt>(Val: II.getArgOperand(i: 0));
1098	// If size argument is undefined, don't do anything.
1099	if (Size->isMinusOne()) return;
1100	// Check that size doesn't saturate uint64_t and can
1101	// be stored in IntptrTy.
1102	const uint64_t SizeValue = Size->getValue().getLimitedValue();
1103	if (SizeValue == ~0ULL ||
1104	!ConstantInt::isValueValidForType(Ty: IntptrTy, V: SizeValue))
1105	return;
1106	// Find alloca instruction that corresponds to llvm.lifetime argument.
1107	// Currently we can only handle lifetime markers pointing to the
1108	// beginning of the alloca.
1109	AllocaInst *AI = findAllocaForValue(V: II.getArgOperand(i: 1), OffsetZero: true);
1110	if (!AI) {
1111	HasUntracedLifetimeIntrinsic = true;
1112	return;
1113	}
1114	// We're interested only in allocas we can handle.
1115	if (!ASan.isInterestingAlloca(AI: *AI))
1116	return;
1117	bool DoPoison = (ID == Intrinsic::lifetime_end);
1118	AllocaPoisonCall APC = {.InsBefore: &II, .AI: AI, .Size: SizeValue, .DoPoison: DoPoison};
1119	if (AI->isStaticAlloca())
1120	StaticAllocaPoisonCallVec.push_back(Elt: APC);
1121	else if (ClInstrumentDynamicAllocas)
1122	DynamicAllocaPoisonCallVec.push_back(Elt: APC);
1123	}
1124
1125	void visitCallBase(CallBase &CB) {
1126	if (CallInst *CI = dyn_cast<CallInst>(Val: &CB)) {
1127	HasInlineAsm |= CI->isInlineAsm() && &CB != ASan.LocalDynamicShadow;
1128	HasReturnsTwiceCall |= CI->canReturnTwice();
1129	}
1130	}
1131
1132	// ---------------------- Helpers.
1133	void initializeCallbacks(Module &M);
1134
1135	// Copies bytes from ShadowBytes into shadow memory for indexes where
1136	// ShadowMask is not zero. If ShadowMask[i] is zero, we assume that
1137	// ShadowBytes[i] is constantly zero and doesn't need to be overwritten.
1138	void copyToShadow(ArrayRef<uint8_t> ShadowMask, ArrayRef<uint8_t> ShadowBytes,
1139	IRBuilder<> &IRB, Value *ShadowBase);
1140	void copyToShadow(ArrayRef<uint8_t> ShadowMask, ArrayRef<uint8_t> ShadowBytes,
1141	size_t Begin, size_t End, IRBuilder<> &IRB,
1142	Value *ShadowBase);
1143	void copyToShadowInline(ArrayRef<uint8_t> ShadowMask,
1144	ArrayRef<uint8_t> ShadowBytes, size_t Begin,
1145	size_t End, IRBuilder<> &IRB, Value *ShadowBase);
1146
1147	void poisonAlloca(Value *V, uint64_t Size, IRBuilder<> &IRB, bool DoPoison);
1148
1149	Value *createAllocaForLayout(IRBuilder<> &IRB, const ASanStackFrameLayout &L,
1150	bool Dynamic);
1151	PHINode *createPHI(IRBuilder<> &IRB, Value *Cond, Value *ValueIfTrue,
1152	Instruction *ThenTerm, Value *ValueIfFalse);
1153	};
1154
1155	} // end anonymous namespace
1156
1157	void AddressSanitizerPass::printPipeline(
1158	raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
1159	static_cast<PassInfoMixin<AddressSanitizerPass> *>(this)->printPipeline(
1160	OS, MapClassName2PassName);
1161	OS << '<';
1162	if (Options.CompileKernel)
1163	OS << "kernel";
1164	OS << '>';
1165	}
1166
1167	AddressSanitizerPass::AddressSanitizerPass(
1168	const AddressSanitizerOptions &Options, bool UseGlobalGC,
1169	bool UseOdrIndicator, AsanDtorKind DestructorKind,
1170	AsanCtorKind ConstructorKind)
1171	: Options(Options), UseGlobalGC(UseGlobalGC),
1172	UseOdrIndicator(UseOdrIndicator), DestructorKind(DestructorKind),
1173	ConstructorKind(ConstructorKind) {}
1174
1175	PreservedAnalyses AddressSanitizerPass::run(Module &M,
1176	ModuleAnalysisManager &MAM) {
1177	ModuleAddressSanitizer ModuleSanitizer(
1178	M, Options.InsertVersionCheck, Options.CompileKernel, Options.Recover,
1179	UseGlobalGC, UseOdrIndicator, DestructorKind, ConstructorKind);
1180	bool Modified = false;
1181	auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
1182	const StackSafetyGlobalInfo *const SSGI =
1183	ClUseStackSafety ? &MAM.getResult<StackSafetyGlobalAnalysis>(IR&: M) : nullptr;
1184	for (Function &F : M) {
1185	AddressSanitizer FunctionSanitizer(
1186	M, SSGI, Options.InstrumentationWithCallsThreshold,
1187	Options.MaxInlinePoisoningSize, Options.CompileKernel, Options.Recover,
1188	Options.UseAfterScope, Options.UseAfterReturn);
1189	const TargetLibraryInfo &TLI = FAM.getResult<TargetLibraryAnalysis>(IR&: F);
1190	Modified |= FunctionSanitizer.instrumentFunction(F, TLI: &TLI);
1191	}
1192	Modified |= ModuleSanitizer.instrumentModule(M);
1193	if (!Modified)
1194	return PreservedAnalyses::all();
1195
1196	PreservedAnalyses PA = PreservedAnalyses::none();
1197	// GlobalsAA is considered stateless and does not get invalidated unless
1198	// explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
1199	// make changes that require GlobalsAA to be invalidated.
1200	PA.abandon<GlobalsAA>();
1201	return PA;
1202	}
1203
1204	static size_t TypeStoreSizeToSizeIndex(uint32_t TypeSize) {
1205	size_t Res = llvm::countr_zero(Val: TypeSize / 8);
1206	assert(Res < kNumberOfAccessSizes);
1207	return Res;
1208	}
1209
1210	/// Check if \p G has been created by a trusted compiler pass.
1211	static bool GlobalWasGeneratedByCompiler(GlobalVariable *G) {
1212	// Do not instrument @llvm.global_ctors, @llvm.used, etc.
1213	if (G->getName().starts_with(Prefix: "llvm.") ||
1214	// Do not instrument gcov counter arrays.
1215	G->getName().starts_with(Prefix: "__llvm_gcov_ctr") ||
1216	// Do not instrument rtti proxy symbols for function sanitizer.
1217	G->getName().starts_with(Prefix: "__llvm_rtti_proxy"))
1218	return true;
1219
1220	// Do not instrument asan globals.
1221	if (G->getName().starts_with(Prefix: kAsanGenPrefix) ||
1222	G->getName().starts_with(Prefix: kSanCovGenPrefix) ||
1223	G->getName().starts_with(Prefix: kODRGenPrefix))
1224	return true;
1225
1226	return false;
1227	}
1228
1229	static bool isUnsupportedAMDGPUAddrspace(Value *Addr) {
1230	Type *PtrTy = cast<PointerType>(Val: Addr->getType()->getScalarType());
1231	unsigned int AddrSpace = PtrTy->getPointerAddressSpace();
1232	if (AddrSpace == 3 || AddrSpace == 5)
1233	return true;
1234	return false;
1235	}
1236
1237	Value *AddressSanitizer::memToShadow(Value *Shadow, IRBuilder<> &IRB) {
1238	// Shadow >> scale
1239	Shadow = IRB.CreateLShr(LHS: Shadow, RHS: Mapping.Scale);
1240	if (Mapping.Offset == 0) return Shadow;
1241	// (Shadow >> scale) | offset
1242	Value *ShadowBase;
1243	if (LocalDynamicShadow)
1244	ShadowBase = LocalDynamicShadow;
1245	else
1246	ShadowBase = ConstantInt::get(Ty: IntptrTy, V: Mapping.Offset);
1247	if (Mapping.OrShadowOffset)
1248	return IRB.CreateOr(LHS: Shadow, RHS: ShadowBase);
1249	else
1250	return IRB.CreateAdd(LHS: Shadow, RHS: ShadowBase);
1251	}
1252
1253	// Instrument memset/memmove/memcpy
1254	void AddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
1255	InstrumentationIRBuilder IRB(MI);
1256	if (isa<MemTransferInst>(Val: MI)) {
1257	IRB.CreateCall(Callee: isa<MemMoveInst>(Val: MI) ? AsanMemmove : AsanMemcpy,
1258	Args: {IRB.CreateAddrSpaceCast(V: MI->getOperand(i_nocapture: 0), DestTy: PtrTy),
1259	IRB.CreateAddrSpaceCast(V: MI->getOperand(i_nocapture: 1), DestTy: PtrTy),
1260	IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 2), DestTy: IntptrTy, isSigned: false)});
1261	} else if (isa<MemSetInst>(Val: MI)) {
1262	IRB.CreateCall(
1263	Callee: AsanMemset,
1264	Args: {IRB.CreateAddrSpaceCast(V: MI->getOperand(i_nocapture: 0), DestTy: PtrTy),
1265	IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 1), DestTy: IRB.getInt32Ty(), isSigned: false),
1266	IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 2), DestTy: IntptrTy, isSigned: false)});
1267	}
1268	MI->eraseFromParent();
1269	}
1270
1271	/// Check if we want (and can) handle this alloca.
1272	bool AddressSanitizer::isInterestingAlloca(const AllocaInst &AI) {
1273	auto PreviouslySeenAllocaInfo = ProcessedAllocas.find(Val: &AI);
1274
1275	if (PreviouslySeenAllocaInfo != ProcessedAllocas.end())
1276	return PreviouslySeenAllocaInfo->getSecond();
1277
1278	bool IsInteresting =
1279	(AI.getAllocatedType()->isSized() &&
1280	// alloca() may be called with 0 size, ignore it.
1281	((!AI.isStaticAlloca()) || !getAllocaSizeInBytes(AI).isZero()) &&
1282	// We are only interested in allocas not promotable to registers.
1283	// Promotable allocas are common under -O0.
1284	(!ClSkipPromotableAllocas || !isAllocaPromotable(AI: &AI)) &&
1285	// inalloca allocas are not treated as static, and we don't want
1286	// dynamic alloca instrumentation for them as well.
1287	!AI.isUsedWithInAlloca() &&
1288	// swifterror allocas are register promoted by ISel
1289	!AI.isSwiftError() &&
1290	// safe allocas are not interesting
1291	!(SSGI && SSGI->isSafe(AI)));
1292
1293	ProcessedAllocas[&AI] = IsInteresting;
1294	return IsInteresting;
1295	}
1296
1297	bool AddressSanitizer::ignoreAccess(Instruction *Inst, Value *Ptr) {
1298	// Instrument accesses from different address spaces only for AMDGPU.
1299	Type *PtrTy = cast<PointerType>(Val: Ptr->getType()->getScalarType());
1300	if (PtrTy->getPointerAddressSpace() != 0 &&
1301	!(TargetTriple.isAMDGPU() && !isUnsupportedAMDGPUAddrspace(Addr: Ptr)))
1302	return true;
1303
1304	// Ignore swifterror addresses.
1305	// swifterror memory addresses are mem2reg promoted by instruction
1306	// selection. As such they cannot have regular uses like an instrumentation
1307	// function and it makes no sense to track them as memory.
1308	if (Ptr->isSwiftError())
1309	return true;
1310
1311	// Treat memory accesses to promotable allocas as non-interesting since they
1312	// will not cause memory violations. This greatly speeds up the instrumented
1313	// executable at -O0.
1314	if (auto AI = dyn_cast_or_null<AllocaInst>(Val: Ptr))
1315	if (ClSkipPromotableAllocas && !isInterestingAlloca(AI: *AI))
1316	return true;
1317
1318	if (SSGI != nullptr && SSGI->stackAccessIsSafe(I: *Inst) &&
1319	findAllocaForValue(V: Ptr))
1320	return true;
1321
1322	return false;
1323	}
1324
1325	void AddressSanitizer::getInterestingMemoryOperands(
1326	Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting) {
1327	// Do not instrument the load fetching the dynamic shadow address.
1328	if (LocalDynamicShadow == I)
1329	return;
1330
1331	if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
1332	if (!ClInstrumentReads || ignoreAccess(Inst: I, Ptr: LI->getPointerOperand()))
1333	return;
1334	Interesting.emplace_back(Args&: I, Args: LI->getPointerOperandIndex(), Args: false,
1335	Args: LI->getType(), Args: LI->getAlign());
1336	} else if (StoreInst *SI = dyn_cast<StoreInst>(Val: I)) {
1337	if (!ClInstrumentWrites || ignoreAccess(Inst: I, Ptr: SI->getPointerOperand()))
1338	return;
1339	Interesting.emplace_back(Args&: I, Args: SI->getPointerOperandIndex(), Args: true,
1340	Args: SI->getValueOperand()->getType(), Args: SI->getAlign());
1341	} else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Val: I)) {
1342	if (!ClInstrumentAtomics || ignoreAccess(Inst: I, Ptr: RMW->getPointerOperand()))
1343	return;
1344	Interesting.emplace_back(Args&: I, Args: RMW->getPointerOperandIndex(), Args: true,
1345	Args: RMW->getValOperand()->getType(), Args: std::nullopt);
1346	} else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
1347	if (!ClInstrumentAtomics || ignoreAccess(Inst: I, Ptr: XCHG->getPointerOperand()))
1348	return;
1349	Interesting.emplace_back(Args&: I, Args: XCHG->getPointerOperandIndex(), Args: true,
1350	Args: XCHG->getCompareOperand()->getType(),
1351	Args: std::nullopt);
1352	} else if (auto CI = dyn_cast<CallInst>(Val: I)) {
1353	switch (CI->getIntrinsicID()) {
1354	case Intrinsic::masked_load:
1355	case Intrinsic::masked_store:
1356	case Intrinsic::masked_gather:
1357	case Intrinsic::masked_scatter: {
1358	bool IsWrite = CI->getType()->isVoidTy();
1359	// Masked store has an initial operand for the value.
1360	unsigned OpOffset = IsWrite ? 1 : 0;
1361	if (IsWrite ? !ClInstrumentWrites : !ClInstrumentReads)
1362	return;
1363
1364	auto BasePtr = CI->getOperand(i_nocapture: OpOffset);
1365	if (ignoreAccess(Inst: I, Ptr: BasePtr))
1366	return;
1367	Type *Ty = IsWrite ? CI->getArgOperand(i: 0)->getType() : CI->getType();
1368	MaybeAlign Alignment = Align(1);
1369	// Otherwise no alignment guarantees. We probably got Undef.
1370	if (auto *Op = dyn_cast<ConstantInt>(Val: CI->getOperand(i_nocapture: 1 + OpOffset)))
1371	Alignment = Op->getMaybeAlignValue();
1372	Value *Mask = CI->getOperand(i_nocapture: 2 + OpOffset);
1373	Interesting.emplace_back(Args&: I, Args&: OpOffset, Args&: IsWrite, Args&: Ty, Args&: Alignment, Args&: Mask);
1374	break;
1375	}
1376	case Intrinsic::masked_expandload:
1377	case Intrinsic::masked_compressstore: {
1378	bool IsWrite = CI->getIntrinsicID() == Intrinsic::masked_compressstore;
1379	unsigned OpOffset = IsWrite ? 1 : 0;
1380	if (IsWrite ? !ClInstrumentWrites : !ClInstrumentReads)
1381	return;
1382	auto BasePtr = CI->getOperand(i_nocapture: OpOffset);
1383	if (ignoreAccess(Inst: I, Ptr: BasePtr))
1384	return;
1385	MaybeAlign Alignment = BasePtr->getPointerAlignment(*DL);
1386	Type *Ty = IsWrite ? CI->getArgOperand(i: 0)->getType() : CI->getType();
1387
1388	IRBuilder IB(I);
1389	Value *Mask = CI->getOperand(i_nocapture: 1 + OpOffset);
1390	// Use the popcount of Mask as the effective vector length.
1391	Type *ExtTy = VectorType::get(ElementType: IntptrTy, Other: cast<VectorType>(Val: Ty));
1392	Value *ExtMask = IB.CreateZExt(V: Mask, DestTy: ExtTy);
1393	Value *EVL = IB.CreateAddReduce(Src: ExtMask);
1394	Value *TrueMask = ConstantInt::get(Ty: Mask->getType(), V: 1);
1395	Interesting.emplace_back(Args&: I, Args&: OpOffset, Args&: IsWrite, Args&: Ty, Args&: Alignment, Args&: TrueMask,
1396	Args&: EVL);
1397	break;
1398	}
1399	case Intrinsic::vp_load:
1400	case Intrinsic::vp_store:
1401	case Intrinsic::experimental_vp_strided_load:
1402	case Intrinsic::experimental_vp_strided_store: {
1403	auto *VPI = cast<VPIntrinsic>(Val: CI);
1404	unsigned IID = CI->getIntrinsicID();
1405	bool IsWrite = CI->getType()->isVoidTy();
1406	if (IsWrite ? !ClInstrumentWrites : !ClInstrumentReads)
1407	return;
1408	unsigned PtrOpNo = *VPI->getMemoryPointerParamPos(IID);
1409	Type *Ty = IsWrite ? CI->getArgOperand(i: 0)->getType() : CI->getType();
1410	MaybeAlign Alignment = VPI->getOperand(i_nocapture: PtrOpNo)->getPointerAlignment(DL: *DL);
1411	Value *Stride = nullptr;
1412	if (IID == Intrinsic::experimental_vp_strided_store ||
1413	IID == Intrinsic::experimental_vp_strided_load) {
1414	Stride = VPI->getOperand(i_nocapture: PtrOpNo + 1);
1415	// Use the pointer alignment as the element alignment if the stride is a
1416	// mutiple of the pointer alignment. Otherwise, the element alignment
1417	// should be Align(1).
1418	unsigned PointerAlign = Alignment.valueOrOne().value();
1419	if (!isa<ConstantInt>(Val: Stride) ||
1420	cast<ConstantInt>(Val: Stride)->getZExtValue() % PointerAlign != 0)
1421	Alignment = Align(1);
1422	}
1423	Interesting.emplace_back(Args&: I, Args&: PtrOpNo, Args&: IsWrite, Args&: Ty, Args&: Alignment,
1424	Args: VPI->getMaskParam(), Args: VPI->getVectorLengthParam(),
1425	Args&: Stride);
1426	break;
1427	}
1428	case Intrinsic::vp_gather:
1429	case Intrinsic::vp_scatter: {
1430	auto *VPI = cast<VPIntrinsic>(Val: CI);
1431	unsigned IID = CI->getIntrinsicID();
1432	bool IsWrite = IID == Intrinsic::vp_scatter;
1433	if (IsWrite ? !ClInstrumentWrites : !ClInstrumentReads)
1434	return;
1435	unsigned PtrOpNo = *VPI->getMemoryPointerParamPos(IID);
1436	Type *Ty = IsWrite ? CI->getArgOperand(i: 0)->getType() : CI->getType();
1437	MaybeAlign Alignment = VPI->getPointerAlignment();
1438	Interesting.emplace_back(Args&: I, Args&: PtrOpNo, Args&: IsWrite, Args&: Ty, Args&: Alignment,
1439	Args: VPI->getMaskParam(),
1440	Args: VPI->getVectorLengthParam());
1441	break;
1442	}
1443	default:
1444	for (unsigned ArgNo = 0; ArgNo < CI->arg_size(); ArgNo++) {
1445	if (!ClInstrumentByval || !CI->isByValArgument(ArgNo) ||
1446	ignoreAccess(Inst: I, Ptr: CI->getArgOperand(i: ArgNo)))
1447	continue;
1448	Type *Ty = CI->getParamByValType(ArgNo);
1449	Interesting.emplace_back(Args&: I, Args&: ArgNo, Args: false, Args&: Ty, Args: Align(1));
1450	}
1451	}
1452	}
1453	}
1454
1455	static bool isPointerOperand(Value *V) {
1456	return V->getType()->isPointerTy() || isa<PtrToIntInst>(Val: V);
1457	}
1458
1459	// This is a rough heuristic; it may cause both false positives and
1460	// false negatives. The proper implementation requires cooperation with
1461	// the frontend.
1462	static bool isInterestingPointerComparison(Instruction *I) {
1463	if (ICmpInst *Cmp = dyn_cast<ICmpInst>(Val: I)) {
1464	if (!Cmp->isRelational())
1465	return false;
1466	} else {
1467	return false;
1468	}
1469	return isPointerOperand(V: I->getOperand(i: 0)) &&
1470	isPointerOperand(V: I->getOperand(i: 1));
1471	}
1472
1473	// This is a rough heuristic; it may cause both false positives and
1474	// false negatives. The proper implementation requires cooperation with
1475	// the frontend.
1476	static bool isInterestingPointerSubtraction(Instruction *I) {
1477	if (BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: I)) {
1478	if (BO->getOpcode() != Instruction::Sub)
1479	return false;
1480	} else {
1481	return false;
1482	}
1483	return isPointerOperand(V: I->getOperand(i: 0)) &&
1484	isPointerOperand(V: I->getOperand(i: 1));
1485	}
1486
1487	bool AddressSanitizer::GlobalIsLinkerInitialized(GlobalVariable *G) {
1488	// If a global variable does not have dynamic initialization we don't
1489	// have to instrument it. However, if a global does not have initializer
1490	// at all, we assume it has dynamic initializer (in other TU).
1491	if (!G->hasInitializer())
1492	return false;
1493
1494	if (G->hasSanitizerMetadata() && G->getSanitizerMetadata().IsDynInit)
1495	return false;
1496
1497	return true;
1498	}
1499
1500	void AddressSanitizer::instrumentPointerComparisonOrSubtraction(
1501	Instruction *I) {
1502	IRBuilder<> IRB(I);
1503	FunctionCallee F = isa<ICmpInst>(Val: I) ? AsanPtrCmpFunction : AsanPtrSubFunction;
1504	Value *Param[2] = {I->getOperand(i: 0), I->getOperand(i: 1)};
1505	for (Value *&i : Param) {
1506	if (i->getType()->isPointerTy())
1507	i = IRB.CreatePointerCast(V: i, DestTy: IntptrTy);
1508	}
1509	IRB.CreateCall(Callee: F, Args: Param);
1510	}
1511
1512	static void doInstrumentAddress(AddressSanitizer *Pass, Instruction *I,
1513	Instruction *InsertBefore, Value *Addr,
1514	MaybeAlign Alignment, unsigned Granularity,
1515	TypeSize TypeStoreSize, bool IsWrite,
1516	Value *SizeArgument, bool UseCalls,
1517	uint32_t Exp) {
1518	// Instrument a 1-, 2-, 4-, 8-, or 16- byte access with one check
1519	// if the data is properly aligned.
1520	if (!TypeStoreSize.isScalable()) {
1521	const auto FixedSize = TypeStoreSize.getFixedValue();
1522	switch (FixedSize) {
1523	case 8:
1524	case 16:
1525	case 32:
1526	case 64:
1527	case 128:
1528	if (!Alignment || *Alignment >= Granularity ||
1529	*Alignment >= FixedSize / 8)
1530	return Pass->instrumentAddress(OrigIns: I, InsertBefore, Addr, Alignment,
1531	TypeStoreSize: FixedSize, IsWrite, SizeArgument: nullptr, UseCalls,
1532	Exp);
1533	}
1534	}
1535	Pass->instrumentUnusualSizeOrAlignment(I, InsertBefore, Addr, TypeStoreSize,
1536	IsWrite, SizeArgument: nullptr, UseCalls, Exp);
1537	}
1538
1539	void AddressSanitizer::instrumentMaskedLoadOrStore(
1540	AddressSanitizer *Pass, const DataLayout &DL, Type *IntptrTy, Value *Mask,
1541	Value *EVL, Value *Stride, Instruction *I, Value *Addr,
1542	MaybeAlign Alignment, unsigned Granularity, Type *OpType, bool IsWrite,
1543	Value *SizeArgument, bool UseCalls, uint32_t Exp) {
1544	auto *VTy = cast<VectorType>(Val: OpType);
1545	TypeSize ElemTypeSize = DL.getTypeStoreSizeInBits(Ty: VTy->getScalarType());
1546	auto Zero = ConstantInt::get(Ty: IntptrTy, V: 0);
1547
1548	IRBuilder IB(I);
1549	Instruction *LoopInsertBefore = I;
1550	if (EVL) {
1551	// The end argument of SplitBlockAndInsertForLane is assumed bigger
1552	// than zero, so we should check whether EVL is zero here.
1553	Type *EVLType = EVL->getType();
1554	Value *IsEVLZero = IB.CreateICmpNE(LHS: EVL, RHS: ConstantInt::get(Ty: EVLType, V: 0));
1555	LoopInsertBefore = SplitBlockAndInsertIfThen(Cond: IsEVLZero, SplitBefore: I, Unreachable: false);
1556	IB.SetInsertPoint(LoopInsertBefore);
1557	// Cast EVL to IntptrTy.
1558	EVL = IB.CreateZExtOrTrunc(V: EVL, DestTy: IntptrTy);
1559	// To avoid undefined behavior for extracting with out of range index, use
1560	// the minimum of evl and element count as trip count.
1561	Value *EC = IB.CreateElementCount(DstType: IntptrTy, EC: VTy->getElementCount());
1562	EVL = IB.CreateBinaryIntrinsic(Intrinsic::ID: umin, LHS: EVL, RHS: EC);
1563	} else {
1564	EVL = IB.CreateElementCount(DstType: IntptrTy, EC: VTy->getElementCount());
1565	}
1566
1567	// Cast Stride to IntptrTy.
1568	if (Stride)
1569	Stride = IB.CreateZExtOrTrunc(V: Stride, DestTy: IntptrTy);
1570
1571	SplitBlockAndInsertForEachLane(End: EVL, InsertBefore: LoopInsertBefore,
1572	Func: [&](IRBuilderBase &IRB, Value *Index) {
1573	Value *MaskElem = IRB.CreateExtractElement(Vec: Mask, Idx: Index);
1574	if (auto *MaskElemC = dyn_cast<ConstantInt>(Val: MaskElem)) {
1575	if (MaskElemC->isZero())
1576	// No check
1577	return;
1578	// Unconditional check
1579	} else {
1580	// Conditional check
1581	Instruction *ThenTerm = SplitBlockAndInsertIfThen(
1582	Cond: MaskElem, SplitBefore: &*IRB.GetInsertPoint(), Unreachable: false);
1583	IRB.SetInsertPoint(ThenTerm);
1584	}
1585
1586	Value *InstrumentedAddress;
1587	if (isa<VectorType>(Val: Addr->getType())) {
1588	assert(
1589	cast<VectorType>(Addr->getType())->getElementType()->isPointerTy() &&
1590	"Expected vector of pointer.");
1591	InstrumentedAddress = IRB.CreateExtractElement(Vec: Addr, Idx: Index);
1592	} else if (Stride) {
1593	Index = IRB.CreateMul(LHS: Index, RHS: Stride);
1594	InstrumentedAddress = IRB.CreatePtrAdd(Ptr: Addr, Offset: Index);
1595	} else {
1596	InstrumentedAddress = IRB.CreateGEP(Ty: VTy, Ptr: Addr, IdxList: {Zero, Index});
1597	}
1598	doInstrumentAddress(Pass, I, InsertBefore: &*IRB.GetInsertPoint(),
1599	Addr: InstrumentedAddress, Alignment, Granularity,
1600	TypeStoreSize: ElemTypeSize, IsWrite, SizeArgument, UseCalls, Exp);
1601	});
1602	}
1603
1604	void AddressSanitizer::instrumentMop(ObjectSizeOffsetVisitor &ObjSizeVis,
1605	InterestingMemoryOperand &O, bool UseCalls,
1606	const DataLayout &DL) {
1607	Value *Addr = O.getPtr();
1608
1609	// Optimization experiments.
1610	// The experiments can be used to evaluate potential optimizations that remove
1611	// instrumentation (assess false negatives). Instead of completely removing
1612	// some instrumentation, you set Exp to a non-zero value (mask of optimization
1613	// experiments that want to remove instrumentation of this instruction).
1614	// If Exp is non-zero, this pass will emit special calls into runtime
1615	// (e.g. __asan_report_exp_load1 instead of __asan_report_load1). These calls
1616	// make runtime terminate the program in a special way (with a different
1617	// exit status). Then you run the new compiler on a buggy corpus, collect
1618	// the special terminations (ideally, you don't see them at all -- no false
1619	// negatives) and make the decision on the optimization.
1620	uint32_t Exp = ClForceExperiment;
1621
1622	if (ClOpt && ClOptGlobals) {
1623	// If initialization order checking is disabled, a simple access to a
1624	// dynamically initialized global is always valid.
1625	GlobalVariable *G = dyn_cast<GlobalVariable>(Val: getUnderlyingObject(V: Addr));
1626	if (G && (!ClInitializers || GlobalIsLinkerInitialized(G)) &&
1627	isSafeAccess(ObjSizeVis, Addr, TypeStoreSize: O.TypeStoreSize)) {
1628	NumOptimizedAccessesToGlobalVar++;
1629	return;
1630	}
1631	}
1632
1633	if (ClOpt && ClOptStack) {
1634	// A direct inbounds access to a stack variable is always valid.
1635	if (isa<AllocaInst>(Val: getUnderlyingObject(V: Addr)) &&
1636	isSafeAccess(ObjSizeVis, Addr, TypeStoreSize: O.TypeStoreSize)) {
1637	NumOptimizedAccessesToStackVar++;
1638	return;
1639	}
1640	}
1641
1642	if (O.IsWrite)
1643	NumInstrumentedWrites++;
1644	else
1645	NumInstrumentedReads++;
1646
1647	unsigned Granularity = 1 << Mapping.Scale;
1648	if (O.MaybeMask) {
1649	instrumentMaskedLoadOrStore(Pass: this, DL, IntptrTy, Mask: O.MaybeMask, EVL: O.MaybeEVL,
1650	Stride: O.MaybeStride, I: O.getInsn(), Addr, Alignment: O.Alignment,
1651	Granularity, OpType: O.OpType, IsWrite: O.IsWrite, SizeArgument: nullptr,
1652	UseCalls, Exp);
1653	} else {
1654	doInstrumentAddress(Pass: this, I: O.getInsn(), InsertBefore: O.getInsn(), Addr, Alignment: O.Alignment,
1655	Granularity, TypeStoreSize: O.TypeStoreSize, IsWrite: O.IsWrite, SizeArgument: nullptr, UseCalls,
1656	Exp);
1657	}
1658	}
1659
1660	Instruction *AddressSanitizer::generateCrashCode(Instruction *InsertBefore,
1661	Value *Addr, bool IsWrite,
1662	size_t AccessSizeIndex,
1663	Value *SizeArgument,
1664	uint32_t Exp) {
1665	InstrumentationIRBuilder IRB(InsertBefore);
1666	Value *ExpVal = Exp == 0 ? nullptr : ConstantInt::get(Ty: IRB.getInt32Ty(), V: Exp);
1667	CallInst *Call = nullptr;
1668	if (SizeArgument) {
1669	if (Exp == 0)
1670	Call = IRB.CreateCall(Callee: AsanErrorCallbackSized[IsWrite][0],
1671	Args: {Addr, SizeArgument});
1672	else
1673	Call = IRB.CreateCall(Callee: AsanErrorCallbackSized[IsWrite][1],
1674	Args: {Addr, SizeArgument, ExpVal});
1675	} else {
1676	if (Exp == 0)
1677	Call =
1678	IRB.CreateCall(Callee: AsanErrorCallback[IsWrite][0][AccessSizeIndex], Args: Addr);
1679	else
1680	Call = IRB.CreateCall(Callee: AsanErrorCallback[IsWrite][1][AccessSizeIndex],
1681	Args: {Addr, ExpVal});
1682	}
1683
1684	Call->setCannotMerge();
1685	return Call;
1686	}
1687
1688	Value *AddressSanitizer::createSlowPathCmp(IRBuilder<> &IRB, Value *AddrLong,
1689	Value *ShadowValue,
1690	uint32_t TypeStoreSize) {
1691	size_t Granularity = static_cast<size_t>(1) << Mapping.Scale;
1692	// Addr & (Granularity - 1)
1693	Value *LastAccessedByte =
1694	IRB.CreateAnd(LHS: AddrLong, RHS: ConstantInt::get(Ty: IntptrTy, V: Granularity - 1));
1695	// (Addr & (Granularity - 1)) + size - 1
1696	if (TypeStoreSize / 8 > 1)
1697	LastAccessedByte = IRB.CreateAdd(
1698	LHS: LastAccessedByte, RHS: ConstantInt::get(Ty: IntptrTy, V: TypeStoreSize / 8 - 1));
1699	// (uint8_t) ((Addr & (Granularity-1)) + size - 1)
1700	LastAccessedByte =
1701	IRB.CreateIntCast(V: LastAccessedByte, DestTy: ShadowValue->getType(), isSigned: false);
1702	// ((uint8_t) ((Addr & (Granularity-1)) + size - 1)) >= ShadowValue
1703	return IRB.CreateICmpSGE(LHS: LastAccessedByte, RHS: ShadowValue);
1704	}
1705
1706	Instruction *AddressSanitizer::instrumentAMDGPUAddress(
1707	Instruction *OrigIns, Instruction *InsertBefore, Value *Addr,
1708	uint32_t TypeStoreSize, bool IsWrite, Value *SizeArgument) {
1709	// Do not instrument unsupported addrspaces.
1710	if (isUnsupportedAMDGPUAddrspace(Addr))
1711	return nullptr;
1712	Type *PtrTy = cast<PointerType>(Val: Addr->getType()->getScalarType());
1713	// Follow host instrumentation for global and constant addresses.
1714	if (PtrTy->getPointerAddressSpace() != 0)
1715	return InsertBefore;
1716	// Instrument generic addresses in supported addressspaces.
1717	IRBuilder<> IRB(InsertBefore);
1718	Value *IsShared = IRB.CreateCall(Callee: AMDGPUAddressShared, Args: {Addr});
1719	Value *IsPrivate = IRB.CreateCall(Callee: AMDGPUAddressPrivate, Args: {Addr});
1720	Value *IsSharedOrPrivate = IRB.CreateOr(LHS: IsShared, RHS: IsPrivate);
1721	Value *Cmp = IRB.CreateNot(V: IsSharedOrPrivate);
1722	Value *AddrSpaceZeroLanding =
1723	SplitBlockAndInsertIfThen(Cond: Cmp, SplitBefore: InsertBefore, Unreachable: false);
1724	InsertBefore = cast<Instruction>(Val: AddrSpaceZeroLanding);
1725	return InsertBefore;
1726	}
1727
1728	Instruction *AddressSanitizer::genAMDGPUReportBlock(IRBuilder<> &IRB,
1729	Value *Cond, bool Recover) {
1730	Module &M = *IRB.GetInsertBlock()->getModule();
1731	Value *ReportCond = Cond;
1732	if (!Recover) {
1733	auto Ballot = M.getOrInsertFunction(Name: kAMDGPUBallotName, RetTy: IRB.getInt64Ty(),
1734	Args: IRB.getInt1Ty());
1735	ReportCond = IRB.CreateIsNotNull(Arg: IRB.CreateCall(Callee: Ballot, Args: {Cond}));
1736	}
1737
1738	auto *Trm =
1739	SplitBlockAndInsertIfThen(Cond: ReportCond, SplitBefore: &*IRB.GetInsertPoint(), Unreachable: false,
1740	BranchWeights: MDBuilder(*C).createBranchWeights(TrueWeight: 1, FalseWeight: 100000));
1741	Trm->getParent()->setName("asan.report");
1742
1743	if (Recover)
1744	return Trm;
1745
1746	Trm = SplitBlockAndInsertIfThen(Cond, SplitBefore: Trm, Unreachable: false);
1747	IRB.SetInsertPoint(Trm);
1748	return IRB.CreateCall(
1749	Callee: M.getOrInsertFunction(Name: kAMDGPUUnreachableName, RetTy: IRB.getVoidTy()), Args: {});
1750	}
1751
1752	void AddressSanitizer::instrumentAddress(Instruction *OrigIns,
1753	Instruction *InsertBefore, Value *Addr,
1754	MaybeAlign Alignment,
1755	uint32_t TypeStoreSize, bool IsWrite,
1756	Value *SizeArgument, bool UseCalls,
1757	uint32_t Exp) {
1758	if (TargetTriple.isAMDGPU()) {
1759	InsertBefore = instrumentAMDGPUAddress(OrigIns, InsertBefore, Addr,
1760	TypeStoreSize, IsWrite, SizeArgument);
1761	if (!InsertBefore)
1762	return;
1763	}
1764
1765	InstrumentationIRBuilder IRB(InsertBefore);
1766	size_t AccessSizeIndex = TypeStoreSizeToSizeIndex(TypeSize: TypeStoreSize);
1767	const ASanAccessInfo AccessInfo(IsWrite, CompileKernel, AccessSizeIndex);
1768
1769	if (UseCalls && ClOptimizeCallbacks) {
1770	const ASanAccessInfo AccessInfo(IsWrite, CompileKernel, AccessSizeIndex);
1771	Module *M = IRB.GetInsertBlock()->getParent()->getParent();
1772	IRB.CreateCall(
1773	Intrinsic::getDeclaration(M, Intrinsic::id: asan_check_memaccess),
1774	{IRB.CreatePointerCast(V: Addr, DestTy: PtrTy),
1775	ConstantInt::get(Ty: Int32Ty, V: AccessInfo.Packed)});
1776	return;
1777	}
1778
1779	Value *AddrLong = IRB.CreatePointerCast(V: Addr, DestTy: IntptrTy);
1780	if (UseCalls) {
1781	if (Exp == 0)
1782	IRB.CreateCall(Callee: AsanMemoryAccessCallback[IsWrite][0][AccessSizeIndex],
1783	Args: AddrLong);
1784	else
1785	IRB.CreateCall(Callee: AsanMemoryAccessCallback[IsWrite][1][AccessSizeIndex],
1786	Args: {AddrLong, ConstantInt::get(Ty: IRB.getInt32Ty(), V: Exp)});
1787	return;
1788	}
1789
1790	Type *ShadowTy =
1791	IntegerType::get(C&: *C, NumBits: std::max(a: 8U, b: TypeStoreSize >> Mapping.Scale));
1792	Type *ShadowPtrTy = PointerType::get(ElementType: ShadowTy, AddressSpace: 0);
1793	Value *ShadowPtr = memToShadow(Shadow: AddrLong, IRB);
1794	const uint64_t ShadowAlign =
1795	std::max<uint64_t>(a: Alignment.valueOrOne().value() >> Mapping.Scale, b: 1);
1796	Value *ShadowValue = IRB.CreateAlignedLoad(
1797	Ty: ShadowTy, Ptr: IRB.CreateIntToPtr(V: ShadowPtr, DestTy: ShadowPtrTy), Align: Align(ShadowAlign));
1798
1799	Value *Cmp = IRB.CreateIsNotNull(Arg: ShadowValue);
1800	size_t Granularity = 1ULL << Mapping.Scale;
1801	Instruction *CrashTerm = nullptr;
1802
1803	bool GenSlowPath = (ClAlwaysSlowPath || (TypeStoreSize < 8 * Granularity));
1804
1805	if (TargetTriple.isAMDGCN()) {
1806	if (GenSlowPath) {
1807	auto *Cmp2 = createSlowPathCmp(IRB, AddrLong, ShadowValue, TypeStoreSize);
1808	Cmp = IRB.CreateAnd(LHS: Cmp, RHS: Cmp2);
1809	}
1810	CrashTerm = genAMDGPUReportBlock(IRB, Cond: Cmp, Recover);
1811	} else if (GenSlowPath) {
1812	// We use branch weights for the slow path check, to indicate that the slow
1813	// path is rarely taken. This seems to be the case for SPEC benchmarks.
1814	Instruction *CheckTerm = SplitBlockAndInsertIfThen(
1815	Cond: Cmp, SplitBefore: InsertBefore, Unreachable: false, BranchWeights: MDBuilder(*C).createBranchWeights(TrueWeight: 1, FalseWeight: 100000));
1816	assert(cast<BranchInst>(CheckTerm)->isUnconditional());
1817	BasicBlock *NextBB = CheckTerm->getSuccessor(Idx: 0);
1818	IRB.SetInsertPoint(CheckTerm);
1819	Value *Cmp2 = createSlowPathCmp(IRB, AddrLong, ShadowValue, TypeStoreSize);
1820	if (Recover) {
1821	CrashTerm = SplitBlockAndInsertIfThen(Cond: Cmp2, SplitBefore: CheckTerm, Unreachable: false);
1822	} else {
1823	BasicBlock *CrashBlock =
1824	BasicBlock::Create(Context&: *C, Name: "", Parent: NextBB->getParent(), InsertBefore: NextBB);
1825	CrashTerm = new UnreachableInst(*C, CrashBlock);
1826	BranchInst *NewTerm = BranchInst::Create(IfTrue: CrashBlock, IfFalse: NextBB, Cond: Cmp2);
1827	ReplaceInstWithInst(From: CheckTerm, To: NewTerm);
1828	}
1829	} else {
1830	CrashTerm = SplitBlockAndInsertIfThen(Cond: Cmp, SplitBefore: InsertBefore, Unreachable: !Recover);
1831	}
1832
1833	Instruction *Crash = generateCrashCode(InsertBefore: CrashTerm, Addr: AddrLong, IsWrite,
1834	AccessSizeIndex, SizeArgument, Exp);
1835	if (OrigIns->getDebugLoc())
1836	Crash->setDebugLoc(OrigIns->getDebugLoc());
1837	}
1838
1839	// Instrument unusual size or unusual alignment.
1840	// We can not do it with a single check, so we do 1-byte check for the first
1841	// and the last bytes. We call __asan_report_*_n(addr, real_size) to be able
1842	// to report the actual access size.
1843	void AddressSanitizer::instrumentUnusualSizeOrAlignment(
1844	Instruction *I, Instruction *InsertBefore, Value *Addr, TypeSize TypeStoreSize,
1845	bool IsWrite, Value *SizeArgument, bool UseCalls, uint32_t Exp) {
1846	InstrumentationIRBuilder IRB(InsertBefore);
1847	Value *NumBits = IRB.CreateTypeSize(DstType: IntptrTy, Size: TypeStoreSize);
1848	Value *Size = IRB.CreateLShr(LHS: NumBits, RHS: ConstantInt::get(Ty: IntptrTy, V: 3));
1849
1850	Value *AddrLong = IRB.CreatePointerCast(V: Addr, DestTy: IntptrTy);
1851	if (UseCalls) {
1852	if (Exp == 0)
1853	IRB.CreateCall(Callee: AsanMemoryAccessCallbackSized[IsWrite][0],
1854	Args: {AddrLong, Size});
1855	else
1856	IRB.CreateCall(Callee: AsanMemoryAccessCallbackSized[IsWrite][1],
1857	Args: {AddrLong, Size, ConstantInt::get(Ty: IRB.getInt32Ty(), V: Exp)});
1858	} else {
1859	Value *SizeMinusOne = IRB.CreateSub(LHS: Size, RHS: ConstantInt::get(Ty: IntptrTy, V: 1));
1860	Value *LastByte = IRB.CreateIntToPtr(
1861	V: IRB.CreateAdd(LHS: AddrLong, RHS: SizeMinusOne),
1862	DestTy: Addr->getType());
1863	instrumentAddress(OrigIns: I, InsertBefore, Addr, Alignment: {}, TypeStoreSize: 8, IsWrite, SizeArgument: Size, UseCalls: false, Exp);
1864	instrumentAddress(OrigIns: I, InsertBefore, Addr: LastByte, Alignment: {}, TypeStoreSize: 8, IsWrite, SizeArgument: Size, UseCalls: false,
1865	Exp);
1866	}
1867	}
1868
1869	void ModuleAddressSanitizer::poisonOneInitializer(Function &GlobalInit,
1870	GlobalValue *ModuleName) {
1871	// Set up the arguments to our poison/unpoison functions.
1872	IRBuilder<> IRB(&GlobalInit.front(),
1873	GlobalInit.front().getFirstInsertionPt());
1874
1875	// Add a call to poison all external globals before the given function starts.
1876	Value *ModuleNameAddr = ConstantExpr::getPointerCast(C: ModuleName, Ty: IntptrTy);
1877	IRB.CreateCall(Callee: AsanPoisonGlobals, Args: ModuleNameAddr);
1878
1879	// Add calls to unpoison all globals before each return instruction.
1880	for (auto &BB : GlobalInit)
1881	if (ReturnInst *RI = dyn_cast<ReturnInst>(Val: BB.getTerminator()))
1882	CallInst::Create(Func: AsanUnpoisonGlobals, NameStr: "", InsertBefore: RI);
1883	}
1884
1885	void ModuleAddressSanitizer::createInitializerPoisonCalls(
1886	Module &M, GlobalValue *ModuleName) {
1887	GlobalVariable *GV = M.getGlobalVariable(Name: "llvm.global_ctors");
1888	if (!GV)
1889	return;
1890
1891	ConstantArray *CA = dyn_cast<ConstantArray>(Val: GV->getInitializer());
1892	if (!CA)
1893	return;
1894
1895	for (Use &OP : CA->operands()) {
1896	if (isa<ConstantAggregateZero>(Val: OP)) continue;
1897	ConstantStruct *CS = cast<ConstantStruct>(Val&: OP);
1898
1899	// Must have a function or null ptr.
1900	if (Function *F = dyn_cast<Function>(Val: CS->getOperand(i_nocapture: 1))) {
1901	if (F->getName() == kAsanModuleCtorName) continue;
1902	auto *Priority = cast<ConstantInt>(Val: CS->getOperand(i_nocapture: 0));
1903	// Don't instrument CTORs that will run before asan.module_ctor.
1904	if (Priority->getLimitedValue() <= GetCtorAndDtorPriority(TargetTriple))
1905	continue;
1906	poisonOneInitializer(GlobalInit&: *F, ModuleName);
1907	}
1908	}
1909	}
1910
1911	const GlobalVariable *
1912	ModuleAddressSanitizer::getExcludedAliasedGlobal(const GlobalAlias &GA) const {
1913	// In case this function should be expanded to include rules that do not just
1914	// apply when CompileKernel is true, either guard all existing rules with an
1915	// 'if (CompileKernel) { ... }' or be absolutely sure that all these rules
1916	// should also apply to user space.
1917	assert(CompileKernel && "Only expecting to be called when compiling kernel");
1918
1919	const Constant *C = GA.getAliasee();
1920
1921	// When compiling the kernel, globals that are aliased by symbols prefixed
1922	// by "__" are special and cannot be padded with a redzone.
1923	if (GA.getName().starts_with(Prefix: "__"))
1924	return dyn_cast<GlobalVariable>(Val: C->stripPointerCastsAndAliases());
1925
1926	return nullptr;
1927	}
1928
1929	bool ModuleAddressSanitizer::shouldInstrumentGlobal(GlobalVariable *G) const {
1930	Type *Ty = G->getValueType();
1931	LLVM_DEBUG(dbgs() << "GLOBAL: " << *G << "\n");
1932
1933	if (G->hasSanitizerMetadata() && G->getSanitizerMetadata().NoAddress)
1934	return false;
1935	if (!Ty->isSized()) return false;
1936	if (!G->hasInitializer()) return false;
1937	// Globals in address space 1 and 4 are supported for AMDGPU.
1938	if (G->getAddressSpace() &&
1939	!(TargetTriple.isAMDGPU() && !isUnsupportedAMDGPUAddrspace(Addr: G)))
1940	return false;
1941	if (GlobalWasGeneratedByCompiler(G)) return false; // Our own globals.
1942	// Two problems with thread-locals:
1943	// - The address of the main thread's copy can't be computed at link-time.
1944	// - Need to poison all copies, not just the main thread's one.
1945	if (G->isThreadLocal()) return false;
1946	// For now, just ignore this Global if the alignment is large.
1947	if (G->getAlign() && *G->getAlign() > getMinRedzoneSizeForGlobal()) return false;
1948
1949	// For non-COFF targets, only instrument globals known to be defined by this
1950	// TU.
1951	// FIXME: We can instrument comdat globals on ELF if we are using the
1952	// GC-friendly metadata scheme.
1953	if (!TargetTriple.isOSBinFormatCOFF()) {
1954	if (!G->hasExactDefinition() || G->hasComdat())
1955	return false;
1956	} else {
1957	// On COFF, don't instrument non-ODR linkages.
1958	if (G->isInterposable())
1959	return false;
1960	}
1961
1962	// If a comdat is present, it must have a selection kind that implies ODR
1963	// semantics: no duplicates, any, or exact match.
1964	if (Comdat *C = G->getComdat()) {
1965	switch (C->getSelectionKind()) {
1966	case Comdat::Any:
1967	case Comdat::ExactMatch:
1968	case Comdat::NoDeduplicate:
1969	break;
1970	case Comdat::Largest:
1971	case Comdat::SameSize:
1972	return false;
1973	}
1974	}
1975
1976	if (G->hasSection()) {
1977	// The kernel uses explicit sections for mostly special global variables
1978	// that we should not instrument. E.g. the kernel may rely on their layout
1979	// without redzones, or remove them at link time ("discard.*"), etc.
1980	if (CompileKernel)
1981	return false;
1982
1983	StringRef Section = G->getSection();
1984
1985	// Globals from llvm.metadata aren't emitted, do not instrument them.
1986	if (Section == "llvm.metadata") return false;
1987	// Do not instrument globals from special LLVM sections.
1988	if (Section.contains(Other: "__llvm") || Section.contains(Other: "__LLVM"))
1989	return false;
1990
1991	// Do not instrument function pointers to initialization and termination
1992	// routines: dynamic linker will not properly handle redzones.
1993	if (Section.starts_with(Prefix: ".preinit_array") ||
1994	Section.starts_with(Prefix: ".init_array") ||
1995	Section.starts_with(Prefix: ".fini_array")) {
1996	return false;
1997	}
1998
1999	// Do not instrument user-defined sections (with names resembling
2000	// valid C identifiers)
2001	if (TargetTriple.isOSBinFormatELF()) {
2002	if (llvm::all_of(Range&: Section,
2003	P: [](char c) { return llvm::isAlnum(C: c) || c == '_'; }))
2004	return false;
2005	}
2006
2007	// On COFF, if the section name contains '$', it is highly likely that the
2008	// user is using section sorting to create an array of globals similar to
2009	// the way initialization callbacks are registered in .init_array and
2010	// .CRT$XCU. The ATL also registers things in .ATL$__[azm]. Adding redzones
2011	// to such globals is counterproductive, because the intent is that they
2012	// will form an array, and out-of-bounds accesses are expected.
2013	// See https://github.com/google/sanitizers/issues/305
2014	// and http://msdn.microsoft.com/en-US/en-en/library/bb918180(v=vs.120).aspx
2015	if (TargetTriple.isOSBinFormatCOFF() && Section.contains(C: '$')) {
2016	LLVM_DEBUG(dbgs() << "Ignoring global in sorted section (contains '$'): "
2017	<< *G << "\n");
2018	return false;
2019	}
2020
2021	if (TargetTriple.isOSBinFormatMachO()) {
2022	StringRef ParsedSegment, ParsedSection;
2023	unsigned TAA = 0, StubSize = 0;
2024	bool TAAParsed;
2025	cantFail(Err: MCSectionMachO::ParseSectionSpecifier(
2026	Spec: Section, Segment&: ParsedSegment, Section&: ParsedSection, TAA, TAAParsed, StubSize));
2027
2028	// Ignore the globals from the __OBJC section. The ObjC runtime assumes
2029	// those conform to /usr/lib/objc/runtime.h, so we can't add redzones to
2030	// them.
2031	if (ParsedSegment == "__OBJC" ||
2032	(ParsedSegment == "__DATA" && ParsedSection.starts_with(Prefix: "__objc_"))) {
2033	LLVM_DEBUG(dbgs() << "Ignoring ObjC runtime global: " << *G << "\n");
2034	return false;
2035	}
2036	// See https://github.com/google/sanitizers/issues/32
2037	// Constant CFString instances are compiled in the following way:
2038	// -- the string buffer is emitted into
2039	// __TEXT,__cstring,cstring_literals
2040	// -- the constant NSConstantString structure referencing that buffer
2041	// is placed into __DATA,__cfstring
2042	// Therefore there's no point in placing redzones into __DATA,__cfstring.
2043	// Moreover, it causes the linker to crash on OS X 10.7
2044	if (ParsedSegment == "__DATA" && ParsedSection == "__cfstring") {
2045	LLVM_DEBUG(dbgs() << "Ignoring CFString: " << *G << "\n");
2046	return false;
2047	}
2048	// The linker merges the contents of cstring_literals and removes the
2049	// trailing zeroes.
2050	if (ParsedSegment == "__TEXT" && (TAA & MachO::S_CSTRING_LITERALS)) {
2051	LLVM_DEBUG(dbgs() << "Ignoring a cstring literal: " << *G << "\n");
2052	return false;
2053	}
2054	}
2055	}
2056
2057	if (CompileKernel) {
2058	// Globals that prefixed by "__" are special and cannot be padded with a
2059	// redzone.
2060	if (G->getName().starts_with(Prefix: "__"))
2061	return false;
2062	}
2063
2064	return true;
2065	}
2066
2067	// On Mach-O platforms, we emit global metadata in a separate section of the
2068	// binary in order to allow the linker to properly dead strip. This is only
2069	// supported on recent versions of ld64.
2070	bool ModuleAddressSanitizer::ShouldUseMachOGlobalsSection() const {
2071	if (!TargetTriple.isOSBinFormatMachO())
2072	return false;
2073
2074	if (TargetTriple.isMacOSX() && !TargetTriple.isMacOSXVersionLT(Major: 10, Minor: 11))
2075	return true;
2076	if (TargetTriple.isiOS() /* or tvOS */ && !TargetTriple.isOSVersionLT(Major: 9))
2077	return true;
2078	if (TargetTriple.isWatchOS() && !TargetTriple.isOSVersionLT(Major: 2))
2079	return true;
2080	if (TargetTriple.isDriverKit())
2081	return true;
2082	if (TargetTriple.isXROS())
2083	return true;
2084
2085	return false;
2086	}
2087
2088	StringRef ModuleAddressSanitizer::getGlobalMetadataSection() const {
2089	switch (TargetTriple.getObjectFormat()) {
2090	case Triple::COFF: return ".ASAN$GL";
2091	case Triple::ELF: return "asan_globals";
2092	case Triple::MachO: return "__DATA,__asan_globals,regular";
2093	case Triple::Wasm:
2094	case Triple::GOFF:
2095	case Triple::SPIRV:
2096	case Triple::XCOFF:
2097	case Triple::DXContainer:
2098	report_fatal_error(
2099	reason: "ModuleAddressSanitizer not implemented for object file format");
2100	case Triple::UnknownObjectFormat:
2101	break;
2102	}
2103	llvm_unreachable("unsupported object format");
2104	}
2105
2106	void ModuleAddressSanitizer::initializeCallbacks(Module &M) {
2107	IRBuilder<> IRB(*C);
2108
2109	// Declare our poisoning and unpoisoning functions.
2110	AsanPoisonGlobals =
2111	M.getOrInsertFunction(Name: kAsanPoisonGlobalsName, RetTy: IRB.getVoidTy(), Args: IntptrTy);
2112	AsanUnpoisonGlobals =
2113	M.getOrInsertFunction(Name: kAsanUnpoisonGlobalsName, RetTy: IRB.getVoidTy());
2114
2115	// Declare functions that register/unregister globals.
2116	AsanRegisterGlobals = M.getOrInsertFunction(
2117	Name: kAsanRegisterGlobalsName, RetTy: IRB.getVoidTy(), Args: IntptrTy, Args: IntptrTy);
2118	AsanUnregisterGlobals = M.getOrInsertFunction(
2119	Name: kAsanUnregisterGlobalsName, RetTy: IRB.getVoidTy(), Args: IntptrTy, Args: IntptrTy);
2120
2121	// Declare the functions that find globals in a shared object and then invoke
2122	// the (un)register function on them.
2123	AsanRegisterImageGlobals = M.getOrInsertFunction(
2124	Name: kAsanRegisterImageGlobalsName, RetTy: IRB.getVoidTy(), Args: IntptrTy);
2125	AsanUnregisterImageGlobals = M.getOrInsertFunction(
2126	Name: kAsanUnregisterImageGlobalsName, RetTy: IRB.getVoidTy(), Args: IntptrTy);
2127
2128	AsanRegisterElfGlobals =
2129	M.getOrInsertFunction(Name: kAsanRegisterElfGlobalsName, RetTy: IRB.getVoidTy(),
2130	Args: IntptrTy, Args: IntptrTy, Args: IntptrTy);
2131	AsanUnregisterElfGlobals =
2132	M.getOrInsertFunction(Name: kAsanUnregisterElfGlobalsName, RetTy: IRB.getVoidTy(),
2133	Args: IntptrTy, Args: IntptrTy, Args: IntptrTy);
2134	}
2135
2136	// Put the metadata and the instrumented global in the same group. This ensures
2137	// that the metadata is discarded if the instrumented global is discarded.
2138	void ModuleAddressSanitizer::SetComdatForGlobalMetadata(
2139	GlobalVariable *G, GlobalVariable *Metadata, StringRef InternalSuffix) {
2140	Module &M = *G->getParent();
2141	Comdat *C = G->getComdat();
2142	if (!C) {
2143	if (!G->hasName()) {
2144	// If G is unnamed, it must be internal. Give it an artificial name
2145	// so we can put it in a comdat.
2146	assert(G->hasLocalLinkage());
2147	G->setName(Twine(kAsanGenPrefix) + "_anon_global");
2148	}
2149
2150	if (!InternalSuffix.empty() && G->hasLocalLinkage()) {
2151	std::string Name = std::string(G->getName());
2152	Name += InternalSuffix;
2153	C = M.getOrInsertComdat(Name);
2154	} else {
2155	C = M.getOrInsertComdat(Name: G->getName());
2156	}
2157
2158	// Make this IMAGE_COMDAT_SELECT_NODUPLICATES on COFF. Also upgrade private
2159	// linkage to internal linkage so that a symbol table entry is emitted. This
2160	// is necessary in order to create the comdat group.
2161	if (TargetTriple.isOSBinFormatCOFF()) {
2162	C->setSelectionKind(Comdat::NoDeduplicate);
2163	if (G->hasPrivateLinkage())
2164	G->setLinkage(GlobalValue::InternalLinkage);
2165	}
2166	G->setComdat(C);
2167	}
2168
2169	assert(G->hasComdat());
2170	Metadata->setComdat(G->getComdat());
2171	}
2172
2173	// Create a separate metadata global and put it in the appropriate ASan
2174	// global registration section.
2175	GlobalVariable *
2176	ModuleAddressSanitizer::CreateMetadataGlobal(Module &M, Constant *Initializer,
2177	StringRef OriginalName) {
2178	auto Linkage = TargetTriple.isOSBinFormatMachO()
2179	? GlobalVariable::InternalLinkage
2180	: GlobalVariable::PrivateLinkage;
2181	GlobalVariable *Metadata = new GlobalVariable(
2182	M, Initializer->getType(), false, Linkage, Initializer,
2183	Twine("__asan_global_") + GlobalValue::dropLLVMManglingEscape(Name: OriginalName));
2184	Metadata->setSection(getGlobalMetadataSection());
2185	// Place metadata in a large section for x86-64 ELF binaries to mitigate
2186	// relocation pressure.
2187	setGlobalVariableLargeSection(TargetTriple, GV&: *Metadata);
2188	return Metadata;
2189	}
2190
2191	Instruction *ModuleAddressSanitizer::CreateAsanModuleDtor(Module &M) {
2192	AsanDtorFunction = Function::createWithDefaultAttr(
2193	Ty: FunctionType::get(Result: Type::getVoidTy(C&: *C), isVarArg: false),
2194	Linkage: GlobalValue::InternalLinkage, AddrSpace: 0, N: kAsanModuleDtorName, M: &M);
2195	AsanDtorFunction->addFnAttr(Attribute::NoUnwind);
2196	// Ensure Dtor cannot be discarded, even if in a comdat.
2197	appendToUsed(M, Values: {AsanDtorFunction});
2198	BasicBlock *AsanDtorBB = BasicBlock::Create(Context&: *C, Name: "", Parent: AsanDtorFunction);
2199
2200	return ReturnInst::Create(C&: *C, InsertAtEnd: AsanDtorBB);
2201	}
2202
2203	void ModuleAddressSanitizer::InstrumentGlobalsCOFF(
2204	IRBuilder<> &IRB, Module &M, ArrayRef<GlobalVariable *> ExtendedGlobals,
2205	ArrayRef<Constant *> MetadataInitializers) {
2206	assert(ExtendedGlobals.size() == MetadataInitializers.size());
2207	auto &DL = M.getDataLayout();
2208
2209	SmallVector<GlobalValue *, 16> MetadataGlobals(ExtendedGlobals.size());
2210	for (size_t i = 0; i < ExtendedGlobals.size(); i++) {
2211	Constant *Initializer = MetadataInitializers[i];
2212	GlobalVariable *G = ExtendedGlobals[i];
2213	GlobalVariable *Metadata =
2214	CreateMetadataGlobal(M, Initializer, OriginalName: G->getName());
2215	MDNode *MD = MDNode::get(Context&: M.getContext(), MDs: ValueAsMetadata::get(V: G));
2216	Metadata->setMetadata(KindID: LLVMContext::MD_associated, Node: MD);
2217	MetadataGlobals[i] = Metadata;
2218
2219	// The MSVC linker always inserts padding when linking incrementally. We
2220	// cope with that by aligning each struct to its size, which must be a power
2221	// of two.
2222	unsigned SizeOfGlobalStruct = DL.getTypeAllocSize(Ty: Initializer->getType());
2223	assert(isPowerOf2_32(SizeOfGlobalStruct) &&
2224	"global metadata will not be padded appropriately");
2225	Metadata->setAlignment(assumeAligned(Value: SizeOfGlobalStruct));
2226
2227	SetComdatForGlobalMetadata(G, Metadata, InternalSuffix: "");
2228	}
2229
2230	// Update llvm.compiler.used, adding the new metadata globals. This is
2231	// needed so that during LTO these variables stay alive.
2232	if (!MetadataGlobals.empty())
2233	appendToCompilerUsed(M, Values: MetadataGlobals);
2234	}
2235
2236	void ModuleAddressSanitizer::instrumentGlobalsELF(
2237	IRBuilder<> &IRB, Module &M, ArrayRef<GlobalVariable *> ExtendedGlobals,
2238	ArrayRef<Constant *> MetadataInitializers,
2239	const std::string &UniqueModuleId) {
2240	assert(ExtendedGlobals.size() == MetadataInitializers.size());
2241
2242	// Putting globals in a comdat changes the semantic and potentially cause
2243	// false negative odr violations at link time. If odr indicators are used, we
2244	// keep the comdat sections, as link time odr violations will be dectected on
2245	// the odr indicator symbols.
2246	bool UseComdatForGlobalsGC = UseOdrIndicator && !UniqueModuleId.empty();
2247
2248	SmallVector<GlobalValue *, 16> MetadataGlobals(ExtendedGlobals.size());
2249	for (size_t i = 0; i < ExtendedGlobals.size(); i++) {
2250	GlobalVariable *G = ExtendedGlobals[i];
2251	GlobalVariable *Metadata =
2252	CreateMetadataGlobal(M, Initializer: MetadataInitializers[i], OriginalName: G->getName());
2253	MDNode *MD = MDNode::get(Context&: M.getContext(), MDs: ValueAsMetadata::get(V: G));
2254	Metadata->setMetadata(KindID: LLVMContext::MD_associated, Node: MD);
2255	MetadataGlobals[i] = Metadata;
2256
2257	if (UseComdatForGlobalsGC)
2258	SetComdatForGlobalMetadata(G, Metadata, InternalSuffix: UniqueModuleId);
2259	}
2260
2261	// Update llvm.compiler.used, adding the new metadata globals. This is
2262	// needed so that during LTO these variables stay alive.
2263	if (!MetadataGlobals.empty())
2264	appendToCompilerUsed(M, Values: MetadataGlobals);
2265
2266	// RegisteredFlag serves two purposes. First, we can pass it to dladdr()
2267	// to look up the loaded image that contains it. Second, we can store in it
2268	// whether registration has already occurred, to prevent duplicate
2269	// registration.
2270	//
2271	// Common linkage ensures that there is only one global per shared library.
2272	GlobalVariable *RegisteredFlag = new GlobalVariable(
2273	M, IntptrTy, false, GlobalVariable::CommonLinkage,
2274	ConstantInt::get(Ty: IntptrTy, V: 0), kAsanGlobalsRegisteredFlagName);
2275	RegisteredFlag->setVisibility(GlobalVariable::HiddenVisibility);
2276
2277	// Create start and stop symbols.
2278	GlobalVariable *StartELFMetadata = new GlobalVariable(
2279	M, IntptrTy, false, GlobalVariable::ExternalWeakLinkage, nullptr,
2280	"__start_" + getGlobalMetadataSection());
2281	StartELFMetadata->setVisibility(GlobalVariable::HiddenVisibility);
2282	GlobalVariable *StopELFMetadata = new GlobalVariable(
2283	M, IntptrTy, false, GlobalVariable::ExternalWeakLinkage, nullptr,
2284	"__stop_" + getGlobalMetadataSection());
2285	StopELFMetadata->setVisibility(GlobalVariable::HiddenVisibility);
2286
2287	// Create a call to register the globals with the runtime.
2288	if (ConstructorKind == AsanCtorKind::Global)
2289	IRB.CreateCall(Callee: AsanRegisterElfGlobals,
2290	Args: {IRB.CreatePointerCast(V: RegisteredFlag, DestTy: IntptrTy),
2291	IRB.CreatePointerCast(V: StartELFMetadata, DestTy: IntptrTy),
2292	IRB.CreatePointerCast(V: StopELFMetadata, DestTy: IntptrTy)});
2293
2294	// We also need to unregister globals at the end, e.g., when a shared library
2295	// gets closed.
2296	if (DestructorKind != AsanDtorKind::None && !MetadataGlobals.empty()) {
2297	IRBuilder<> IrbDtor(CreateAsanModuleDtor(M));
2298	IrbDtor.CreateCall(Callee: AsanUnregisterElfGlobals,
2299	Args: {IRB.CreatePointerCast(V: RegisteredFlag, DestTy: IntptrTy),
2300	IRB.CreatePointerCast(V: StartELFMetadata, DestTy: IntptrTy),
2301	IRB.CreatePointerCast(V: StopELFMetadata, DestTy: IntptrTy)});
2302	}
2303	}
2304
2305	void ModuleAddressSanitizer::InstrumentGlobalsMachO(
2306	IRBuilder<> &IRB, Module &M, ArrayRef<GlobalVariable *> ExtendedGlobals,
2307	ArrayRef<Constant *> MetadataInitializers) {
2308	assert(ExtendedGlobals.size() == MetadataInitializers.size());
2309
2310	// On recent Mach-O platforms, use a structure which binds the liveness of
2311	// the global variable to the metadata struct. Keep the list of "Liveness" GV
2312	// created to be added to llvm.compiler.used
2313	StructType *LivenessTy = StructType::get(elt1: IntptrTy, elts: IntptrTy);
2314	SmallVector<GlobalValue *, 16> LivenessGlobals(ExtendedGlobals.size());
2315
2316	for (size_t i = 0; i < ExtendedGlobals.size(); i++) {
2317	Constant *Initializer = MetadataInitializers[i];
2318	GlobalVariable *G = ExtendedGlobals[i];
2319	GlobalVariable *Metadata =
2320	CreateMetadataGlobal(M, Initializer, OriginalName: G->getName());
2321
2322	// On recent Mach-O platforms, we emit the global metadata in a way that
2323	// allows the linker to properly strip dead globals.
2324	auto LivenessBinder =
2325	ConstantStruct::get(T: LivenessTy, Vs: Initializer->getAggregateElement(Elt: 0u),
2326	Vs: ConstantExpr::getPointerCast(C: Metadata, Ty: IntptrTy));
2327	GlobalVariable *Liveness = new GlobalVariable(
2328	M, LivenessTy, false, GlobalVariable::InternalLinkage, LivenessBinder,
2329	Twine("__asan_binder_") + G->getName());
2330	Liveness->setSection("__DATA,__asan_liveness,regular,live_support");
2331	LivenessGlobals[i] = Liveness;
2332	}
2333
2334	// Update llvm.compiler.used, adding the new liveness globals. This is
2335	// needed so that during LTO these variables stay alive. The alternative
2336	// would be to have the linker handling the LTO symbols, but libLTO
2337	// current API does not expose access to the section for each symbol.
2338	if (!LivenessGlobals.empty())
2339	appendToCompilerUsed(M, Values: LivenessGlobals);
2340
2341	// RegisteredFlag serves two purposes. First, we can pass it to dladdr()
2342	// to look up the loaded image that contains it. Second, we can store in it
2343	// whether registration has already occurred, to prevent duplicate
2344	// registration.
2345	//
2346	// common linkage ensures that there is only one global per shared library.
2347	GlobalVariable *RegisteredFlag = new GlobalVariable(
2348	M, IntptrTy, false, GlobalVariable::CommonLinkage,
2349	ConstantInt::get(Ty: IntptrTy, V: 0), kAsanGlobalsRegisteredFlagName);
2350	RegisteredFlag->setVisibility(GlobalVariable::HiddenVisibility);
2351
2352	if (ConstructorKind == AsanCtorKind::Global)
2353	IRB.CreateCall(Callee: AsanRegisterImageGlobals,
2354	Args: {IRB.CreatePointerCast(V: RegisteredFlag, DestTy: IntptrTy)});
2355
2356	// We also need to unregister globals at the end, e.g., when a shared library
2357	// gets closed.
2358	if (DestructorKind != AsanDtorKind::None) {
2359	IRBuilder<> IrbDtor(CreateAsanModuleDtor(M));
2360	IrbDtor.CreateCall(Callee: AsanUnregisterImageGlobals,
2361	Args: {IRB.CreatePointerCast(V: RegisteredFlag, DestTy: IntptrTy)});
2362	}
2363	}
2364
2365	void ModuleAddressSanitizer::InstrumentGlobalsWithMetadataArray(
2366	IRBuilder<> &IRB, Module &M, ArrayRef<GlobalVariable *> ExtendedGlobals,
2367	ArrayRef<Constant *> MetadataInitializers) {
2368	assert(ExtendedGlobals.size() == MetadataInitializers.size());
2369	unsigned N = ExtendedGlobals.size();
2370	assert(N > 0);
2371
2372	// On platforms that don't have a custom metadata section, we emit an array
2373	// of global metadata structures.
2374	ArrayType *ArrayOfGlobalStructTy =
2375	ArrayType::get(ElementType: MetadataInitializers[0]->getType(), NumElements: N);
2376	auto AllGlobals = new GlobalVariable(
2377	M, ArrayOfGlobalStructTy, false, GlobalVariable::InternalLinkage,
2378	ConstantArray::get(T: ArrayOfGlobalStructTy, V: MetadataInitializers), "");
2379	if (Mapping.Scale > 3)
2380	AllGlobals->setAlignment(Align(1ULL << Mapping.Scale));
2381
2382	if (ConstructorKind == AsanCtorKind::Global)
2383	IRB.CreateCall(Callee: AsanRegisterGlobals,
2384	Args: {IRB.CreatePointerCast(V: AllGlobals, DestTy: IntptrTy),
2385	ConstantInt::get(Ty: IntptrTy, V: N)});
2386
2387	// We also need to unregister globals at the end, e.g., when a shared library
2388	// gets closed.
2389	if (DestructorKind != AsanDtorKind::None) {
2390	IRBuilder<> IrbDtor(CreateAsanModuleDtor(M));
2391	IrbDtor.CreateCall(Callee: AsanUnregisterGlobals,
2392	Args: {IRB.CreatePointerCast(V: AllGlobals, DestTy: IntptrTy),
2393	ConstantInt::get(Ty: IntptrTy, V: N)});
2394	}
2395	}
2396
2397	// This function replaces all global variables with new variables that have
2398	// trailing redzones. It also creates a function that poisons
2399	// redzones and inserts this function into llvm.global_ctors.
2400	// Sets *CtorComdat to true if the global registration code emitted into the
2401	// asan constructor is comdat-compatible.
2402	void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB, Module &M,
2403	bool *CtorComdat) {
2404	// Build set of globals that are aliased by some GA, where
2405	// getExcludedAliasedGlobal(GA) returns the relevant GlobalVariable.
2406	SmallPtrSet<const GlobalVariable *, 16> AliasedGlobalExclusions;
2407	if (CompileKernel) {
2408	for (auto &GA : M.aliases()) {
2409	if (const GlobalVariable *GV = getExcludedAliasedGlobal(GA))
2410	AliasedGlobalExclusions.insert(Ptr: GV);
2411	}
2412	}
2413
2414	SmallVector<GlobalVariable *, 16> GlobalsToChange;
2415	for (auto &G : M.globals()) {
2416	if (!AliasedGlobalExclusions.count(Ptr: &G) && shouldInstrumentGlobal(G: &G))
2417	GlobalsToChange.push_back(Elt: &G);
2418	}
2419
2420	size_t n = GlobalsToChange.size();
2421	auto &DL = M.getDataLayout();
2422
2423	// A global is described by a structure
2424	// size_t beg;
2425	// size_t size;
2426	// size_t size_with_redzone;
2427	// const char *name;
2428	// const char *module_name;
2429	// size_t has_dynamic_init;
2430	// size_t padding_for_windows_msvc_incremental_link;
2431	// size_t odr_indicator;
2432	// We initialize an array of such structures and pass it to a run-time call.
2433	StructType *GlobalStructTy =
2434	StructType::get(elt1: IntptrTy, elts: IntptrTy, elts: IntptrTy, elts: IntptrTy, elts: IntptrTy,
2435	elts: IntptrTy, elts: IntptrTy, elts: IntptrTy);
2436	SmallVector<GlobalVariable *, 16> NewGlobals(n);
2437	SmallVector<Constant *, 16> Initializers(n);
2438
2439	bool HasDynamicallyInitializedGlobals = false;
2440
2441	// We shouldn't merge same module names, as this string serves as unique
2442	// module ID in runtime.
2443	GlobalVariable *ModuleName =
2444	n != 0
2445	? createPrivateGlobalForString(M, Str: M.getModuleIdentifier(),
2446	/*AllowMerging*/ false, NamePrefix: kAsanGenPrefix)
2447	: nullptr;
2448
2449	for (size_t i = 0; i < n; i++) {
2450	GlobalVariable *G = GlobalsToChange[i];
2451
2452	GlobalValue::SanitizerMetadata MD;
2453	if (G->hasSanitizerMetadata())
2454	MD = G->getSanitizerMetadata();
2455
2456	// The runtime library tries demangling symbol names in the descriptor but
2457	// functionality like __cxa_demangle may be unavailable (e.g.
2458	// -static-libstdc++). So we demangle the symbol names here.
2459	std::string NameForGlobal = G->getName().str();
2460	GlobalVariable *Name =
2461	createPrivateGlobalForString(M, Str: llvm::demangle(MangledName: NameForGlobal),
2462	/*AllowMerging*/ true, NamePrefix: kAsanGenPrefix);
2463
2464	Type *Ty = G->getValueType();
2465	const uint64_t SizeInBytes = DL.getTypeAllocSize(Ty);
2466	const uint64_t RightRedzoneSize = getRedzoneSizeForGlobal(SizeInBytes);
2467	Type *RightRedZoneTy = ArrayType::get(ElementType: IRB.getInt8Ty(), NumElements: RightRedzoneSize);
2468
2469	StructType *NewTy = StructType::get(elt1: Ty, elts: RightRedZoneTy);
2470	Constant *NewInitializer = ConstantStruct::get(
2471	T: NewTy, Vs: G->getInitializer(), Vs: Constant::getNullValue(Ty: RightRedZoneTy));
2472
2473	// Create a new global variable with enough space for a redzone.
2474	GlobalValue::LinkageTypes Linkage = G->getLinkage();
2475	if (G->isConstant() && Linkage == GlobalValue::PrivateLinkage)
2476	Linkage = GlobalValue::InternalLinkage;
2477	GlobalVariable *NewGlobal = new GlobalVariable(
2478	M, NewTy, G->isConstant(), Linkage, NewInitializer, "", G,
2479	G->getThreadLocalMode(), G->getAddressSpace());
2480	NewGlobal->copyAttributesFrom(Src: G);
2481	NewGlobal->setComdat(G->getComdat());
2482	NewGlobal->setAlignment(Align(getMinRedzoneSizeForGlobal()));
2483	// Don't fold globals with redzones. ODR violation detector and redzone
2484	// poisoning implicitly creates a dependence on the global's address, so it
2485	// is no longer valid for it to be marked unnamed_addr.
2486	NewGlobal->setUnnamedAddr(GlobalValue::UnnamedAddr::None);
2487
2488	// Move null-terminated C strings to "__asan_cstring" section on Darwin.
2489	if (TargetTriple.isOSBinFormatMachO() && !G->hasSection() &&
2490	G->isConstant()) {
2491	auto Seq = dyn_cast<ConstantDataSequential>(Val: G->getInitializer());
2492	if (Seq && Seq->isCString())
2493	NewGlobal->setSection("__TEXT,__asan_cstring,regular");
2494	}
2495
2496	// Transfer the debug info and type metadata. The payload starts at offset
2497	// zero so we can copy the metadata over as is.
2498	NewGlobal->copyMetadata(Src: G, Offset: 0);
2499
2500	Value *Indices2[2];
2501	Indices2[0] = IRB.getInt32(C: 0);
2502	Indices2[1] = IRB.getInt32(C: 0);
2503
2504	G->replaceAllUsesWith(
2505	V: ConstantExpr::getGetElementPtr(Ty: NewTy, C: NewGlobal, IdxList: Indices2, InBounds: true));
2506	NewGlobal->takeName(V: G);
2507	G->eraseFromParent();
2508	NewGlobals[i] = NewGlobal;
2509
2510	Constant *ODRIndicator = ConstantPointerNull::get(T: PtrTy);
2511	GlobalValue *InstrumentedGlobal = NewGlobal;
2512
2513	bool CanUsePrivateAliases =
2514	TargetTriple.isOSBinFormatELF() || TargetTriple.isOSBinFormatMachO() ||
2515	TargetTriple.isOSBinFormatWasm();
2516	if (CanUsePrivateAliases && UsePrivateAlias) {
2517	// Create local alias for NewGlobal to avoid crash on ODR between
2518	// instrumented and non-instrumented libraries.
2519	InstrumentedGlobal =
2520	GlobalAlias::create(Linkage: GlobalValue::PrivateLinkage, Name: "", Aliasee: NewGlobal);
2521	}
2522
2523	// ODR should not happen for local linkage.
2524	if (NewGlobal->hasLocalLinkage()) {
2525	ODRIndicator =
2526	ConstantExpr::getIntToPtr(C: ConstantInt::get(Ty: IntptrTy, V: -1), Ty: PtrTy);
2527	} else if (UseOdrIndicator) {
2528	// With local aliases, we need to provide another externally visible
2529	// symbol __odr_asan_XXX to detect ODR violation.
2530	auto *ODRIndicatorSym =
2531	new GlobalVariable(M, IRB.getInt8Ty(), false, Linkage,
2532	Constant::getNullValue(Ty: IRB.getInt8Ty()),
2533	kODRGenPrefix + NameForGlobal, nullptr,
2534	NewGlobal->getThreadLocalMode());
2535
2536	// Set meaningful attributes for indicator symbol.
2537	ODRIndicatorSym->setVisibility(NewGlobal->getVisibility());
2538	ODRIndicatorSym->setDLLStorageClass(NewGlobal->getDLLStorageClass());
2539	ODRIndicatorSym->setAlignment(Align(1));
2540	ODRIndicator = ODRIndicatorSym;
2541	}
2542
2543	Constant *Initializer = ConstantStruct::get(
2544	T: GlobalStructTy,
2545	Vs: ConstantExpr::getPointerCast(C: InstrumentedGlobal, Ty: IntptrTy),
2546	Vs: ConstantInt::get(Ty: IntptrTy, V: SizeInBytes),
2547	Vs: ConstantInt::get(Ty: IntptrTy, V: SizeInBytes + RightRedzoneSize),
2548	Vs: ConstantExpr::getPointerCast(C: Name, Ty: IntptrTy),
2549	Vs: ConstantExpr::getPointerCast(C: ModuleName, Ty: IntptrTy),
2550	Vs: ConstantInt::get(Ty: IntptrTy, V: MD.IsDynInit),
2551	Vs: Constant::getNullValue(Ty: IntptrTy),
2552	Vs: ConstantExpr::getPointerCast(C: ODRIndicator, Ty: IntptrTy));
2553
2554	if (ClInitializers && MD.IsDynInit)
2555	HasDynamicallyInitializedGlobals = true;
2556
2557	LLVM_DEBUG(dbgs() << "NEW GLOBAL: " << *NewGlobal << "\n");
2558
2559	Initializers[i] = Initializer;
2560	}
2561
2562	// Add instrumented globals to llvm.compiler.used list to avoid LTO from
2563	// ConstantMerge'ing them.
2564	SmallVector<GlobalValue *, 16> GlobalsToAddToUsedList;
2565	for (size_t i = 0; i < n; i++) {
2566	GlobalVariable *G = NewGlobals[i];
2567	if (G->getName().empty()) continue;
2568	GlobalsToAddToUsedList.push_back(Elt: G);
2569	}
2570	appendToCompilerUsed(M, Values: ArrayRef<GlobalValue *>(GlobalsToAddToUsedList));
2571
2572	if (UseGlobalsGC && TargetTriple.isOSBinFormatELF()) {
2573	// Use COMDAT and register globals even if n == 0 to ensure that (a) the
2574	// linkage unit will only have one module constructor, and (b) the register
2575	// function will be called. The module destructor is not created when n ==
2576	// 0.
2577	*CtorComdat = true;
2578	instrumentGlobalsELF(IRB, M, ExtendedGlobals: NewGlobals, MetadataInitializers: Initializers,
2579	UniqueModuleId: getUniqueModuleId(M: &M));
2580	} else if (n == 0) {
2581	// When UseGlobalsGC is false, COMDAT can still be used if n == 0, because
2582	// all compile units will have identical module constructor/destructor.
2583	*CtorComdat = TargetTriple.isOSBinFormatELF();
2584	} else {
2585	*CtorComdat = false;
2586	if (UseGlobalsGC && TargetTriple.isOSBinFormatCOFF()) {
2587	InstrumentGlobalsCOFF(IRB, M, ExtendedGlobals: NewGlobals, MetadataInitializers: Initializers);
2588	} else if (UseGlobalsGC && ShouldUseMachOGlobalsSection()) {
2589	InstrumentGlobalsMachO(IRB, M, ExtendedGlobals: NewGlobals, MetadataInitializers: Initializers);
2590	} else {
2591	InstrumentGlobalsWithMetadataArray(IRB, M, ExtendedGlobals: NewGlobals, MetadataInitializers: Initializers);
2592	}
2593	}
2594
2595	// Create calls for poisoning before initializers run and unpoisoning after.
2596	if (HasDynamicallyInitializedGlobals)
2597	createInitializerPoisonCalls(M, ModuleName);
2598
2599	LLVM_DEBUG(dbgs() << M);
2600	}
2601
2602	uint64_t
2603	ModuleAddressSanitizer::getRedzoneSizeForGlobal(uint64_t SizeInBytes) const {
2604	constexpr uint64_t kMaxRZ = 1 << 18;
2605	const uint64_t MinRZ = getMinRedzoneSizeForGlobal();
2606
2607	uint64_t RZ = 0;
2608	if (SizeInBytes <= MinRZ / 2) {
2609	// Reduce redzone size for small size objects, e.g. int, char[1]. MinRZ is
2610	// at least 32 bytes, optimize when SizeInBytes is less than or equal to
2611	// half of MinRZ.
2612	RZ = MinRZ - SizeInBytes;
2613	} else {
2614	// Calculate RZ, where MinRZ <= RZ <= MaxRZ, and RZ ~ 1/4 * SizeInBytes.
2615	RZ = std::clamp(val: (SizeInBytes / MinRZ / 4) * MinRZ, lo: MinRZ, hi: kMaxRZ);
2616
2617	// Round up to multiple of MinRZ.
2618	if (SizeInBytes % MinRZ)
2619	RZ += MinRZ - (SizeInBytes % MinRZ);
2620	}
2621
2622	assert((RZ + SizeInBytes) % MinRZ == 0);
2623
2624	return RZ;
2625	}
2626
2627	int ModuleAddressSanitizer::GetAsanVersion(const Module &M) const {
2628	int LongSize = M.getDataLayout().getPointerSizeInBits();
2629	bool isAndroid = Triple(M.getTargetTriple()).isAndroid();
2630	int Version = 8;
2631	// 32-bit Android is one version ahead because of the switch to dynamic
2632	// shadow.
2633	Version += (LongSize == 32 && isAndroid);
2634	return Version;
2635	}
2636
2637	bool ModuleAddressSanitizer::instrumentModule(Module &M) {
2638	initializeCallbacks(M);
2639
2640	// Create a module constructor. A destructor is created lazily because not all
2641	// platforms, and not all modules need it.
2642	if (ConstructorKind == AsanCtorKind::Global) {
2643	if (CompileKernel) {
2644	// The kernel always builds with its own runtime, and therefore does not
2645	// need the init and version check calls.
2646	AsanCtorFunction = createSanitizerCtor(M, CtorName: kAsanModuleCtorName);
2647	} else {
2648	std::string AsanVersion = std::to_string(val: GetAsanVersion(M));
2649	std::string VersionCheckName =
2650	InsertVersionCheck ? (kAsanVersionCheckNamePrefix + AsanVersion) : "";
2651	std::tie(args&: AsanCtorFunction, args: std::ignore) =
2652	createSanitizerCtorAndInitFunctions(M, CtorName: kAsanModuleCtorName,
2653	InitName: kAsanInitName, /*InitArgTypes=*/{},
2654	/*InitArgs=*/{}, VersionCheckName);
2655	}
2656	}
2657
2658	bool CtorComdat = true;
2659	if (ClGlobals) {
2660	assert(AsanCtorFunction || ConstructorKind == AsanCtorKind::None);
2661	if (AsanCtorFunction) {
2662	IRBuilder<> IRB(AsanCtorFunction->getEntryBlock().getTerminator());
2663	instrumentGlobals(IRB, M, CtorComdat: &CtorComdat);
2664	} else {
2665	IRBuilder<> IRB(*C);
2666	instrumentGlobals(IRB, M, CtorComdat: &CtorComdat);
2667	}
2668	}
2669
2670	const uint64_t Priority = GetCtorAndDtorPriority(TargetTriple);
2671
2672	// Put the constructor and destructor in comdat if both
2673	// (1) global instrumentation is not TU-specific
2674	// (2) target is ELF.
2675	if (UseCtorComdat && TargetTriple.isOSBinFormatELF() && CtorComdat) {
2676	if (AsanCtorFunction) {
2677	AsanCtorFunction->setComdat(M.getOrInsertComdat(Name: kAsanModuleCtorName));
2678	appendToGlobalCtors(M, F: AsanCtorFunction, Priority, Data: AsanCtorFunction);
2679	}
2680	if (AsanDtorFunction) {
2681	AsanDtorFunction->setComdat(M.getOrInsertComdat(Name: kAsanModuleDtorName));
2682	appendToGlobalDtors(M, F: AsanDtorFunction, Priority, Data: AsanDtorFunction);
2683	}
2684	} else {
2685	if (AsanCtorFunction)
2686	appendToGlobalCtors(M, F: AsanCtorFunction, Priority);
2687	if (AsanDtorFunction)
2688	appendToGlobalDtors(M, F: AsanDtorFunction, Priority);
2689	}
2690
2691	return true;
2692	}
2693
2694	void AddressSanitizer::initializeCallbacks(Module &M, const TargetLibraryInfo *TLI) {
2695	IRBuilder<> IRB(*C);
2696	// Create __asan_report* callbacks.
2697	// IsWrite, TypeSize and Exp are encoded in the function name.
2698	for (int Exp = 0; Exp < 2; Exp++) {
2699	for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
2700	const std::string TypeStr = AccessIsWrite ? "store" : "load";
2701	const std::string ExpStr = Exp ? "exp_" : "";
2702	const std::string EndingStr = Recover ? "_noabort" : "";
2703
2704	SmallVector<Type *, 3> Args2 = {IntptrTy, IntptrTy};
2705	SmallVector<Type *, 2> Args1{1, IntptrTy};
2706	AttributeList AL2;
2707	AttributeList AL1;
2708	if (Exp) {
2709	Type *ExpType = Type::getInt32Ty(C&: *C);
2710	Args2.push_back(Elt: ExpType);
2711	Args1.push_back(Elt: ExpType);
2712	if (auto AK = TLI->getExtAttrForI32Param(Signed: false)) {
2713	AL2 = AL2.addParamAttribute(C&: *C, ArgNo: 2, Kind: AK);
2714	AL1 = AL1.addParamAttribute(C&: *C, ArgNo: 1, Kind: AK);
2715	}
2716	}
2717	AsanErrorCallbackSized[AccessIsWrite][Exp] = M.getOrInsertFunction(
2718	Name: kAsanReportErrorTemplate + ExpStr + TypeStr + "_n" + EndingStr,
2719	T: FunctionType::get(Result: IRB.getVoidTy(), Params: Args2, isVarArg: false), AttributeList: AL2);
2720
2721	AsanMemoryAccessCallbackSized[AccessIsWrite][Exp] = M.getOrInsertFunction(
2722	Name: ClMemoryAccessCallbackPrefix + ExpStr + TypeStr + "N" + EndingStr,
2723	T: FunctionType::get(Result: IRB.getVoidTy(), Params: Args2, isVarArg: false), AttributeList: AL2);
2724
2725	for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
2726	AccessSizeIndex++) {
2727	const std::string Suffix = TypeStr + itostr(X: 1ULL << AccessSizeIndex);
2728	AsanErrorCallback[AccessIsWrite][Exp][AccessSizeIndex] =
2729	M.getOrInsertFunction(
2730	Name: kAsanReportErrorTemplate + ExpStr + Suffix + EndingStr,
2731	T: FunctionType::get(Result: IRB.getVoidTy(), Params: Args1, isVarArg: false), AttributeList: AL1);
2732
2733	AsanMemoryAccessCallback[AccessIsWrite][Exp][AccessSizeIndex] =
2734	M.getOrInsertFunction(
2735	Name: ClMemoryAccessCallbackPrefix + ExpStr + Suffix + EndingStr,
2736	T: FunctionType::get(Result: IRB.getVoidTy(), Params: Args1, isVarArg: false), AttributeList: AL1);
2737	}
2738	}
2739	}
2740
2741	const std::string MemIntrinCallbackPrefix =
2742	(CompileKernel && !ClKasanMemIntrinCallbackPrefix)
2743	? std::string("")
2744	: ClMemoryAccessCallbackPrefix;
2745	AsanMemmove = M.getOrInsertFunction(Name: MemIntrinCallbackPrefix + "memmove",
2746	RetTy: PtrTy, Args: PtrTy, Args: PtrTy, Args: IntptrTy);
2747	AsanMemcpy = M.getOrInsertFunction(Name: MemIntrinCallbackPrefix + "memcpy", RetTy: PtrTy,
2748	Args: PtrTy, Args: PtrTy, Args: IntptrTy);
2749	AsanMemset = M.getOrInsertFunction(Name: MemIntrinCallbackPrefix + "memset",
2750	AttributeList: TLI->getAttrList(C, ArgNos: {1}, /*Signed=*/false),
2751	RetTy: PtrTy, Args: PtrTy, Args: IRB.getInt32Ty(), Args: IntptrTy);
2752
2753	AsanHandleNoReturnFunc =
2754	M.getOrInsertFunction(Name: kAsanHandleNoReturnName, RetTy: IRB.getVoidTy());
2755
2756	AsanPtrCmpFunction =
2757	M.getOrInsertFunction(Name: kAsanPtrCmp, RetTy: IRB.getVoidTy(), Args: IntptrTy, Args: IntptrTy);
2758	AsanPtrSubFunction =
2759	M.getOrInsertFunction(Name: kAsanPtrSub, RetTy: IRB.getVoidTy(), Args: IntptrTy, Args: IntptrTy);
2760	if (Mapping.InGlobal)
2761	AsanShadowGlobal = M.getOrInsertGlobal(Name: "__asan_shadow",
2762	Ty: ArrayType::get(ElementType: IRB.getInt8Ty(), NumElements: 0));
2763
2764	AMDGPUAddressShared =
2765	M.getOrInsertFunction(Name: kAMDGPUAddressSharedName, RetTy: IRB.getInt1Ty(), Args: PtrTy);
2766	AMDGPUAddressPrivate =
2767	M.getOrInsertFunction(Name: kAMDGPUAddressPrivateName, RetTy: IRB.getInt1Ty(), Args: PtrTy);
2768	}
2769
2770	bool AddressSanitizer::maybeInsertAsanInitAtFunctionEntry(Function &F) {
2771	// For each NSObject descendant having a +load method, this method is invoked
2772	// by the ObjC runtime before any of the static constructors is called.
2773	// Therefore we need to instrument such methods with a call to __asan_init
2774	// at the beginning in order to initialize our runtime before any access to
2775	// the shadow memory.
2776	// We cannot just ignore these methods, because they may call other
2777	// instrumented functions.
2778	if (F.getName().contains(Other: " load]")) {
2779	FunctionCallee AsanInitFunction =
2780	declareSanitizerInitFunction(M&: *F.getParent(), InitName: kAsanInitName, InitArgTypes: {});
2781	IRBuilder<> IRB(&F.front(), F.front().begin());
2782	IRB.CreateCall(Callee: AsanInitFunction, Args: {});
2783	return true;
2784	}
2785	return false;
2786	}
2787
2788	bool AddressSanitizer::maybeInsertDynamicShadowAtFunctionEntry(Function &F) {
2789	// Generate code only when dynamic addressing is needed.
2790	if (Mapping.Offset != kDynamicShadowSentinel)
2791	return false;
2792
2793	IRBuilder<> IRB(&F.front().front());
2794	if (Mapping.InGlobal) {
2795	if (ClWithIfuncSuppressRemat) {
2796	// An empty inline asm with input reg == output reg.
2797	// An opaque pointer-to-int cast, basically.
2798	InlineAsm *Asm = InlineAsm::get(
2799	Ty: FunctionType::get(Result: IntptrTy, Params: {AsanShadowGlobal->getType()}, isVarArg: false),
2800	AsmString: StringRef(""), Constraints: StringRef("=r,0"),
2801	/*hasSideEffects=*/false);
2802	LocalDynamicShadow =
2803	IRB.CreateCall(Callee: Asm, Args: {AsanShadowGlobal}, Name: ".asan.shadow");
2804	} else {
2805	LocalDynamicShadow =
2806	IRB.CreatePointerCast(V: AsanShadowGlobal, DestTy: IntptrTy, Name: ".asan.shadow");
2807	}
2808	} else {
2809	Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal(
2810	Name: kAsanShadowMemoryDynamicAddress, Ty: IntptrTy);
2811	LocalDynamicShadow = IRB.CreateLoad(Ty: IntptrTy, Ptr: GlobalDynamicAddress);
2812	}
2813	return true;
2814	}
2815
2816	void AddressSanitizer::markEscapedLocalAllocas(Function &F) {
2817	// Find the one possible call to llvm.localescape and pre-mark allocas passed
2818	// to it as uninteresting. This assumes we haven't started processing allocas
2819	// yet. This check is done up front because iterating the use list in
2820	// isInterestingAlloca would be algorithmically slower.
2821	assert(ProcessedAllocas.empty() && "must process localescape before allocas");
2822
2823	// Try to get the declaration of llvm.localescape. If it's not in the module,
2824	// we can exit early.
2825	if (!F.getParent()->getFunction(Name: "llvm.localescape")) return;
2826
2827	// Look for a call to llvm.localescape call in the entry block. It can't be in
2828	// any other block.
2829	for (Instruction &I : F.getEntryBlock()) {
2830	IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: &I);
2831	if (II && II->getIntrinsicID() == Intrinsic::localescape) {
2832	// We found a call. Mark all the allocas passed in as uninteresting.
2833	for (Value *Arg : II->args()) {
2834	AllocaInst *AI = dyn_cast<AllocaInst>(Val: Arg->stripPointerCasts());
2835	assert(AI && AI->isStaticAlloca() &&
2836	"non-static alloca arg to localescape");
2837	ProcessedAllocas[AI] = false;
2838	}
2839	break;
2840	}
2841	}
2842	}
2843
2844	bool AddressSanitizer::suppressInstrumentationSiteForDebug(int &Instrumented) {
2845	bool ShouldInstrument =
2846	ClDebugMin < 0 || ClDebugMax < 0 ||
2847	(Instrumented >= ClDebugMin && Instrumented <= ClDebugMax);
2848	Instrumented++;
2849	return !ShouldInstrument;
2850	}
2851
2852	bool AddressSanitizer::instrumentFunction(Function &F,
2853	const TargetLibraryInfo *TLI) {
2854	if (F.empty())
2855	return false;
2856	if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage) return false;
2857	if (!ClDebugFunc.empty() && ClDebugFunc == F.getName()) return false;
2858	if (F.getName().starts_with(Prefix: "__asan_")) return false;
2859
2860	bool FunctionModified = false;
2861
2862	// If needed, insert __asan_init before checking for SanitizeAddress attr.
2863	// This function needs to be called even if the function body is not
2864	// instrumented.
2865	if (maybeInsertAsanInitAtFunctionEntry(F))
2866	FunctionModified = true;
2867
2868	// Leave if the function doesn't need instrumentation.
2869	if (!F.hasFnAttribute(Attribute::SanitizeAddress)) return FunctionModified;
2870
2871	if (F.hasFnAttribute(Attribute::DisableSanitizerInstrumentation))
2872	return FunctionModified;
2873
2874	LLVM_DEBUG(dbgs() << "ASAN instrumenting:\n" << F << "\n");
2875
2876	initializeCallbacks(M&: *F.getParent(), TLI);
2877
2878	FunctionStateRAII CleanupObj(this);
2879
2880	FunctionModified |= maybeInsertDynamicShadowAtFunctionEntry(F);
2881
2882	// We can't instrument allocas used with llvm.localescape. Only static allocas
2883	// can be passed to that intrinsic.
2884	markEscapedLocalAllocas(F);
2885
2886	// We want to instrument every address only once per basic block (unless there
2887	// are calls between uses).
2888	SmallPtrSet<Value *, 16> TempsToInstrument;
2889	SmallVector<InterestingMemoryOperand, 16> OperandsToInstrument;
2890	SmallVector<MemIntrinsic *, 16> IntrinToInstrument;
2891	SmallVector<Instruction *, 8> NoReturnCalls;
2892	SmallVector<BasicBlock *, 16> AllBlocks;
2893	SmallVector<Instruction *, 16> PointerComparisonsOrSubtracts;
2894
2895	// Fill the set of memory operations to instrument.
2896	for (auto &BB : F) {
2897	AllBlocks.push_back(Elt: &BB);
2898	TempsToInstrument.clear();
2899	int NumInsnsPerBB = 0;
2900	for (auto &Inst : BB) {
2901	if (LooksLikeCodeInBug11395(I: &Inst)) return false;
2902	// Skip instructions inserted by another instrumentation.
2903	if (Inst.hasMetadata(KindID: LLVMContext::MD_nosanitize))
2904	continue;
2905	SmallVector<InterestingMemoryOperand, 1> InterestingOperands;
2906	getInterestingMemoryOperands(I: &Inst, Interesting&: InterestingOperands);
2907
2908	if (!InterestingOperands.empty()) {
2909	for (auto &Operand : InterestingOperands) {
2910	if (ClOpt && ClOptSameTemp) {
2911	Value *Ptr = Operand.getPtr();
2912	// If we have a mask, skip instrumentation if we've already
2913	// instrumented the full object. But don't add to TempsToInstrument
2914	// because we might get another load/store with a different mask.
2915	if (Operand.MaybeMask) {
2916	if (TempsToInstrument.count(Ptr))
2917	continue; // We've seen this (whole) temp in the current BB.
2918	} else {
2919	if (!TempsToInstrument.insert(Ptr).second)
2920	continue; // We've seen this temp in the current BB.
2921	}
2922	}
2923	OperandsToInstrument.push_back(Elt: Operand);
2924	NumInsnsPerBB++;
2925	}
2926	} else if (((ClInvalidPointerPairs || ClInvalidPointerCmp) &&
2927	isInterestingPointerComparison(I: &Inst)) ||
2928	((ClInvalidPointerPairs || ClInvalidPointerSub) &&
2929	isInterestingPointerSubtraction(I: &Inst))) {
2930	PointerComparisonsOrSubtracts.push_back(Elt: &Inst);
2931	} else if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(Val: &Inst)) {
2932	// ok, take it.
2933	IntrinToInstrument.push_back(Elt: MI);
2934	NumInsnsPerBB++;
2935	} else {
2936	if (auto *CB = dyn_cast<CallBase>(Val: &Inst)) {
2937	// A call inside BB.
2938	TempsToInstrument.clear();
2939	if (CB->doesNotReturn())
2940	NoReturnCalls.push_back(Elt: CB);
2941	}
2942	if (CallInst *CI = dyn_cast<CallInst>(Val: &Inst))
2943	maybeMarkSanitizerLibraryCallNoBuiltin(CI, TLI);
2944	}
2945	if (NumInsnsPerBB >= ClMaxInsnsToInstrumentPerBB) break;
2946	}
2947	}
2948
2949	bool UseCalls = (InstrumentationWithCallsThreshold >= 0 &&
2950	OperandsToInstrument.size() + IntrinToInstrument.size() >
2951	(unsigned)InstrumentationWithCallsThreshold);
2952	const DataLayout &DL = F.getParent()->getDataLayout();
2953	ObjectSizeOpts ObjSizeOpts;
2954	ObjSizeOpts.RoundToAlign = true;
2955	ObjectSizeOffsetVisitor ObjSizeVis(DL, TLI, F.getContext(), ObjSizeOpts);
2956
2957	// Instrument.
2958	int NumInstrumented = 0;
2959	for (auto &Operand : OperandsToInstrument) {
2960	if (!suppressInstrumentationSiteForDebug(Instrumented&: NumInstrumented))
2961	instrumentMop(ObjSizeVis, O&: Operand, UseCalls,
2962	DL: F.getParent()->getDataLayout());
2963	FunctionModified = true;
2964	}
2965	for (auto *Inst : IntrinToInstrument) {
2966	if (!suppressInstrumentationSiteForDebug(Instrumented&: NumInstrumented))
2967	instrumentMemIntrinsic(MI: Inst);
2968	FunctionModified = true;
2969	}
2970
2971	FunctionStackPoisoner FSP(F, *this);
2972	bool ChangedStack = FSP.runOnFunction();
2973
2974	// We must unpoison the stack before NoReturn calls (throw, _exit, etc).
2975	// See e.g. https://github.com/google/sanitizers/issues/37
2976	for (auto *CI : NoReturnCalls) {
2977	IRBuilder<> IRB(CI);
2978	IRB.CreateCall(Callee: AsanHandleNoReturnFunc, Args: {});
2979	}
2980
2981	for (auto *Inst : PointerComparisonsOrSubtracts) {
2982	instrumentPointerComparisonOrSubtraction(I: Inst);
2983	FunctionModified = true;
2984	}
2985
2986	if (ChangedStack || !NoReturnCalls.empty())
2987	FunctionModified = true;
2988
2989	LLVM_DEBUG(dbgs() << "ASAN done instrumenting: " << FunctionModified << " "
2990	<< F << "\n");
2991
2992	return FunctionModified;
2993	}
2994
2995	// Workaround for bug 11395: we don't want to instrument stack in functions
2996	// with large assembly blobs (32-bit only), otherwise reg alloc may crash.
2997	// FIXME: remove once the bug 11395 is fixed.
2998	bool AddressSanitizer::LooksLikeCodeInBug11395(Instruction *I) {
2999	if (LongSize != 32) return false;
3000	CallInst *CI = dyn_cast<CallInst>(Val: I);
3001	if (!CI || !CI->isInlineAsm()) return false;
3002	if (CI->arg_size() <= 5)
3003	return false;
3004	// We have inline assembly with quite a few arguments.
3005	return true;
3006	}
3007
3008	void FunctionStackPoisoner::initializeCallbacks(Module &M) {
3009	IRBuilder<> IRB(*C);
3010	if (ASan.UseAfterReturn == AsanDetectStackUseAfterReturnMode::Always ||
3011	ASan.UseAfterReturn == AsanDetectStackUseAfterReturnMode::Runtime) {
3012	const char *MallocNameTemplate =
3013	ASan.UseAfterReturn == AsanDetectStackUseAfterReturnMode::Always
3014	? kAsanStackMallocAlwaysNameTemplate
3015	: kAsanStackMallocNameTemplate;
3016	for (int Index = 0; Index <= kMaxAsanStackMallocSizeClass; Index++) {
3017	std::string Suffix = itostr(X: Index);
3018	AsanStackMallocFunc[Index] = M.getOrInsertFunction(
3019	Name: MallocNameTemplate + Suffix, RetTy: IntptrTy, Args: IntptrTy);
3020	AsanStackFreeFunc[Index] =
3021	M.getOrInsertFunction(Name: kAsanStackFreeNameTemplate + Suffix,
3022	RetTy: IRB.getVoidTy(), Args: IntptrTy, Args: IntptrTy);
3023	}
3024	}
3025	if (ASan.UseAfterScope) {
3026	AsanPoisonStackMemoryFunc = M.getOrInsertFunction(
3027	Name: kAsanPoisonStackMemoryName, RetTy: IRB.getVoidTy(), Args: IntptrTy, Args: IntptrTy);
3028	AsanUnpoisonStackMemoryFunc = M.getOrInsertFunction(
3029	Name: kAsanUnpoisonStackMemoryName, RetTy: IRB.getVoidTy(), Args: IntptrTy, Args: IntptrTy);
3030	}
3031
3032	for (size_t Val : {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0xf1, 0xf2,
3033	0xf3, 0xf5, 0xf8}) {
3034	std::ostringstream Name;
3035	Name << kAsanSetShadowPrefix;
3036	Name << std::setw(2) << std::setfill('0') << std::hex << Val;
3037	AsanSetShadowFunc[Val] =
3038	M.getOrInsertFunction(Name: Name.str(), RetTy: IRB.getVoidTy(), Args: IntptrTy, Args: IntptrTy);
3039	}
3040
3041	AsanAllocaPoisonFunc = M.getOrInsertFunction(
3042	Name: kAsanAllocaPoison, RetTy: IRB.getVoidTy(), Args: IntptrTy, Args: IntptrTy);
3043	AsanAllocasUnpoisonFunc = M.getOrInsertFunction(
3044	Name: kAsanAllocasUnpoison, RetTy: IRB.getVoidTy(), Args: IntptrTy, Args: IntptrTy);
3045	}
3046
3047	void FunctionStackPoisoner::copyToShadowInline(ArrayRef<uint8_t> ShadowMask,
3048	ArrayRef<uint8_t> ShadowBytes,
3049	size_t Begin, size_t End,
3050	IRBuilder<> &IRB,
3051	Value *ShadowBase) {
3052	if (Begin >= End)
3053	return;
3054
3055	const size_t LargestStoreSizeInBytes =
3056	std::min<size_t>(a: sizeof(uint64_t), b: ASan.LongSize / 8);
3057
3058	const bool IsLittleEndian = F.getParent()->getDataLayout().isLittleEndian();
3059
3060	// Poison given range in shadow using larges store size with out leading and
3061	// trailing zeros in ShadowMask. Zeros never change, so they need neither
3062	// poisoning nor up-poisoning. Still we don't mind if some of them get into a
3063	// middle of a store.
3064	for (size_t i = Begin; i < End;) {
3065	if (!ShadowMask[i]) {
3066	assert(!ShadowBytes[i]);
3067	++i;
3068	continue;
3069	}
3070
3071	size_t StoreSizeInBytes = LargestStoreSizeInBytes;
3072	// Fit store size into the range.
3073	while (StoreSizeInBytes > End - i)
3074	StoreSizeInBytes /= 2;
3075
3076	// Minimize store size by trimming trailing zeros.
3077	for (size_t j = StoreSizeInBytes - 1; j && !ShadowMask[i + j]; --j) {
3078	while (j <= StoreSizeInBytes / 2)
3079	StoreSizeInBytes /= 2;
3080	}
3081
3082	uint64_t Val = 0;
3083	for (size_t j = 0; j < StoreSizeInBytes; j++) {
3084	if (IsLittleEndian)
3085	Val |= (uint64_t)ShadowBytes[i + j] << (8 * j);
3086	else
3087	Val = (Val << 8) | ShadowBytes[i + j];
3088	}
3089
3090	Value *Ptr = IRB.CreateAdd(LHS: ShadowBase, RHS: ConstantInt::get(Ty: IntptrTy, V: i));
3091	Value *Poison = IRB.getIntN(N: StoreSizeInBytes * 8, C: Val);
3092	IRB.CreateAlignedStore(
3093	Val: Poison, Ptr: IRB.CreateIntToPtr(V: Ptr, DestTy: PointerType::getUnqual(C&: Poison->getContext())),
3094	Align: Align(1));
3095
3096	i += StoreSizeInBytes;
3097	}
3098	}
3099
3100	void FunctionStackPoisoner::copyToShadow(ArrayRef<uint8_t> ShadowMask,
3101	ArrayRef<uint8_t> ShadowBytes,
3102	IRBuilder<> &IRB, Value *ShadowBase) {
3103	copyToShadow(ShadowMask, ShadowBytes, Begin: 0, End: ShadowMask.size(), IRB, ShadowBase);
3104	}
3105
3106	void FunctionStackPoisoner::copyToShadow(ArrayRef<uint8_t> ShadowMask,
3107	ArrayRef<uint8_t> ShadowBytes,
3108	size_t Begin, size_t End,
3109	IRBuilder<> &IRB, Value *ShadowBase) {
3110	assert(ShadowMask.size() == ShadowBytes.size());
3111	size_t Done = Begin;
3112	for (size_t i = Begin, j = Begin + 1; i < End; i = j++) {
3113	if (!ShadowMask[i]) {
3114	assert(!ShadowBytes[i]);
3115	continue;
3116	}
3117	uint8_t Val = ShadowBytes[i];
3118	if (!AsanSetShadowFunc[Val])
3119	continue;
3120
3121	// Skip same values.
3122	for (; j < End && ShadowMask[j] && Val == ShadowBytes[j]; ++j) {
3123	}
3124
3125	if (j - i >= ASan.MaxInlinePoisoningSize) {
3126	copyToShadowInline(ShadowMask, ShadowBytes, Begin: Done, End: i, IRB, ShadowBase);
3127	IRB.CreateCall(Callee: AsanSetShadowFunc[Val],
3128	Args: {IRB.CreateAdd(LHS: ShadowBase, RHS: ConstantInt::get(Ty: IntptrTy, V: i)),
3129	ConstantInt::get(Ty: IntptrTy, V: j - i)});
3130	Done = j;
3131	}
3132	}
3133
3134	copyToShadowInline(ShadowMask, ShadowBytes, Begin: Done, End, IRB, ShadowBase);
3135	}
3136
3137	// Fake stack allocator (asan_fake_stack.h) has 11 size classes
3138	// for every power of 2 from kMinStackMallocSize to kMaxAsanStackMallocSizeClass
3139	static int StackMallocSizeClass(uint64_t LocalStackSize) {
3140	assert(LocalStackSize <= kMaxStackMallocSize);
3141	uint64_t MaxSize = kMinStackMallocSize;
3142	for (int i = 0;; i++, MaxSize *= 2)
3143	if (LocalStackSize <= MaxSize) return i;
3144	llvm_unreachable("impossible LocalStackSize");
3145	}
3146
3147	void FunctionStackPoisoner::copyArgsPassedByValToAllocas() {
3148	Instruction *CopyInsertPoint = &F.front().front();
3149	if (CopyInsertPoint == ASan.LocalDynamicShadow) {
3150	// Insert after the dynamic shadow location is determined
3151	CopyInsertPoint = CopyInsertPoint->getNextNode();
3152	assert(CopyInsertPoint);
3153	}
3154	IRBuilder<> IRB(CopyInsertPoint);
3155	const DataLayout &DL = F.getParent()->getDataLayout();
3156	for (Argument &Arg : F.args()) {
3157	if (Arg.hasByValAttr()) {
3158	Type *Ty = Arg.getParamByValType();
3159	const Align Alignment =
3160	DL.getValueOrABITypeAlignment(Alignment: Arg.getParamAlign(), Ty);
3161
3162	AllocaInst *AI = IRB.CreateAlloca(
3163	Ty, ArraySize: nullptr,
3164	Name: (Arg.hasName() ? Arg.getName() : "Arg" + Twine(Arg.getArgNo())) +
3165	".byval");
3166	AI->setAlignment(Alignment);
3167	Arg.replaceAllUsesWith(V: AI);
3168
3169	uint64_t AllocSize = DL.getTypeAllocSize(Ty);
3170	IRB.CreateMemCpy(Dst: AI, DstAlign: Alignment, Src: &Arg, SrcAlign: Alignment, Size: AllocSize);
3171	}
3172	}
3173	}
3174
3175	PHINode *FunctionStackPoisoner::createPHI(IRBuilder<> &IRB, Value *Cond,
3176	Value *ValueIfTrue,
3177	Instruction *ThenTerm,
3178	Value *ValueIfFalse) {
3179	PHINode *PHI = IRB.CreatePHI(Ty: IntptrTy, NumReservedValues: 2);
3180	BasicBlock *CondBlock = cast<Instruction>(Val: Cond)->getParent();
3181	PHI->addIncoming(V: ValueIfFalse, BB: CondBlock);
3182	BasicBlock *ThenBlock = ThenTerm->getParent();
3183	PHI->addIncoming(V: ValueIfTrue, BB: ThenBlock);
3184	return PHI;
3185	}
3186
3187	Value *FunctionStackPoisoner::createAllocaForLayout(
3188	IRBuilder<> &IRB, const ASanStackFrameLayout &L, bool Dynamic) {
3189	AllocaInst *Alloca;
3190	if (Dynamic) {
3191	Alloca = IRB.CreateAlloca(Ty: IRB.getInt8Ty(),
3192	ArraySize: ConstantInt::get(Ty: IRB.getInt64Ty(), V: L.FrameSize),
3193	Name: "MyAlloca");
3194	} else {
3195	Alloca = IRB.CreateAlloca(Ty: ArrayType::get(ElementType: IRB.getInt8Ty(), NumElements: L.FrameSize),
3196	ArraySize: nullptr, Name: "MyAlloca");
3197	assert(Alloca->isStaticAlloca());
3198	}
3199	assert((ClRealignStack & (ClRealignStack - 1)) == 0);
3200	uint64_t FrameAlignment = std::max(a: L.FrameAlignment, b: uint64_t(ClRealignStack));
3201	Alloca->setAlignment(Align(FrameAlignment));
3202	return IRB.CreatePointerCast(V: Alloca, DestTy: IntptrTy);
3203	}
3204
3205	void FunctionStackPoisoner::createDynamicAllocasInitStorage() {
3206	BasicBlock &FirstBB = *F.begin();
3207	IRBuilder<> IRB(dyn_cast<Instruction>(Val: FirstBB.begin()));
3208	DynamicAllocaLayout = IRB.CreateAlloca(Ty: IntptrTy, ArraySize: nullptr);
3209	IRB.CreateStore(Val: Constant::getNullValue(Ty: IntptrTy), Ptr: DynamicAllocaLayout);
3210	DynamicAllocaLayout->setAlignment(Align(32));
3211	}
3212
3213	void FunctionStackPoisoner::processDynamicAllocas() {
3214	if (!ClInstrumentDynamicAllocas || DynamicAllocaVec.empty()) {
3215	assert(DynamicAllocaPoisonCallVec.empty());
3216	return;
3217	}
3218
3219	// Insert poison calls for lifetime intrinsics for dynamic allocas.
3220	for (const auto &APC : DynamicAllocaPoisonCallVec) {
3221	assert(APC.InsBefore);
3222	assert(APC.AI);
3223	assert(ASan.isInterestingAlloca(*APC.AI));
3224	assert(!APC.AI->isStaticAlloca());
3225
3226	IRBuilder<> IRB(APC.InsBefore);
3227	poisonAlloca(V: APC.AI, Size: APC.Size, IRB, DoPoison: APC.DoPoison);
3228	// Dynamic allocas will be unpoisoned unconditionally below in
3229	// unpoisonDynamicAllocas.
3230	// Flag that we need unpoison static allocas.
3231	}
3232
3233	// Handle dynamic allocas.
3234	createDynamicAllocasInitStorage();
3235	for (auto &AI : DynamicAllocaVec)
3236	handleDynamicAllocaCall(AI);
3237	unpoisonDynamicAllocas();
3238	}
3239
3240	/// Collect instructions in the entry block after \p InsBefore which initialize
3241	/// permanent storage for a function argument. These instructions must remain in
3242	/// the entry block so that uninitialized values do not appear in backtraces. An
3243	/// added benefit is that this conserves spill slots. This does not move stores
3244	/// before instrumented / "interesting" allocas.
3245	static void findStoresToUninstrumentedArgAllocas(
3246	AddressSanitizer &ASan, Instruction &InsBefore,
3247	SmallVectorImpl<Instruction *> &InitInsts) {
3248	Instruction *Start = InsBefore.getNextNonDebugInstruction();
3249	for (Instruction *It = Start; It; It = It->getNextNonDebugInstruction()) {
3250	// Argument initialization looks like:
3251	// 1) store <Argument>, <Alloca> OR
3252	// 2) <CastArgument> = cast <Argument> to ...
3253	// store <CastArgument> to <Alloca>
3254	// Do not consider any other kind of instruction.
3255	//
3256	// Note: This covers all known cases, but may not be exhaustive. An
3257	// alternative to pattern-matching stores is to DFS over all Argument uses:
3258	// this might be more general, but is probably much more complicated.
3259	if (isa<AllocaInst>(Val: It) || isa<CastInst>(Val: It))
3260	continue;
3261	if (auto *Store = dyn_cast<StoreInst>(Val: It)) {
3262	// The store destination must be an alloca that isn't interesting for
3263	// ASan to instrument. These are moved up before InsBefore, and they're
3264	// not interesting because allocas for arguments can be mem2reg'd.
3265	auto *Alloca = dyn_cast<AllocaInst>(Val: Store->getPointerOperand());
3266	if (!Alloca || ASan.isInterestingAlloca(AI: *Alloca))
3267	continue;
3268
3269	Value *Val = Store->getValueOperand();
3270	bool IsDirectArgInit = isa<Argument>(Val);
3271	bool IsArgInitViaCast =
3272	isa<CastInst>(Val) &&
3273	isa<Argument>(Val: cast<CastInst>(Val)->getOperand(i_nocapture: 0)) &&
3274	// Check that the cast appears directly before the store. Otherwise
3275	// moving the cast before InsBefore may break the IR.
3276	Val == It->getPrevNonDebugInstruction();
3277	bool IsArgInit = IsDirectArgInit || IsArgInitViaCast;
3278	if (!IsArgInit)
3279	continue;
3280
3281	if (IsArgInitViaCast)
3282	InitInsts.push_back(Elt: cast<Instruction>(Val));
3283	InitInsts.push_back(Elt: Store);
3284	continue;
3285	}
3286
3287	// Do not reorder past unknown instructions: argument initialization should
3288	// only involve casts and stores.
3289	return;
3290	}
3291	}
3292
3293	void FunctionStackPoisoner::processStaticAllocas() {
3294	if (AllocaVec.empty()) {
3295	assert(StaticAllocaPoisonCallVec.empty());
3296	return;
3297	}
3298
3299	int StackMallocIdx = -1;
3300	DebugLoc EntryDebugLocation;
3301	if (auto SP = F.getSubprogram())
3302	EntryDebugLocation =
3303	DILocation::get(Context&: SP->getContext(), Line: SP->getScopeLine(), Column: 0, Scope: SP);
3304
3305	Instruction *InsBefore = AllocaVec[0];
3306	IRBuilder<> IRB(InsBefore);
3307
3308	// Make sure non-instrumented allocas stay in the entry block. Otherwise,
3309	// debug info is broken, because only entry-block allocas are treated as
3310	// regular stack slots.
3311	auto InsBeforeB = InsBefore->getParent();
3312	assert(InsBeforeB == &F.getEntryBlock());
3313	for (auto *AI : StaticAllocasToMoveUp)
3314	if (AI->getParent() == InsBeforeB)
3315	AI->moveBefore(MovePos: InsBefore);
3316
3317	// Move stores of arguments into entry-block allocas as well. This prevents
3318	// extra stack slots from being generated (to house the argument values until
3319	// they can be stored into the allocas). This also prevents uninitialized
3320	// values from being shown in backtraces.
3321	SmallVector<Instruction *, 8> ArgInitInsts;
3322	findStoresToUninstrumentedArgAllocas(ASan, InsBefore&: *InsBefore, InitInsts&: ArgInitInsts);
3323	for (Instruction *ArgInitInst : ArgInitInsts)
3324	ArgInitInst->moveBefore(MovePos: InsBefore);
3325
3326	// If we have a call to llvm.localescape, keep it in the entry block.
3327	if (LocalEscapeCall) LocalEscapeCall->moveBefore(MovePos: InsBefore);
3328
3329	SmallVector<ASanStackVariableDescription, 16> SVD;
3330	SVD.reserve(N: AllocaVec.size());
3331	for (AllocaInst *AI : AllocaVec) {
3332	ASanStackVariableDescription D = {.Name: AI->getName().data(),
3333	.Size: ASan.getAllocaSizeInBytes(AI: *AI),
3334	.LifetimeSize: 0,
3335	.Alignment: AI->getAlign().value(),
3336	.AI: AI,
3337	.Offset: 0,
3338	.Line: 0};
3339	SVD.push_back(Elt: D);
3340	}
3341
3342	// Minimal header size (left redzone) is 4 pointers,
3343	// i.e. 32 bytes on 64-bit platforms and 16 bytes in 32-bit platforms.
3344	uint64_t Granularity = 1ULL << Mapping.Scale;
3345	uint64_t MinHeaderSize = std::max(a: (uint64_t)ASan.LongSize / 2, b: Granularity);
3346	const ASanStackFrameLayout &L =
3347	ComputeASanStackFrameLayout(Vars&: SVD, Granularity, MinHeaderSize);
3348
3349	// Build AllocaToSVDMap for ASanStackVariableDescription lookup.
3350	DenseMap<const AllocaInst *, ASanStackVariableDescription *> AllocaToSVDMap;
3351	for (auto &Desc : SVD)
3352	AllocaToSVDMap[Desc.AI] = &Desc;
3353
3354	// Update SVD with information from lifetime intrinsics.
3355	for (const auto &APC : StaticAllocaPoisonCallVec) {
3356	assert(APC.InsBefore);
3357	assert(APC.AI);
3358	assert(ASan.isInterestingAlloca(*APC.AI));
3359	assert(APC.AI->isStaticAlloca());
3360
3361	ASanStackVariableDescription &Desc = *AllocaToSVDMap[APC.AI];
3362	Desc.LifetimeSize = Desc.Size;
3363	if (const DILocation *FnLoc = EntryDebugLocation.get()) {
3364	if (const DILocation *LifetimeLoc = APC.InsBefore->getDebugLoc().get()) {
3365	if (LifetimeLoc->getFile() == FnLoc->getFile())
3366	if (unsigned Line = LifetimeLoc->getLine())
3367	Desc.Line = std::min(a: Desc.Line ? Desc.Line : Line, b: Line);
3368	}
3369	}
3370	}
3371
3372	auto DescriptionString = ComputeASanStackFrameDescription(Vars: SVD);
3373	LLVM_DEBUG(dbgs() << DescriptionString << " --- " << L.FrameSize << "\n");
3374	uint64_t LocalStackSize = L.FrameSize;
3375	bool DoStackMalloc =
3376	ASan.UseAfterReturn != AsanDetectStackUseAfterReturnMode::Never &&
3377	!ASan.CompileKernel && LocalStackSize <= kMaxStackMallocSize;
3378	bool DoDynamicAlloca = ClDynamicAllocaStack;
3379	// Don't do dynamic alloca or stack malloc if:
3380	// 1) There is inline asm: too often it makes assumptions on which registers
3381	// are available.
3382	// 2) There is a returns_twice call (typically setjmp), which is
3383	// optimization-hostile, and doesn't play well with introduced indirect
3384	// register-relative calculation of local variable addresses.
3385	DoDynamicAlloca &= !HasInlineAsm && !HasReturnsTwiceCall;
3386	DoStackMalloc &= !HasInlineAsm && !HasReturnsTwiceCall;
3387
3388	Value *StaticAlloca =
3389	DoDynamicAlloca ? nullptr : createAllocaForLayout(IRB, L, Dynamic: false);
3390
3391	Value *FakeStack;
3392	Value *LocalStackBase;
3393	Value *LocalStackBaseAlloca;
3394	uint8_t DIExprFlags = DIExpression::ApplyOffset;
3395
3396	if (DoStackMalloc) {
3397	LocalStackBaseAlloca =
3398	IRB.CreateAlloca(Ty: IntptrTy, ArraySize: nullptr, Name: "asan_local_stack_base");
3399	if (ASan.UseAfterReturn == AsanDetectStackUseAfterReturnMode::Runtime) {
3400	// void *FakeStack = __asan_option_detect_stack_use_after_return
3401	// ? __asan_stack_malloc_N(LocalStackSize)
3402	// : nullptr;
3403	// void *LocalStackBase = (FakeStack) ? FakeStack :
3404	// alloca(LocalStackSize);
3405	Constant *OptionDetectUseAfterReturn = F.getParent()->getOrInsertGlobal(
3406	Name: kAsanOptionDetectUseAfterReturn, Ty: IRB.getInt32Ty());
3407	Value *UseAfterReturnIsEnabled = IRB.CreateICmpNE(
3408	LHS: IRB.CreateLoad(Ty: IRB.getInt32Ty(), Ptr: OptionDetectUseAfterReturn),
3409	RHS: Constant::getNullValue(Ty: IRB.getInt32Ty()));
3410	Instruction *Term =
3411	SplitBlockAndInsertIfThen(Cond: UseAfterReturnIsEnabled, SplitBefore: InsBefore, Unreachable: false);
3412	IRBuilder<> IRBIf(Term);
3413	StackMallocIdx = StackMallocSizeClass(LocalStackSize);
3414	assert(StackMallocIdx <= kMaxAsanStackMallocSizeClass);
3415	Value *FakeStackValue =
3416	IRBIf.CreateCall(Callee: AsanStackMallocFunc[StackMallocIdx],
3417	Args: ConstantInt::get(Ty: IntptrTy, V: LocalStackSize));
3418	IRB.SetInsertPoint(InsBefore);
3419	FakeStack = createPHI(IRB, Cond: UseAfterReturnIsEnabled, ValueIfTrue: FakeStackValue, ThenTerm: Term,
3420	ValueIfFalse: ConstantInt::get(Ty: IntptrTy, V: 0));
3421	} else {
3422	// assert(ASan.UseAfterReturn == AsanDetectStackUseAfterReturnMode:Always)
3423	// void *FakeStack = __asan_stack_malloc_N(LocalStackSize);
3424	// void *LocalStackBase = (FakeStack) ? FakeStack :
3425	// alloca(LocalStackSize);
3426	StackMallocIdx = StackMallocSizeClass(LocalStackSize);
3427	FakeStack = IRB.CreateCall(Callee: AsanStackMallocFunc[StackMallocIdx],
3428	Args: ConstantInt::get(Ty: IntptrTy, V: LocalStackSize));
3429	}
3430	Value *NoFakeStack =
3431	IRB.CreateICmpEQ(LHS: FakeStack, RHS: Constant::getNullValue(Ty: IntptrTy));
3432	Instruction *Term =
3433	SplitBlockAndInsertIfThen(Cond: NoFakeStack, SplitBefore: InsBefore, Unreachable: false);
3434	IRBuilder<> IRBIf(Term);
3435	Value *AllocaValue =
3436	DoDynamicAlloca ? createAllocaForLayout(IRB&: IRBIf, L, Dynamic: true) : StaticAlloca;
3437
3438	IRB.SetInsertPoint(InsBefore);
3439	LocalStackBase = createPHI(IRB, Cond: NoFakeStack, ValueIfTrue: AllocaValue, ThenTerm: Term, ValueIfFalse: FakeStack);
3440	IRB.CreateStore(Val: LocalStackBase, Ptr: LocalStackBaseAlloca);
3441	DIExprFlags |= DIExpression::DerefBefore;
3442	} else {
3443	// void *FakeStack = nullptr;
3444	// void *LocalStackBase = alloca(LocalStackSize);
3445	FakeStack = ConstantInt::get(Ty: IntptrTy, V: 0);
3446	LocalStackBase =
3447	DoDynamicAlloca ? createAllocaForLayout(IRB, L, Dynamic: true) : StaticAlloca;
3448	LocalStackBaseAlloca = LocalStackBase;
3449	}
3450
3451	// It shouldn't matter whether we pass an `alloca` or a `ptrtoint` as the
3452	// dbg.declare address opereand, but passing a `ptrtoint` seems to confuse
3453	// later passes and can result in dropped variable coverage in debug info.
3454	Value *LocalStackBaseAllocaPtr =
3455	isa<PtrToIntInst>(Val: LocalStackBaseAlloca)
3456	? cast<PtrToIntInst>(Val: LocalStackBaseAlloca)->getPointerOperand()
3457	: LocalStackBaseAlloca;
3458	assert(isa<AllocaInst>(LocalStackBaseAllocaPtr) &&
3459	"Variable descriptions relative to ASan stack base will be dropped");
3460
3461	// Replace Alloca instructions with base+offset.
3462	for (const auto &Desc : SVD) {
3463	AllocaInst *AI = Desc.AI;
3464	replaceDbgDeclare(Address: AI, NewAddress: LocalStackBaseAllocaPtr, Builder&: DIB, DIExprFlags,
3465	Offset: Desc.Offset);
3466	Value *NewAllocaPtr = IRB.CreateIntToPtr(
3467	V: IRB.CreateAdd(LHS: LocalStackBase, RHS: ConstantInt::get(Ty: IntptrTy, V: Desc.Offset)),
3468	DestTy: AI->getType());
3469	AI->replaceAllUsesWith(V: NewAllocaPtr);
3470	}
3471
3472	// The left-most redzone has enough space for at least 4 pointers.
3473	// Write the Magic value to redzone[0].
3474	Value *BasePlus0 = IRB.CreateIntToPtr(V: LocalStackBase, DestTy: IntptrPtrTy);
3475	IRB.CreateStore(Val: ConstantInt::get(Ty: IntptrTy, V: kCurrentStackFrameMagic),
3476	Ptr: BasePlus0);
3477	// Write the frame description constant to redzone[1].
3478	Value *BasePlus1 = IRB.CreateIntToPtr(
3479	V: IRB.CreateAdd(LHS: LocalStackBase,
3480	RHS: ConstantInt::get(Ty: IntptrTy, V: ASan.LongSize / 8)),
3481	DestTy: IntptrPtrTy);
3482	GlobalVariable *StackDescriptionGlobal =
3483	createPrivateGlobalForString(M&: *F.getParent(), Str: DescriptionString,
3484	/*AllowMerging*/ true, NamePrefix: kAsanGenPrefix);
3485	Value *Description = IRB.CreatePointerCast(V: StackDescriptionGlobal, DestTy: IntptrTy);
3486	IRB.CreateStore(Val: Description, Ptr: BasePlus1);
3487	// Write the PC to redzone[2].
3488	Value *BasePlus2 = IRB.CreateIntToPtr(
3489	V: IRB.CreateAdd(LHS: LocalStackBase,
3490	RHS: ConstantInt::get(Ty: IntptrTy, V: 2 * ASan.LongSize / 8)),
3491	DestTy: IntptrPtrTy);
3492	IRB.CreateStore(Val: IRB.CreatePointerCast(V: &F, DestTy: IntptrTy), Ptr: BasePlus2);
3493
3494	const auto &ShadowAfterScope = GetShadowBytesAfterScope(Vars: SVD, Layout: L);
3495
3496	// Poison the stack red zones at the entry.
3497	Value *ShadowBase = ASan.memToShadow(Shadow: LocalStackBase, IRB);
3498	// As mask we must use most poisoned case: red zones and after scope.
3499	// As bytes we can use either the same or just red zones only.
3500	copyToShadow(ShadowMask: ShadowAfterScope, ShadowBytes: ShadowAfterScope, IRB, ShadowBase);
3501
3502	if (!StaticAllocaPoisonCallVec.empty()) {
3503	const auto &ShadowInScope = GetShadowBytes(Vars: SVD, Layout: L);
3504
3505	// Poison static allocas near lifetime intrinsics.
3506	for (const auto &APC : StaticAllocaPoisonCallVec) {
3507	const ASanStackVariableDescription &Desc = *AllocaToSVDMap[APC.AI];
3508	assert(Desc.Offset % L.Granularity == 0);
3509	size_t Begin = Desc.Offset / L.Granularity;
3510	size_t End = Begin + (APC.Size + L.Granularity - 1) / L.Granularity;
3511
3512	IRBuilder<> IRB(APC.InsBefore);
3513	copyToShadow(ShadowMask: ShadowAfterScope,
3514	ShadowBytes: APC.DoPoison ? ShadowAfterScope : ShadowInScope, Begin, End,
3515	IRB, ShadowBase);
3516	}
3517	}
3518
3519	SmallVector<uint8_t, 64> ShadowClean(ShadowAfterScope.size(), 0);
3520	SmallVector<uint8_t, 64> ShadowAfterReturn;
3521
3522	// (Un)poison the stack before all ret instructions.
3523	for (Instruction *Ret : RetVec) {
3524	IRBuilder<> IRBRet(Ret);
3525	// Mark the current frame as retired.
3526	IRBRet.CreateStore(Val: ConstantInt::get(Ty: IntptrTy, V: kRetiredStackFrameMagic),
3527	Ptr: BasePlus0);
3528	if (DoStackMalloc) {
3529	assert(StackMallocIdx >= 0);
3530	// if FakeStack != 0 // LocalStackBase == FakeStack
3531	// // In use-after-return mode, poison the whole stack frame.
3532	// if StackMallocIdx <= 4
3533	// // For small sizes inline the whole thing:
3534	// memset(ShadowBase, kAsanStackAfterReturnMagic, ShadowSize);
3535	// **SavedFlagPtr(FakeStack) = 0
3536	// else
3537	// __asan_stack_free_N(FakeStack, LocalStackSize)
3538	// else
3539	// <This is not a fake stack; unpoison the redzones>
3540	Value *Cmp =
3541	IRBRet.CreateICmpNE(LHS: FakeStack, RHS: Constant::getNullValue(Ty: IntptrTy));
3542	Instruction *ThenTerm, *ElseTerm;
3543	SplitBlockAndInsertIfThenElse(Cond: Cmp, SplitBefore: Ret, ThenTerm: &ThenTerm, ElseTerm: &ElseTerm);
3544
3545	IRBuilder<> IRBPoison(ThenTerm);
3546	if (ASan.MaxInlinePoisoningSize != 0 && StackMallocIdx <= 4) {
3547	int ClassSize = kMinStackMallocSize << StackMallocIdx;
3548	ShadowAfterReturn.resize(N: ClassSize / L.Granularity,
3549	NV: kAsanStackUseAfterReturnMagic);
3550	copyToShadow(ShadowMask: ShadowAfterReturn, ShadowBytes: ShadowAfterReturn, IRB&: IRBPoison,
3551	ShadowBase);
3552	Value *SavedFlagPtrPtr = IRBPoison.CreateAdd(
3553	LHS: FakeStack,
3554	RHS: ConstantInt::get(Ty: IntptrTy, V: ClassSize - ASan.LongSize / 8));
3555	Value *SavedFlagPtr = IRBPoison.CreateLoad(
3556	Ty: IntptrTy, Ptr: IRBPoison.CreateIntToPtr(V: SavedFlagPtrPtr, DestTy: IntptrPtrTy));
3557	IRBPoison.CreateStore(
3558	Val: Constant::getNullValue(Ty: IRBPoison.getInt8Ty()),
3559	Ptr: IRBPoison.CreateIntToPtr(V: SavedFlagPtr, DestTy: IRBPoison.getPtrTy()));
3560	} else {
3561	// For larger frames call __asan_stack_free_*.
3562	IRBPoison.CreateCall(
3563	Callee: AsanStackFreeFunc[StackMallocIdx],
3564	Args: {FakeStack, ConstantInt::get(Ty: IntptrTy, V: LocalStackSize)});
3565	}
3566
3567	IRBuilder<> IRBElse(ElseTerm);
3568	copyToShadow(ShadowMask: ShadowAfterScope, ShadowBytes: ShadowClean, IRB&: IRBElse, ShadowBase);
3569	} else {
3570	copyToShadow(ShadowMask: ShadowAfterScope, ShadowBytes: ShadowClean, IRB&: IRBRet, ShadowBase);
3571	}
3572	}
3573
3574	// We are done. Remove the old unused alloca instructions.
3575	for (auto *AI : AllocaVec)
3576	AI->eraseFromParent();
3577	}
3578
3579	void FunctionStackPoisoner::poisonAlloca(Value *V, uint64_t Size,
3580	IRBuilder<> &IRB, bool DoPoison) {
3581	// For now just insert the call to ASan runtime.
3582	Value *AddrArg = IRB.CreatePointerCast(V, DestTy: IntptrTy);
3583	Value *SizeArg = ConstantInt::get(Ty: IntptrTy, V: Size);
3584	IRB.CreateCall(
3585	Callee: DoPoison ? AsanPoisonStackMemoryFunc : AsanUnpoisonStackMemoryFunc,
3586	Args: {AddrArg, SizeArg});
3587	}
3588
3589	// Handling llvm.lifetime intrinsics for a given %alloca:
3590	// (1) collect all llvm.lifetime.xxx(%size, %value) describing the alloca.
3591	// (2) if %size is constant, poison memory for llvm.lifetime.end (to detect
3592	// invalid accesses) and unpoison it for llvm.lifetime.start (the memory
3593	// could be poisoned by previous llvm.lifetime.end instruction, as the
3594	// variable may go in and out of scope several times, e.g. in loops).
3595	// (3) if we poisoned at least one %alloca in a function,
3596	// unpoison the whole stack frame at function exit.
3597	void FunctionStackPoisoner::handleDynamicAllocaCall(AllocaInst *AI) {
3598	IRBuilder<> IRB(AI);
3599
3600	const Align Alignment = std::max(a: Align(kAllocaRzSize), b: AI->getAlign());
3601	const uint64_t AllocaRedzoneMask = kAllocaRzSize - 1;
3602
3603	Value *Zero = Constant::getNullValue(Ty: IntptrTy);
3604	Value *AllocaRzSize = ConstantInt::get(Ty: IntptrTy, V: kAllocaRzSize);
3605	Value *AllocaRzMask = ConstantInt::get(Ty: IntptrTy, V: AllocaRedzoneMask);
3606
3607	// Since we need to extend alloca with additional memory to locate
3608	// redzones, and OldSize is number of allocated blocks with
3609	// ElementSize size, get allocated memory size in bytes by
3610	// OldSize * ElementSize.
3611	const unsigned ElementSize =
3612	F.getParent()->getDataLayout().getTypeAllocSize(Ty: AI->getAllocatedType());
3613	Value *OldSize =
3614	IRB.CreateMul(LHS: IRB.CreateIntCast(V: AI->getArraySize(), DestTy: IntptrTy, isSigned: false),
3615	RHS: ConstantInt::get(Ty: IntptrTy, V: ElementSize));
3616
3617	// PartialSize = OldSize % 32
3618	Value *PartialSize = IRB.CreateAnd(LHS: OldSize, RHS: AllocaRzMask);
3619
3620	// Misalign = kAllocaRzSize - PartialSize;
3621	Value *Misalign = IRB.CreateSub(LHS: AllocaRzSize, RHS: PartialSize);
3622
3623	// PartialPadding = Misalign != kAllocaRzSize ? Misalign : 0;
3624	Value *Cond = IRB.CreateICmpNE(LHS: Misalign, RHS: AllocaRzSize);
3625	Value *PartialPadding = IRB.CreateSelect(C: Cond, True: Misalign, False: Zero);
3626
3627	// AdditionalChunkSize = Alignment + PartialPadding + kAllocaRzSize
3628	// Alignment is added to locate left redzone, PartialPadding for possible
3629	// partial redzone and kAllocaRzSize for right redzone respectively.
3630	Value *AdditionalChunkSize = IRB.CreateAdd(
3631	LHS: ConstantInt::get(Ty: IntptrTy, V: Alignment.value() + kAllocaRzSize),
3632	RHS: PartialPadding);
3633
3634	Value *NewSize = IRB.CreateAdd(LHS: OldSize, RHS: AdditionalChunkSize);
3635
3636	// Insert new alloca with new NewSize and Alignment params.
3637	AllocaInst *NewAlloca = IRB.CreateAlloca(Ty: IRB.getInt8Ty(), ArraySize: NewSize);
3638	NewAlloca->setAlignment(Alignment);
3639
3640	// NewAddress = Address + Alignment
3641	Value *NewAddress =
3642	IRB.CreateAdd(LHS: IRB.CreatePtrToInt(V: NewAlloca, DestTy: IntptrTy),
3643	RHS: ConstantInt::get(Ty: IntptrTy, V: Alignment.value()));
3644
3645	// Insert __asan_alloca_poison call for new created alloca.
3646	IRB.CreateCall(Callee: AsanAllocaPoisonFunc, Args: {NewAddress, OldSize});
3647
3648	// Store the last alloca's address to DynamicAllocaLayout. We'll need this
3649	// for unpoisoning stuff.
3650	IRB.CreateStore(Val: IRB.CreatePtrToInt(V: NewAlloca, DestTy: IntptrTy), Ptr: DynamicAllocaLayout);
3651
3652	Value *NewAddressPtr = IRB.CreateIntToPtr(V: NewAddress, DestTy: AI->getType());
3653
3654	// Replace all uses of AddessReturnedByAlloca with NewAddressPtr.
3655	AI->replaceAllUsesWith(V: NewAddressPtr);
3656
3657	// We are done. Erase old alloca from parent.
3658	AI->eraseFromParent();
3659	}
3660
3661	// isSafeAccess returns true if Addr is always inbounds with respect to its
3662	// base object. For example, it is a field access or an array access with
3663	// constant inbounds index.
3664	bool AddressSanitizer::isSafeAccess(ObjectSizeOffsetVisitor &ObjSizeVis,
3665	Value *Addr, TypeSize TypeStoreSize) const {
3666	if (TypeStoreSize.isScalable())
3667	// TODO: We can use vscale_range to convert a scalable value to an
3668	// upper bound on the access size.
3669	return false;
3670
3671	SizeOffsetAPInt SizeOffset = ObjSizeVis.compute(V: Addr);
3672	if (!SizeOffset.bothKnown())
3673	return false;
3674
3675	uint64_t Size = SizeOffset.Size.getZExtValue();
3676	int64_t Offset = SizeOffset.Offset.getSExtValue();
3677
3678	// Three checks are required to ensure safety:
3679	// . Offset >= 0 (since the offset is given from the base ptr)
3680	// . Size >= Offset (unsigned)
3681	// . Size - Offset >= NeededSize (unsigned)
3682	return Offset >= 0 && Size >= uint64_t(Offset) &&
3683	Size - uint64_t(Offset) >= TypeStoreSize / 8;
3684	}
3685