1//===-- dfsan_allocator.cpp -------------------------- --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of DataflowSanitizer.
10//
11// DataflowSanitizer allocator.
12//===----------------------------------------------------------------------===//
13
14#include "dfsan_allocator.h"
15
16#include "dfsan.h"
17#include "dfsan_flags.h"
18#include "dfsan_thread.h"
19#include "sanitizer_common/sanitizer_allocator.h"
20#include "sanitizer_common/sanitizer_allocator_checks.h"
21#include "sanitizer_common/sanitizer_allocator_interface.h"
22#include "sanitizer_common/sanitizer_allocator_report.h"
23#include "sanitizer_common/sanitizer_errno.h"
24
25namespace __dfsan {
26
27struct Metadata {
28 uptr requested_size;
29};
30
31struct DFsanMapUnmapCallback {
32 void OnMap(uptr p, uptr size) const { dfsan_set_label(label: 0, addr: (void *)p, size); }
33 void OnMapSecondary(uptr p, uptr size, uptr user_begin,
34 uptr user_size) const {
35 OnMap(p, size);
36 }
37 void OnUnmap(uptr p, uptr size) const { dfsan_set_label(label: 0, addr: (void *)p, size); }
38};
39
40#if defined(__aarch64__)
41const uptr kAllocatorSpace = 0xE00000000000ULL;
42#else
43const uptr kAllocatorSpace = 0x700000000000ULL;
44#endif
45const uptr kMaxAllowedMallocSize = 8UL << 30;
46
47struct AP64 { // Allocator64 parameters. Deliberately using a short name.
48 static const uptr kSpaceBeg = kAllocatorSpace;
49 static const uptr kSpaceSize = 0x40000000000; // 4T.
50 static const uptr kMetadataSize = sizeof(Metadata);
51 typedef DefaultSizeClassMap SizeClassMap;
52 typedef DFsanMapUnmapCallback MapUnmapCallback;
53 static const uptr kFlags = 0;
54 using AddressSpaceView = LocalAddressSpaceView;
55};
56
57typedef SizeClassAllocator64<AP64> PrimaryAllocator;
58
59typedef CombinedAllocator<PrimaryAllocator> Allocator;
60typedef Allocator::AllocatorCache AllocatorCache;
61
62static Allocator allocator;
63static AllocatorCache fallback_allocator_cache;
64static StaticSpinMutex fallback_mutex;
65
66static uptr max_malloc_size;
67
68void dfsan_allocator_init() {
69 SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
70 allocator.Init(release_to_os_interval_ms: common_flags()->allocator_release_to_os_interval_ms);
71 if (common_flags()->max_allocation_size_mb)
72 max_malloc_size = Min(a: common_flags()->max_allocation_size_mb << 20,
73 b: kMaxAllowedMallocSize);
74 else
75 max_malloc_size = kMaxAllowedMallocSize;
76}
77
78AllocatorCache *GetAllocatorCache(DFsanThreadLocalMallocStorage *ms) {
79 CHECK(ms);
80 CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache));
81 return reinterpret_cast<AllocatorCache *>(ms->allocator_cache);
82}
83
84void DFsanThreadLocalMallocStorage::CommitBack() {
85 allocator.SwallowCache(cache: GetAllocatorCache(ms: this));
86}
87
88static void *DFsanAllocate(uptr size, uptr alignment, bool zeroise) {
89 if (size > max_malloc_size) {
90 if (AllocatorMayReturnNull()) {
91 Report(format: "WARNING: DataflowSanitizer failed to allocate 0x%zx bytes\n",
92 size);
93 return nullptr;
94 }
95 BufferedStackTrace stack;
96 ReportAllocationSizeTooBig(user_size: size, max_size: max_malloc_size, stack: &stack);
97 }
98 if (UNLIKELY(IsRssLimitExceeded())) {
99 if (AllocatorMayReturnNull())
100 return nullptr;
101 BufferedStackTrace stack;
102 ReportRssLimitExceeded(stack: &stack);
103 }
104 DFsanThread *t = GetCurrentThread();
105 void *allocated;
106 if (t) {
107 AllocatorCache *cache = GetAllocatorCache(ms: &t->malloc_storage());
108 allocated = allocator.Allocate(cache, size, alignment);
109 } else {
110 SpinMutexLock l(&fallback_mutex);
111 AllocatorCache *cache = &fallback_allocator_cache;
112 allocated = allocator.Allocate(cache, size, alignment);
113 }
114 if (UNLIKELY(!allocated)) {
115 SetAllocatorOutOfMemory();
116 if (AllocatorMayReturnNull())
117 return nullptr;
118 BufferedStackTrace stack;
119 ReportOutOfMemory(requested_size: size, stack: &stack);
120 }
121 Metadata *meta =
122 reinterpret_cast<Metadata *>(allocator.GetMetaData(p: allocated));
123 meta->requested_size = size;
124 if (zeroise) {
125 internal_memset(s: allocated, c: 0, n: size);
126 dfsan_set_label(label: 0, addr: allocated, size);
127 } else if (flags().zero_in_malloc) {
128 dfsan_set_label(label: 0, addr: allocated, size);
129 }
130 return allocated;
131}
132
133void dfsan_deallocate(void *p) {
134 CHECK(p);
135 Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p));
136 uptr size = meta->requested_size;
137 meta->requested_size = 0;
138 if (flags().zero_in_free)
139 dfsan_set_label(label: 0, addr: p, size);
140 DFsanThread *t = GetCurrentThread();
141 if (t) {
142 AllocatorCache *cache = GetAllocatorCache(ms: &t->malloc_storage());
143 allocator.Deallocate(cache, p);
144 } else {
145 SpinMutexLock l(&fallback_mutex);
146 AllocatorCache *cache = &fallback_allocator_cache;
147 allocator.Deallocate(cache, p);
148 }
149}
150
151void *DFsanReallocate(void *old_p, uptr new_size, uptr alignment) {
152 Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p: old_p));
153 uptr old_size = meta->requested_size;
154 uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(p: old_p);
155 if (new_size <= actually_allocated_size) {
156 // We are not reallocating here.
157 meta->requested_size = new_size;
158 if (new_size > old_size && flags().zero_in_malloc)
159 dfsan_set_label(label: 0, addr: (char *)old_p + old_size, size: new_size - old_size);
160 return old_p;
161 }
162 uptr memcpy_size = Min(a: new_size, b: old_size);
163 void *new_p = DFsanAllocate(size: new_size, alignment, zeroise: false /*zeroise*/);
164 if (new_p) {
165 dfsan_copy_memory(dst: new_p, src: old_p, size: memcpy_size);
166 dfsan_deallocate(p: old_p);
167 }
168 return new_p;
169}
170
171void *DFsanCalloc(uptr nmemb, uptr size) {
172 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
173 if (AllocatorMayReturnNull())
174 return nullptr;
175 BufferedStackTrace stack;
176 ReportCallocOverflow(count: nmemb, size, stack: &stack);
177 }
178 return DFsanAllocate(size: nmemb * size, alignment: sizeof(u64), zeroise: true /*zeroise*/);
179}
180
181static const void *AllocationBegin(const void *p) {
182 if (!p)
183 return nullptr;
184 void *beg = allocator.GetBlockBegin(p);
185 if (!beg)
186 return nullptr;
187 Metadata *b = (Metadata *)allocator.GetMetaData(p: beg);
188 if (!b)
189 return nullptr;
190 if (b->requested_size == 0)
191 return nullptr;
192 return (const void *)beg;
193}
194
195static uptr AllocationSize(const void *p) {
196 if (!p)
197 return 0;
198 const void *beg = allocator.GetBlockBegin(p);
199 if (beg != p)
200 return 0;
201 Metadata *b = (Metadata *)allocator.GetMetaData(p);
202 return b->requested_size;
203}
204
205static uptr AllocationSizeFast(const void *p) {
206 return reinterpret_cast<Metadata *>(allocator.GetMetaData(p))->requested_size;
207}
208
209void *dfsan_malloc(uptr size) {
210 return SetErrnoOnNull(DFsanAllocate(size, alignment: sizeof(u64), zeroise: false /*zeroise*/));
211}
212
213void *dfsan_calloc(uptr nmemb, uptr size) {
214 return SetErrnoOnNull(DFsanCalloc(nmemb, size));
215}
216
217void *dfsan_realloc(void *ptr, uptr size) {
218 if (!ptr)
219 return SetErrnoOnNull(DFsanAllocate(size, alignment: sizeof(u64), zeroise: false /*zeroise*/));
220 if (size == 0) {
221 dfsan_deallocate(p: ptr);
222 return nullptr;
223 }
224 return SetErrnoOnNull(DFsanReallocate(old_p: ptr, new_size: size, alignment: sizeof(u64)));
225}
226
227void *dfsan_reallocarray(void *ptr, uptr nmemb, uptr size) {
228 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
229 errno = errno_ENOMEM;
230 if (AllocatorMayReturnNull())
231 return nullptr;
232 BufferedStackTrace stack;
233 ReportReallocArrayOverflow(count: nmemb, size, stack: &stack);
234 }
235 return dfsan_realloc(ptr, size: nmemb * size);
236}
237
238void *dfsan_valloc(uptr size) {
239 return SetErrnoOnNull(
240 DFsanAllocate(size, alignment: GetPageSizeCached(), zeroise: false /*zeroise*/));
241}
242
243void *dfsan_pvalloc(uptr size) {
244 uptr PageSize = GetPageSizeCached();
245 if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
246 errno = errno_ENOMEM;
247 if (AllocatorMayReturnNull())
248 return nullptr;
249 BufferedStackTrace stack;
250 ReportPvallocOverflow(size, stack: &stack);
251 }
252 // pvalloc(0) should allocate one page.
253 size = size ? RoundUpTo(size, boundary: PageSize) : PageSize;
254 return SetErrnoOnNull(DFsanAllocate(size, alignment: PageSize, zeroise: false /*zeroise*/));
255}
256
257void *dfsan_aligned_alloc(uptr alignment, uptr size) {
258 if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
259 errno = errno_EINVAL;
260 if (AllocatorMayReturnNull())
261 return nullptr;
262 BufferedStackTrace stack;
263 ReportInvalidAlignedAllocAlignment(size, alignment, stack: &stack);
264 }
265 return SetErrnoOnNull(DFsanAllocate(size, alignment, zeroise: false /*zeroise*/));
266}
267
268void *dfsan_memalign(uptr alignment, uptr size) {
269 if (UNLIKELY(!IsPowerOfTwo(alignment))) {
270 errno = errno_EINVAL;
271 if (AllocatorMayReturnNull())
272 return nullptr;
273 BufferedStackTrace stack;
274 ReportInvalidAllocationAlignment(alignment, stack: &stack);
275 }
276 return SetErrnoOnNull(DFsanAllocate(size, alignment, zeroise: false /*zeroise*/));
277}
278
279int dfsan_posix_memalign(void **memptr, uptr alignment, uptr size) {
280 if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
281 if (AllocatorMayReturnNull())
282 return errno_EINVAL;
283 BufferedStackTrace stack;
284 ReportInvalidPosixMemalignAlignment(alignment, stack: &stack);
285 }
286 void *ptr = DFsanAllocate(size, alignment, zeroise: false /*zeroise*/);
287 if (UNLIKELY(!ptr))
288 // OOM error is already taken care of by DFsanAllocate.
289 return errno_ENOMEM;
290 CHECK(IsAligned((uptr)ptr, alignment));
291 *memptr = ptr;
292 return 0;
293}
294
295} // namespace __dfsan
296
297using namespace __dfsan;
298
299uptr __sanitizer_get_current_allocated_bytes() {
300 uptr stats[AllocatorStatCount];
301 allocator.GetStats(s: stats);
302 return stats[AllocatorStatAllocated];
303}
304
305uptr __sanitizer_get_heap_size() {
306 uptr stats[AllocatorStatCount];
307 allocator.GetStats(s: stats);
308 return stats[AllocatorStatMapped];
309}
310
311uptr __sanitizer_get_free_bytes() { return 1; }
312
313uptr __sanitizer_get_unmapped_bytes() { return 1; }
314
315uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
316
317int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; }
318
319const void *__sanitizer_get_allocated_begin(const void *p) {
320 return AllocationBegin(p);
321}
322
323uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); }
324
325uptr __sanitizer_get_allocated_size_fast(const void *p) {
326 DCHECK_EQ(p, __sanitizer_get_allocated_begin(p));
327 uptr ret = AllocationSizeFast(p);
328 DCHECK_EQ(ret, __sanitizer_get_allocated_size(p));
329 return ret;
330}
331

source code of compiler-rt/lib/dfsan/dfsan_allocator.cpp