1 | //===-- dfsan_allocator.cpp -------------------------- --------------------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file is a part of DataflowSanitizer. |
10 | // |
11 | // DataflowSanitizer allocator. |
12 | //===----------------------------------------------------------------------===// |
13 | |
14 | #include "dfsan_allocator.h" |
15 | |
16 | #include "dfsan.h" |
17 | #include "dfsan_flags.h" |
18 | #include "dfsan_thread.h" |
19 | #include "sanitizer_common/sanitizer_allocator.h" |
20 | #include "sanitizer_common/sanitizer_allocator_checks.h" |
21 | #include "sanitizer_common/sanitizer_allocator_interface.h" |
22 | #include "sanitizer_common/sanitizer_allocator_report.h" |
23 | #include "sanitizer_common/sanitizer_errno.h" |
24 | |
25 | namespace __dfsan { |
26 | |
27 | struct Metadata { |
28 | uptr requested_size; |
29 | }; |
30 | |
31 | struct DFsanMapUnmapCallback { |
32 | void OnMap(uptr p, uptr size) const { dfsan_set_label(label: 0, addr: (void *)p, size); } |
33 | void OnMapSecondary(uptr p, uptr size, uptr user_begin, |
34 | uptr user_size) const { |
35 | OnMap(p, size); |
36 | } |
37 | void OnUnmap(uptr p, uptr size) const { dfsan_set_label(label: 0, addr: (void *)p, size); } |
38 | }; |
39 | |
40 | #if defined(__aarch64__) |
41 | const uptr kAllocatorSpace = 0xE00000000000ULL; |
42 | #else |
43 | const uptr kAllocatorSpace = 0x700000000000ULL; |
44 | #endif |
45 | const uptr kMaxAllowedMallocSize = 8UL << 30; |
46 | |
47 | struct AP64 { // Allocator64 parameters. Deliberately using a short name. |
48 | static const uptr kSpaceBeg = kAllocatorSpace; |
49 | static const uptr kSpaceSize = 0x40000000000; // 4T. |
50 | static const uptr kMetadataSize = sizeof(Metadata); |
51 | typedef DefaultSizeClassMap SizeClassMap; |
52 | typedef DFsanMapUnmapCallback MapUnmapCallback; |
53 | static const uptr kFlags = 0; |
54 | using AddressSpaceView = LocalAddressSpaceView; |
55 | }; |
56 | |
57 | typedef SizeClassAllocator64<AP64> PrimaryAllocator; |
58 | |
59 | typedef CombinedAllocator<PrimaryAllocator> Allocator; |
60 | typedef Allocator::AllocatorCache AllocatorCache; |
61 | |
62 | static Allocator allocator; |
63 | static AllocatorCache fallback_allocator_cache; |
64 | static StaticSpinMutex fallback_mutex; |
65 | |
66 | static uptr max_malloc_size; |
67 | |
68 | void dfsan_allocator_init() { |
69 | SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); |
70 | allocator.Init(release_to_os_interval_ms: common_flags()->allocator_release_to_os_interval_ms); |
71 | if (common_flags()->max_allocation_size_mb) |
72 | max_malloc_size = Min(a: common_flags()->max_allocation_size_mb << 20, |
73 | b: kMaxAllowedMallocSize); |
74 | else |
75 | max_malloc_size = kMaxAllowedMallocSize; |
76 | } |
77 | |
78 | AllocatorCache *GetAllocatorCache(DFsanThreadLocalMallocStorage *ms) { |
79 | CHECK(ms); |
80 | CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache)); |
81 | return reinterpret_cast<AllocatorCache *>(ms->allocator_cache); |
82 | } |
83 | |
84 | void DFsanThreadLocalMallocStorage::CommitBack() { |
85 | allocator.SwallowCache(cache: GetAllocatorCache(ms: this)); |
86 | } |
87 | |
88 | static void *DFsanAllocate(uptr size, uptr alignment, bool zeroise) { |
89 | if (size > max_malloc_size) { |
90 | if (AllocatorMayReturnNull()) { |
91 | Report(format: "WARNING: DataflowSanitizer failed to allocate 0x%zx bytes\n" , |
92 | size); |
93 | return nullptr; |
94 | } |
95 | BufferedStackTrace stack; |
96 | ReportAllocationSizeTooBig(user_size: size, max_size: max_malloc_size, stack: &stack); |
97 | } |
98 | if (UNLIKELY(IsRssLimitExceeded())) { |
99 | if (AllocatorMayReturnNull()) |
100 | return nullptr; |
101 | BufferedStackTrace stack; |
102 | ReportRssLimitExceeded(stack: &stack); |
103 | } |
104 | DFsanThread *t = GetCurrentThread(); |
105 | void *allocated; |
106 | if (t) { |
107 | AllocatorCache *cache = GetAllocatorCache(ms: &t->malloc_storage()); |
108 | allocated = allocator.Allocate(cache, size, alignment); |
109 | } else { |
110 | SpinMutexLock l(&fallback_mutex); |
111 | AllocatorCache *cache = &fallback_allocator_cache; |
112 | allocated = allocator.Allocate(cache, size, alignment); |
113 | } |
114 | if (UNLIKELY(!allocated)) { |
115 | SetAllocatorOutOfMemory(); |
116 | if (AllocatorMayReturnNull()) |
117 | return nullptr; |
118 | BufferedStackTrace stack; |
119 | ReportOutOfMemory(requested_size: size, stack: &stack); |
120 | } |
121 | Metadata *meta = |
122 | reinterpret_cast<Metadata *>(allocator.GetMetaData(p: allocated)); |
123 | meta->requested_size = size; |
124 | if (zeroise) { |
125 | internal_memset(s: allocated, c: 0, n: size); |
126 | dfsan_set_label(label: 0, addr: allocated, size); |
127 | } else if (flags().zero_in_malloc) { |
128 | dfsan_set_label(label: 0, addr: allocated, size); |
129 | } |
130 | return allocated; |
131 | } |
132 | |
133 | void dfsan_deallocate(void *p) { |
134 | CHECK(p); |
135 | Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p)); |
136 | uptr size = meta->requested_size; |
137 | meta->requested_size = 0; |
138 | if (flags().zero_in_free) |
139 | dfsan_set_label(label: 0, addr: p, size); |
140 | DFsanThread *t = GetCurrentThread(); |
141 | if (t) { |
142 | AllocatorCache *cache = GetAllocatorCache(ms: &t->malloc_storage()); |
143 | allocator.Deallocate(cache, p); |
144 | } else { |
145 | SpinMutexLock l(&fallback_mutex); |
146 | AllocatorCache *cache = &fallback_allocator_cache; |
147 | allocator.Deallocate(cache, p); |
148 | } |
149 | } |
150 | |
151 | void *DFsanReallocate(void *old_p, uptr new_size, uptr alignment) { |
152 | Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p: old_p)); |
153 | uptr old_size = meta->requested_size; |
154 | uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(p: old_p); |
155 | if (new_size <= actually_allocated_size) { |
156 | // We are not reallocating here. |
157 | meta->requested_size = new_size; |
158 | if (new_size > old_size && flags().zero_in_malloc) |
159 | dfsan_set_label(label: 0, addr: (char *)old_p + old_size, size: new_size - old_size); |
160 | return old_p; |
161 | } |
162 | uptr memcpy_size = Min(a: new_size, b: old_size); |
163 | void *new_p = DFsanAllocate(size: new_size, alignment, zeroise: false /*zeroise*/); |
164 | if (new_p) { |
165 | dfsan_copy_memory(dst: new_p, src: old_p, size: memcpy_size); |
166 | dfsan_deallocate(p: old_p); |
167 | } |
168 | return new_p; |
169 | } |
170 | |
171 | void *DFsanCalloc(uptr nmemb, uptr size) { |
172 | if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { |
173 | if (AllocatorMayReturnNull()) |
174 | return nullptr; |
175 | BufferedStackTrace stack; |
176 | ReportCallocOverflow(count: nmemb, size, stack: &stack); |
177 | } |
178 | return DFsanAllocate(size: nmemb * size, alignment: sizeof(u64), zeroise: true /*zeroise*/); |
179 | } |
180 | |
181 | static const void *AllocationBegin(const void *p) { |
182 | if (!p) |
183 | return nullptr; |
184 | void *beg = allocator.GetBlockBegin(p); |
185 | if (!beg) |
186 | return nullptr; |
187 | Metadata *b = (Metadata *)allocator.GetMetaData(p: beg); |
188 | if (!b) |
189 | return nullptr; |
190 | if (b->requested_size == 0) |
191 | return nullptr; |
192 | return (const void *)beg; |
193 | } |
194 | |
195 | static uptr AllocationSize(const void *p) { |
196 | if (!p) |
197 | return 0; |
198 | const void *beg = allocator.GetBlockBegin(p); |
199 | if (beg != p) |
200 | return 0; |
201 | Metadata *b = (Metadata *)allocator.GetMetaData(p); |
202 | return b->requested_size; |
203 | } |
204 | |
205 | static uptr AllocationSizeFast(const void *p) { |
206 | return reinterpret_cast<Metadata *>(allocator.GetMetaData(p))->requested_size; |
207 | } |
208 | |
209 | void *dfsan_malloc(uptr size) { |
210 | return SetErrnoOnNull(DFsanAllocate(size, alignment: sizeof(u64), zeroise: false /*zeroise*/)); |
211 | } |
212 | |
213 | void *dfsan_calloc(uptr nmemb, uptr size) { |
214 | return SetErrnoOnNull(DFsanCalloc(nmemb, size)); |
215 | } |
216 | |
217 | void *dfsan_realloc(void *ptr, uptr size) { |
218 | if (!ptr) |
219 | return SetErrnoOnNull(DFsanAllocate(size, alignment: sizeof(u64), zeroise: false /*zeroise*/)); |
220 | if (size == 0) { |
221 | dfsan_deallocate(p: ptr); |
222 | return nullptr; |
223 | } |
224 | return SetErrnoOnNull(DFsanReallocate(old_p: ptr, new_size: size, alignment: sizeof(u64))); |
225 | } |
226 | |
227 | void *dfsan_reallocarray(void *ptr, uptr nmemb, uptr size) { |
228 | if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { |
229 | errno = errno_ENOMEM; |
230 | if (AllocatorMayReturnNull()) |
231 | return nullptr; |
232 | BufferedStackTrace stack; |
233 | ReportReallocArrayOverflow(count: nmemb, size, stack: &stack); |
234 | } |
235 | return dfsan_realloc(ptr, size: nmemb * size); |
236 | } |
237 | |
238 | void *dfsan_valloc(uptr size) { |
239 | return SetErrnoOnNull( |
240 | DFsanAllocate(size, alignment: GetPageSizeCached(), zeroise: false /*zeroise*/)); |
241 | } |
242 | |
243 | void *dfsan_pvalloc(uptr size) { |
244 | uptr PageSize = GetPageSizeCached(); |
245 | if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) { |
246 | errno = errno_ENOMEM; |
247 | if (AllocatorMayReturnNull()) |
248 | return nullptr; |
249 | BufferedStackTrace stack; |
250 | ReportPvallocOverflow(size, stack: &stack); |
251 | } |
252 | // pvalloc(0) should allocate one page. |
253 | size = size ? RoundUpTo(size, boundary: PageSize) : PageSize; |
254 | return SetErrnoOnNull(DFsanAllocate(size, alignment: PageSize, zeroise: false /*zeroise*/)); |
255 | } |
256 | |
257 | void *dfsan_aligned_alloc(uptr alignment, uptr size) { |
258 | if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) { |
259 | errno = errno_EINVAL; |
260 | if (AllocatorMayReturnNull()) |
261 | return nullptr; |
262 | BufferedStackTrace stack; |
263 | ReportInvalidAlignedAllocAlignment(size, alignment, stack: &stack); |
264 | } |
265 | return SetErrnoOnNull(DFsanAllocate(size, alignment, zeroise: false /*zeroise*/)); |
266 | } |
267 | |
268 | void *dfsan_memalign(uptr alignment, uptr size) { |
269 | if (UNLIKELY(!IsPowerOfTwo(alignment))) { |
270 | errno = errno_EINVAL; |
271 | if (AllocatorMayReturnNull()) |
272 | return nullptr; |
273 | BufferedStackTrace stack; |
274 | ReportInvalidAllocationAlignment(alignment, stack: &stack); |
275 | } |
276 | return SetErrnoOnNull(DFsanAllocate(size, alignment, zeroise: false /*zeroise*/)); |
277 | } |
278 | |
279 | int dfsan_posix_memalign(void **memptr, uptr alignment, uptr size) { |
280 | if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) { |
281 | if (AllocatorMayReturnNull()) |
282 | return errno_EINVAL; |
283 | BufferedStackTrace stack; |
284 | ReportInvalidPosixMemalignAlignment(alignment, stack: &stack); |
285 | } |
286 | void *ptr = DFsanAllocate(size, alignment, zeroise: false /*zeroise*/); |
287 | if (UNLIKELY(!ptr)) |
288 | // OOM error is already taken care of by DFsanAllocate. |
289 | return errno_ENOMEM; |
290 | CHECK(IsAligned((uptr)ptr, alignment)); |
291 | *memptr = ptr; |
292 | return 0; |
293 | } |
294 | |
295 | } // namespace __dfsan |
296 | |
297 | using namespace __dfsan; |
298 | |
299 | uptr __sanitizer_get_current_allocated_bytes() { |
300 | uptr stats[AllocatorStatCount]; |
301 | allocator.GetStats(s: stats); |
302 | return stats[AllocatorStatAllocated]; |
303 | } |
304 | |
305 | uptr __sanitizer_get_heap_size() { |
306 | uptr stats[AllocatorStatCount]; |
307 | allocator.GetStats(s: stats); |
308 | return stats[AllocatorStatMapped]; |
309 | } |
310 | |
311 | uptr __sanitizer_get_free_bytes() { return 1; } |
312 | |
313 | uptr __sanitizer_get_unmapped_bytes() { return 1; } |
314 | |
315 | uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; } |
316 | |
317 | int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; } |
318 | |
319 | const void *__sanitizer_get_allocated_begin(const void *p) { |
320 | return AllocationBegin(p); |
321 | } |
322 | |
323 | uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); } |
324 | |
325 | uptr __sanitizer_get_allocated_size_fast(const void *p) { |
326 | DCHECK_EQ(p, __sanitizer_get_allocated_begin(p)); |
327 | uptr ret = AllocationSizeFast(p); |
328 | DCHECK_EQ(ret, __sanitizer_get_allocated_size(p)); |
329 | return ret; |
330 | } |
331 | |