1/* Malloc implementation for multiple threads without lock contention.
2 Copyright (C) 1996-2022 Free Software Foundation, Inc.
3 Copyright The GNU Toolchain Authors.
4 This file is part of the GNU C Library.
5
6 The GNU C Library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Lesser General Public License as
8 published by the Free Software Foundation; either version 2.1 of the
9 License, or (at your option) any later version.
10
11 The GNU C Library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
15
16 You should have received a copy of the GNU Lesser General Public
17 License along with the GNU C Library; see the file COPYING.LIB. If
18 not, see <https://www.gnu.org/licenses/>. */
19
20/*
21 This is a version (aka ptmalloc2) of malloc/free/realloc written by
22 Doug Lea and adapted to multiple threads/arenas by Wolfram Gloger.
23
24 There have been substantial changes made after the integration into
25 glibc in all parts of the code. Do not look for much commonality
26 with the ptmalloc2 version.
27
28* Version ptmalloc2-20011215
29 based on:
30 VERSION 2.7.0 Sun Mar 11 14:14:06 2001 Doug Lea (dl at gee)
31
32* Quickstart
33
34 In order to compile this implementation, a Makefile is provided with
35 the ptmalloc2 distribution, which has pre-defined targets for some
36 popular systems (e.g. "make posix" for Posix threads). All that is
37 typically required with regard to compiler flags is the selection of
38 the thread package via defining one out of USE_PTHREADS, USE_THR or
39 USE_SPROC. Check the thread-m.h file for what effects this has.
40 Many/most systems will additionally require USE_TSD_DATA_HACK to be
41 defined, so this is the default for "make posix".
42
43* Why use this malloc?
44
45 This is not the fastest, most space-conserving, most portable, or
46 most tunable malloc ever written. However it is among the fastest
47 while also being among the most space-conserving, portable and tunable.
48 Consistent balance across these factors results in a good general-purpose
49 allocator for malloc-intensive programs.
50
51 The main properties of the algorithms are:
52 * For large (>= 512 bytes) requests, it is a pure best-fit allocator,
53 with ties normally decided via FIFO (i.e. least recently used).
54 * For small (<= 64 bytes by default) requests, it is a caching
55 allocator, that maintains pools of quickly recycled chunks.
56 * In between, and for combinations of large and small requests, it does
57 the best it can trying to meet both goals at once.
58 * For very large requests (>= 128KB by default), it relies on system
59 memory mapping facilities, if supported.
60
61 For a longer but slightly out of date high-level description, see
62 http://gee.cs.oswego.edu/dl/html/malloc.html
63
64 You may already by default be using a C library containing a malloc
65 that is based on some version of this malloc (for example in
66 linux). You might still want to use the one in this file in order to
67 customize settings or to avoid overheads associated with library
68 versions.
69
70* Contents, described in more detail in "description of public routines" below.
71
72 Standard (ANSI/SVID/...) functions:
73 malloc(size_t n);
74 calloc(size_t n_elements, size_t element_size);
75 free(void* p);
76 realloc(void* p, size_t n);
77 memalign(size_t alignment, size_t n);
78 valloc(size_t n);
79 mallinfo()
80 mallopt(int parameter_number, int parameter_value)
81
82 Additional functions:
83 independent_calloc(size_t n_elements, size_t size, void* chunks[]);
84 independent_comalloc(size_t n_elements, size_t sizes[], void* chunks[]);
85 pvalloc(size_t n);
86 malloc_trim(size_t pad);
87 malloc_usable_size(void* p);
88 malloc_stats();
89
90* Vital statistics:
91
92 Supported pointer representation: 4 or 8 bytes
93 Supported size_t representation: 4 or 8 bytes
94 Note that size_t is allowed to be 4 bytes even if pointers are 8.
95 You can adjust this by defining INTERNAL_SIZE_T
96
97 Alignment: 2 * sizeof(size_t) (default)
98 (i.e., 8 byte alignment with 4byte size_t). This suffices for
99 nearly all current machines and C compilers. However, you can
100 define MALLOC_ALIGNMENT to be wider than this if necessary.
101
102 Minimum overhead per allocated chunk: 4 or 8 bytes
103 Each malloced chunk has a hidden word of overhead holding size
104 and status information.
105
106 Minimum allocated size: 4-byte ptrs: 16 bytes (including 4 overhead)
107 8-byte ptrs: 24/32 bytes (including, 4/8 overhead)
108
109 When a chunk is freed, 12 (for 4byte ptrs) or 20 (for 8 byte
110 ptrs but 4 byte size) or 24 (for 8/8) additional bytes are
111 needed; 4 (8) for a trailing size field and 8 (16) bytes for
112 free list pointers. Thus, the minimum allocatable size is
113 16/24/32 bytes.
114
115 Even a request for zero bytes (i.e., malloc(0)) returns a
116 pointer to something of the minimum allocatable size.
117
118 The maximum overhead wastage (i.e., number of extra bytes
119 allocated than were requested in malloc) is less than or equal
120 to the minimum size, except for requests >= mmap_threshold that
121 are serviced via mmap(), where the worst case wastage is 2 *
122 sizeof(size_t) bytes plus the remainder from a system page (the
123 minimal mmap unit); typically 4096 or 8192 bytes.
124
125 Maximum allocated size: 4-byte size_t: 2^32 minus about two pages
126 8-byte size_t: 2^64 minus about two pages
127
128 It is assumed that (possibly signed) size_t values suffice to
129 represent chunk sizes. `Possibly signed' is due to the fact
130 that `size_t' may be defined on a system as either a signed or
131 an unsigned type. The ISO C standard says that it must be
132 unsigned, but a few systems are known not to adhere to this.
133 Additionally, even when size_t is unsigned, sbrk (which is by
134 default used to obtain memory from system) accepts signed
135 arguments, and may not be able to handle size_t-wide arguments
136 with negative sign bit. Generally, values that would
137 appear as negative after accounting for overhead and alignment
138 are supported only via mmap(), which does not have this
139 limitation.
140
141 Requests for sizes outside the allowed range will perform an optional
142 failure action and then return null. (Requests may also
143 also fail because a system is out of memory.)
144
145 Thread-safety: thread-safe
146
147 Compliance: I believe it is compliant with the 1997 Single Unix Specification
148 Also SVID/XPG, ANSI C, and probably others as well.
149
150* Synopsis of compile-time options:
151
152 People have reported using previous versions of this malloc on all
153 versions of Unix, sometimes by tweaking some of the defines
154 below. It has been tested most extensively on Solaris and Linux.
155 People also report using it in stand-alone embedded systems.
156
157 The implementation is in straight, hand-tuned ANSI C. It is not
158 at all modular. (Sorry!) It uses a lot of macros. To be at all
159 usable, this code should be compiled using an optimizing compiler
160 (for example gcc -O3) that can simplify expressions and control
161 paths. (FAQ: some macros import variables as arguments rather than
162 declare locals because people reported that some debuggers
163 otherwise get confused.)
164
165 OPTION DEFAULT VALUE
166
167 Compilation Environment options:
168
169 HAVE_MREMAP 0
170
171 Changing default word sizes:
172
173 INTERNAL_SIZE_T size_t
174
175 Configuration and functionality options:
176
177 USE_PUBLIC_MALLOC_WRAPPERS NOT defined
178 USE_MALLOC_LOCK NOT defined
179 MALLOC_DEBUG NOT defined
180 REALLOC_ZERO_BYTES_FREES 1
181 TRIM_FASTBINS 0
182
183 Options for customizing MORECORE:
184
185 MORECORE sbrk
186 MORECORE_FAILURE -1
187 MORECORE_CONTIGUOUS 1
188 MORECORE_CANNOT_TRIM NOT defined
189 MORECORE_CLEARS 1
190 MMAP_AS_MORECORE_SIZE (1024 * 1024)
191
192 Tuning options that are also dynamically changeable via mallopt:
193
194 DEFAULT_MXFAST 64 (for 32bit), 128 (for 64bit)
195 DEFAULT_TRIM_THRESHOLD 128 * 1024
196 DEFAULT_TOP_PAD 0
197 DEFAULT_MMAP_THRESHOLD 128 * 1024
198 DEFAULT_MMAP_MAX 65536
199
200 There are several other #defined constants and macros that you
201 probably don't want to touch unless you are extending or adapting malloc. */
202
203/*
204 void* is the pointer type that malloc should say it returns
205*/
206
207#ifndef void
208#define void void
209#endif /*void*/
210
211#include <stddef.h> /* for size_t */
212#include <stdlib.h> /* for getenv(), abort() */
213#include <unistd.h> /* for __libc_enable_secure */
214
215#include <atomic.h>
216#include <_itoa.h>
217#include <bits/wordsize.h>
218#include <sys/sysinfo.h>
219
220#include <ldsodefs.h>
221
222#include <unistd.h>
223#include <stdio.h> /* needed for malloc_stats */
224#include <errno.h>
225#include <assert.h>
226
227#include <shlib-compat.h>
228
229/* For uintptr_t. */
230#include <stdint.h>
231
232/* For va_arg, va_start, va_end. */
233#include <stdarg.h>
234
235/* For MIN, MAX, powerof2. */
236#include <sys/param.h>
237
238/* For ALIGN_UP et. al. */
239#include <libc-pointer-arith.h>
240
241/* For DIAG_PUSH/POP_NEEDS_COMMENT et al. */
242#include <libc-diag.h>
243
244/* For memory tagging. */
245#include <libc-mtag.h>
246
247#include <malloc/malloc-internal.h>
248
249/* For SINGLE_THREAD_P. */
250#include <sysdep-cancel.h>
251
252#include <libc-internal.h>
253
254/* For tcache double-free check. */
255#include <random-bits.h>
256#include <sys/random.h>
257
258/*
259 Debugging:
260
261 Because freed chunks may be overwritten with bookkeeping fields, this
262 malloc will often die when freed memory is overwritten by user
263 programs. This can be very effective (albeit in an annoying way)
264 in helping track down dangling pointers.
265
266 If you compile with -DMALLOC_DEBUG, a number of assertion checks are
267 enabled that will catch more memory errors. You probably won't be
268 able to make much sense of the actual assertion errors, but they
269 should help you locate incorrectly overwritten memory. The checking
270 is fairly extensive, and will slow down execution
271 noticeably. Calling malloc_stats or mallinfo with MALLOC_DEBUG set
272 will attempt to check every non-mmapped allocated and free chunk in
273 the course of computing the summmaries. (By nature, mmapped regions
274 cannot be checked very much automatically.)
275
276 Setting MALLOC_DEBUG may also be helpful if you are trying to modify
277 this code. The assertions in the check routines spell out in more
278 detail the assumptions and invariants underlying the algorithms.
279
280 Setting MALLOC_DEBUG does NOT provide an automated mechanism for
281 checking that all accesses to malloced memory stay within their
282 bounds. However, there are several add-ons and adaptations of this
283 or other mallocs available that do this.
284*/
285
286#ifndef MALLOC_DEBUG
287#define MALLOC_DEBUG 0
288#endif
289
290#if IS_IN (libc)
291#ifndef NDEBUG
292# define __assert_fail(assertion, file, line, function) \
293 __malloc_assert(assertion, file, line, function)
294
295_Noreturn static void
296__malloc_assert (const char *assertion, const char *file, unsigned int line,
297 const char *function)
298{
299 __libc_message (do_abort, "\
300Fatal glibc error: malloc assertion failure in %s: %s\n",
301 function, assertion);
302 __builtin_unreachable ();
303}
304#endif
305#endif
306
307#if USE_TCACHE
308/* We want 64 entries. This is an arbitrary limit, which tunables can reduce. */
309# define TCACHE_MAX_BINS 64
310# define MAX_TCACHE_SIZE tidx2usize (TCACHE_MAX_BINS-1)
311
312/* Only used to pre-fill the tunables. */
313# define tidx2usize(idx) (((size_t) idx) * MALLOC_ALIGNMENT + MINSIZE - SIZE_SZ)
314
315/* When "x" is from chunksize(). */
316# define csize2tidx(x) (((x) - MINSIZE + MALLOC_ALIGNMENT - 1) / MALLOC_ALIGNMENT)
317/* When "x" is a user-provided size. */
318# define usize2tidx(x) csize2tidx (request2size (x))
319
320/* With rounding and alignment, the bins are...
321 idx 0 bytes 0..24 (64-bit) or 0..12 (32-bit)
322 idx 1 bytes 25..40 or 13..20
323 idx 2 bytes 41..56 or 21..28
324 etc. */
325
326/* This is another arbitrary limit, which tunables can change. Each
327 tcache bin will hold at most this number of chunks. */
328# define TCACHE_FILL_COUNT 7
329
330/* Maximum chunks in tcache bins for tunables. This value must fit the range
331 of tcache->counts[] entries, else they may overflow. */
332# define MAX_TCACHE_COUNT UINT16_MAX
333#endif
334
335/* Safe-Linking:
336 Use randomness from ASLR (mmap_base) to protect single-linked lists
337 of Fast-Bins and TCache. That is, mask the "next" pointers of the
338 lists' chunks, and also perform allocation alignment checks on them.
339 This mechanism reduces the risk of pointer hijacking, as was done with
340 Safe-Unlinking in the double-linked lists of Small-Bins.
341 It assumes a minimum page size of 4096 bytes (12 bits). Systems with
342 larger pages provide less entropy, although the pointer mangling
343 still works. */
344#define PROTECT_PTR(pos, ptr) \
345 ((__typeof (ptr)) ((((size_t) pos) >> 12) ^ ((size_t) ptr)))
346#define REVEAL_PTR(ptr) PROTECT_PTR (&ptr, ptr)
347
348/*
349 The REALLOC_ZERO_BYTES_FREES macro controls the behavior of realloc (p, 0)
350 when p is nonnull. If the macro is nonzero, the realloc call returns NULL;
351 otherwise, the call returns what malloc (0) would. In either case,
352 p is freed. Glibc uses a nonzero REALLOC_ZERO_BYTES_FREES, which
353 implements common historical practice.
354
355 ISO C17 says the realloc call has implementation-defined behavior,
356 and it might not even free p.
357*/
358
359#ifndef REALLOC_ZERO_BYTES_FREES
360#define REALLOC_ZERO_BYTES_FREES 1
361#endif
362
363/*
364 TRIM_FASTBINS controls whether free() of a very small chunk can
365 immediately lead to trimming. Setting to true (1) can reduce memory
366 footprint, but will almost always slow down programs that use a lot
367 of small chunks.
368
369 Define this only if you are willing to give up some speed to more
370 aggressively reduce system-level memory footprint when releasing
371 memory in programs that use many small chunks. You can get
372 essentially the same effect by setting MXFAST to 0, but this can
373 lead to even greater slowdowns in programs using many small chunks.
374 TRIM_FASTBINS is an in-between compile-time option, that disables
375 only those chunks bordering topmost memory from being placed in
376 fastbins.
377*/
378
379#ifndef TRIM_FASTBINS
380#define TRIM_FASTBINS 0
381#endif
382
383/* Definition for getting more memory from the OS. */
384#include "morecore.c"
385
386#define MORECORE (*__glibc_morecore)
387#define MORECORE_FAILURE 0
388
389/* Memory tagging. */
390
391/* Some systems support the concept of tagging (sometimes known as
392 coloring) memory locations on a fine grained basis. Each memory
393 location is given a color (normally allocated randomly) and
394 pointers are also colored. When the pointer is dereferenced, the
395 pointer's color is checked against the memory's color and if they
396 differ the access is faulted (sometimes lazily).
397
398 We use this in glibc by maintaining a single color for the malloc
399 data structures that are interleaved with the user data and then
400 assigning separate colors for each block allocation handed out. In
401 this way simple buffer overruns will be rapidly detected. When
402 memory is freed, the memory is recolored back to the glibc default
403 so that simple use-after-free errors can also be detected.
404
405 If memory is reallocated the buffer is recolored even if the
406 address remains the same. This has a performance impact, but
407 guarantees that the old pointer cannot mistakenly be reused (code
408 that compares old against new will see a mismatch and will then
409 need to behave as though realloc moved the data to a new location).
410
411 Internal API for memory tagging support.
412
413 The aim is to keep the code for memory tagging support as close to
414 the normal APIs in glibc as possible, so that if tagging is not
415 enabled in the library, or is disabled at runtime then standard
416 operations can continue to be used. Support macros are used to do
417 this:
418
419 void *tag_new_zero_region (void *ptr, size_t size)
420
421 Allocates a new tag, colors the memory with that tag, zeros the
422 memory and returns a pointer that is correctly colored for that
423 location. The non-tagging version will simply call memset with 0.
424
425 void *tag_region (void *ptr, size_t size)
426
427 Color the region of memory pointed to by PTR and size SIZE with
428 the color of PTR. Returns the original pointer.
429
430 void *tag_new_usable (void *ptr)
431
432 Allocate a new random color and use it to color the user region of
433 a chunk; this may include data from the subsequent chunk's header
434 if tagging is sufficiently fine grained. Returns PTR suitably
435 recolored for accessing the memory there.
436
437 void *tag_at (void *ptr)
438
439 Read the current color of the memory at the address pointed to by
440 PTR (ignoring it's current color) and return PTR recolored to that
441 color. PTR must be valid address in all other respects. When
442 tagging is not enabled, it simply returns the original pointer.
443*/
444
445#ifdef USE_MTAG
446static bool mtag_enabled = false;
447static int mtag_mmap_flags = 0;
448#else
449# define mtag_enabled false
450# define mtag_mmap_flags 0
451#endif
452
453static __always_inline void *
454tag_region (void *ptr, size_t size)
455{
456 if (__glibc_unlikely (mtag_enabled))
457 return __libc_mtag_tag_region (p: ptr, n: size);
458 return ptr;
459}
460
461static __always_inline void *
462tag_new_zero_region (void *ptr, size_t size)
463{
464 if (__glibc_unlikely (mtag_enabled))
465 return __libc_mtag_tag_zero_region (p: __libc_mtag_new_tag (p: ptr), n: size);
466 return memset (s: ptr, c: 0, n: size);
467}
468
469/* Defined later. */
470static void *
471tag_new_usable (void *ptr);
472
473static __always_inline void *
474tag_at (void *ptr)
475{
476 if (__glibc_unlikely (mtag_enabled))
477 return __libc_mtag_address_get_tag (p: ptr);
478 return ptr;
479}
480
481#include <string.h>
482
483/*
484 MORECORE-related declarations. By default, rely on sbrk
485*/
486
487
488/*
489 MORECORE is the name of the routine to call to obtain more memory
490 from the system. See below for general guidance on writing
491 alternative MORECORE functions, as well as a version for WIN32 and a
492 sample version for pre-OSX macos.
493*/
494
495#ifndef MORECORE
496#define MORECORE sbrk
497#endif
498
499/*
500 MORECORE_FAILURE is the value returned upon failure of MORECORE
501 as well as mmap. Since it cannot be an otherwise valid memory address,
502 and must reflect values of standard sys calls, you probably ought not
503 try to redefine it.
504*/
505
506#ifndef MORECORE_FAILURE
507#define MORECORE_FAILURE (-1)
508#endif
509
510/*
511 If MORECORE_CONTIGUOUS is true, take advantage of fact that
512 consecutive calls to MORECORE with positive arguments always return
513 contiguous increasing addresses. This is true of unix sbrk. Even
514 if not defined, when regions happen to be contiguous, malloc will
515 permit allocations spanning regions obtained from different
516 calls. But defining this when applicable enables some stronger
517 consistency checks and space efficiencies.
518*/
519
520#ifndef MORECORE_CONTIGUOUS
521#define MORECORE_CONTIGUOUS 1
522#endif
523
524/*
525 Define MORECORE_CANNOT_TRIM if your version of MORECORE
526 cannot release space back to the system when given negative
527 arguments. This is generally necessary only if you are using
528 a hand-crafted MORECORE function that cannot handle negative arguments.
529*/
530
531/* #define MORECORE_CANNOT_TRIM */
532
533/* MORECORE_CLEARS (default 1)
534 The degree to which the routine mapped to MORECORE zeroes out
535 memory: never (0), only for newly allocated space (1) or always
536 (2). The distinction between (1) and (2) is necessary because on
537 some systems, if the application first decrements and then
538 increments the break value, the contents of the reallocated space
539 are unspecified.
540 */
541
542#ifndef MORECORE_CLEARS
543# define MORECORE_CLEARS 1
544#endif
545
546
547/*
548 MMAP_AS_MORECORE_SIZE is the minimum mmap size argument to use if
549 sbrk fails, and mmap is used as a backup. The value must be a
550 multiple of page size. This backup strategy generally applies only
551 when systems have "holes" in address space, so sbrk cannot perform
552 contiguous expansion, but there is still space available on system.
553 On systems for which this is known to be useful (i.e. most linux
554 kernels), this occurs only when programs allocate huge amounts of
555 memory. Between this, and the fact that mmap regions tend to be
556 limited, the size should be large, to avoid too many mmap calls and
557 thus avoid running out of kernel resources. */
558
559#ifndef MMAP_AS_MORECORE_SIZE
560#define MMAP_AS_MORECORE_SIZE (1024 * 1024)
561#endif
562
563/*
564 Define HAVE_MREMAP to make realloc() use mremap() to re-allocate
565 large blocks.
566*/
567
568#ifndef HAVE_MREMAP
569#define HAVE_MREMAP 0
570#endif
571
572/*
573 This version of malloc supports the standard SVID/XPG mallinfo
574 routine that returns a struct containing usage properties and
575 statistics. It should work on any SVID/XPG compliant system that has
576 a /usr/include/malloc.h defining struct mallinfo. (If you'd like to
577 install such a thing yourself, cut out the preliminary declarations
578 as described above and below and save them in a malloc.h file. But
579 there's no compelling reason to bother to do this.)
580
581 The main declaration needed is the mallinfo struct that is returned
582 (by-copy) by mallinfo(). The SVID/XPG malloinfo struct contains a
583 bunch of fields that are not even meaningful in this version of
584 malloc. These fields are are instead filled by mallinfo() with
585 other numbers that might be of interest.
586*/
587
588
589/* ---------- description of public routines ------------ */
590
591#if IS_IN (libc)
592/*
593 malloc(size_t n)
594 Returns a pointer to a newly allocated chunk of at least n bytes, or null
595 if no space is available. Additionally, on failure, errno is
596 set to ENOMEM on ANSI C systems.
597
598 If n is zero, malloc returns a minimum-sized chunk. (The minimum
599 size is 16 bytes on most 32bit systems, and 24 or 32 bytes on 64bit
600 systems.) On most systems, size_t is an unsigned type, so calls
601 with negative arguments are interpreted as requests for huge amounts
602 of space, which will often fail. The maximum supported value of n
603 differs across systems, but is in all cases less than the maximum
604 representable value of a size_t.
605*/
606void* __libc_malloc(size_t);
607libc_hidden_proto (__libc_malloc)
608
609/*
610 free(void* p)
611 Releases the chunk of memory pointed to by p, that had been previously
612 allocated using malloc or a related routine such as realloc.
613 It has no effect if p is null. It can have arbitrary (i.e., bad!)
614 effects if p has already been freed.
615
616 Unless disabled (using mallopt), freeing very large spaces will
617 when possible, automatically trigger operations that give
618 back unused memory to the system, thus reducing program footprint.
619*/
620void __libc_free(void*);
621libc_hidden_proto (__libc_free)
622
623/*
624 calloc(size_t n_elements, size_t element_size);
625 Returns a pointer to n_elements * element_size bytes, with all locations
626 set to zero.
627*/
628void* __libc_calloc(size_t, size_t);
629
630/*
631 realloc(void* p, size_t n)
632 Returns a pointer to a chunk of size n that contains the same data
633 as does chunk p up to the minimum of (n, p's size) bytes, or null
634 if no space is available.
635
636 The returned pointer may or may not be the same as p. The algorithm
637 prefers extending p when possible, otherwise it employs the
638 equivalent of a malloc-copy-free sequence.
639
640 If p is null, realloc is equivalent to malloc.
641
642 If space is not available, realloc returns null, errno is set (if on
643 ANSI) and p is NOT freed.
644
645 if n is for fewer bytes than already held by p, the newly unused
646 space is lopped off and freed if possible. Unless the #define
647 REALLOC_ZERO_BYTES_FREES is set, realloc with a size argument of
648 zero (re)allocates a minimum-sized chunk.
649
650 Large chunks that were internally obtained via mmap will always be
651 grown using malloc-copy-free sequences unless the system supports
652 MREMAP (currently only linux).
653
654 The old unix realloc convention of allowing the last-free'd chunk
655 to be used as an argument to realloc is not supported.
656*/
657void* __libc_realloc(void*, size_t);
658libc_hidden_proto (__libc_realloc)
659
660/*
661 memalign(size_t alignment, size_t n);
662 Returns a pointer to a newly allocated chunk of n bytes, aligned
663 in accord with the alignment argument.
664
665 The alignment argument should be a power of two. If the argument is
666 not a power of two, the nearest greater power is used.
667 8-byte alignment is guaranteed by normal malloc calls, so don't
668 bother calling memalign with an argument of 8 or less.
669
670 Overreliance on memalign is a sure way to fragment space.
671*/
672void* __libc_memalign(size_t, size_t);
673libc_hidden_proto (__libc_memalign)
674
675/*
676 valloc(size_t n);
677 Equivalent to memalign(pagesize, n), where pagesize is the page
678 size of the system. If the pagesize is unknown, 4096 is used.
679*/
680void* __libc_valloc(size_t);
681
682
683
684/*
685 mallinfo()
686 Returns (by copy) a struct containing various summary statistics:
687
688 arena: current total non-mmapped bytes allocated from system
689 ordblks: the number of free chunks
690 smblks: the number of fastbin blocks (i.e., small chunks that
691 have been freed but not use resused or consolidated)
692 hblks: current number of mmapped regions
693 hblkhd: total bytes held in mmapped regions
694 usmblks: always 0
695 fsmblks: total bytes held in fastbin blocks
696 uordblks: current total allocated space (normal or mmapped)
697 fordblks: total free space
698 keepcost: the maximum number of bytes that could ideally be released
699 back to system via malloc_trim. ("ideally" means that
700 it ignores page restrictions etc.)
701
702 Because these fields are ints, but internal bookkeeping may
703 be kept as longs, the reported values may wrap around zero and
704 thus be inaccurate.
705*/
706struct mallinfo2 __libc_mallinfo2(void);
707libc_hidden_proto (__libc_mallinfo2)
708
709struct mallinfo __libc_mallinfo(void);
710
711
712/*
713 pvalloc(size_t n);
714 Equivalent to valloc(minimum-page-that-holds(n)), that is,
715 round up n to nearest pagesize.
716 */
717void* __libc_pvalloc(size_t);
718
719/*
720 malloc_trim(size_t pad);
721
722 If possible, gives memory back to the system (via negative
723 arguments to sbrk) if there is unused memory at the `high' end of
724 the malloc pool. You can call this after freeing large blocks of
725 memory to potentially reduce the system-level memory requirements
726 of a program. However, it cannot guarantee to reduce memory. Under
727 some allocation patterns, some large free blocks of memory will be
728 locked between two used chunks, so they cannot be given back to
729 the system.
730
731 The `pad' argument to malloc_trim represents the amount of free
732 trailing space to leave untrimmed. If this argument is zero,
733 only the minimum amount of memory to maintain internal data
734 structures will be left (one page or less). Non-zero arguments
735 can be supplied to maintain enough trailing space to service
736 future expected allocations without having to re-obtain memory
737 from the system.
738
739 Malloc_trim returns 1 if it actually released any memory, else 0.
740 On systems that do not support "negative sbrks", it will always
741 return 0.
742*/
743int __malloc_trim(size_t);
744
745/*
746 malloc_usable_size(void* p);
747
748 Returns the number of bytes you can actually use in
749 an allocated chunk, which may be more than you requested (although
750 often not) due to alignment and minimum size constraints.
751 You can use this many bytes without worrying about
752 overwriting other allocated objects. This is not a particularly great
753 programming practice. malloc_usable_size can be more useful in
754 debugging and assertions, for example:
755
756 p = malloc(n);
757 assert(malloc_usable_size(p) >= 256);
758
759*/
760size_t __malloc_usable_size(void*);
761
762/*
763 malloc_stats();
764 Prints on stderr the amount of space obtained from the system (both
765 via sbrk and mmap), the maximum amount (which may be more than
766 current if malloc_trim and/or munmap got called), and the current
767 number of bytes allocated via malloc (or realloc, etc) but not yet
768 freed. Note that this is the number of bytes allocated, not the
769 number requested. It will be larger than the number requested
770 because of alignment and bookkeeping overhead. Because it includes
771 alignment wastage as being in use, this figure may be greater than
772 zero even when no user-level chunks are allocated.
773
774 The reported current and maximum system memory can be inaccurate if
775 a program makes other calls to system memory allocation functions
776 (normally sbrk) outside of malloc.
777
778 malloc_stats prints only the most commonly interesting statistics.
779 More information can be obtained by calling mallinfo.
780
781*/
782void __malloc_stats(void);
783
784/*
785 posix_memalign(void **memptr, size_t alignment, size_t size);
786
787 POSIX wrapper like memalign(), checking for validity of size.
788*/
789int __posix_memalign(void **, size_t, size_t);
790#endif /* IS_IN (libc) */
791
792/*
793 mallopt(int parameter_number, int parameter_value)
794 Sets tunable parameters The format is to provide a
795 (parameter-number, parameter-value) pair. mallopt then sets the
796 corresponding parameter to the argument value if it can (i.e., so
797 long as the value is meaningful), and returns 1 if successful else
798 0. SVID/XPG/ANSI defines four standard param numbers for mallopt,
799 normally defined in malloc.h. Only one of these (M_MXFAST) is used
800 in this malloc. The others (M_NLBLKS, M_GRAIN, M_KEEP) don't apply,
801 so setting them has no effect. But this malloc also supports four
802 other options in mallopt. See below for details. Briefly, supported
803 parameters are as follows (listed defaults are for "typical"
804 configurations).
805
806 Symbol param # default allowed param values
807 M_MXFAST 1 64 0-80 (0 disables fastbins)
808 M_TRIM_THRESHOLD -1 128*1024 any (-1U disables trimming)
809 M_TOP_PAD -2 0 any
810 M_MMAP_THRESHOLD -3 128*1024 any (or 0 if no MMAP support)
811 M_MMAP_MAX -4 65536 any (0 disables use of mmap)
812*/
813int __libc_mallopt(int, int);
814#if IS_IN (libc)
815libc_hidden_proto (__libc_mallopt)
816#endif
817
818/* mallopt tuning options */
819
820/*
821 M_MXFAST is the maximum request size used for "fastbins", special bins
822 that hold returned chunks without consolidating their spaces. This
823 enables future requests for chunks of the same size to be handled
824 very quickly, but can increase fragmentation, and thus increase the
825 overall memory footprint of a program.
826
827 This malloc manages fastbins very conservatively yet still
828 efficiently, so fragmentation is rarely a problem for values less
829 than or equal to the default. The maximum supported value of MXFAST
830 is 80. You wouldn't want it any higher than this anyway. Fastbins
831 are designed especially for use with many small structs, objects or
832 strings -- the default handles structs/objects/arrays with sizes up
833 to 8 4byte fields, or small strings representing words, tokens,
834 etc. Using fastbins for larger objects normally worsens
835 fragmentation without improving speed.
836
837 M_MXFAST is set in REQUEST size units. It is internally used in
838 chunksize units, which adds padding and alignment. You can reduce
839 M_MXFAST to 0 to disable all use of fastbins. This causes the malloc
840 algorithm to be a closer approximation of fifo-best-fit in all cases,
841 not just for larger requests, but will generally cause it to be
842 slower.
843*/
844
845
846/* M_MXFAST is a standard SVID/XPG tuning option, usually listed in malloc.h */
847#ifndef M_MXFAST
848#define M_MXFAST 1
849#endif
850
851#ifndef DEFAULT_MXFAST
852#define DEFAULT_MXFAST (64 * SIZE_SZ / 4)
853#endif
854
855
856/*
857 M_TRIM_THRESHOLD is the maximum amount of unused top-most memory
858 to keep before releasing via malloc_trim in free().
859
860 Automatic trimming is mainly useful in long-lived programs.
861 Because trimming via sbrk can be slow on some systems, and can
862 sometimes be wasteful (in cases where programs immediately
863 afterward allocate more large chunks) the value should be high
864 enough so that your overall system performance would improve by
865 releasing this much memory.
866
867 The trim threshold and the mmap control parameters (see below)
868 can be traded off with one another. Trimming and mmapping are
869 two different ways of releasing unused memory back to the
870 system. Between these two, it is often possible to keep
871 system-level demands of a long-lived program down to a bare
872 minimum. For example, in one test suite of sessions measuring
873 the XF86 X server on Linux, using a trim threshold of 128K and a
874 mmap threshold of 192K led to near-minimal long term resource
875 consumption.
876
877 If you are using this malloc in a long-lived program, it should
878 pay to experiment with these values. As a rough guide, you
879 might set to a value close to the average size of a process
880 (program) running on your system. Releasing this much memory
881 would allow such a process to run in memory. Generally, it's
882 worth it to tune for trimming rather tham memory mapping when a
883 program undergoes phases where several large chunks are
884 allocated and released in ways that can reuse each other's
885 storage, perhaps mixed with phases where there are no such
886 chunks at all. And in well-behaved long-lived programs,
887 controlling release of large blocks via trimming versus mapping
888 is usually faster.
889
890 However, in most programs, these parameters serve mainly as
891 protection against the system-level effects of carrying around
892 massive amounts of unneeded memory. Since frequent calls to
893 sbrk, mmap, and munmap otherwise degrade performance, the default
894 parameters are set to relatively high values that serve only as
895 safeguards.
896
897 The trim value It must be greater than page size to have any useful
898 effect. To disable trimming completely, you can set to
899 (unsigned long)(-1)
900
901 Trim settings interact with fastbin (MXFAST) settings: Unless
902 TRIM_FASTBINS is defined, automatic trimming never takes place upon
903 freeing a chunk with size less than or equal to MXFAST. Trimming is
904 instead delayed until subsequent freeing of larger chunks. However,
905 you can still force an attempted trim by calling malloc_trim.
906
907 Also, trimming is not generally possible in cases where
908 the main arena is obtained via mmap.
909
910 Note that the trick some people use of mallocing a huge space and
911 then freeing it at program startup, in an attempt to reserve system
912 memory, doesn't have the intended effect under automatic trimming,
913 since that memory will immediately be returned to the system.
914*/
915
916#define M_TRIM_THRESHOLD -1
917
918#ifndef DEFAULT_TRIM_THRESHOLD
919#define DEFAULT_TRIM_THRESHOLD (128 * 1024)
920#endif
921
922/*
923 M_TOP_PAD is the amount of extra `padding' space to allocate or
924 retain whenever sbrk is called. It is used in two ways internally:
925
926 * When sbrk is called to extend the top of the arena to satisfy
927 a new malloc request, this much padding is added to the sbrk
928 request.
929
930 * When malloc_trim is called automatically from free(),
931 it is used as the `pad' argument.
932
933 In both cases, the actual amount of padding is rounded
934 so that the end of the arena is always a system page boundary.
935
936 The main reason for using padding is to avoid calling sbrk so
937 often. Having even a small pad greatly reduces the likelihood
938 that nearly every malloc request during program start-up (or
939 after trimming) will invoke sbrk, which needlessly wastes
940 time.
941
942 Automatic rounding-up to page-size units is normally sufficient
943 to avoid measurable overhead, so the default is 0. However, in
944 systems where sbrk is relatively slow, it can pay to increase
945 this value, at the expense of carrying around more memory than
946 the program needs.
947*/
948
949#define M_TOP_PAD -2
950
951#ifndef DEFAULT_TOP_PAD
952#define DEFAULT_TOP_PAD (0)
953#endif
954
955/*
956 MMAP_THRESHOLD_MAX and _MIN are the bounds on the dynamically
957 adjusted MMAP_THRESHOLD.
958*/
959
960#ifndef DEFAULT_MMAP_THRESHOLD_MIN
961#define DEFAULT_MMAP_THRESHOLD_MIN (128 * 1024)
962#endif
963
964#ifndef DEFAULT_MMAP_THRESHOLD_MAX
965 /* For 32-bit platforms we cannot increase the maximum mmap
966 threshold much because it is also the minimum value for the
967 maximum heap size and its alignment. Going above 512k (i.e., 1M
968 for new heaps) wastes too much address space. */
969# if __WORDSIZE == 32
970# define DEFAULT_MMAP_THRESHOLD_MAX (512 * 1024)
971# else
972# define DEFAULT_MMAP_THRESHOLD_MAX (4 * 1024 * 1024 * sizeof(long))
973# endif
974#endif
975
976/*
977 M_MMAP_THRESHOLD is the request size threshold for using mmap()
978 to service a request. Requests of at least this size that cannot
979 be allocated using already-existing space will be serviced via mmap.
980 (If enough normal freed space already exists it is used instead.)
981
982 Using mmap segregates relatively large chunks of memory so that
983 they can be individually obtained and released from the host
984 system. A request serviced through mmap is never reused by any
985 other request (at least not directly; the system may just so
986 happen to remap successive requests to the same locations).
987
988 Segregating space in this way has the benefits that:
989
990 1. Mmapped space can ALWAYS be individually released back
991 to the system, which helps keep the system level memory
992 demands of a long-lived program low.
993 2. Mapped memory can never become `locked' between
994 other chunks, as can happen with normally allocated chunks, which
995 means that even trimming via malloc_trim would not release them.
996 3. On some systems with "holes" in address spaces, mmap can obtain
997 memory that sbrk cannot.
998
999 However, it has the disadvantages that:
1000
1001 1. The space cannot be reclaimed, consolidated, and then
1002 used to service later requests, as happens with normal chunks.
1003 2. It can lead to more wastage because of mmap page alignment
1004 requirements
1005 3. It causes malloc performance to be more dependent on host
1006 system memory management support routines which may vary in
1007 implementation quality and may impose arbitrary
1008 limitations. Generally, servicing a request via normal
1009 malloc steps is faster than going through a system's mmap.
1010
1011 The advantages of mmap nearly always outweigh disadvantages for
1012 "large" chunks, but the value of "large" varies across systems. The
1013 default is an empirically derived value that works well in most
1014 systems.
1015
1016
1017 Update in 2006:
1018 The above was written in 2001. Since then the world has changed a lot.
1019 Memory got bigger. Applications got bigger. The virtual address space
1020 layout in 32 bit linux changed.
1021
1022 In the new situation, brk() and mmap space is shared and there are no
1023 artificial limits on brk size imposed by the kernel. What is more,
1024 applications have started using transient allocations larger than the
1025 128Kb as was imagined in 2001.
1026
1027 The price for mmap is also high now; each time glibc mmaps from the
1028 kernel, the kernel is forced to zero out the memory it gives to the
1029 application. Zeroing memory is expensive and eats a lot of cache and
1030 memory bandwidth. This has nothing to do with the efficiency of the
1031 virtual memory system, by doing mmap the kernel just has no choice but
1032 to zero.
1033
1034 In 2001, the kernel had a maximum size for brk() which was about 800
1035 megabytes on 32 bit x86, at that point brk() would hit the first
1036 mmaped shared libaries and couldn't expand anymore. With current 2.6
1037 kernels, the VA space layout is different and brk() and mmap
1038 both can span the entire heap at will.
1039
1040 Rather than using a static threshold for the brk/mmap tradeoff,
1041 we are now using a simple dynamic one. The goal is still to avoid
1042 fragmentation. The old goals we kept are
1043 1) try to get the long lived large allocations to use mmap()
1044 2) really large allocations should always use mmap()
1045 and we're adding now:
1046 3) transient allocations should use brk() to avoid forcing the kernel
1047 having to zero memory over and over again
1048
1049 The implementation works with a sliding threshold, which is by default
1050 limited to go between 128Kb and 32Mb (64Mb for 64 bitmachines) and starts
1051 out at 128Kb as per the 2001 default.
1052
1053 This allows us to satisfy requirement 1) under the assumption that long
1054 lived allocations are made early in the process' lifespan, before it has
1055 started doing dynamic allocations of the same size (which will
1056 increase the threshold).
1057
1058 The upperbound on the threshold satisfies requirement 2)
1059
1060 The threshold goes up in value when the application frees memory that was
1061 allocated with the mmap allocator. The idea is that once the application
1062 starts freeing memory of a certain size, it's highly probable that this is
1063 a size the application uses for transient allocations. This estimator
1064 is there to satisfy the new third requirement.
1065
1066*/
1067
1068#define M_MMAP_THRESHOLD -3
1069
1070#ifndef DEFAULT_MMAP_THRESHOLD
1071#define DEFAULT_MMAP_THRESHOLD DEFAULT_MMAP_THRESHOLD_MIN
1072#endif
1073
1074/*
1075 M_MMAP_MAX is the maximum number of requests to simultaneously
1076 service using mmap. This parameter exists because
1077 some systems have a limited number of internal tables for
1078 use by mmap, and using more than a few of them may degrade
1079 performance.
1080
1081 The default is set to a value that serves only as a safeguard.
1082 Setting to 0 disables use of mmap for servicing large requests.
1083*/
1084
1085#define M_MMAP_MAX -4
1086
1087#ifndef DEFAULT_MMAP_MAX
1088#define DEFAULT_MMAP_MAX (65536)
1089#endif
1090
1091#include <malloc.h>
1092
1093#ifndef RETURN_ADDRESS
1094#define RETURN_ADDRESS(X_) (NULL)
1095#endif
1096
1097/* Forward declarations. */
1098struct malloc_chunk;
1099typedef struct malloc_chunk* mchunkptr;
1100
1101/* Internal routines. */
1102
1103static void* _int_malloc(mstate, size_t);
1104static void _int_free(mstate, mchunkptr, int);
1105static void* _int_realloc(mstate, mchunkptr, INTERNAL_SIZE_T,
1106 INTERNAL_SIZE_T);
1107static void* _int_memalign(mstate, size_t, size_t);
1108#if IS_IN (libc)
1109static void* _mid_memalign(size_t, size_t, void *);
1110#endif
1111
1112static void malloc_printerr(const char *str) __attribute__ ((noreturn));
1113
1114static void munmap_chunk(mchunkptr p);
1115#if HAVE_MREMAP
1116static mchunkptr mremap_chunk(mchunkptr p, size_t new_size);
1117#endif
1118
1119/* ------------------ MMAP support ------------------ */
1120
1121
1122#include <fcntl.h>
1123#include <sys/mman.h>
1124
1125#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
1126# define MAP_ANONYMOUS MAP_ANON
1127#endif
1128
1129#ifndef MAP_NORESERVE
1130# define MAP_NORESERVE 0
1131#endif
1132
1133#define MMAP(addr, size, prot, flags) \
1134 __mmap((addr), (size), (prot), (flags)|MAP_ANONYMOUS|MAP_PRIVATE, -1, 0)
1135
1136
1137/*
1138 ----------------------- Chunk representations -----------------------
1139*/
1140
1141
1142/*
1143 This struct declaration is misleading (but accurate and necessary).
1144 It declares a "view" into memory allowing access to necessary
1145 fields at known offsets from a given base. See explanation below.
1146*/
1147
1148struct malloc_chunk {
1149
1150 INTERNAL_SIZE_T mchunk_prev_size; /* Size of previous chunk (if free). */
1151 INTERNAL_SIZE_T mchunk_size; /* Size in bytes, including overhead. */
1152
1153 struct malloc_chunk* fd; /* double links -- used only if free. */
1154 struct malloc_chunk* bk;
1155
1156 /* Only used for large blocks: pointer to next larger size. */
1157 struct malloc_chunk* fd_nextsize; /* double links -- used only if free. */
1158 struct malloc_chunk* bk_nextsize;
1159};
1160
1161
1162/*
1163 malloc_chunk details:
1164
1165 (The following includes lightly edited explanations by Colin Plumb.)
1166
1167 Chunks of memory are maintained using a `boundary tag' method as
1168 described in e.g., Knuth or Standish. (See the paper by Paul
1169 Wilson ftp://ftp.cs.utexas.edu/pub/garbage/allocsrv.ps for a
1170 survey of such techniques.) Sizes of free chunks are stored both
1171 in the front of each chunk and at the end. This makes
1172 consolidating fragmented chunks into bigger chunks very fast. The
1173 size fields also hold bits representing whether chunks are free or
1174 in use.
1175
1176 An allocated chunk looks like this:
1177
1178
1179 chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1180 | Size of previous chunk, if unallocated (P clear) |
1181 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1182 | Size of chunk, in bytes |A|M|P|
1183 mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1184 | User data starts here... .
1185 . .
1186 . (malloc_usable_size() bytes) .
1187 . |
1188nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1189 | (size of chunk, but used for application data) |
1190 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1191 | Size of next chunk, in bytes |A|0|1|
1192 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1193
1194 Where "chunk" is the front of the chunk for the purpose of most of
1195 the malloc code, but "mem" is the pointer that is returned to the
1196 user. "Nextchunk" is the beginning of the next contiguous chunk.
1197
1198 Chunks always begin on even word boundaries, so the mem portion
1199 (which is returned to the user) is also on an even word boundary, and
1200 thus at least double-word aligned.
1201
1202 Free chunks are stored in circular doubly-linked lists, and look like this:
1203
1204 chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1205 | Size of previous chunk, if unallocated (P clear) |
1206 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1207 `head:' | Size of chunk, in bytes |A|0|P|
1208 mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1209 | Forward pointer to next chunk in list |
1210 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1211 | Back pointer to previous chunk in list |
1212 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1213 | Unused space (may be 0 bytes long) .
1214 . .
1215 . |
1216nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1217 `foot:' | Size of chunk, in bytes |
1218 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1219 | Size of next chunk, in bytes |A|0|0|
1220 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1221
1222 The P (PREV_INUSE) bit, stored in the unused low-order bit of the
1223 chunk size (which is always a multiple of two words), is an in-use
1224 bit for the *previous* chunk. If that bit is *clear*, then the
1225 word before the current chunk size contains the previous chunk
1226 size, and can be used to find the front of the previous chunk.
1227 The very first chunk allocated always has this bit set,
1228 preventing access to non-existent (or non-owned) memory. If
1229 prev_inuse is set for any given chunk, then you CANNOT determine
1230 the size of the previous chunk, and might even get a memory
1231 addressing fault when trying to do so.
1232
1233 The A (NON_MAIN_ARENA) bit is cleared for chunks on the initial,
1234 main arena, described by the main_arena variable. When additional
1235 threads are spawned, each thread receives its own arena (up to a
1236 configurable limit, after which arenas are reused for multiple
1237 threads), and the chunks in these arenas have the A bit set. To
1238 find the arena for a chunk on such a non-main arena, heap_for_ptr
1239 performs a bit mask operation and indirection through the ar_ptr
1240 member of the per-heap header heap_info (see arena.c).
1241
1242 Note that the `foot' of the current chunk is actually represented
1243 as the prev_size of the NEXT chunk. This makes it easier to
1244 deal with alignments etc but can be very confusing when trying
1245 to extend or adapt this code.
1246
1247 The three exceptions to all this are:
1248
1249 1. The special chunk `top' doesn't bother using the
1250 trailing size field since there is no next contiguous chunk
1251 that would have to index off it. After initialization, `top'
1252 is forced to always exist. If it would become less than
1253 MINSIZE bytes long, it is replenished.
1254
1255 2. Chunks allocated via mmap, which have the second-lowest-order
1256 bit M (IS_MMAPPED) set in their size fields. Because they are
1257 allocated one-by-one, each must contain its own trailing size
1258 field. If the M bit is set, the other bits are ignored
1259 (because mmapped chunks are neither in an arena, nor adjacent
1260 to a freed chunk). The M bit is also used for chunks which
1261 originally came from a dumped heap via malloc_set_state in
1262 hooks.c.
1263
1264 3. Chunks in fastbins are treated as allocated chunks from the
1265 point of view of the chunk allocator. They are consolidated
1266 with their neighbors only in bulk, in malloc_consolidate.
1267*/
1268
1269/*
1270 ---------- Size and alignment checks and conversions ----------
1271*/
1272
1273/* Conversion from malloc headers to user pointers, and back. When
1274 using memory tagging the user data and the malloc data structure
1275 headers have distinct tags. Converting fully from one to the other
1276 involves extracting the tag at the other address and creating a
1277 suitable pointer using it. That can be quite expensive. There are
1278 cases when the pointers are not dereferenced (for example only used
1279 for alignment check) so the tags are not relevant, and there are
1280 cases when user data is not tagged distinctly from malloc headers
1281 (user data is untagged because tagging is done late in malloc and
1282 early in free). User memory tagging across internal interfaces:
1283
1284 sysmalloc: Returns untagged memory.
1285 _int_malloc: Returns untagged memory.
1286 _int_free: Takes untagged memory.
1287 _int_memalign: Returns untagged memory.
1288 _int_memalign: Returns untagged memory.
1289 _mid_memalign: Returns tagged memory.
1290 _int_realloc: Takes and returns tagged memory.
1291*/
1292
1293/* The chunk header is two SIZE_SZ elements, but this is used widely, so
1294 we define it here for clarity later. */
1295#define CHUNK_HDR_SZ (2 * SIZE_SZ)
1296
1297/* Convert a chunk address to a user mem pointer without correcting
1298 the tag. */
1299#define chunk2mem(p) ((void*)((char*)(p) + CHUNK_HDR_SZ))
1300
1301/* Convert a chunk address to a user mem pointer and extract the right tag. */
1302#define chunk2mem_tag(p) ((void*)tag_at ((char*)(p) + CHUNK_HDR_SZ))
1303
1304/* Convert a user mem pointer to a chunk address and extract the right tag. */
1305#define mem2chunk(mem) ((mchunkptr)tag_at (((char*)(mem) - CHUNK_HDR_SZ)))
1306
1307/* The smallest possible chunk */
1308#define MIN_CHUNK_SIZE (offsetof(struct malloc_chunk, fd_nextsize))
1309
1310/* The smallest size we can malloc is an aligned minimal chunk */
1311
1312#define MINSIZE \
1313 (unsigned long)(((MIN_CHUNK_SIZE+MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK))
1314
1315/* Check if m has acceptable alignment */
1316
1317#define aligned_OK(m) (((unsigned long)(m) & MALLOC_ALIGN_MASK) == 0)
1318
1319#define misaligned_chunk(p) \
1320 ((uintptr_t)(MALLOC_ALIGNMENT == CHUNK_HDR_SZ ? (p) : chunk2mem (p)) \
1321 & MALLOC_ALIGN_MASK)
1322
1323/* pad request bytes into a usable size -- internal version */
1324/* Note: This must be a macro that evaluates to a compile time constant
1325 if passed a literal constant. */
1326#define request2size(req) \
1327 (((req) + SIZE_SZ + MALLOC_ALIGN_MASK < MINSIZE) ? \
1328 MINSIZE : \
1329 ((req) + SIZE_SZ + MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK)
1330
1331/* Check if REQ overflows when padded and aligned and if the resulting value
1332 is less than PTRDIFF_T. Returns TRUE and the requested size or MINSIZE in
1333 case the value is less than MINSIZE on SZ or false if any of the previous
1334 check fail. */
1335static inline bool
1336checked_request2size (size_t req, size_t *sz) __nonnull (1)
1337{
1338 if (__glibc_unlikely (req > PTRDIFF_MAX))
1339 return false;
1340
1341 /* When using tagged memory, we cannot share the end of the user
1342 block with the header for the next chunk, so ensure that we
1343 allocate blocks that are rounded up to the granule size. Take
1344 care not to overflow from close to MAX_SIZE_T to a small
1345 number. Ideally, this would be part of request2size(), but that
1346 must be a macro that produces a compile time constant if passed
1347 a constant literal. */
1348 if (__glibc_unlikely (mtag_enabled))
1349 {
1350 /* Ensure this is not evaluated if !mtag_enabled, see gcc PR 99551. */
1351 asm ("");
1352
1353 req = (req + (__MTAG_GRANULE_SIZE - 1)) &
1354 ~(size_t)(__MTAG_GRANULE_SIZE - 1);
1355 }
1356
1357 *sz = request2size (req);
1358 return true;
1359}
1360
1361/*
1362 --------------- Physical chunk operations ---------------
1363 */
1364
1365
1366/* size field is or'ed with PREV_INUSE when previous adjacent chunk in use */
1367#define PREV_INUSE 0x1
1368
1369/* extract inuse bit of previous chunk */
1370#define prev_inuse(p) ((p)->mchunk_size & PREV_INUSE)
1371
1372
1373/* size field is or'ed with IS_MMAPPED if the chunk was obtained with mmap() */
1374#define IS_MMAPPED 0x2
1375
1376/* check for mmap()'ed chunk */
1377#define chunk_is_mmapped(p) ((p)->mchunk_size & IS_MMAPPED)
1378
1379
1380/* size field is or'ed with NON_MAIN_ARENA if the chunk was obtained
1381 from a non-main arena. This is only set immediately before handing
1382 the chunk to the user, if necessary. */
1383#define NON_MAIN_ARENA 0x4
1384
1385/* Check for chunk from main arena. */
1386#define chunk_main_arena(p) (((p)->mchunk_size & NON_MAIN_ARENA) == 0)
1387
1388/* Mark a chunk as not being on the main arena. */
1389#define set_non_main_arena(p) ((p)->mchunk_size |= NON_MAIN_ARENA)
1390
1391
1392/*
1393 Bits to mask off when extracting size
1394
1395 Note: IS_MMAPPED is intentionally not masked off from size field in
1396 macros for which mmapped chunks should never be seen. This should
1397 cause helpful core dumps to occur if it is tried by accident by
1398 people extending or adapting this malloc.
1399 */
1400#define SIZE_BITS (PREV_INUSE | IS_MMAPPED | NON_MAIN_ARENA)
1401
1402/* Get size, ignoring use bits */
1403#define chunksize(p) (chunksize_nomask (p) & ~(SIZE_BITS))
1404
1405/* Like chunksize, but do not mask SIZE_BITS. */
1406#define chunksize_nomask(p) ((p)->mchunk_size)
1407
1408/* Ptr to next physical malloc_chunk. */
1409#define next_chunk(p) ((mchunkptr) (((char *) (p)) + chunksize (p)))
1410
1411/* Size of the chunk below P. Only valid if !prev_inuse (P). */
1412#define prev_size(p) ((p)->mchunk_prev_size)
1413
1414/* Set the size of the chunk below P. Only valid if !prev_inuse (P). */
1415#define set_prev_size(p, sz) ((p)->mchunk_prev_size = (sz))
1416
1417/* Ptr to previous physical malloc_chunk. Only valid if !prev_inuse (P). */
1418#define prev_chunk(p) ((mchunkptr) (((char *) (p)) - prev_size (p)))
1419
1420/* Treat space at ptr + offset as a chunk */
1421#define chunk_at_offset(p, s) ((mchunkptr) (((char *) (p)) + (s)))
1422
1423/* extract p's inuse bit */
1424#define inuse(p) \
1425 ((((mchunkptr) (((char *) (p)) + chunksize (p)))->mchunk_size) & PREV_INUSE)
1426
1427/* set/clear chunk as being inuse without otherwise disturbing */
1428#define set_inuse(p) \
1429 ((mchunkptr) (((char *) (p)) + chunksize (p)))->mchunk_size |= PREV_INUSE
1430
1431#define clear_inuse(p) \
1432 ((mchunkptr) (((char *) (p)) + chunksize (p)))->mchunk_size &= ~(PREV_INUSE)
1433
1434
1435/* check/set/clear inuse bits in known places */
1436#define inuse_bit_at_offset(p, s) \
1437 (((mchunkptr) (((char *) (p)) + (s)))->mchunk_size & PREV_INUSE)
1438
1439#define set_inuse_bit_at_offset(p, s) \
1440 (((mchunkptr) (((char *) (p)) + (s)))->mchunk_size |= PREV_INUSE)
1441
1442#define clear_inuse_bit_at_offset(p, s) \
1443 (((mchunkptr) (((char *) (p)) + (s)))->mchunk_size &= ~(PREV_INUSE))
1444
1445
1446/* Set size at head, without disturbing its use bit */
1447#define set_head_size(p, s) ((p)->mchunk_size = (((p)->mchunk_size & SIZE_BITS) | (s)))
1448
1449/* Set size/use field */
1450#define set_head(p, s) ((p)->mchunk_size = (s))
1451
1452/* Set size at footer (only when chunk is not in use) */
1453#define set_foot(p, s) (((mchunkptr) ((char *) (p) + (s)))->mchunk_prev_size = (s))
1454
1455#pragma GCC poison mchunk_size
1456#pragma GCC poison mchunk_prev_size
1457
1458/* This is the size of the real usable data in the chunk. Not valid for
1459 dumped heap chunks. */
1460#define memsize(p) \
1461 (__MTAG_GRANULE_SIZE > SIZE_SZ && __glibc_unlikely (mtag_enabled) ? \
1462 chunksize (p) - CHUNK_HDR_SZ : \
1463 chunksize (p) - CHUNK_HDR_SZ + (chunk_is_mmapped (p) ? 0 : SIZE_SZ))
1464
1465/* If memory tagging is enabled the layout changes to accommodate the granule
1466 size, this is wasteful for small allocations so not done by default.
1467 Both the chunk header and user data has to be granule aligned. */
1468_Static_assert (__MTAG_GRANULE_SIZE <= CHUNK_HDR_SZ,
1469 "memory tagging is not supported with large granule.");
1470
1471static __always_inline void *
1472tag_new_usable (void *ptr)
1473{
1474 if (__glibc_unlikely (mtag_enabled) && ptr)
1475 {
1476 mchunkptr cp = mem2chunk(ptr);
1477 ptr = __libc_mtag_tag_region (p: __libc_mtag_new_tag (p: ptr), memsize (cp));
1478 }
1479 return ptr;
1480}
1481
1482/*
1483 -------------------- Internal data structures --------------------
1484
1485 All internal state is held in an instance of malloc_state defined
1486 below. There are no other static variables, except in two optional
1487 cases:
1488 * If USE_MALLOC_LOCK is defined, the mALLOC_MUTEx declared above.
1489 * If mmap doesn't support MAP_ANONYMOUS, a dummy file descriptor
1490 for mmap.
1491
1492 Beware of lots of tricks that minimize the total bookkeeping space
1493 requirements. The result is a little over 1K bytes (for 4byte
1494 pointers and size_t.)
1495 */
1496
1497/*
1498 Bins
1499
1500 An array of bin headers for free chunks. Each bin is doubly
1501 linked. The bins are approximately proportionally (log) spaced.
1502 There are a lot of these bins (128). This may look excessive, but
1503 works very well in practice. Most bins hold sizes that are
1504 unusual as malloc request sizes, but are more usual for fragments
1505 and consolidated sets of chunks, which is what these bins hold, so
1506 they can be found quickly. All procedures maintain the invariant
1507 that no consolidated chunk physically borders another one, so each
1508 chunk in a list is known to be preceeded and followed by either
1509 inuse chunks or the ends of memory.
1510
1511 Chunks in bins are kept in size order, with ties going to the
1512 approximately least recently used chunk. Ordering isn't needed
1513 for the small bins, which all contain the same-sized chunks, but
1514 facilitates best-fit allocation for larger chunks. These lists
1515 are just sequential. Keeping them in order almost never requires
1516 enough traversal to warrant using fancier ordered data
1517 structures.
1518
1519 Chunks of the same size are linked with the most
1520 recently freed at the front, and allocations are taken from the
1521 back. This results in LRU (FIFO) allocation order, which tends
1522 to give each chunk an equal opportunity to be consolidated with
1523 adjacent freed chunks, resulting in larger free chunks and less
1524 fragmentation.
1525
1526 To simplify use in double-linked lists, each bin header acts
1527 as a malloc_chunk. This avoids special-casing for headers.
1528 But to conserve space and improve locality, we allocate
1529 only the fd/bk pointers of bins, and then use repositioning tricks
1530 to treat these as the fields of a malloc_chunk*.
1531 */
1532
1533typedef struct malloc_chunk *mbinptr;
1534
1535/* addressing -- note that bin_at(0) does not exist */
1536#define bin_at(m, i) \
1537 (mbinptr) (((char *) &((m)->bins[((i) - 1) * 2])) \
1538 - offsetof (struct malloc_chunk, fd))
1539
1540/* analog of ++bin */
1541#define next_bin(b) ((mbinptr) ((char *) (b) + (sizeof (mchunkptr) << 1)))
1542
1543/* Reminders about list directionality within bins */
1544#define first(b) ((b)->fd)
1545#define last(b) ((b)->bk)
1546
1547/*
1548 Indexing
1549
1550 Bins for sizes < 512 bytes contain chunks of all the same size, spaced
1551 8 bytes apart. Larger bins are approximately logarithmically spaced:
1552
1553 64 bins of size 8
1554 32 bins of size 64
1555 16 bins of size 512
1556 8 bins of size 4096
1557 4 bins of size 32768
1558 2 bins of size 262144
1559 1 bin of size what's left
1560
1561 There is actually a little bit of slop in the numbers in bin_index
1562 for the sake of speed. This makes no difference elsewhere.
1563
1564 The bins top out around 1MB because we expect to service large
1565 requests via mmap.
1566
1567 Bin 0 does not exist. Bin 1 is the unordered list; if that would be
1568 a valid chunk size the small bins are bumped up one.
1569 */
1570
1571#define NBINS 128
1572#define NSMALLBINS 64
1573#define SMALLBIN_WIDTH MALLOC_ALIGNMENT
1574#define SMALLBIN_CORRECTION (MALLOC_ALIGNMENT > CHUNK_HDR_SZ)
1575#define MIN_LARGE_SIZE ((NSMALLBINS - SMALLBIN_CORRECTION) * SMALLBIN_WIDTH)
1576
1577#define in_smallbin_range(sz) \
1578 ((unsigned long) (sz) < (unsigned long) MIN_LARGE_SIZE)
1579
1580#define smallbin_index(sz) \
1581 ((SMALLBIN_WIDTH == 16 ? (((unsigned) (sz)) >> 4) : (((unsigned) (sz)) >> 3))\
1582 + SMALLBIN_CORRECTION)
1583
1584#define largebin_index_32(sz) \
1585 (((((unsigned long) (sz)) >> 6) <= 38) ? 56 + (((unsigned long) (sz)) >> 6) :\
1586 ((((unsigned long) (sz)) >> 9) <= 20) ? 91 + (((unsigned long) (sz)) >> 9) :\
1587 ((((unsigned long) (sz)) >> 12) <= 10) ? 110 + (((unsigned long) (sz)) >> 12) :\
1588 ((((unsigned long) (sz)) >> 15) <= 4) ? 119 + (((unsigned long) (sz)) >> 15) :\
1589 ((((unsigned long) (sz)) >> 18) <= 2) ? 124 + (((unsigned long) (sz)) >> 18) :\
1590 126)
1591
1592#define largebin_index_32_big(sz) \
1593 (((((unsigned long) (sz)) >> 6) <= 45) ? 49 + (((unsigned long) (sz)) >> 6) :\
1594 ((((unsigned long) (sz)) >> 9) <= 20) ? 91 + (((unsigned long) (sz)) >> 9) :\
1595 ((((unsigned long) (sz)) >> 12) <= 10) ? 110 + (((unsigned long) (sz)) >> 12) :\
1596 ((((unsigned long) (sz)) >> 15) <= 4) ? 119 + (((unsigned long) (sz)) >> 15) :\
1597 ((((unsigned long) (sz)) >> 18) <= 2) ? 124 + (((unsigned long) (sz)) >> 18) :\
1598 126)
1599
1600// XXX It remains to be seen whether it is good to keep the widths of
1601// XXX the buckets the same or whether it should be scaled by a factor
1602// XXX of two as well.
1603#define largebin_index_64(sz) \
1604 (((((unsigned long) (sz)) >> 6) <= 48) ? 48 + (((unsigned long) (sz)) >> 6) :\
1605 ((((unsigned long) (sz)) >> 9) <= 20) ? 91 + (((unsigned long) (sz)) >> 9) :\
1606 ((((unsigned long) (sz)) >> 12) <= 10) ? 110 + (((unsigned long) (sz)) >> 12) :\
1607 ((((unsigned long) (sz)) >> 15) <= 4) ? 119 + (((unsigned long) (sz)) >> 15) :\
1608 ((((unsigned long) (sz)) >> 18) <= 2) ? 124 + (((unsigned long) (sz)) >> 18) :\
1609 126)
1610
1611#define largebin_index(sz) \
1612 (SIZE_SZ == 8 ? largebin_index_64 (sz) \
1613 : MALLOC_ALIGNMENT == 16 ? largebin_index_32_big (sz) \
1614 : largebin_index_32 (sz))
1615
1616#define bin_index(sz) \
1617 ((in_smallbin_range (sz)) ? smallbin_index (sz) : largebin_index (sz))
1618
1619/* Take a chunk off a bin list. */
1620static void
1621unlink_chunk (mstate av, mchunkptr p)
1622{
1623 if (chunksize (p) != prev_size (next_chunk (p)))
1624 malloc_printerr (str: "corrupted size vs. prev_size");
1625
1626 mchunkptr fd = p->fd;
1627 mchunkptr bk = p->bk;
1628
1629 if (__builtin_expect (fd->bk != p || bk->fd != p, 0))
1630 malloc_printerr (str: "corrupted double-linked list");
1631
1632 fd->bk = bk;
1633 bk->fd = fd;
1634 if (!in_smallbin_range (chunksize_nomask (p)) && p->fd_nextsize != NULL)
1635 {
1636 if (p->fd_nextsize->bk_nextsize != p
1637 || p->bk_nextsize->fd_nextsize != p)
1638 malloc_printerr (str: "corrupted double-linked list (not small)");
1639
1640 if (fd->fd_nextsize == NULL)
1641 {
1642 if (p->fd_nextsize == p)
1643 fd->fd_nextsize = fd->bk_nextsize = fd;
1644 else
1645 {
1646 fd->fd_nextsize = p->fd_nextsize;
1647 fd->bk_nextsize = p->bk_nextsize;
1648 p->fd_nextsize->bk_nextsize = fd;
1649 p->bk_nextsize->fd_nextsize = fd;
1650 }
1651 }
1652 else
1653 {
1654 p->fd_nextsize->bk_nextsize = p->bk_nextsize;
1655 p->bk_nextsize->fd_nextsize = p->fd_nextsize;
1656 }
1657 }
1658}
1659
1660/*
1661 Unsorted chunks
1662
1663 All remainders from chunk splits, as well as all returned chunks,
1664 are first placed in the "unsorted" bin. They are then placed
1665 in regular bins after malloc gives them ONE chance to be used before
1666 binning. So, basically, the unsorted_chunks list acts as a queue,
1667 with chunks being placed on it in free (and malloc_consolidate),
1668 and taken off (to be either used or placed in bins) in malloc.
1669
1670 The NON_MAIN_ARENA flag is never set for unsorted chunks, so it
1671 does not have to be taken into account in size comparisons.
1672 */
1673
1674/* The otherwise unindexable 1-bin is used to hold unsorted chunks. */
1675#define unsorted_chunks(M) (bin_at (M, 1))
1676
1677/*
1678 Top
1679
1680 The top-most available chunk (i.e., the one bordering the end of
1681 available memory) is treated specially. It is never included in
1682 any bin, is used only if no other chunk is available, and is
1683 released back to the system if it is very large (see
1684 M_TRIM_THRESHOLD). Because top initially
1685 points to its own bin with initial zero size, thus forcing
1686 extension on the first malloc request, we avoid having any special
1687 code in malloc to check whether it even exists yet. But we still
1688 need to do so when getting memory from system, so we make
1689 initial_top treat the bin as a legal but unusable chunk during the
1690 interval between initialization and the first call to
1691 sysmalloc. (This is somewhat delicate, since it relies on
1692 the 2 preceding words to be zero during this interval as well.)
1693 */
1694
1695/* Conveniently, the unsorted bin can be used as dummy top on first call */
1696#define initial_top(M) (unsorted_chunks (M))
1697
1698/*
1699 Binmap
1700
1701 To help compensate for the large number of bins, a one-level index
1702 structure is used for bin-by-bin searching. `binmap' is a
1703 bitvector recording whether bins are definitely empty so they can
1704 be skipped over during during traversals. The bits are NOT always
1705 cleared as soon as bins are empty, but instead only
1706 when they are noticed to be empty during traversal in malloc.
1707 */
1708
1709/* Conservatively use 32 bits per map word, even if on 64bit system */
1710#define BINMAPSHIFT 5
1711#define BITSPERMAP (1U << BINMAPSHIFT)
1712#define BINMAPSIZE (NBINS / BITSPERMAP)
1713
1714#define idx2block(i) ((i) >> BINMAPSHIFT)
1715#define idx2bit(i) ((1U << ((i) & ((1U << BINMAPSHIFT) - 1))))
1716
1717#define mark_bin(m, i) ((m)->binmap[idx2block (i)] |= idx2bit (i))
1718#define unmark_bin(m, i) ((m)->binmap[idx2block (i)] &= ~(idx2bit (i)))
1719#define get_binmap(m, i) ((m)->binmap[idx2block (i)] & idx2bit (i))
1720
1721/*
1722 Fastbins
1723
1724 An array of lists holding recently freed small chunks. Fastbins
1725 are not doubly linked. It is faster to single-link them, and
1726 since chunks are never removed from the middles of these lists,
1727 double linking is not necessary. Also, unlike regular bins, they
1728 are not even processed in FIFO order (they use faster LIFO) since
1729 ordering doesn't much matter in the transient contexts in which
1730 fastbins are normally used.
1731
1732 Chunks in fastbins keep their inuse bit set, so they cannot
1733 be consolidated with other free chunks. malloc_consolidate
1734 releases all chunks in fastbins and consolidates them with
1735 other free chunks.
1736 */
1737
1738typedef struct malloc_chunk *mfastbinptr;
1739#define fastbin(ar_ptr, idx) ((ar_ptr)->fastbinsY[idx])
1740
1741/* offset 2 to use otherwise unindexable first 2 bins */
1742#define fastbin_index(sz) \
1743 ((((unsigned int) (sz)) >> (SIZE_SZ == 8 ? 4 : 3)) - 2)
1744
1745
1746/* The maximum fastbin request size we support */
1747#define MAX_FAST_SIZE (80 * SIZE_SZ / 4)
1748
1749#define NFASTBINS (fastbin_index (request2size (MAX_FAST_SIZE)) + 1)
1750
1751/*
1752 FASTBIN_CONSOLIDATION_THRESHOLD is the size of a chunk in free()
1753 that triggers automatic consolidation of possibly-surrounding
1754 fastbin chunks. This is a heuristic, so the exact value should not
1755 matter too much. It is defined at half the default trim threshold as a
1756 compromise heuristic to only attempt consolidation if it is likely
1757 to lead to trimming. However, it is not dynamically tunable, since
1758 consolidation reduces fragmentation surrounding large chunks even
1759 if trimming is not used.
1760 */
1761
1762#define FASTBIN_CONSOLIDATION_THRESHOLD (65536UL)
1763
1764/*
1765 NONCONTIGUOUS_BIT indicates that MORECORE does not return contiguous
1766 regions. Otherwise, contiguity is exploited in merging together,
1767 when possible, results from consecutive MORECORE calls.
1768
1769 The initial value comes from MORECORE_CONTIGUOUS, but is
1770 changed dynamically if mmap is ever used as an sbrk substitute.
1771 */
1772
1773#define NONCONTIGUOUS_BIT (2U)
1774
1775#define contiguous(M) (((M)->flags & NONCONTIGUOUS_BIT) == 0)
1776#define noncontiguous(M) (((M)->flags & NONCONTIGUOUS_BIT) != 0)
1777#define set_noncontiguous(M) ((M)->flags |= NONCONTIGUOUS_BIT)
1778#define set_contiguous(M) ((M)->flags &= ~NONCONTIGUOUS_BIT)
1779
1780/* Maximum size of memory handled in fastbins. */
1781static INTERNAL_SIZE_T global_max_fast;
1782
1783/*
1784 Set value of max_fast.
1785 Use impossibly small value if 0.
1786 Precondition: there are no existing fastbin chunks in the main arena.
1787 Since do_check_malloc_state () checks this, we call malloc_consolidate ()
1788 before changing max_fast. Note other arenas will leak their fast bin
1789 entries if max_fast is reduced.
1790 */
1791
1792#define set_max_fast(s) \
1793 global_max_fast = (((size_t) (s) <= MALLOC_ALIGN_MASK - SIZE_SZ) \
1794 ? MIN_CHUNK_SIZE / 2 : ((s + SIZE_SZ) & ~MALLOC_ALIGN_MASK))
1795
1796static inline INTERNAL_SIZE_T
1797get_max_fast (void)
1798{
1799 /* Tell the GCC optimizers that global_max_fast is never larger
1800 than MAX_FAST_SIZE. This avoids out-of-bounds array accesses in
1801 _int_malloc after constant propagation of the size parameter.
1802 (The code never executes because malloc preserves the
1803 global_max_fast invariant, but the optimizers may not recognize
1804 this.) */
1805 if (global_max_fast > MAX_FAST_SIZE)
1806 __builtin_unreachable ();
1807 return global_max_fast;
1808}
1809
1810/*
1811 ----------- Internal state representation and initialization -----------
1812 */
1813
1814/*
1815 have_fastchunks indicates that there are probably some fastbin chunks.
1816 It is set true on entering a chunk into any fastbin, and cleared early in
1817 malloc_consolidate. The value is approximate since it may be set when there
1818 are no fastbin chunks, or it may be clear even if there are fastbin chunks
1819 available. Given it's sole purpose is to reduce number of redundant calls to
1820 malloc_consolidate, it does not affect correctness. As a result we can safely
1821 use relaxed atomic accesses.
1822 */
1823
1824
1825struct malloc_state
1826{
1827 /* Serialize access. */
1828 __libc_lock_define (, mutex);
1829
1830 /* Flags (formerly in max_fast). */
1831 int flags;
1832
1833 /* Set if the fastbin chunks contain recently inserted free blocks. */
1834 /* Note this is a bool but not all targets support atomics on booleans. */
1835 int have_fastchunks;
1836
1837 /* Fastbins */
1838 mfastbinptr fastbinsY[NFASTBINS];
1839
1840 /* Base of the topmost chunk -- not otherwise kept in a bin */
1841 mchunkptr top;
1842
1843 /* The remainder from the most recent split of a small request */
1844 mchunkptr last_remainder;
1845
1846 /* Normal bins packed as described above */
1847 mchunkptr bins[NBINS * 2 - 2];
1848
1849 /* Bitmap of bins */
1850 unsigned int binmap[BINMAPSIZE];
1851
1852 /* Linked list */
1853 struct malloc_state *next;
1854
1855 /* Linked list for free arenas. Access to this field is serialized
1856 by free_list_lock in arena.c. */
1857 struct malloc_state *next_free;
1858
1859 /* Number of threads attached to this arena. 0 if the arena is on
1860 the free list. Access to this field is serialized by
1861 free_list_lock in arena.c. */
1862 INTERNAL_SIZE_T attached_threads;
1863
1864 /* Memory allocated from the system in this arena. */
1865 INTERNAL_SIZE_T system_mem;
1866 INTERNAL_SIZE_T max_system_mem;
1867};
1868
1869struct malloc_par
1870{
1871 /* Tunable parameters */
1872 unsigned long trim_threshold;
1873 INTERNAL_SIZE_T top_pad;
1874 INTERNAL_SIZE_T mmap_threshold;
1875 INTERNAL_SIZE_T arena_test;
1876 INTERNAL_SIZE_T arena_max;
1877
1878#if HAVE_TUNABLES
1879 /* Transparent Large Page support. */
1880 INTERNAL_SIZE_T thp_pagesize;
1881 /* A value different than 0 means to align mmap allocation to hp_pagesize
1882 add hp_flags on flags. */
1883 INTERNAL_SIZE_T hp_pagesize;
1884 int hp_flags;
1885#endif
1886
1887 /* Memory map support */
1888 int n_mmaps;
1889 int n_mmaps_max;
1890 int max_n_mmaps;
1891 /* the mmap_threshold is dynamic, until the user sets
1892 it manually, at which point we need to disable any
1893 dynamic behavior. */
1894 int no_dyn_threshold;
1895
1896 /* Statistics */
1897 INTERNAL_SIZE_T mmapped_mem;
1898 INTERNAL_SIZE_T max_mmapped_mem;
1899
1900 /* First address handed out by MORECORE/sbrk. */
1901 char *sbrk_base;
1902
1903#if USE_TCACHE
1904 /* Maximum number of buckets to use. */
1905 size_t tcache_bins;
1906 size_t tcache_max_bytes;
1907 /* Maximum number of chunks in each bucket. */
1908 size_t tcache_count;
1909 /* Maximum number of chunks to remove from the unsorted list, which
1910 aren't used to prefill the cache. */
1911 size_t tcache_unsorted_limit;
1912#endif
1913};
1914
1915/* There are several instances of this struct ("arenas") in this
1916 malloc. If you are adapting this malloc in a way that does NOT use
1917 a static or mmapped malloc_state, you MUST explicitly zero-fill it
1918 before using. This malloc relies on the property that malloc_state
1919 is initialized to all zeroes (as is true of C statics). */
1920
1921static struct malloc_state main_arena =
1922{
1923 .mutex = _LIBC_LOCK_INITIALIZER,
1924 .next = &main_arena,
1925 .attached_threads = 1
1926};
1927
1928/* There is only one instance of the malloc parameters. */
1929
1930static struct malloc_par mp_ =
1931{
1932 .top_pad = DEFAULT_TOP_PAD,
1933 .n_mmaps_max = DEFAULT_MMAP_MAX,
1934 .mmap_threshold = DEFAULT_MMAP_THRESHOLD,
1935 .trim_threshold = DEFAULT_TRIM_THRESHOLD,
1936#define NARENAS_FROM_NCORES(n) ((n) * (sizeof (long) == 4 ? 2 : 8))
1937 .arena_test = NARENAS_FROM_NCORES (1)
1938#if USE_TCACHE
1939 ,
1940 .tcache_count = TCACHE_FILL_COUNT,
1941 .tcache_bins = TCACHE_MAX_BINS,
1942 .tcache_max_bytes = tidx2usize (TCACHE_MAX_BINS-1),
1943 .tcache_unsorted_limit = 0 /* No limit. */
1944#endif
1945};
1946
1947/*
1948 Initialize a malloc_state struct.
1949
1950 This is called from ptmalloc_init () or from _int_new_arena ()
1951 when creating a new arena.
1952 */
1953
1954static void
1955malloc_init_state (mstate av)
1956{
1957 int i;
1958 mbinptr bin;
1959
1960 /* Establish circular links for normal bins */
1961 for (i = 1; i < NBINS; ++i)
1962 {
1963 bin = bin_at (av, i);
1964 bin->fd = bin->bk = bin;
1965 }
1966
1967#if MORECORE_CONTIGUOUS
1968 if (av != &main_arena)
1969#endif
1970 set_noncontiguous (av);
1971 if (av == &main_arena)
1972 set_max_fast (DEFAULT_MXFAST);
1973 atomic_store_relaxed (&av->have_fastchunks, false);
1974
1975 av->top = initial_top (av);
1976}
1977
1978/*
1979 Other internal utilities operating on mstates
1980 */
1981
1982static void *sysmalloc (INTERNAL_SIZE_T, mstate);
1983static int systrim (size_t, mstate);
1984static void malloc_consolidate (mstate);
1985
1986
1987/* -------------- Early definitions for debugging hooks ---------------- */
1988
1989/* This function is called from the arena shutdown hook, to free the
1990 thread cache (if it exists). */
1991static void tcache_thread_shutdown (void);
1992
1993/* ------------------ Testing support ----------------------------------*/
1994
1995static int perturb_byte;
1996
1997static void
1998alloc_perturb (char *p, size_t n)
1999{
2000 if (__glibc_unlikely (perturb_byte))
2001 memset (s: p, c: perturb_byte ^ 0xff, n: n);
2002}
2003
2004static void
2005free_perturb (char *p, size_t n)
2006{
2007 if (__glibc_unlikely (perturb_byte))
2008 memset (s: p, c: perturb_byte, n: n);
2009}
2010
2011
2012
2013#include <stap-probe.h>
2014
2015/* ----------- Routines dealing with transparent huge pages ----------- */
2016
2017static inline void
2018madvise_thp (void *p, INTERNAL_SIZE_T size)
2019{
2020#if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
2021 /* Do not consider areas smaller than a huge page or if the tunable is
2022 not active. */
2023 if (mp_.thp_pagesize == 0 || size < mp_.thp_pagesize)
2024 return;
2025
2026 /* Linux requires the input address to be page-aligned, and unaligned
2027 inputs happens only for initial data segment. */
2028 if (__glibc_unlikely (!PTR_IS_ALIGNED (p, GLRO (dl_pagesize))))
2029 {
2030 void *q = PTR_ALIGN_DOWN (p, GLRO (dl_pagesize));
2031 size += PTR_DIFF (p, q);
2032 p = q;
2033 }
2034
2035 __madvise (addr: p, len: size, MADV_HUGEPAGE);
2036#endif
2037}
2038
2039/* ------------------- Support for multiple arenas -------------------- */
2040#include "arena.c"
2041
2042/*
2043 Debugging support
2044
2045 These routines make a number of assertions about the states
2046 of data structures that should be true at all times. If any
2047 are not true, it's very likely that a user program has somehow
2048 trashed memory. (It's also possible that there is a coding error
2049 in malloc. In which case, please report it!)
2050 */
2051
2052#if !MALLOC_DEBUG
2053
2054# define check_chunk(A, P)
2055# define check_free_chunk(A, P)
2056# define check_inuse_chunk(A, P)
2057# define check_remalloced_chunk(A, P, N)
2058# define check_malloced_chunk(A, P, N)
2059# define check_malloc_state(A)
2060
2061#else
2062
2063# define check_chunk(A, P) do_check_chunk (A, P)
2064# define check_free_chunk(A, P) do_check_free_chunk (A, P)
2065# define check_inuse_chunk(A, P) do_check_inuse_chunk (A, P)
2066# define check_remalloced_chunk(A, P, N) do_check_remalloced_chunk (A, P, N)
2067# define check_malloced_chunk(A, P, N) do_check_malloced_chunk (A, P, N)
2068# define check_malloc_state(A) do_check_malloc_state (A)
2069
2070/*
2071 Properties of all chunks
2072 */
2073
2074static void
2075do_check_chunk (mstate av, mchunkptr p)
2076{
2077 unsigned long sz = chunksize (p);
2078 /* min and max possible addresses assuming contiguous allocation */
2079 char *max_address = (char *) (av->top) + chunksize (av->top);
2080 char *min_address = max_address - av->system_mem;
2081
2082 if (!chunk_is_mmapped (p))
2083 {
2084 /* Has legal address ... */
2085 if (p != av->top)
2086 {
2087 if (contiguous (av))
2088 {
2089 assert (((char *) p) >= min_address);
2090 assert (((char *) p + sz) <= ((char *) (av->top)));
2091 }
2092 }
2093 else
2094 {
2095 /* top size is always at least MINSIZE */
2096 assert ((unsigned long) (sz) >= MINSIZE);
2097 /* top predecessor always marked inuse */
2098 assert (prev_inuse (p));
2099 }
2100 }
2101 else
2102 {
2103 /* address is outside main heap */
2104 if (contiguous (av) && av->top != initial_top (av))
2105 {
2106 assert (((char *) p) < min_address || ((char *) p) >= max_address);
2107 }
2108 /* chunk is page-aligned */
2109 assert (((prev_size (p) + sz) & (GLRO (dl_pagesize) - 1)) == 0);
2110 /* mem is aligned */
2111 assert (aligned_OK (chunk2mem (p)));
2112 }
2113}
2114
2115/*
2116 Properties of free chunks
2117 */
2118
2119static void
2120do_check_free_chunk (mstate av, mchunkptr p)
2121{
2122 INTERNAL_SIZE_T sz = chunksize_nomask (p) & ~(PREV_INUSE | NON_MAIN_ARENA);
2123 mchunkptr next = chunk_at_offset (p, sz);
2124
2125 do_check_chunk (av, p);
2126
2127 /* Chunk must claim to be free ... */
2128 assert (!inuse (p));
2129 assert (!chunk_is_mmapped (p));
2130
2131 /* Unless a special marker, must have OK fields */
2132 if ((unsigned long) (sz) >= MINSIZE)
2133 {
2134 assert ((sz & MALLOC_ALIGN_MASK) == 0);
2135 assert (aligned_OK (chunk2mem (p)));
2136 /* ... matching footer field */
2137 assert (prev_size (next_chunk (p)) == sz);
2138 /* ... and is fully consolidated */
2139 assert (prev_inuse (p));
2140 assert (next == av->top || inuse (next));
2141
2142 /* ... and has minimally sane links */
2143 assert (p->fd->bk == p);
2144 assert (p->bk->fd == p);
2145 }
2146 else /* markers are always of size SIZE_SZ */
2147 assert (sz == SIZE_SZ);
2148}
2149
2150/*
2151 Properties of inuse chunks
2152 */
2153
2154static void
2155do_check_inuse_chunk (mstate av, mchunkptr p)
2156{
2157 mchunkptr next;
2158
2159 do_check_chunk (av, p);
2160
2161 if (chunk_is_mmapped (p))
2162 return; /* mmapped chunks have no next/prev */
2163
2164 /* Check whether it claims to be in use ... */
2165 assert (inuse (p));
2166
2167 next = next_chunk (p);
2168
2169 /* ... and is surrounded by OK chunks.
2170 Since more things can be checked with free chunks than inuse ones,
2171 if an inuse chunk borders them and debug is on, it's worth doing them.
2172 */
2173 if (!prev_inuse (p))
2174 {
2175 /* Note that we cannot even look at prev unless it is not inuse */
2176 mchunkptr prv = prev_chunk (p);
2177 assert (next_chunk (prv) == p);
2178 do_check_free_chunk (av, prv);
2179 }
2180
2181 if (next == av->top)
2182 {
2183 assert (prev_inuse (next));
2184 assert (chunksize (next) >= MINSIZE);
2185 }
2186 else if (!inuse (next))
2187 do_check_free_chunk (av, next);
2188}
2189
2190/*
2191 Properties of chunks recycled from fastbins
2192 */
2193
2194static void
2195do_check_remalloced_chunk (mstate av, mchunkptr p, INTERNAL_SIZE_T s)
2196{
2197 INTERNAL_SIZE_T sz = chunksize_nomask (p) & ~(PREV_INUSE | NON_MAIN_ARENA);
2198
2199 if (!chunk_is_mmapped (p))
2200 {
2201 assert (av == arena_for_chunk (p));
2202 if (chunk_main_arena (p))
2203 assert (av == &main_arena);
2204 else
2205 assert (av != &main_arena);
2206 }
2207
2208 do_check_inuse_chunk (av, p);
2209
2210 /* Legal size ... */
2211 assert ((sz & MALLOC_ALIGN_MASK) == 0);
2212 assert ((unsigned long) (sz) >= MINSIZE);
2213 /* ... and alignment */
2214 assert (aligned_OK (chunk2mem (p)));
2215 /* chunk is less than MINSIZE more than request */
2216 assert ((long) (sz) - (long) (s) >= 0);
2217 assert ((long) (sz) - (long) (s + MINSIZE) < 0);
2218}
2219
2220/*
2221 Properties of nonrecycled chunks at the point they are malloced
2222 */
2223
2224static void
2225do_check_malloced_chunk (mstate av, mchunkptr p, INTERNAL_SIZE_T s)
2226{
2227 /* same as recycled case ... */
2228 do_check_remalloced_chunk (av, p, s);
2229
2230 /*
2231 ... plus, must obey implementation invariant that prev_inuse is
2232 always true of any allocated chunk; i.e., that each allocated
2233 chunk borders either a previously allocated and still in-use
2234 chunk, or the base of its memory arena. This is ensured
2235 by making all allocations from the `lowest' part of any found
2236 chunk. This does not necessarily hold however for chunks
2237 recycled via fastbins.
2238 */
2239
2240 assert (prev_inuse (p));
2241}
2242
2243
2244/*
2245 Properties of malloc_state.
2246
2247 This may be useful for debugging malloc, as well as detecting user
2248 programmer errors that somehow write into malloc_state.
2249
2250 If you are extending or experimenting with this malloc, you can
2251 probably figure out how to hack this routine to print out or
2252 display chunk addresses, sizes, bins, and other instrumentation.
2253 */
2254
2255static void
2256do_check_malloc_state (mstate av)
2257{
2258 int i;
2259 mchunkptr p;
2260 mchunkptr q;
2261 mbinptr b;
2262 unsigned int idx;
2263 INTERNAL_SIZE_T size;
2264 unsigned long total = 0;
2265 int max_fast_bin;
2266
2267 /* internal size_t must be no wider than pointer type */
2268 assert (sizeof (INTERNAL_SIZE_T) <= sizeof (char *));
2269
2270 /* alignment is a power of 2 */
2271 assert ((MALLOC_ALIGNMENT & (MALLOC_ALIGNMENT - 1)) == 0);
2272
2273 /* Check the arena is initialized. */
2274 assert (av->top != 0);
2275
2276 /* No memory has been allocated yet, so doing more tests is not possible. */
2277 if (av->top == initial_top (av))
2278 return;
2279
2280 /* pagesize is a power of 2 */
2281 assert (powerof2(GLRO (dl_pagesize)));
2282
2283 /* A contiguous main_arena is consistent with sbrk_base. */
2284 if (av == &main_arena && contiguous (av))
2285 assert ((char *) mp_.sbrk_base + av->system_mem ==
2286 (char *) av->top + chunksize (av->top));
2287
2288 /* properties of fastbins */
2289
2290 /* max_fast is in allowed range */
2291 assert ((get_max_fast () & ~1) <= request2size (MAX_FAST_SIZE));
2292
2293 max_fast_bin = fastbin_index (get_max_fast ());
2294
2295 for (i = 0; i < NFASTBINS; ++i)
2296 {
2297 p = fastbin (av, i);
2298
2299 /* The following test can only be performed for the main arena.
2300 While mallopt calls malloc_consolidate to get rid of all fast
2301 bins (especially those larger than the new maximum) this does
2302 only happen for the main arena. Trying to do this for any
2303 other arena would mean those arenas have to be locked and
2304 malloc_consolidate be called for them. This is excessive. And
2305 even if this is acceptable to somebody it still cannot solve
2306 the problem completely since if the arena is locked a
2307 concurrent malloc call might create a new arena which then
2308 could use the newly invalid fast bins. */
2309
2310 /* all bins past max_fast are empty */
2311 if (av == &main_arena && i > max_fast_bin)
2312 assert (p == 0);
2313
2314 while (p != 0)
2315 {
2316 if (__glibc_unlikely (misaligned_chunk (p)))
2317 malloc_printerr ("do_check_malloc_state(): "
2318 "unaligned fastbin chunk detected");
2319 /* each chunk claims to be inuse */
2320 do_check_inuse_chunk (av, p);
2321 total += chunksize (p);
2322 /* chunk belongs in this bin */
2323 assert (fastbin_index (chunksize (p)) == i);
2324 p = REVEAL_PTR (p->fd);
2325 }
2326 }
2327
2328 /* check normal bins */
2329 for (i = 1; i < NBINS; ++i)
2330 {
2331 b = bin_at (av, i);
2332
2333 /* binmap is accurate (except for bin 1 == unsorted_chunks) */
2334 if (i >= 2)
2335 {
2336 unsigned int binbit = get_binmap (av, i);
2337 int empty = last (b) == b;
2338 if (!binbit)
2339 assert (empty);
2340 else if (!empty)
2341 assert (binbit);
2342 }
2343
2344 for (p = last (b); p != b; p = p->bk)
2345 {
2346 /* each chunk claims to be free */
2347 do_check_free_chunk (av, p);
2348 size = chunksize (p);
2349 total += size;
2350 if (i >= 2)
2351 {
2352 /* chunk belongs in bin */
2353 idx = bin_index (size);
2354 assert (idx == i);
2355 /* lists are sorted */
2356 assert (p->bk == b ||
2357 (unsigned long) chunksize (p->bk) >= (unsigned long) chunksize (p));
2358
2359 if (!in_smallbin_range (size))
2360 {
2361 if (p->fd_nextsize != NULL)
2362 {
2363 if (p->fd_nextsize == p)
2364 assert (p->bk_nextsize == p);
2365 else
2366 {
2367 if (p->fd_nextsize == first (b))
2368 assert (chunksize (p) < chunksize (p->fd_nextsize));
2369 else
2370 assert (chunksize (p) > chunksize (p->fd_nextsize));
2371
2372 if (p == first (b))
2373 assert (chunksize (p) > chunksize (p->bk_nextsize));
2374 else
2375 assert (chunksize (p) < chunksize (p->bk_nextsize));
2376 }
2377 }
2378 else
2379 assert (p->bk_nextsize == NULL);
2380 }
2381 }
2382 else if (!in_smallbin_range (size))
2383 assert (p->fd_nextsize == NULL && p->bk_nextsize == NULL);
2384 /* chunk is followed by a legal chain of inuse chunks */
2385 for (q = next_chunk (p);
2386 (q != av->top && inuse (q) &&
2387 (unsigned long) (chunksize (q)) >= MINSIZE);
2388 q = next_chunk (q))
2389 do_check_inuse_chunk (av, q);
2390 }
2391 }
2392
2393 /* top chunk is OK */
2394 check_chunk (av, av->top);
2395}
2396#endif
2397
2398
2399/* ----------------- Support for debugging hooks -------------------- */
2400#if IS_IN (libc)
2401#include "hooks.c"
2402#endif
2403
2404
2405/* ----------- Routines dealing with system allocation -------------- */
2406
2407/*
2408 sysmalloc handles malloc cases requiring more memory from the system.
2409 On entry, it is assumed that av->top does not have enough
2410 space to service request for nb bytes, thus requiring that av->top
2411 be extended or replaced.
2412 */
2413
2414static void *
2415sysmalloc_mmap (INTERNAL_SIZE_T nb, size_t pagesize, int extra_flags, mstate av)
2416{
2417 long int size;
2418
2419 /*
2420 Round up size to nearest page. For mmapped chunks, the overhead is one
2421 SIZE_SZ unit larger than for normal chunks, because there is no
2422 following chunk whose prev_size field could be used.
2423
2424 See the front_misalign handling below, for glibc there is no need for
2425 further alignments unless we have have high alignment.
2426 */
2427 if (MALLOC_ALIGNMENT == CHUNK_HDR_SZ)
2428 size = ALIGN_UP (nb + SIZE_SZ, pagesize);
2429 else
2430 size = ALIGN_UP (nb + SIZE_SZ + MALLOC_ALIGN_MASK, pagesize);
2431
2432 /* Don't try if size wraps around 0. */
2433 if ((unsigned long) (size) <= (unsigned long) (nb))
2434 return MAP_FAILED;
2435
2436 char *mm = (char *) MMAP (0, size,
2437 mtag_mmap_flags | PROT_READ | PROT_WRITE,
2438 extra_flags);
2439 if (mm == MAP_FAILED)
2440 return mm;
2441
2442#ifdef MAP_HUGETLB
2443 if (!(extra_flags & MAP_HUGETLB))
2444 madvise_thp (p: mm, size);
2445#endif
2446
2447 /*
2448 The offset to the start of the mmapped region is stored in the prev_size
2449 field of the chunk. This allows us to adjust returned start address to
2450 meet alignment requirements here and in memalign(), and still be able to
2451 compute proper address argument for later munmap in free() and realloc().
2452 */
2453
2454 INTERNAL_SIZE_T front_misalign; /* unusable bytes at front of new space */
2455
2456 if (MALLOC_ALIGNMENT == CHUNK_HDR_SZ)
2457 {
2458 /* For glibc, chunk2mem increases the address by CHUNK_HDR_SZ and
2459 MALLOC_ALIGN_MASK is CHUNK_HDR_SZ-1. Each mmap'ed area is page
2460 aligned and therefore definitely MALLOC_ALIGN_MASK-aligned. */
2461 assert (((INTERNAL_SIZE_T) chunk2mem (mm) & MALLOC_ALIGN_MASK) == 0);
2462 front_misalign = 0;
2463 }
2464 else
2465 front_misalign = (INTERNAL_SIZE_T) chunk2mem (mm) & MALLOC_ALIGN_MASK;
2466
2467 mchunkptr p; /* the allocated/returned chunk */
2468
2469 if (front_misalign > 0)
2470 {
2471 ptrdiff_t correction = MALLOC_ALIGNMENT - front_misalign;
2472 p = (mchunkptr) (mm + correction);
2473 set_prev_size (p, correction);
2474 set_head (p, (size - correction) | IS_MMAPPED);
2475 }
2476 else
2477 {
2478 p = (mchunkptr) mm;
2479 set_prev_size (p, 0);
2480 set_head (p, size | IS_MMAPPED);
2481 }
2482
2483 /* update statistics */
2484 int new = atomic_exchange_and_add (&mp_.n_mmaps, 1) + 1;
2485 atomic_max (&mp_.max_n_mmaps, new);
2486
2487 unsigned long sum;
2488 sum = atomic_exchange_and_add (&mp_.mmapped_mem, size) + size;
2489 atomic_max (&mp_.max_mmapped_mem, sum);
2490
2491 check_chunk (av, p);
2492
2493 return chunk2mem (p);
2494}
2495
2496/*
2497 Allocate memory using mmap() based on S and NB requested size, aligning to
2498 PAGESIZE if required. The EXTRA_FLAGS is used on mmap() call. If the call
2499 succeedes S is updated with the allocated size. This is used as a fallback
2500 if MORECORE fails.
2501 */
2502static void *
2503sysmalloc_mmap_fallback (long int *s, INTERNAL_SIZE_T nb,
2504 INTERNAL_SIZE_T old_size, size_t minsize,
2505 size_t pagesize, int extra_flags, mstate av)
2506{
2507 long int size = *s;
2508
2509 /* Cannot merge with old top, so add its size back in */
2510 if (contiguous (av))
2511 size = ALIGN_UP (size + old_size, pagesize);
2512
2513 /* If we are relying on mmap as backup, then use larger units */
2514 if ((unsigned long) (size) < minsize)
2515 size = minsize;
2516
2517 /* Don't try if size wraps around 0 */
2518 if ((unsigned long) (size) <= (unsigned long) (nb))
2519 return MORECORE_FAILURE;
2520
2521 char *mbrk = (char *) (MMAP (0, size,
2522 mtag_mmap_flags | PROT_READ | PROT_WRITE,
2523 extra_flags));
2524 if (mbrk == MAP_FAILED)
2525 return MAP_FAILED;
2526
2527#ifdef MAP_HUGETLB
2528 if (!(extra_flags & MAP_HUGETLB))
2529 madvise_thp (p: mbrk, size);
2530#endif
2531
2532 /* Record that we no longer have a contiguous sbrk region. After the first
2533 time mmap is used as backup, we do not ever rely on contiguous space
2534 since this could incorrectly bridge regions. */
2535 set_noncontiguous (av);
2536
2537 *s = size;
2538 return mbrk;
2539}
2540
2541static void *
2542sysmalloc (INTERNAL_SIZE_T nb, mstate av)
2543{
2544 mchunkptr old_top; /* incoming value of av->top */
2545 INTERNAL_SIZE_T old_size; /* its size */
2546 char *old_end; /* its end address */
2547
2548 long size; /* arg to first MORECORE or mmap call */
2549 char *brk; /* return value from MORECORE */
2550
2551 long correction; /* arg to 2nd MORECORE call */
2552 char *snd_brk; /* 2nd return val */
2553
2554 INTERNAL_SIZE_T front_misalign; /* unusable bytes at front of new space */
2555 INTERNAL_SIZE_T end_misalign; /* partial page left at end of new space */
2556 char *aligned_brk; /* aligned offset into brk */
2557
2558 mchunkptr p; /* the allocated/returned chunk */
2559 mchunkptr remainder; /* remainder from allocation */
2560 unsigned long remainder_size; /* its size */
2561
2562
2563 size_t pagesize = GLRO (dl_pagesize);
2564 bool tried_mmap = false;
2565
2566
2567 /*
2568 If have mmap, and the request size meets the mmap threshold, and
2569 the system supports mmap, and there are few enough currently
2570 allocated mmapped regions, try to directly map this request
2571 rather than expanding top.
2572 */
2573
2574 if (av == NULL
2575 || ((unsigned long) (nb) >= (unsigned long) (mp_.mmap_threshold)
2576 && (mp_.n_mmaps < mp_.n_mmaps_max)))
2577 {
2578 char *mm;
2579#if HAVE_TUNABLES
2580 if (mp_.hp_pagesize > 0 && nb >= mp_.hp_pagesize)
2581 {
2582 /* There is no need to isse the THP madvise call if Huge Pages are
2583 used directly. */
2584 mm = sysmalloc_mmap (nb, pagesize: mp_.hp_pagesize, extra_flags: mp_.hp_flags, av);
2585 if (mm != MAP_FAILED)
2586 return mm;
2587 }
2588#endif
2589 mm = sysmalloc_mmap (nb, pagesize, extra_flags: 0, av);
2590 if (mm != MAP_FAILED)
2591 return mm;
2592 tried_mmap = true;
2593 }
2594
2595 /* There are no usable arenas and mmap also failed. */
2596 if (av == NULL)
2597 return 0;
2598
2599 /* Record incoming configuration of top */
2600
2601 old_top = av->top;
2602 old_size = chunksize (old_top);
2603 old_end = (char *) (chunk_at_offset (old_top, old_size));
2604
2605 brk = snd_brk = (char *) (MORECORE_FAILURE);
2606
2607 /*
2608 If not the first time through, we require old_size to be
2609 at least MINSIZE and to have prev_inuse set.
2610 */
2611
2612 assert ((old_top == initial_top (av) && old_size == 0) ||
2613 ((unsigned long) (old_size) >= MINSIZE &&
2614 prev_inuse (old_top) &&
2615 ((unsigned long) old_end & (pagesize - 1)) == 0));
2616
2617 /* Precondition: not enough current space to satisfy nb request */
2618 assert ((unsigned long) (old_size) < (unsigned long) (nb + MINSIZE));
2619
2620
2621 if (av != &main_arena)
2622 {
2623 heap_info *old_heap, *heap;
2624 size_t old_heap_size;
2625
2626 /* First try to extend the current heap. */
2627 old_heap = heap_for_ptr (ptr: old_top);
2628 old_heap_size = old_heap->size;
2629 if ((long) (MINSIZE + nb - old_size) > 0
2630 && grow_heap (h: old_heap, MINSIZE + nb - old_size) == 0)
2631 {
2632 av->system_mem += old_heap->size - old_heap_size;
2633 set_head (old_top, (((char *) old_heap + old_heap->size) - (char *) old_top)
2634 | PREV_INUSE);
2635 }
2636 else if ((heap = new_heap (size: nb + (MINSIZE + sizeof (*heap)), top_pad: mp_.top_pad)))
2637 {
2638 /* Use a newly allocated heap. */
2639 heap->ar_ptr = av;
2640 heap->prev = old_heap;
2641 av->system_mem += heap->size;
2642 /* Set up the new top. */
2643 top (av) = chunk_at_offset (heap, sizeof (*heap));
2644 set_head (top (av), (heap->size - sizeof (*heap)) | PREV_INUSE);
2645
2646 /* Setup fencepost and free the old top chunk with a multiple of
2647 MALLOC_ALIGNMENT in size. */
2648 /* The fencepost takes at least MINSIZE bytes, because it might
2649 become the top chunk again later. Note that a footer is set
2650 up, too, although the chunk is marked in use. */
2651 old_size = (old_size - MINSIZE) & ~MALLOC_ALIGN_MASK;
2652 set_head (chunk_at_offset (old_top, old_size + CHUNK_HDR_SZ),
2653 0 | PREV_INUSE);
2654 if (old_size >= MINSIZE)
2655 {
2656 set_head (chunk_at_offset (old_top, old_size),
2657 CHUNK_HDR_SZ | PREV_INUSE);
2658 set_foot (chunk_at_offset (old_top, old_size), CHUNK_HDR_SZ);
2659 set_head (old_top, old_size | PREV_INUSE | NON_MAIN_ARENA);
2660 _int_free (av, old_top, 1);
2661 }
2662 else
2663 {
2664 set_head (old_top, (old_size + CHUNK_HDR_SZ) | PREV_INUSE);
2665 set_foot (old_top, (old_size + CHUNK_HDR_SZ));
2666 }
2667 }
2668 else if (!tried_mmap)
2669 {
2670 /* We can at least try to use to mmap memory. If new_heap fails
2671 it is unlikely that trying to allocate huge pages will
2672 succeed. */
2673 char *mm = sysmalloc_mmap (nb, pagesize, extra_flags: 0, av);
2674 if (mm != MAP_FAILED)
2675 return mm;
2676 }
2677 }
2678 else /* av == main_arena */
2679
2680
2681 { /* Request enough space for nb + pad + overhead */
2682 size = nb + mp_.top_pad + MINSIZE;
2683
2684 /*
2685 If contiguous, we can subtract out existing space that we hope to
2686 combine with new space. We add it back later only if
2687 we don't actually get contiguous space.
2688 */
2689
2690 if (contiguous (av))
2691 size -= old_size;
2692
2693 /*
2694 Round to a multiple of page size or huge page size.
2695 If MORECORE is not contiguous, this ensures that we only call it
2696 with whole-page arguments. And if MORECORE is contiguous and
2697 this is not first time through, this preserves page-alignment of
2698 previous calls. Otherwise, we correct to page-align below.
2699 */
2700
2701#if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
2702 /* Defined in brk.c. */
2703 extern void *__curbrk;
2704 if (__glibc_unlikely (mp_.thp_pagesize != 0))
2705 {
2706 uintptr_t top = ALIGN_UP ((uintptr_t) __curbrk + size,
2707 mp_.thp_pagesize);
2708 size = top - (uintptr_t) __curbrk;
2709 }
2710 else
2711#endif
2712 size = ALIGN_UP (size, GLRO(dl_pagesize));
2713
2714 /*
2715 Don't try to call MORECORE if argument is so big as to appear
2716 negative. Note that since mmap takes size_t arg, it may succeed
2717 below even if we cannot call MORECORE.
2718 */
2719
2720 if (size > 0)
2721 {
2722 brk = (char *) (MORECORE (increment: size));
2723 if (brk != (char *) (MORECORE_FAILURE))
2724 madvise_thp (p: brk, size);
2725 LIBC_PROBE (memory_sbrk_more, 2, brk, size);
2726 }
2727
2728 if (brk == (char *) (MORECORE_FAILURE))
2729 {
2730 /*
2731 If have mmap, try using it as a backup when MORECORE fails or
2732 cannot be used. This is worth doing on systems that have "holes" in
2733 address space, so sbrk cannot extend to give contiguous space, but
2734 space is available elsewhere. Note that we ignore mmap max count
2735 and threshold limits, since the space will not be used as a
2736 segregated mmap region.
2737 */
2738
2739 char *mbrk = MAP_FAILED;
2740#if HAVE_TUNABLES
2741 if (mp_.hp_pagesize > 0)
2742 mbrk = sysmalloc_mmap_fallback (s: &size, nb, old_size,
2743 minsize: mp_.hp_pagesize, pagesize: mp_.hp_pagesize,
2744 extra_flags: mp_.hp_flags, av);
2745#endif
2746 if (mbrk == MAP_FAILED)
2747 mbrk = sysmalloc_mmap_fallback (s: &size, nb, old_size, minsize: pagesize,
2748 MMAP_AS_MORECORE_SIZE, extra_flags: 0, av);
2749 if (mbrk != MAP_FAILED)
2750 {
2751 /* We do not need, and cannot use, another sbrk call to find end */
2752 brk = mbrk;
2753 snd_brk = brk + size;
2754 }
2755 }
2756
2757 if (brk != (char *) (MORECORE_FAILURE))
2758 {
2759 if (mp_.sbrk_base == 0)
2760 mp_.sbrk_base = brk;
2761 av->system_mem += size;
2762
2763 /*
2764 If MORECORE extends previous space, we can likewise extend top size.
2765 */
2766
2767 if (brk == old_end && snd_brk == (char *) (MORECORE_FAILURE))
2768 set_head (old_top, (size + old_size) | PREV_INUSE);
2769
2770 else if (contiguous (av) && old_size && brk < old_end)
2771 /* Oops! Someone else killed our space.. Can't touch anything. */
2772 malloc_printerr (str: "break adjusted to free malloc space");
2773
2774 /*
2775 Otherwise, make adjustments:
2776
2777 * If the first time through or noncontiguous, we need to call sbrk
2778 just to find out where the end of memory lies.
2779
2780 * We need to ensure that all returned chunks from malloc will meet
2781 MALLOC_ALIGNMENT
2782
2783 * If there was an intervening foreign sbrk, we need to adjust sbrk
2784 request size to account for fact that we will not be able to
2785 combine new space with existing space in old_top.
2786
2787 * Almost all systems internally allocate whole pages at a time, in
2788 which case we might as well use the whole last page of request.
2789 So we allocate enough more memory to hit a page boundary now,
2790 which in turn causes future contiguous calls to page-align.
2791 */
2792
2793 else
2794 {
2795 front_misalign = 0;
2796 end_misalign = 0;
2797 correction = 0;
2798 aligned_brk = brk;
2799
2800 /* handle contiguous cases */
2801 if (contiguous (av))
2802 {
2803 /* Count foreign sbrk as system_mem. */
2804 if (old_size)
2805 av->system_mem += brk - old_end;
2806
2807 /* Guarantee alignment of first new chunk made from this space */
2808
2809 front_misalign = (INTERNAL_SIZE_T) chunk2mem (brk) & MALLOC_ALIGN_MASK;
2810 if (front_misalign > 0)
2811 {
2812 /*
2813 Skip over some bytes to arrive at an aligned position.
2814 We don't need to specially mark these wasted front bytes.
2815 They will never be accessed anyway because
2816 prev_inuse of av->top (and any chunk created from its start)
2817 is always true after initialization.
2818 */
2819
2820 correction = MALLOC_ALIGNMENT - front_misalign;
2821 aligned_brk += correction;
2822 }
2823
2824 /*
2825 If this isn't adjacent to existing space, then we will not
2826 be able to merge with old_top space, so must add to 2nd request.
2827 */
2828
2829 correction += old_size;
2830
2831 /* Extend the end address to hit a page boundary */
2832 end_misalign = (INTERNAL_SIZE_T) (brk + size + correction);
2833 correction += (ALIGN_UP (end_misalign, pagesize)) - end_misalign;
2834
2835 assert (correction >= 0);
2836 snd_brk = (char *) (MORECORE (increment: correction));
2837
2838 /*
2839 If can't allocate correction, try to at least find out current
2840 brk. It might be enough to proceed without failing.
2841
2842 Note that if second sbrk did NOT fail, we assume that space
2843 is contiguous with first sbrk. This is a safe assumption unless
2844 program is multithreaded but doesn't use locks and a foreign sbrk
2845 occurred between our first and second calls.
2846 */
2847
2848 if (snd_brk == (char *) (MORECORE_FAILURE))
2849 {
2850 correction = 0;
2851 snd_brk = (char *) (MORECORE (increment: 0));
2852 }
2853 else
2854 madvise_thp (p: snd_brk, size: correction);
2855 }
2856
2857 /* handle non-contiguous cases */
2858 else
2859 {
2860 if (MALLOC_ALIGNMENT == CHUNK_HDR_SZ)
2861 /* MORECORE/mmap must correctly align */
2862 assert (((unsigned long) chunk2mem (brk) & MALLOC_ALIGN_MASK) == 0);
2863 else
2864 {
2865 front_misalign = (INTERNAL_SIZE_T) chunk2mem (brk) & MALLOC_ALIGN_MASK;
2866 if (front_misalign > 0)
2867 {
2868 /*
2869 Skip over some bytes to arrive at an aligned position.
2870 We don't need to specially mark these wasted front bytes.
2871 They will never be accessed anyway because
2872 prev_inuse of av->top (and any chunk created from its start)
2873 is always true after initialization.
2874 */
2875
2876 aligned_brk += MALLOC_ALIGNMENT - front_misalign;
2877 }
2878 }
2879
2880 /* Find out current end of memory */
2881 if (snd_brk == (char *) (MORECORE_FAILURE))
2882 {
2883 snd_brk = (char *) (MORECORE (increment: 0));
2884 }
2885 }
2886
2887 /* Adjust top based on results of second sbrk */
2888 if (snd_brk != (char *) (MORECORE_FAILURE))
2889 {
2890 av->top = (mchunkptr) aligned_brk;
2891 set_head (av->top, (snd_brk - aligned_brk + correction) | PREV_INUSE);
2892 av->system_mem += correction;
2893
2894 /*
2895 If not the first time through, we either have a
2896 gap due to foreign sbrk or a non-contiguous region. Insert a
2897 double fencepost at old_top to prevent consolidation with space
2898 we don't own. These fenceposts are artificial chunks that are
2899 marked as inuse and are in any case too small to use. We need
2900 two to make sizes and alignments work out.
2901 */
2902
2903 if (old_size != 0)
2904 {
2905 /*
2906 Shrink old_top to insert fenceposts, keeping size a
2907 multiple of MALLOC_ALIGNMENT. We know there is at least
2908 enough space in old_top to do this.
2909 */
2910 old_size = (old_size - 2 * CHUNK_HDR_SZ) & ~MALLOC_ALIGN_MASK;
2911 set_head (old_top, old_size | PREV_INUSE);
2912
2913 /*
2914 Note that the following assignments completely overwrite
2915 old_top when old_size was previously MINSIZE. This is
2916 intentional. We need the fencepost, even if old_top otherwise gets
2917 lost.
2918 */
2919 set_head (chunk_at_offset (old_top, old_size),
2920 CHUNK_HDR_SZ | PREV_INUSE);
2921 set_head (chunk_at_offset (old_top,
2922 old_size + CHUNK_HDR_SZ),
2923 CHUNK_HDR_SZ | PREV_INUSE);
2924
2925 /* If possible, release the rest. */
2926 if (old_size >= MINSIZE)
2927 {
2928 _int_free (av, old_top, 1);
2929 }
2930 }
2931 }
2932 }
2933 }
2934 } /* if (av != &main_arena) */
2935
2936 if ((unsigned long) av->system_mem > (unsigned long) (av->max_system_mem))
2937 av->max_system_mem = av->system_mem;
2938 check_malloc_state (av);
2939
2940 /* finally, do the allocation */
2941 p = av->top;
2942 size = chunksize (p);
2943
2944 /* check that one of the above allocation paths succeeded */
2945 if ((unsigned long) (size) >= (unsigned long) (nb + MINSIZE))
2946 {
2947 remainder_size = size - nb;
2948 remainder = chunk_at_offset (p, nb);
2949 av->top = remainder;
2950 set_head (p, nb | PREV_INUSE | (av != &main_arena ? NON_MAIN_ARENA : 0));
2951 set_head (remainder, remainder_size | PREV_INUSE);
2952 check_malloced_chunk (av, p, nb);
2953 return chunk2mem (p);
2954 }
2955
2956 /* catch all failure paths */
2957 __set_errno (ENOMEM);
2958 return 0;
2959}
2960
2961
2962/*
2963 systrim is an inverse of sorts to sysmalloc. It gives memory back
2964 to the system (via negative arguments to sbrk) if there is unused
2965 memory at the `high' end of the malloc pool. It is called
2966 automatically by free() when top space exceeds the trim
2967 threshold. It is also called by the public malloc_trim routine. It
2968 returns 1 if it actually released any memory, else 0.
2969 */
2970
2971static int
2972systrim (size_t pad, mstate av)
2973{
2974 long top_size; /* Amount of top-most memory */
2975 long extra; /* Amount to release */
2976 long released; /* Amount actually released */
2977 char *current_brk; /* address returned by pre-check sbrk call */
2978 char *new_brk; /* address returned by post-check sbrk call */
2979 long top_area;
2980
2981 top_size = chunksize (av->top);
2982
2983 top_area = top_size - MINSIZE - 1;
2984 if (top_area <= pad)
2985 return 0;
2986
2987 /* Release in pagesize units and round down to the nearest page. */
2988#if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
2989 if (__glibc_unlikely (mp_.thp_pagesize != 0))
2990 extra = ALIGN_DOWN (top_area - pad, mp_.thp_pagesize);
2991 else
2992#endif
2993 extra = ALIGN_DOWN (top_area - pad, GLRO(dl_pagesize));
2994
2995 if (extra == 0)
2996 return 0;
2997
2998 /*
2999 Only proceed if end of memory is where we last set it.
3000 This avoids problems if there were foreign sbrk calls.
3001 */
3002 current_brk = (char *) (MORECORE (increment: 0));
3003 if (current_brk == (char *) (av->top) + top_size)
3004 {
3005 /*
3006 Attempt to release memory. We ignore MORECORE return value,
3007 and instead call again to find out where new end of memory is.
3008 This avoids problems if first call releases less than we asked,
3009 of if failure somehow altered brk value. (We could still
3010 encounter problems if it altered brk in some very bad way,
3011 but the only thing we can do is adjust anyway, which will cause
3012 some downstream failure.)
3013 */
3014
3015 MORECORE (increment: -extra);
3016 new_brk = (char *) (MORECORE (increment: 0));
3017
3018 LIBC_PROBE (memory_sbrk_less, 2, new_brk, extra);
3019
3020 if (new_brk != (char *) MORECORE_FAILURE)
3021 {
3022 released = (long) (current_brk - new_brk);
3023
3024 if (released != 0)
3025 {
3026 /* Success. Adjust top. */
3027 av->system_mem -= released;
3028 set_head (av->top, (top_size - released) | PREV_INUSE);
3029 check_malloc_state (av);
3030 return 1;
3031 }
3032 }
3033 }
3034 return 0;
3035}
3036
3037static void
3038munmap_chunk (mchunkptr p)
3039{
3040 size_t pagesize = GLRO (dl_pagesize);
3041 INTERNAL_SIZE_T size = chunksize (p);
3042
3043 assert (chunk_is_mmapped (p));
3044
3045 uintptr_t mem = (uintptr_t) chunk2mem (p);
3046 uintptr_t block = (uintptr_t) p - prev_size (p);
3047 size_t total_size = prev_size (p) + size;
3048 /* Unfortunately we have to do the compilers job by hand here. Normally
3049 we would test BLOCK and TOTAL-SIZE separately for compliance with the
3050 page size. But gcc does not recognize the optimization possibility
3051 (in the moment at least) so we combine the two values into one before
3052 the bit test. */
3053 if (__glibc_unlikely ((block | total_size) & (pagesize - 1)) != 0
3054 || __glibc_unlikely (!powerof2 (mem & (pagesize - 1))))
3055 malloc_printerr (str: "munmap_chunk(): invalid pointer");
3056
3057 atomic_decrement (&mp_.n_mmaps);
3058 atomic_add (&mp_.mmapped_mem, -total_size);
3059
3060 /* If munmap failed the process virtual memory address space is in a
3061 bad shape. Just leave the block hanging around, the process will
3062 terminate shortly anyway since not much can be done. */
3063 __munmap (addr: (char *) block, len: total_size);
3064}
3065
3066#if HAVE_MREMAP
3067
3068static mchunkptr
3069mremap_chunk (mchunkptr p, size_t new_size)
3070{
3071 size_t pagesize = GLRO (dl_pagesize);
3072 INTERNAL_SIZE_T offset = prev_size (p);
3073 INTERNAL_SIZE_T size = chunksize (p);
3074 char *cp;
3075
3076 assert (chunk_is_mmapped (p));
3077
3078 uintptr_t block = (uintptr_t) p - offset;
3079 uintptr_t mem = (uintptr_t) chunk2mem(p);
3080 size_t total_size = offset + size;
3081 if (__glibc_unlikely ((block | total_size) & (pagesize - 1)) != 0
3082 || __glibc_unlikely (!powerof2 (mem & (pagesize - 1))))
3083 malloc_printerr(str: "mremap_chunk(): invalid pointer");
3084
3085 /* Note the extra SIZE_SZ overhead as in mmap_chunk(). */
3086 new_size = ALIGN_UP (new_size + offset + SIZE_SZ, pagesize);
3087
3088 /* No need to remap if the number of pages does not change. */
3089 if (total_size == new_size)
3090 return p;
3091
3092 cp = (char *) __mremap (addr: (char *) block, old_len: total_size, new_len: new_size,
3093 MREMAP_MAYMOVE);
3094
3095 if (cp == MAP_FAILED)
3096 return 0;
3097
3098 madvise_thp (p: cp, size: new_size);
3099
3100 p = (mchunkptr) (cp + offset);
3101
3102 assert (aligned_OK (chunk2mem (p)));
3103
3104 assert (prev_size (p) == offset);
3105 set_head (p, (new_size - offset) | IS_MMAPPED);
3106
3107 INTERNAL_SIZE_T new;
3108 new = atomic_exchange_and_add (&mp_.mmapped_mem, new_size - size - offset)
3109 + new_size - size - offset;
3110 atomic_max (&mp_.max_mmapped_mem, new);
3111 return p;
3112}
3113#endif /* HAVE_MREMAP */
3114
3115/*------------------------ Public wrappers. --------------------------------*/
3116
3117#if USE_TCACHE
3118
3119/* We overlay this structure on the user-data portion of a chunk when
3120 the chunk is stored in the per-thread cache. */
3121typedef struct tcache_entry
3122{
3123 struct tcache_entry *next;
3124 /* This field exists to detect double frees. */
3125 uintptr_t key;
3126} tcache_entry;
3127
3128/* There is one of these for each thread, which contains the
3129 per-thread cache (hence "tcache_perthread_struct"). Keeping
3130 overall size low is mildly important. Note that COUNTS and ENTRIES
3131 are redundant (we could have just counted the linked list each
3132 time), this is for performance reasons. */
3133typedef struct tcache_perthread_struct
3134{
3135 uint16_t counts[TCACHE_MAX_BINS];
3136 tcache_entry *entries[TCACHE_MAX_BINS];
3137} tcache_perthread_struct;
3138
3139static __thread bool tcache_shutting_down = false;
3140static __thread tcache_perthread_struct *tcache = NULL;
3141
3142/* Process-wide key to try and catch a double-free in the same thread. */
3143static uintptr_t tcache_key;
3144
3145/* The value of tcache_key does not really have to be a cryptographically
3146 secure random number. It only needs to be arbitrary enough so that it does
3147 not collide with values present in applications. If a collision does happen
3148 consistently enough, it could cause a degradation in performance since the
3149 entire list is checked to check if the block indeed has been freed the
3150 second time. The odds of this happening are exceedingly low though, about 1
3151 in 2^wordsize. There is probably a higher chance of the performance
3152 degradation being due to a double free where the first free happened in a
3153 different thread; that's a case this check does not cover. */
3154static void
3155tcache_key_initialize (void)
3156{
3157 if (__getrandom (&tcache_key, sizeof(tcache_key), GRND_NONBLOCK)
3158 != sizeof (tcache_key))
3159 {
3160 tcache_key = random_bits ();
3161#if __WORDSIZE == 64
3162 tcache_key = (tcache_key << 32) | random_bits ();
3163#endif
3164 }
3165}
3166
3167/* Caller must ensure that we know tc_idx is valid and there's room
3168 for more chunks. */
3169static __always_inline void
3170tcache_put (mchunkptr chunk, size_t tc_idx)
3171{
3172 tcache_entry *e = (tcache_entry *) chunk2mem (chunk);
3173
3174 /* Mark this chunk as "in the tcache" so the test in _int_free will
3175 detect a double free. */
3176 e->key = tcache_key;
3177
3178 e->next = PROTECT_PTR (&e->next, tcache->entries[tc_idx]);
3179 tcache->entries[tc_idx] = e;
3180 ++(tcache->counts[tc_idx]);
3181}
3182
3183/* Caller must ensure that we know tc_idx is valid and there's
3184 available chunks to remove. */
3185static __always_inline void *
3186tcache_get (size_t tc_idx)
3187{
3188 tcache_entry *e = tcache->entries[tc_idx];
3189 if (__glibc_unlikely (!aligned_OK (e)))
3190 malloc_printerr ("malloc(): unaligned tcache chunk detected");
3191 tcache->entries[tc_idx] = REVEAL_PTR (e->next);
3192 --(tcache->counts[tc_idx]);
3193 e->key = 0;
3194 return (void *) e;
3195}
3196
3197static void
3198tcache_thread_shutdown (void)
3199{
3200 int i;
3201 tcache_perthread_struct *tcache_tmp = tcache;
3202
3203 tcache_shutting_down = true;
3204
3205 if (!tcache)
3206 return;
3207
3208 /* Disable the tcache and prevent it from being reinitialized. */
3209 tcache = NULL;
3210
3211 /* Free all of the entries and the tcache itself back to the arena
3212 heap for coalescing. */
3213 for (i = 0; i < TCACHE_MAX_BINS; ++i)
3214 {
3215 while (tcache_tmp->entries[i])
3216 {
3217 tcache_entry *e = tcache_tmp->entries[i];
3218 if (__glibc_unlikely (!aligned_OK (e)))
3219 malloc_printerr ("tcache_thread_shutdown(): "
3220 "unaligned tcache chunk detected");
3221 tcache_tmp->entries[i] = REVEAL_PTR (e->next);
3222 __libc_free (e);
3223 }
3224 }
3225
3226 __libc_free (tcache_tmp);
3227}
3228
3229static void
3230tcache_init(void)
3231{
3232 mstate ar_ptr;
3233 void *victim = 0;
3234 const size_t bytes = sizeof (tcache_perthread_struct);
3235
3236 if (tcache_shutting_down)
3237 return;
3238
3239 arena_get (ar_ptr, bytes);
3240 victim = _int_malloc (ar_ptr, bytes);
3241 if (!victim && ar_ptr != NULL)
3242 {
3243 ar_ptr = arena_get_retry (ar_ptr, bytes);
3244 victim = _int_malloc (ar_ptr, bytes);
3245 }
3246
3247
3248 if (ar_ptr != NULL)
3249 __libc_lock_unlock (ar_ptr->mutex);
3250
3251 /* In a low memory situation, we may not be able to allocate memory
3252 - in which case, we just keep trying later. However, we
3253 typically do this very early, so either there is sufficient
3254 memory, or there isn't enough memory to do non-trivial
3255 allocations anyway. */
3256 if (victim)
3257 {
3258 tcache = (tcache_perthread_struct *) victim;
3259 memset (tcache, 0, sizeof (tcache_perthread_struct));
3260 }
3261
3262}
3263
3264# define MAYBE_INIT_TCACHE() \
3265 if (__glibc_unlikely (tcache == NULL)) \
3266 tcache_init();
3267
3268#else /* !USE_TCACHE */
3269# define MAYBE_INIT_TCACHE()
3270
3271static void
3272tcache_thread_shutdown (void)
3273{
3274 /* Nothing to do if there is no thread cache. */
3275}
3276
3277#endif /* !USE_TCACHE */
3278
3279#if IS_IN (libc)
3280void *
3281__libc_malloc (size_t bytes)
3282{
3283 mstate ar_ptr;
3284 void *victim;
3285
3286 _Static_assert (PTRDIFF_MAX <= SIZE_MAX / 2,
3287 "PTRDIFF_MAX is not more than half of SIZE_MAX");
3288
3289 if (!__malloc_initialized)
3290 ptmalloc_init ();
3291#if USE_TCACHE
3292 /* int_free also calls request2size, be careful to not pad twice. */
3293 size_t tbytes;
3294 if (!checked_request2size (bytes, &tbytes))
3295 {
3296 __set_errno (ENOMEM);
3297 return NULL;
3298 }
3299 size_t tc_idx = csize2tidx (tbytes);
3300
3301 MAYBE_INIT_TCACHE ();
3302
3303 DIAG_PUSH_NEEDS_COMMENT;
3304 if (tc_idx < mp_.tcache_bins
3305 && tcache
3306 && tcache->counts[tc_idx] > 0)
3307 {
3308 victim = tcache_get (tc_idx);
3309 return tag_new_usable (victim);
3310 }
3311 DIAG_POP_NEEDS_COMMENT;
3312#endif
3313
3314 if (SINGLE_THREAD_P)
3315 {
3316 victim = tag_new_usable (_int_malloc (&main_arena, bytes));
3317 assert (!victim || chunk_is_mmapped (mem2chunk (victim)) ||
3318 &main_arena == arena_for_chunk (mem2chunk (victim)));
3319 return victim;
3320 }
3321
3322 arena_get (ar_ptr, bytes);
3323
3324 victim = _int_malloc (ar_ptr, bytes);
3325 /* Retry with another arena only if we were able to find a usable arena
3326 before. */
3327 if (!victim && ar_ptr != NULL)
3328 {
3329 LIBC_PROBE (memory_malloc_retry, 1, bytes);
3330 ar_ptr = arena_get_retry (ar_ptr, bytes);
3331 victim = _int_malloc (ar_ptr, bytes);
3332 }
3333
3334 if (ar_ptr != NULL)
3335 __libc_lock_unlock (ar_ptr->mutex);
3336
3337 victim = tag_new_usable (victim);
3338
3339 assert (!victim || chunk_is_mmapped (mem2chunk (victim)) ||
3340 ar_ptr == arena_for_chunk (mem2chunk (victim)));
3341 return victim;
3342}
3343libc_hidden_def (__libc_malloc)
3344
3345void
3346__libc_free (void *mem)
3347{
3348 mstate ar_ptr;
3349 mchunkptr p; /* chunk corresponding to mem */
3350
3351 if (mem == 0) /* free(0) has no effect */
3352 return;
3353
3354 /* Quickly check that the freed pointer matches the tag for the memory.
3355 This gives a useful double-free detection. */
3356 if (__glibc_unlikely (mtag_enabled))
3357 *(volatile char *)mem;
3358
3359 int err = errno;
3360
3361 p = mem2chunk (mem);
3362
3363 if (chunk_is_mmapped (p)) /* release mmapped memory. */
3364 {
3365 /* See if the dynamic brk/mmap threshold needs adjusting.
3366 Dumped fake mmapped chunks do not affect the threshold. */
3367 if (!mp_.no_dyn_threshold
3368 && chunksize_nomask (p) > mp_.mmap_threshold
3369 && chunksize_nomask (p) <= DEFAULT_MMAP_THRESHOLD_MAX)
3370 {
3371 mp_.mmap_threshold = chunksize (p);
3372 mp_.trim_threshold = 2 * mp_.mmap_threshold;
3373 LIBC_PROBE (memory_mallopt_free_dyn_thresholds, 2,
3374 mp_.mmap_threshold, mp_.trim_threshold);
3375 }
3376 munmap_chunk (p);
3377 }
3378 else
3379 {
3380 MAYBE_INIT_TCACHE ();
3381
3382 /* Mark the chunk as belonging to the library again. */
3383 (void)tag_region (chunk2mem (p), memsize (p));
3384
3385 ar_ptr = arena_for_chunk (p);
3386 _int_free (ar_ptr, p, 0);
3387 }
3388
3389 __set_errno (err);
3390}
3391libc_hidden_def (__libc_free)
3392
3393void *
3394__libc_realloc (void *oldmem, size_t bytes)
3395{
3396 mstate ar_ptr;
3397 INTERNAL_SIZE_T nb; /* padded request size */
3398
3399 void *newp; /* chunk to return */
3400
3401 if (!__malloc_initialized)
3402 ptmalloc_init ();
3403
3404#if REALLOC_ZERO_BYTES_FREES
3405 if (bytes == 0 && oldmem != NULL)
3406 {
3407 __libc_free (oldmem); return 0;
3408 }
3409#endif
3410
3411 /* realloc of null is supposed to be same as malloc */
3412 if (oldmem == 0)
3413 return __libc_malloc (bytes);
3414
3415 /* Perform a quick check to ensure that the pointer's tag matches the
3416 memory's tag. */
3417 if (__glibc_unlikely (mtag_enabled))
3418 *(volatile char*) oldmem;
3419
3420 /* chunk corresponding to oldmem */
3421 const mchunkptr oldp = mem2chunk (oldmem);
3422 /* its size */
3423 const INTERNAL_SIZE_T oldsize = chunksize (oldp);
3424
3425 if (chunk_is_mmapped (oldp))
3426 ar_ptr = NULL;
3427 else
3428 {
3429 MAYBE_INIT_TCACHE ();
3430 ar_ptr = arena_for_chunk (oldp);
3431 }
3432
3433 /* Little security check which won't hurt performance: the allocator
3434 never wrapps around at the end of the address space. Therefore
3435 we can exclude some size values which might appear here by
3436 accident or by "design" from some intruder. */
3437 if ((__builtin_expect ((uintptr_t) oldp > (uintptr_t) -oldsize, 0)
3438 || __builtin_expect (misaligned_chunk (oldp), 0)))
3439 malloc_printerr ("realloc(): invalid pointer");
3440
3441 if (!checked_request2size (bytes, &nb))
3442 {
3443 __set_errno (ENOMEM);
3444 return NULL;
3445 }
3446
3447 if (chunk_is_mmapped (oldp))
3448 {
3449 void *newmem;
3450
3451#if HAVE_MREMAP
3452 newp = mremap_chunk (oldp, nb);
3453 if (newp)
3454 {
3455 void *newmem = chunk2mem_tag (newp);
3456 /* Give the new block a different tag. This helps to ensure
3457 that stale handles to the previous mapping are not
3458 reused. There's a performance hit for both us and the
3459 caller for doing this, so we might want to
3460 reconsider. */
3461 return tag_new_usable (newmem);
3462 }
3463#endif
3464 /* Note the extra SIZE_SZ overhead. */
3465 if (oldsize - SIZE_SZ >= nb)
3466 return oldmem; /* do nothing */
3467
3468 /* Must alloc, copy, free. */
3469 newmem = __libc_malloc (bytes);
3470 if (newmem == 0)
3471 return 0; /* propagate failure */
3472
3473 memcpy (newmem, oldmem, oldsize - CHUNK_HDR_SZ);
3474 munmap_chunk (oldp);
3475 return newmem;
3476 }
3477
3478 if (SINGLE_THREAD_P)
3479 {
3480 newp = _int_realloc (ar_ptr, oldp, oldsize, nb);
3481 assert (!newp || chunk_is_mmapped (mem2chunk (newp)) ||
3482 ar_ptr == arena_for_chunk (mem2chunk (newp)));
3483
3484 return newp;
3485 }
3486
3487 __libc_lock_lock (ar_ptr->mutex);
3488
3489 newp = _int_realloc (ar_ptr, oldp, oldsize, nb);
3490
3491 __libc_lock_unlock (ar_ptr->mutex);
3492 assert (!newp || chunk_is_mmapped (mem2chunk (newp)) ||
3493 ar_ptr == arena_for_chunk (mem2chunk (newp)));
3494
3495 if (newp == NULL)
3496 {
3497 /* Try harder to allocate memory in other arenas. */
3498 LIBC_PROBE (memory_realloc_retry, 2, bytes, oldmem);
3499 newp = __libc_malloc (bytes);
3500 if (newp != NULL)
3501 {
3502 size_t sz = memsize (oldp);
3503 memcpy (newp, oldmem, sz);
3504 (void) tag_region (chunk2mem (oldp), sz);
3505 _int_free (ar_ptr, oldp, 0);
3506 }
3507 }
3508
3509 return newp;
3510}
3511libc_hidden_def (__libc_realloc)
3512
3513void *
3514__libc_memalign (size_t alignment, size_t bytes)
3515{
3516 if (!__malloc_initialized)
3517 ptmalloc_init ();
3518
3519 void *address = RETURN_ADDRESS (0);
3520 return _mid_memalign (alignment, bytes, address);
3521}
3522
3523static void *
3524_mid_memalign (size_t alignment, size_t bytes, void *address)
3525{
3526 mstate ar_ptr;
3527 void *p;
3528
3529 /* If we need less alignment than we give anyway, just relay to malloc. */
3530 if (alignment <= MALLOC_ALIGNMENT)
3531 return __libc_malloc (bytes);
3532
3533 /* Otherwise, ensure that it is at least a minimum chunk size */
3534 if (alignment < MINSIZE)
3535 alignment = MINSIZE;
3536
3537 /* If the alignment is greater than SIZE_MAX / 2 + 1 it cannot be a
3538 power of 2 and will cause overflow in the check below. */
3539 if (alignment > SIZE_MAX / 2 + 1)
3540 {
3541 __set_errno (EINVAL);
3542 return 0;
3543 }
3544
3545
3546 /* Make sure alignment is power of 2. */
3547 if (!powerof2 (alignment))
3548 {
3549 size_t a = MALLOC_ALIGNMENT * 2;
3550 while (a < alignment)
3551 a <<= 1;
3552 alignment = a;
3553 }
3554
3555 if (SINGLE_THREAD_P)
3556 {
3557 p = _int_memalign (&main_arena, alignment, bytes);
3558 assert (!p || chunk_is_mmapped (mem2chunk (p)) ||
3559 &main_arena == arena_for_chunk (mem2chunk (p)));
3560 return tag_new_usable (p);
3561 }
3562
3563 arena_get (ar_ptr, bytes + alignment + MINSIZE);
3564
3565 p = _int_memalign (ar_ptr, alignment, bytes);
3566 if (!p && ar_ptr != NULL)
3567 {
3568 LIBC_PROBE (memory_memalign_retry, 2, bytes, alignment);
3569 ar_ptr = arena_get_retry (ar_ptr, bytes);
3570 p = _int_memalign (ar_ptr, alignment, bytes);
3571 }
3572
3573 if (ar_ptr != NULL)
3574 __libc_lock_unlock (ar_ptr->mutex);
3575
3576 assert (!p || chunk_is_mmapped (mem2chunk (p)) ||
3577 ar_ptr == arena_for_chunk (mem2chunk (p)));
3578 return tag_new_usable (p);
3579}
3580/* For ISO C11. */
3581weak_alias (__libc_memalign, aligned_alloc)
3582libc_hidden_def (__libc_memalign)
3583
3584void *
3585__libc_valloc (size_t bytes)
3586{
3587 if (!__malloc_initialized)
3588 ptmalloc_init ();
3589
3590 void *address = RETURN_ADDRESS (0);
3591 size_t pagesize = GLRO (dl_pagesize);
3592 return _mid_memalign (pagesize, bytes, address);
3593}
3594
3595void *
3596__libc_pvalloc (size_t bytes)
3597{
3598 if (!__malloc_initialized)
3599 ptmalloc_init ();
3600
3601 void *address = RETURN_ADDRESS (0);
3602 size_t pagesize = GLRO (dl_pagesize);
3603 size_t rounded_bytes;
3604 /* ALIGN_UP with overflow check. */
3605 if (__glibc_unlikely (__builtin_add_overflow (bytes,
3606 pagesize - 1,
3607 &rounded_bytes)))
3608 {
3609 __set_errno (ENOMEM);
3610 return 0;
3611 }
3612 rounded_bytes = rounded_bytes & -(pagesize - 1);
3613
3614 return _mid_memalign (pagesize, rounded_bytes, address);
3615}
3616
3617void *
3618__libc_calloc (size_t n, size_t elem_size)
3619{
3620 mstate av;
3621 mchunkptr oldtop;
3622 INTERNAL_SIZE_T sz, oldtopsize;
3623 void *mem;
3624 unsigned long clearsize;
3625 unsigned long nclears;
3626 INTERNAL_SIZE_T *d;
3627 ptrdiff_t bytes;
3628
3629 if (__glibc_unlikely (__builtin_mul_overflow (n, elem_size, &bytes)))
3630 {
3631 __set_errno (ENOMEM);
3632 return NULL;
3633 }
3634
3635 sz = bytes;
3636
3637 if (!__malloc_initialized)
3638 ptmalloc_init ();
3639
3640 MAYBE_INIT_TCACHE ();
3641
3642 if (SINGLE_THREAD_P)
3643 av = &main_arena;
3644 else
3645 arena_get (av, sz);
3646
3647 if (av)
3648 {
3649 /* Check if we hand out the top chunk, in which case there may be no
3650 need to clear. */
3651#if MORECORE_CLEARS
3652 oldtop = top (av);
3653 oldtopsize = chunksize (top (av));
3654# if MORECORE_CLEARS < 2
3655 /* Only newly allocated memory is guaranteed to be cleared. */
3656 if (av == &main_arena &&
3657 oldtopsize < mp_.sbrk_base + av->max_system_mem - (char *) oldtop)
3658 oldtopsize = (mp_.sbrk_base + av->max_system_mem - (char *) oldtop);
3659# endif
3660 if (av != &main_arena)
3661 {
3662 heap_info *heap = heap_for_ptr (oldtop);
3663 if (oldtopsize < (char *) heap + heap->mprotect_size - (char *) oldtop)
3664 oldtopsize = (char *) heap + heap->mprotect_size - (char *) oldtop;
3665 }
3666#endif
3667 }
3668 else
3669 {
3670 /* No usable arenas. */
3671 oldtop = 0;
3672 oldtopsize = 0;
3673 }
3674 mem = _int_malloc (av, sz);
3675
3676 assert (!mem || chunk_is_mmapped (mem2chunk (mem)) ||
3677 av == arena_for_chunk (mem2chunk (mem)));
3678
3679 if (!SINGLE_THREAD_P)
3680 {
3681 if (mem == 0 && av != NULL)
3682 {
3683 LIBC_PROBE (memory_calloc_retry, 1, sz);
3684 av = arena_get_retry (av, sz);
3685 mem = _int_malloc (av, sz);
3686 }
3687
3688 if (av != NULL)
3689 __libc_lock_unlock (av->mutex);
3690 }
3691
3692 /* Allocation failed even after a retry. */
3693 if (mem == 0)
3694 return 0;
3695
3696 mchunkptr p = mem2chunk (mem);
3697
3698 /* If we are using memory tagging, then we need to set the tags
3699 regardless of MORECORE_CLEARS, so we zero the whole block while
3700 doing so. */
3701 if (__glibc_unlikely (mtag_enabled))
3702 return tag_new_zero_region (mem, memsize (p));
3703
3704 INTERNAL_SIZE_T csz = chunksize (p);
3705
3706 /* Two optional cases in which clearing not necessary */
3707 if (chunk_is_mmapped (p))
3708 {
3709 if (__builtin_expect (perturb_byte, 0))
3710 return memset (mem, 0, sz);
3711
3712 return mem;
3713 }
3714
3715#if MORECORE_CLEARS
3716 if (perturb_byte == 0 && (p == oldtop && csz > oldtopsize))
3717 {
3718 /* clear only the bytes from non-freshly-sbrked memory */
3719 csz = oldtopsize;
3720 }
3721#endif
3722
3723 /* Unroll clear of <= 36 bytes (72 if 8byte sizes). We know that
3724 contents have an odd number of INTERNAL_SIZE_T-sized words;
3725 minimally 3. */
3726 d = (INTERNAL_SIZE_T *) mem;
3727 clearsize = csz - SIZE_SZ;
3728 nclears = clearsize / sizeof (INTERNAL_SIZE_T);
3729 assert (nclears >= 3);
3730
3731 if (nclears > 9)
3732 return memset (d, 0, clearsize);
3733
3734 else
3735 {
3736 *(d + 0) = 0;
3737 *(d + 1) = 0;
3738 *(d + 2) = 0;
3739 if (nclears > 4)
3740 {
3741 *(d + 3) = 0;
3742 *(d + 4) = 0;
3743 if (nclears > 6)
3744 {
3745 *(d + 5) = 0;
3746 *(d + 6) = 0;
3747 if (nclears > 8)
3748 {
3749 *(d + 7) = 0;
3750 *(d + 8) = 0;
3751 }
3752 }
3753 }
3754 }
3755
3756 return mem;
3757}
3758#endif /* IS_IN (libc) */
3759
3760/*
3761 ------------------------------ malloc ------------------------------
3762 */
3763
3764static void *
3765_int_malloc (mstate av, size_t bytes)
3766{
3767 INTERNAL_SIZE_T nb; /* normalized request size */
3768 unsigned int idx; /* associated bin index */
3769 mbinptr bin; /* associated bin */
3770
3771 mchunkptr victim; /* inspected/selected chunk */
3772 INTERNAL_SIZE_T size; /* its size */
3773 int victim_index; /* its bin index */
3774
3775 mchunkptr remainder; /* remainder from a split */
3776 unsigned long remainder_size; /* its size */
3777
3778 unsigned int block; /* bit map traverser */
3779 unsigned int bit; /* bit map traverser */
3780 unsigned int map; /* current word of binmap */
3781
3782 mchunkptr fwd; /* misc temp for linking */
3783 mchunkptr bck; /* misc temp for linking */
3784
3785#if USE_TCACHE
3786 size_t tcache_unsorted_count; /* count of unsorted chunks processed */
3787#endif
3788
3789 /*
3790 Convert request size to internal form by adding SIZE_SZ bytes
3791 overhead plus possibly more to obtain necessary alignment and/or
3792 to obtain a size of at least MINSIZE, the smallest allocatable
3793 size. Also, checked_request2size returns false for request sizes
3794 that are so large that they wrap around zero when padded and
3795 aligned.
3796 */
3797
3798 if (!checked_request2size (req: bytes, sz: &nb))
3799 {
3800 __set_errno (ENOMEM);
3801 return NULL;
3802 }
3803
3804 /* There are no usable arenas. Fall back to sysmalloc to get a chunk from
3805 mmap. */
3806 if (__glibc_unlikely (av == NULL))
3807 {
3808 void *p = sysmalloc (nb, av);
3809 if (p != NULL)
3810 alloc_perturb (p, n: bytes);
3811 return p;
3812 }
3813
3814 /*
3815 If the size qualifies as a fastbin, first check corresponding bin.
3816 This code is safe to execute even if av is not yet initialized, so we
3817 can try it without checking, which saves some time on this fast path.
3818 */
3819
3820#define REMOVE_FB(fb, victim, pp) \
3821 do \
3822 { \
3823 victim = pp; \
3824 if (victim == NULL) \
3825 break; \
3826 pp = REVEAL_PTR (victim->fd); \
3827 if (__glibc_unlikely (pp != NULL && misaligned_chunk (pp))) \
3828 malloc_printerr ("malloc(): unaligned fastbin chunk detected"); \
3829 } \
3830 while ((pp = catomic_compare_and_exchange_val_acq (fb, pp, victim)) \
3831 != victim); \
3832
3833 if ((unsigned long) (nb) <= (unsigned long) (get_max_fast ()))
3834 {
3835 idx = fastbin_index (nb);
3836 mfastbinptr *fb = &fastbin (av, idx);
3837 mchunkptr pp;
3838 victim = *fb;
3839
3840 if (victim != NULL)
3841 {
3842 if (__glibc_unlikely (misaligned_chunk (victim)))
3843 malloc_printerr (str: "malloc(): unaligned fastbin chunk detected 2");
3844
3845 if (SINGLE_THREAD_P)
3846 *fb = REVEAL_PTR (victim->fd);
3847 else
3848 REMOVE_FB (fb, pp, victim);
3849 if (__glibc_likely (victim != NULL))
3850 {
3851 size_t victim_idx = fastbin_index (chunksize (victim));
3852 if (__builtin_expect (victim_idx != idx, 0))
3853 malloc_printerr (str: "malloc(): memory corruption (fast)");
3854 check_remalloced_chunk (av, victim, nb);
3855#if USE_TCACHE
3856 /* While we're here, if we see other chunks of the same size,
3857 stash them in the tcache. */
3858 size_t tc_idx = csize2tidx (nb);
3859 if (tcache && tc_idx < mp_.tcache_bins)
3860 {
3861 mchunkptr tc_victim;
3862
3863 /* While bin not empty and tcache not full, copy chunks. */
3864 while (tcache->counts[tc_idx] < mp_.tcache_count
3865 && (tc_victim = *fb) != NULL)
3866 {
3867 if (__glibc_unlikely (misaligned_chunk (tc_victim)))
3868 malloc_printerr ("malloc(): unaligned fastbin chunk detected 3");
3869 if (SINGLE_THREAD_P)
3870 *fb = REVEAL_PTR (tc_victim->fd);
3871 else
3872 {
3873 REMOVE_FB (fb, pp, tc_victim);
3874 if (__glibc_unlikely (tc_victim == NULL))
3875 break;
3876 }
3877 tcache_put (tc_victim, tc_idx);
3878 }
3879 }
3880#endif
3881 void *p = chunk2mem (victim);
3882 alloc_perturb (p, n: bytes);
3883 return p;
3884 }
3885 }
3886 }
3887
3888 /*
3889 If a small request, check regular bin. Since these "smallbins"
3890 hold one size each, no searching within bins is necessary.
3891 (For a large request, we need to wait until unsorted chunks are
3892 processed to find best fit. But for small ones, fits are exact
3893 anyway, so we can check now, which is faster.)
3894 */
3895
3896 if (in_smallbin_range (nb))
3897 {
3898 idx = smallbin_index (nb);
3899 bin = bin_at (av, idx);
3900
3901 if ((victim = last (bin)) != bin)
3902 {
3903 bck = victim->bk;
3904 if (__glibc_unlikely (bck->fd != victim))
3905 malloc_printerr (str: "malloc(): smallbin double linked list corrupted");
3906 set_inuse_bit_at_offset (victim, nb);
3907 bin->bk = bck;
3908 bck->fd = bin;
3909
3910 if (av != &main_arena)
3911 set_non_main_arena (victim);
3912 check_malloced_chunk (av, victim, nb);
3913#if USE_TCACHE
3914 /* While we're here, if we see other chunks of the same size,
3915 stash them in the tcache. */
3916 size_t tc_idx = csize2tidx (nb);
3917 if (tcache && tc_idx < mp_.tcache_bins)
3918 {
3919 mchunkptr tc_victim;
3920
3921 /* While bin not empty and tcache not full, copy chunks over. */
3922 while (tcache->counts[tc_idx] < mp_.tcache_count
3923 && (tc_victim = last (bin)) != bin)
3924 {
3925 if (tc_victim != 0)
3926 {
3927 bck = tc_victim->bk;
3928 set_inuse_bit_at_offset (tc_victim, nb);
3929 if (av != &main_arena)
3930 set_non_main_arena (tc_victim);
3931 bin->bk = bck;
3932 bck->fd = bin;
3933
3934 tcache_put (tc_victim, tc_idx);
3935 }
3936 }
3937 }
3938#endif
3939 void *p = chunk2mem (victim);
3940 alloc_perturb (p, n: bytes);
3941 return p;
3942 }
3943 }
3944
3945 /*
3946 If this is a large request, consolidate fastbins before continuing.
3947 While it might look excessive to kill all fastbins before
3948 even seeing if there is space available, this avoids
3949 fragmentation problems normally associated with fastbins.
3950 Also, in practice, programs tend to have runs of either small or
3951 large requests, but less often mixtures, so consolidation is not
3952 invoked all that often in most programs. And the programs that
3953 it is called frequently in otherwise tend to fragment.
3954 */
3955
3956 else
3957 {
3958 idx = largebin_index (nb);
3959 if (atomic_load_relaxed (&av->have_fastchunks))
3960 malloc_consolidate (av);
3961 }
3962
3963 /*
3964 Process recently freed or remaindered chunks, taking one only if
3965 it is exact fit, or, if this a small request, the chunk is remainder from
3966 the most recent non-exact fit. Place other traversed chunks in
3967 bins. Note that this step is the only place in any routine where
3968 chunks are placed in bins.
3969
3970 The outer loop here is needed because we might not realize until
3971 near the end of malloc that we should have consolidated, so must
3972 do so and retry. This happens at most once, and only when we would
3973 otherwise need to expand memory to service a "small" request.
3974 */
3975
3976#if USE_TCACHE
3977 INTERNAL_SIZE_T tcache_nb = 0;
3978 size_t tc_idx = csize2tidx (nb);
3979 if (tcache && tc_idx < mp_.tcache_bins)
3980 tcache_nb = nb;
3981 int return_cached = 0;
3982
3983 tcache_unsorted_count = 0;
3984#endif
3985
3986 for (;; )
3987 {
3988 int iters = 0;
3989 while ((victim = unsorted_chunks (av)->bk) != unsorted_chunks (av))
3990 {
3991 bck = victim->bk;
3992 size = chunksize (victim);
3993 mchunkptr next = chunk_at_offset (victim, size);
3994
3995 if (__glibc_unlikely (size <= CHUNK_HDR_SZ)
3996 || __glibc_unlikely (size > av->system_mem))
3997 malloc_printerr (str: "malloc(): invalid size (unsorted)");
3998 if (__glibc_unlikely (chunksize_nomask (next) < CHUNK_HDR_SZ)
3999 || __glibc_unlikely (chunksize_nomask (next) > av->system_mem))
4000 malloc_printerr (str: "malloc(): invalid next size (unsorted)");
4001 if (__glibc_unlikely ((prev_size (next) & ~(SIZE_BITS)) != size))
4002 malloc_printerr (str: "malloc(): mismatching next->prev_size (unsorted)");
4003 if (__glibc_unlikely (bck->fd != victim)
4004 || __glibc_unlikely (victim->fd != unsorted_chunks (av)))
4005 malloc_printerr (str: "malloc(): unsorted double linked list corrupted");
4006 if (__glibc_unlikely (prev_inuse (next)))
4007 malloc_printerr (str: "malloc(): invalid next->prev_inuse (unsorted)");
4008
4009 /*
4010 If a small request, try to use last remainder if it is the
4011 only chunk in unsorted bin. This helps promote locality for
4012 runs of consecutive small requests. This is the only
4013 exception to best-fit, and applies only when there is
4014 no exact fit for a small chunk.
4015 */
4016
4017 if (in_smallbin_range (nb) &&
4018 bck == unsorted_chunks (av) &&
4019 victim == av->last_remainder &&
4020 (unsigned long) (size) > (unsigned long) (nb + MINSIZE))
4021 {
4022 /* split and reattach remainder */
4023 remainder_size = size - nb;
4024 remainder = chunk_at_offset (victim, nb);
4025 unsorted_chunks (av)->bk = unsorted_chunks (av)->fd = remainder;
4026 av->last_remainder = remainder;
4027 remainder->bk = remainder->fd = unsorted_chunks (av);
4028 if (!in_smallbin_range (remainder_size))
4029 {
4030 remainder->fd_nextsize = NULL;
4031 remainder->bk_nextsize = NULL;
4032 }
4033
4034 set_head (victim, nb | PREV_INUSE |
4035 (av != &main_arena ? NON_MAIN_ARENA : 0));
4036 set_head (remainder, remainder_size | PREV_INUSE);
4037 set_foot (remainder, remainder_size);
4038
4039 check_malloced_chunk (av, victim, nb);
4040 void *p = chunk2mem (victim);
4041 alloc_perturb (p, n: bytes);
4042 return p;
4043 }
4044
4045 /* remove from unsorted list */
4046 if (__glibc_unlikely (bck->fd != victim))
4047 malloc_printerr (str: "malloc(): corrupted unsorted chunks 3");
4048 unsorted_chunks (av)->bk = bck;
4049 bck->fd = unsorted_chunks (av);
4050
4051 /* Take now instead of binning if exact fit */
4052
4053 if (size == nb)
4054 {
4055 set_inuse_bit_at_offset (victim, size);
4056 if (av != &main_arena)
4057 set_non_main_arena (victim);
4058#if USE_TCACHE
4059 /* Fill cache first, return to user only if cache fills.
4060 We may return one of these chunks later. */
4061 if (tcache_nb
4062 && tcache->counts[tc_idx] < mp_.tcache_count)
4063 {
4064 tcache_put (victim, tc_idx);
4065 return_cached = 1;
4066 continue;
4067 }
4068 else
4069 {
4070#endif
4071 check_malloced_chunk (av, victim, nb);
4072 void *p = chunk2mem (victim);
4073 alloc_perturb (p, n: bytes);
4074 return p;
4075#if USE_TCACHE
4076 }
4077#endif
4078 }
4079
4080 /* place chunk in bin */
4081
4082 if (in_smallbin_range (size))
4083 {
4084 victim_index = smallbin_index (size);
4085 bck = bin_at (av, victim_index);
4086 fwd = bck->fd;
4087 }
4088 else
4089 {
4090 victim_index = largebin_index (size);
4091 bck = bin_at (av, victim_index);
4092 fwd = bck->fd;
4093
4094 /* maintain large bins in sorted order */
4095 if (fwd != bck)
4096 {
4097 /* Or with inuse bit to speed comparisons */
4098 size |= PREV_INUSE;
4099 /* if smaller than smallest, bypass loop below */
4100 assert (chunk_main_arena (bck->bk));
4101 if ((unsigned long) (size)
4102 < (unsigned long) chunksize_nomask (bck->bk))
4103 {
4104 fwd = bck;
4105 bck = bck->bk;
4106
4107 victim->fd_nextsize = fwd->fd;
4108 victim->bk_nextsize = fwd->fd->bk_nextsize;
4109 fwd->fd->bk_nextsize = victim->bk_nextsize->fd_nextsize = victim;
4110 }
4111 else
4112 {
4113 assert (chunk_main_arena (fwd));
4114 while ((unsigned long) size < chunksize_nomask (fwd))
4115 {
4116 fwd = fwd->fd_nextsize;
4117 assert (chunk_main_arena (fwd));
4118 }
4119
4120 if ((unsigned long) size
4121 == (unsigned long) chunksize_nomask (fwd))
4122 /* Always insert in the second position. */
4123 fwd = fwd->fd;
4124 else
4125 {
4126 victim->fd_nextsize = fwd;
4127 victim->bk_nextsize = fwd->bk_nextsize;
4128 if (__glibc_unlikely (fwd->bk_nextsize->fd_nextsize != fwd))
4129 malloc_printerr (str: "malloc(): largebin double linked list corrupted (nextsize)");
4130 fwd->bk_nextsize = victim;
4131 victim->bk_nextsize->fd_nextsize = victim;
4132 }
4133 bck = fwd->bk;
4134 if (bck->fd != fwd)
4135 malloc_printerr (str: "malloc(): largebin double linked list corrupted (bk)");
4136 }
4137 }
4138 else
4139 victim->fd_nextsize = victim->bk_nextsize = victim;
4140 }
4141
4142 mark_bin (av, victim_index);
4143 victim->bk = bck;
4144 victim->fd = fwd;
4145 fwd->bk = victim;
4146 bck->fd = victim;
4147
4148#if USE_TCACHE
4149 /* If we've processed as many chunks as we're allowed while
4150 filling the cache, return one of the cached ones. */
4151 ++tcache_unsorted_count;
4152 if (return_cached
4153 && mp_.tcache_unsorted_limit > 0
4154 && tcache_unsorted_count > mp_.tcache_unsorted_limit)
4155 {
4156 return tcache_get (tc_idx);
4157 }
4158#endif
4159
4160#define MAX_ITERS 10000
4161 if (++iters >= MAX_ITERS)
4162 break;
4163 }
4164
4165#if USE_TCACHE
4166 /* If all the small chunks we found ended up cached, return one now. */
4167 if (return_cached)
4168 {
4169 return tcache_get (tc_idx);
4170 }
4171#endif
4172
4173 /*
4174 If a large request, scan through the chunks of current bin in
4175 sorted order to find smallest that fits. Use the skip list for this.
4176 */
4177
4178 if (!in_smallbin_range (nb))
4179 {
4180 bin = bin_at (av, idx);
4181
4182 /* skip scan if empty or largest chunk is too small */
4183 if ((victim = first (bin)) != bin
4184 && (unsigned long) chunksize_nomask (victim)
4185 >= (unsigned long) (nb))
4186 {
4187 victim = victim->bk_nextsize;
4188 while (((unsigned long) (size = chunksize (victim)) <
4189 (unsigned long) (nb)))
4190 victim = victim->bk_nextsize;
4191
4192 /* Avoid removing the first entry for a size so that the skip
4193 list does not have to be rerouted. */
4194 if (victim != last (bin)
4195 && chunksize_nomask (victim)
4196 == chunksize_nomask (victim->fd))
4197 victim = victim->fd;
4198
4199 remainder_size = size - nb;
4200 unlink_chunk (av, p: victim);
4201
4202 /* Exhaust */
4203 if (remainder_size < MINSIZE)
4204 {
4205 set_inuse_bit_at_offset (victim, size);
4206 if (av != &main_arena)
4207 set_non_main_arena (victim);
4208 }
4209 /* Split */
4210 else
4211 {
4212 remainder = chunk_at_offset (victim, nb);
4213 /* We cannot assume the unsorted list is empty and therefore
4214 have to perform a complete insert here. */
4215 bck = unsorted_chunks (av);
4216 fwd = bck->fd;
4217 if (__glibc_unlikely (fwd->bk != bck))
4218 malloc_printerr (str: "malloc(): corrupted unsorted chunks");
4219 remainder->bk = bck;
4220 remainder->fd = fwd;
4221 bck->fd = remainder;
4222 fwd->bk = remainder;
4223 if (!in_smallbin_range (remainder_size))
4224 {
4225 remainder->fd_nextsize = NULL;
4226 remainder->bk_nextsize = NULL;
4227 }
4228 set_head (victim, nb | PREV_INUSE |
4229 (av != &main_arena ? NON_MAIN_ARENA : 0));
4230 set_head (remainder, remainder_size | PREV_INUSE);
4231 set_foot (remainder, remainder_size);
4232 }
4233 check_malloced_chunk (av, victim, nb);
4234 void *p = chunk2mem (victim);
4235 alloc_perturb (p, n: bytes);
4236 return p;
4237 }
4238 }
4239
4240 /*
4241 Search for a chunk by scanning bins, starting with next largest
4242 bin. This search is strictly by best-fit; i.e., the smallest
4243 (with ties going to approximately the least recently used) chunk
4244 that fits is selected.
4245
4246 The bitmap avoids needing to check that most blocks are nonempty.
4247 The particular case of skipping all bins during warm-up phases
4248 when no chunks have been returned yet is faster than it might look.
4249 */
4250
4251 ++idx;
4252 bin = bin_at (av, idx);
4253 block = idx2block (idx);
4254 map = av->binmap[block];
4255 bit = idx2bit (idx);
4256
4257 for (;; )
4258 {
4259 /* Skip rest of block if there are no more set bits in this block. */
4260 if (bit > map || bit == 0)
4261 {
4262 do
4263 {
4264 if (++block >= BINMAPSIZE) /* out of bins */
4265 goto use_top;
4266 }
4267 while ((map = av->binmap[block]) == 0);
4268
4269 bin = bin_at (av, (block << BINMAPSHIFT));
4270 bit = 1;
4271 }
4272
4273 /* Advance to bin with set bit. There must be one. */
4274 while ((bit & map) == 0)
4275 {
4276 bin = next_bin (bin);
4277 bit <<= 1;
4278 assert (bit != 0);
4279 }
4280
4281 /* Inspect the bin. It is likely to be non-empty */
4282 victim = last (bin);
4283
4284 /* If a false alarm (empty bin), clear the bit. */
4285 if (victim == bin)
4286 {
4287 av->binmap[block] = map &= ~bit; /* Write through */
4288 bin = next_bin (bin);
4289 bit <<= 1;
4290 }
4291
4292 else
4293 {
4294 size = chunksize (victim);
4295
4296 /* We know the first chunk in this bin is big enough to use. */
4297 assert ((unsigned long) (size) >= (unsigned long) (nb));
4298
4299 remainder_size = size - nb;
4300
4301 /* unlink */
4302 unlink_chunk (av, p: victim);
4303
4304 /* Exhaust */
4305 if (remainder_size < MINSIZE)
4306 {
4307 set_inuse_bit_at_offset (victim, size);
4308 if (av != &main_arena)
4309 set_non_main_arena (victim);
4310 }
4311
4312 /* Split */
4313 else
4314 {
4315 remainder = chunk_at_offset (victim, nb);
4316
4317 /* We cannot assume the unsorted list is empty and therefore
4318 have to perform a complete insert here. */
4319 bck = unsorted_chunks (av);
4320 fwd = bck->fd;
4321 if (__glibc_unlikely (fwd->bk != bck))
4322 malloc_printerr (str: "malloc(): corrupted unsorted chunks 2");
4323 remainder->bk = bck;
4324 remainder->fd = fwd;
4325 bck->fd = remainder;
4326 fwd->bk = remainder;
4327
4328 /* advertise as last remainder */
4329 if (in_smallbin_range (nb))
4330 av->last_remainder = remainder;
4331 if (!in_smallbin_range (remainder_size))
4332 {
4333 remainder->fd_nextsize = NULL;
4334 remainder->bk_nextsize = NULL;
4335 }
4336 set_head (victim, nb | PREV_INUSE |
4337 (av != &main_arena ? NON_MAIN_ARENA : 0));
4338 set_head (remainder, remainder_size | PREV_INUSE);
4339 set_foot (remainder, remainder_size);
4340 }
4341 check_malloced_chunk (av, victim, nb);
4342 void *p = chunk2mem (victim);
4343 alloc_perturb (p, n: bytes);
4344 return p;
4345 }
4346 }
4347
4348 use_top:
4349 /*
4350 If large enough, split off the chunk bordering the end of memory
4351 (held in av->top). Note that this is in accord with the best-fit
4352 search rule. In effect, av->top is treated as larger (and thus
4353 less well fitting) than any other available chunk since it can
4354 be extended to be as large as necessary (up to system
4355 limitations).
4356
4357 We require that av->top always exists (i.e., has size >=
4358 MINSIZE) after initialization, so if it would otherwise be
4359 exhausted by current request, it is replenished. (The main
4360 reason for ensuring it exists is that we may need MINSIZE space
4361 to put in fenceposts in sysmalloc.)
4362 */
4363
4364 victim = av->top;
4365 size = chunksize (victim);
4366
4367 if (__glibc_unlikely (size > av->system_mem))
4368 malloc_printerr (str: "malloc(): corrupted top size");
4369
4370 if ((unsigned long) (size) >= (unsigned long) (nb + MINSIZE))
4371 {
4372 remainder_size = size - nb;
4373 remainder = chunk_at_offset (victim, nb);
4374 av->top = remainder;
4375 set_head (victim, nb | PREV_INUSE |
4376 (av != &main_arena ? NON_MAIN_ARENA : 0));
4377 set_head (remainder, remainder_size | PREV_INUSE);
4378
4379 check_malloced_chunk (av, victim, nb);
4380 void *p = chunk2mem (victim);
4381 alloc_perturb (p, n: bytes);
4382 return p;
4383 }
4384
4385 /* When we are using atomic ops to free fast chunks we can get
4386 here for all block sizes. */
4387 else if (atomic_load_relaxed (&av->have_fastchunks))
4388 {
4389 malloc_consolidate (av);
4390 /* restore original bin index */
4391 if (in_smallbin_range (nb))
4392 idx = smallbin_index (nb);
4393 else
4394 idx = largebin_index (nb);
4395 }
4396
4397 /*
4398 Otherwise, relay to handle system-dependent cases
4399 */
4400 else
4401 {
4402 void *p = sysmalloc (nb, av);
4403 if (p != NULL)
4404 alloc_perturb (p, n: bytes);
4405 return p;
4406 }
4407 }
4408}
4409
4410/*
4411 ------------------------------ free ------------------------------
4412 */
4413
4414static void
4415_int_free (mstate av, mchunkptr p, int have_lock)
4416{
4417 INTERNAL_SIZE_T size; /* its size */
4418 mfastbinptr *fb; /* associated fastbin */
4419 mchunkptr nextchunk; /* next contiguous chunk */
4420 INTERNAL_SIZE_T nextsize; /* its size */
4421 int nextinuse; /* true if nextchunk is used */
4422 INTERNAL_SIZE_T prevsize; /* size of previous contiguous chunk */
4423 mchunkptr bck; /* misc temp for linking */
4424 mchunkptr fwd; /* misc temp for linking */
4425
4426 size = chunksize (p);
4427
4428 /* Little security check which won't hurt performance: the
4429 allocator never wrapps around at the end of the address space.
4430 Therefore we can exclude some size values which might appear
4431 here by accident or by "design" from some intruder. */
4432 if (__builtin_expect ((uintptr_t) p > (uintptr_t) -size, 0)
4433 || __builtin_expect (misaligned_chunk (p), 0))
4434 malloc_printerr (str: "free(): invalid pointer");
4435 /* We know that each chunk is at least MINSIZE bytes in size or a
4436 multiple of MALLOC_ALIGNMENT. */
4437 if (__glibc_unlikely (size < MINSIZE || !aligned_OK (size)))
4438 malloc_printerr (str: "free(): invalid size");
4439
4440 check_inuse_chunk(av, p);
4441
4442#if USE_TCACHE
4443 {
4444 size_t tc_idx = csize2tidx (size);
4445 if (tcache != NULL && tc_idx < mp_.tcache_bins)
4446 {
4447 /* Check to see if it's already in the tcache. */
4448 tcache_entry *e = (tcache_entry *) chunk2mem (p);
4449
4450 /* This test succeeds on double free. However, we don't 100%
4451 trust it (it also matches random payload data at a 1 in
4452 2^<size_t> chance), so verify it's not an unlikely
4453 coincidence before aborting. */
4454 if (__glibc_unlikely (e->key == tcache_key))
4455 {
4456 tcache_entry *tmp;
4457 size_t cnt = 0;
4458 LIBC_PROBE (memory_tcache_double_free, 2, e, tc_idx);
4459 for (tmp = tcache->entries[tc_idx];
4460 tmp;
4461 tmp = REVEAL_PTR (tmp->next), ++cnt)
4462 {
4463 if (cnt >= mp_.tcache_count)
4464 malloc_printerr ("free(): too many chunks detected in tcache");
4465 if (__glibc_unlikely (!aligned_OK (tmp)))
4466 malloc_printerr ("free(): unaligned chunk detected in tcache 2");
4467 if (tmp == e)
4468 malloc_printerr ("free(): double free detected in tcache 2");
4469 /* If we get here, it was a coincidence. We've wasted a
4470 few cycles, but don't abort. */
4471 }
4472 }
4473
4474 if (tcache->counts[tc_idx] < mp_.tcache_count)
4475 {
4476 tcache_put (p, tc_idx);
4477 return;
4478 }
4479 }
4480 }
4481#endif
4482
4483 /*
4484 If eligible, place chunk on a fastbin so it can be found
4485 and used quickly in malloc.
4486 */
4487
4488 if ((unsigned long)(size) <= (unsigned long)(get_max_fast ())
4489
4490#if TRIM_FASTBINS
4491 /*
4492 If TRIM_FASTBINS set, don't place chunks
4493 bordering top into fastbins
4494 */
4495 && (chunk_at_offset(p, size) != av->top)
4496#endif
4497 ) {
4498
4499 if (__builtin_expect (chunksize_nomask (chunk_at_offset (p, size))
4500 <= CHUNK_HDR_SZ, 0)
4501 || __builtin_expect (chunksize (chunk_at_offset (p, size))
4502 >= av->system_mem, 0))
4503 {
4504 bool fail = true;
4505 /* We might not have a lock at this point and concurrent modifications
4506 of system_mem might result in a false positive. Redo the test after
4507 getting the lock. */
4508 if (!have_lock)
4509 {
4510 __libc_lock_lock (av->mutex);
4511 fail = (chunksize_nomask (chunk_at_offset (p, size)) <= CHUNK_HDR_SZ
4512 || chunksize (chunk_at_offset (p, size)) >= av->system_mem);
4513 __libc_lock_unlock (av->mutex);
4514 }
4515
4516 if (fail)
4517 malloc_printerr (str: "free(): invalid next size (fast)");
4518 }
4519
4520 free_perturb (chunk2mem(p), n: size - CHUNK_HDR_SZ);
4521
4522 atomic_store_relaxed (&av->have_fastchunks, true);
4523 unsigned int idx = fastbin_index(size);
4524 fb = &fastbin (av, idx);
4525
4526 /* Atomically link P to its fastbin: P->FD = *FB; *FB = P; */
4527 mchunkptr old = *fb, old2;
4528
4529 if (SINGLE_THREAD_P)
4530 {
4531 /* Check that the top of the bin is not the record we are going to
4532 add (i.e., double free). */
4533 if (__builtin_expect (old == p, 0))
4534 malloc_printerr (str: "double free or corruption (fasttop)");
4535 p->fd = PROTECT_PTR (&p->fd, old);
4536 *fb = p;
4537 }
4538 else
4539 do
4540 {
4541 /* Check that the top of the bin is not the record we are going to
4542 add (i.e., double free). */
4543 if (__builtin_expect (old == p, 0))
4544 malloc_printerr (str: "double free or corruption (fasttop)");
4545 old2 = old;
4546 p->fd = PROTECT_PTR (&p->fd, old);
4547 }
4548 while ((old = catomic_compare_and_exchange_val_rel (fb, p, old2))
4549 != old2);
4550
4551 /* Check that size of fastbin chunk at the top is the same as
4552 size of the chunk that we are adding. We can dereference OLD
4553 only if we have the lock, otherwise it might have already been
4554 allocated again. */
4555 if (have_lock && old != NULL
4556 && __builtin_expect (fastbin_index (chunksize (old)) != idx, 0))
4557 malloc_printerr (str: "invalid fastbin entry (free)");
4558 }
4559
4560 /*
4561 Consolidate other non-mmapped chunks as they arrive.
4562 */
4563
4564 else if (!chunk_is_mmapped(p)) {
4565
4566 /* If we're single-threaded, don't lock the arena. */
4567 if (SINGLE_THREAD_P)
4568 have_lock = true;
4569
4570 if (!have_lock)
4571 __libc_lock_lock (av->mutex);
4572
4573 nextchunk = chunk_at_offset(p, size);
4574
4575 /* Lightweight tests: check whether the block is already the
4576 top block. */
4577 if (__glibc_unlikely (p == av->top))
4578 malloc_printerr (str: "double free or corruption (top)");
4579 /* Or whether the next chunk is beyond the boundaries of the arena. */
4580 if (__builtin_expect (contiguous (av)
4581 && (char *) nextchunk
4582 >= ((char *) av->top + chunksize(av->top)), 0))
4583 malloc_printerr (str: "double free or corruption (out)");
4584 /* Or whether the block is actually not marked used. */
4585 if (__glibc_unlikely (!prev_inuse(nextchunk)))
4586 malloc_printerr (str: "double free or corruption (!prev)");
4587
4588 nextsize = chunksize(nextchunk);
4589 if (__builtin_expect (chunksize_nomask (nextchunk) <= CHUNK_HDR_SZ, 0)
4590 || __builtin_expect (nextsize >= av->system_mem, 0))
4591 malloc_printerr (str: "free(): invalid next size (normal)");
4592
4593 free_perturb (chunk2mem(p), n: size - CHUNK_HDR_SZ);
4594
4595 /* consolidate backward */
4596 if (!prev_inuse(p)) {
4597 prevsize = prev_size (p);
4598 size += prevsize;
4599 p = chunk_at_offset(p, -((long) prevsize));
4600 if (__glibc_unlikely (chunksize(p) != prevsize))
4601 malloc_printerr (str: "corrupted size vs. prev_size while consolidating");
4602 unlink_chunk (av, p);
4603 }
4604
4605 if (nextchunk != av->top) {
4606 /* get and clear inuse bit */
4607 nextinuse = inuse_bit_at_offset(nextchunk, nextsize);
4608
4609 /* consolidate forward */
4610 if (!nextinuse) {
4611 unlink_chunk (av, p: nextchunk);
4612 size += nextsize;
4613 } else
4614 clear_inuse_bit_at_offset(nextchunk, 0);
4615
4616 /*
4617 Place the chunk in unsorted chunk list. Chunks are
4618 not placed into regular bins until after they have
4619 been given one chance to be used in malloc.
4620 */
4621
4622 bck = unsorted_chunks(av);
4623 fwd = bck->fd;
4624 if (__glibc_unlikely (fwd->bk != bck))
4625 malloc_printerr (str: "free(): corrupted unsorted chunks");
4626 p->fd = fwd;
4627 p->bk = bck;
4628 if (!in_smallbin_range(size))
4629 {
4630 p->fd_nextsize = NULL;
4631 p->bk_nextsize = NULL;
4632 }
4633 bck->fd = p;
4634 fwd->bk = p;
4635
4636 set_head(p, size | PREV_INUSE);
4637 set_foot(p, size);
4638
4639 check_free_chunk(av, p);
4640 }
4641
4642 /*
4643 If the chunk borders the current high end of memory,
4644 consolidate into top
4645 */
4646
4647 else {
4648 size += nextsize;
4649 set_head(p, size | PREV_INUSE);
4650 av->top = p;
4651 check_chunk(av, p);
4652 }
4653
4654 /*
4655 If freeing a large space, consolidate possibly-surrounding
4656 chunks. Then, if the total unused topmost memory exceeds trim
4657 threshold, ask malloc_trim to reduce top.
4658
4659 Unless max_fast is 0, we don't know if there are fastbins
4660 bordering top, so we cannot tell for sure whether threshold
4661 has been reached unless fastbins are consolidated. But we
4662 don't want to consolidate on each free. As a compromise,
4663 consolidation is performed if FASTBIN_CONSOLIDATION_THRESHOLD
4664 is reached.
4665 */
4666
4667 if ((unsigned long)(size) >= FASTBIN_CONSOLIDATION_THRESHOLD) {
4668 if (atomic_load_relaxed (&av->have_fastchunks))
4669 malloc_consolidate(av);
4670
4671 if (av == &main_arena) {
4672#ifndef MORECORE_CANNOT_TRIM
4673 if ((unsigned long)(chunksize(av->top)) >=
4674 (unsigned long)(mp_.trim_threshold))
4675 systrim(pad: mp_.top_pad, av);
4676#endif
4677 } else {
4678 /* Always try heap_trim(), even if the top chunk is not
4679 large, because the corresponding heap might go away. */
4680 heap_info *heap = heap_for_ptr(top(av));
4681
4682 assert(heap->ar_ptr == av);
4683 heap_trim(heap, pad: mp_.top_pad);
4684 }
4685 }
4686
4687 if (!have_lock)
4688 __libc_lock_unlock (av->mutex);
4689 }
4690 /*
4691 If the chunk was allocated via mmap, release via munmap().
4692 */
4693
4694 else {
4695 munmap_chunk (p);
4696 }
4697}
4698
4699/*
4700 ------------------------- malloc_consolidate -------------------------
4701
4702 malloc_consolidate is a specialized version of free() that tears
4703 down chunks held in fastbins. Free itself cannot be used for this
4704 purpose since, among other things, it might place chunks back onto
4705 fastbins. So, instead, we need to use a minor variant of the same
4706 code.
4707*/
4708
4709static void malloc_consolidate(mstate av)
4710{
4711 mfastbinptr* fb; /* current fastbin being consolidated */
4712 mfastbinptr* maxfb; /* last fastbin (for loop control) */
4713 mchunkptr p; /* current chunk being consolidated */
4714 mchunkptr nextp; /* next chunk to consolidate */
4715 mchunkptr unsorted_bin; /* bin header */
4716 mchunkptr first_unsorted; /* chunk to link to */
4717
4718 /* These have same use as in free() */
4719 mchunkptr nextchunk;
4720 INTERNAL_SIZE_T size;
4721 INTERNAL_SIZE_T nextsize;
4722 INTERNAL_SIZE_T prevsize;
4723 int nextinuse;
4724
4725 atomic_store_relaxed (&av->have_fastchunks, false);
4726
4727 unsorted_bin = unsorted_chunks(av);
4728
4729 /*
4730 Remove each chunk from fast bin and consolidate it, placing it
4731 then in unsorted bin. Among other reasons for doing this,
4732 placing in unsorted bin avoids needing to calculate actual bins
4733 until malloc is sure that chunks aren't immediately going to be
4734 reused anyway.
4735 */
4736
4737 maxfb = &fastbin (av, NFASTBINS - 1);
4738 fb = &fastbin (av, 0);
4739 do {
4740 p = atomic_exchange_acq (fb, NULL);
4741 if (p != 0) {
4742 do {
4743 {
4744 if (__glibc_unlikely (misaligned_chunk (p)))
4745 malloc_printerr (str: "malloc_consolidate(): "
4746 "unaligned fastbin chunk detected");
4747
4748 unsigned int idx = fastbin_index (chunksize (p));
4749 if ((&fastbin (av, idx)) != fb)
4750 malloc_printerr (str: "malloc_consolidate(): invalid chunk size");
4751 }
4752
4753 check_inuse_chunk(av, p);
4754 nextp = REVEAL_PTR (p->fd);
4755
4756 /* Slightly streamlined version of consolidation code in free() */
4757 size = chunksize (p);
4758 nextchunk = chunk_at_offset(p, size);
4759 nextsize = chunksize(nextchunk);
4760
4761 if (!prev_inuse(p)) {
4762 prevsize = prev_size (p);
4763 size += prevsize;
4764 p = chunk_at_offset(p, -((long) prevsize));
4765 if (__glibc_unlikely (chunksize(p) != prevsize))
4766 malloc_printerr (str: "corrupted size vs. prev_size in fastbins");
4767 unlink_chunk (av, p);
4768 }
4769
4770 if (nextchunk != av->top) {
4771 nextinuse = inuse_bit_at_offset(nextchunk, nextsize);
4772
4773 if (!nextinuse) {
4774 size += nextsize;
4775 unlink_chunk (av, p: nextchunk);
4776 } else
4777 clear_inuse_bit_at_offset(nextchunk, 0);
4778
4779 first_unsorted = unsorted_bin->fd;
4780 unsorted_bin->fd = p;
4781 first_unsorted->bk = p;
4782
4783 if (!in_smallbin_range (size)) {
4784 p->fd_nextsize = NULL;
4785 p->bk_nextsize = NULL;
4786 }
4787
4788 set_head(p, size | PREV_INUSE);
4789 p->bk = unsorted_bin;
4790 p->fd = first_unsorted;
4791 set_foot(p, size);
4792 }
4793
4794 else {
4795 size += nextsize;
4796 set_head(p, size | PREV_INUSE);
4797 av->top = p;
4798 }
4799
4800 } while ( (p = nextp) != 0);
4801
4802 }
4803 } while (fb++ != maxfb);
4804}
4805
4806/*
4807 ------------------------------ realloc ------------------------------
4808*/
4809
4810static void *
4811_int_realloc (mstate av, mchunkptr oldp, INTERNAL_SIZE_T oldsize,
4812 INTERNAL_SIZE_T nb)
4813{
4814 mchunkptr newp; /* chunk to return */
4815 INTERNAL_SIZE_T newsize; /* its size */
4816 void* newmem; /* corresponding user mem */
4817
4818 mchunkptr next; /* next contiguous chunk after oldp */
4819
4820 mchunkptr remainder; /* extra space at end of newp */
4821 unsigned long remainder_size; /* its size */
4822
4823 /* oldmem size */
4824 if (__builtin_expect (chunksize_nomask (oldp) <= CHUNK_HDR_SZ, 0)
4825 || __builtin_expect (oldsize >= av->system_mem, 0))
4826 malloc_printerr (str: "realloc(): invalid old size");
4827
4828 check_inuse_chunk (av, oldp);
4829
4830 /* All callers already filter out mmap'ed chunks. */
4831 assert (!chunk_is_mmapped (oldp));
4832
4833 next = chunk_at_offset (oldp, oldsize);
4834 INTERNAL_SIZE_T nextsize = chunksize (next);
4835 if (__builtin_expect (chunksize_nomask (next) <= CHUNK_HDR_SZ, 0)
4836 || __builtin_expect (nextsize >= av->system_mem, 0))
4837 malloc_printerr (str: "realloc(): invalid next size");
4838
4839 if ((unsigned long) (oldsize) >= (unsigned long) (nb))
4840 {
4841 /* already big enough; split below */
4842 newp = oldp;
4843 newsize = oldsize;
4844 }
4845
4846 else
4847 {
4848 /* Try to expand forward into top */
4849 if (next == av->top &&
4850 (unsigned long) (newsize = oldsize + nextsize) >=
4851 (unsigned long) (nb + MINSIZE))
4852 {
4853 set_head_size (oldp, nb | (av != &main_arena ? NON_MAIN_ARENA : 0));
4854 av->top = chunk_at_offset (oldp, nb);
4855 set_head (av->top, (newsize - nb) | PREV_INUSE);
4856 check_inuse_chunk (av, oldp);
4857 return tag_new_usable (chunk2mem (oldp));
4858 }
4859
4860 /* Try to expand forward into next chunk; split off remainder below */
4861 else if (next != av->top &&
4862 !inuse (next) &&
4863 (unsigned long) (newsize = oldsize + nextsize) >=
4864 (unsigned long) (nb))
4865 {
4866 newp = oldp;
4867 unlink_chunk (av, p: next);
4868 }
4869
4870 /* allocate, copy, free */
4871 else
4872 {
4873 newmem = _int_malloc (av, bytes: nb - MALLOC_ALIGN_MASK);
4874 if (newmem == 0)
4875 return 0; /* propagate failure */
4876
4877 newp = mem2chunk (newmem);
4878 newsize = chunksize (newp);
4879
4880 /*
4881 Avoid copy if newp is next chunk after oldp.
4882 */
4883 if (newp == next)
4884 {
4885 newsize += oldsize;
4886 newp = oldp;
4887 }
4888 else
4889 {
4890 void *oldmem = chunk2mem (oldp);
4891 size_t sz = memsize (oldp);
4892 (void) tag_region (ptr: oldmem, size: sz);
4893 newmem = tag_new_usable (ptr: newmem);
4894 memcpy (dest: newmem, src: oldmem, n: sz);
4895 _int_free (av, p: oldp, have_lock: 1);
4896 check_inuse_chunk (av, newp);
4897 return newmem;
4898 }
4899 }
4900 }
4901
4902 /* If possible, free extra space in old or extended chunk */
4903
4904 assert ((unsigned long) (newsize) >= (unsigned long) (nb));
4905
4906 remainder_size = newsize - nb;
4907
4908 if (remainder_size < MINSIZE) /* not enough extra to split off */
4909 {
4910 set_head_size (newp, newsize | (av != &main_arena ? NON_MAIN_ARENA : 0));
4911 set_inuse_bit_at_offset (newp, newsize);
4912 }
4913 else /* split remainder */
4914 {
4915 remainder = chunk_at_offset (newp, nb);
4916 /* Clear any user-space tags before writing the header. */
4917 remainder = tag_region (ptr: remainder, size: remainder_size);
4918 set_head_size (newp, nb | (av != &main_arena ? NON_MAIN_ARENA : 0));
4919 set_head (remainder, remainder_size | PREV_INUSE |
4920 (av != &main_arena ? NON_MAIN_ARENA : 0));
4921 /* Mark remainder as inuse so free() won't complain */
4922 set_inuse_bit_at_offset (remainder, remainder_size);
4923 _int_free (av, p: remainder, have_lock: 1);
4924 }
4925
4926 check_inuse_chunk (av, newp);
4927 return tag_new_usable (chunk2mem (newp));
4928}
4929
4930/*
4931 ------------------------------ memalign ------------------------------
4932 */
4933
4934static void *
4935_int_memalign (mstate av, size_t alignment, size_t bytes)
4936{
4937 INTERNAL_SIZE_T nb; /* padded request size */
4938 char *m; /* memory returned by malloc call */
4939 mchunkptr p; /* corresponding chunk */
4940 char *brk; /* alignment point within p */
4941 mchunkptr newp; /* chunk to return */
4942 INTERNAL_SIZE_T newsize; /* its size */
4943 INTERNAL_SIZE_T leadsize; /* leading space before alignment point */
4944 mchunkptr remainder; /* spare room at end to split off */
4945 unsigned long remainder_size; /* its size */
4946 INTERNAL_SIZE_T size;
4947
4948
4949
4950 if (!checked_request2size (req: bytes, sz: &nb))
4951 {
4952 __set_errno (ENOMEM);
4953 return NULL;
4954 }
4955
4956 /*
4957 Strategy: find a spot within that chunk that meets the alignment
4958 request, and then possibly free the leading and trailing space.
4959 */
4960
4961 /* Call malloc with worst case padding to hit alignment. */
4962
4963 m = (char *) (_int_malloc (av, bytes: nb + alignment + MINSIZE));
4964
4965 if (m == 0)
4966 return 0; /* propagate failure */
4967
4968 p = mem2chunk (m);
4969
4970 if ((((unsigned long) (m)) % alignment) != 0) /* misaligned */
4971
4972 { /*
4973 Find an aligned spot inside chunk. Since we need to give back
4974 leading space in a chunk of at least MINSIZE, if the first
4975 calculation places us at a spot with less than MINSIZE leader,
4976 we can move to the next aligned spot -- we've allocated enough
4977 total room so that this is always possible.
4978 */
4979 brk = (char *) mem2chunk (((unsigned long) (m + alignment - 1)) &
4980 - ((signed long) alignment));
4981 if ((unsigned long) (brk - (char *) (p)) < MINSIZE)
4982 brk += alignment;
4983
4984 newp = (mchunkptr) brk;
4985 leadsize = brk - (char *) (p);
4986 newsize = chunksize (p) - leadsize;
4987
4988 /* For mmapped chunks, just adjust offset */
4989 if (chunk_is_mmapped (p))
4990 {
4991 set_prev_size (newp, prev_size (p) + leadsize);
4992 set_head (newp, newsize | IS_MMAPPED);
4993 return chunk2mem (newp);
4994 }
4995
4996 /* Otherwise, give back leader, use the rest */
4997 set_head (newp, newsize | PREV_INUSE |
4998 (av != &main_arena ? NON_MAIN_ARENA : 0));
4999 set_inuse_bit_at_offset (newp, newsize);
5000 set_head_size (p, leadsize | (av != &main_arena ? NON_MAIN_ARENA : 0));
5001 _int_free (av, p, have_lock: 1);
5002 p = newp;
5003
5004 assert (newsize >= nb &&
5005 (((unsigned long) (chunk2mem (p))) % alignment) == 0);
5006 }
5007
5008 /* Also give back spare room at the end */
5009 if (!chunk_is_mmapped (p))
5010 {
5011 size = chunksize (p);
5012 if ((unsigned long) (size) > (unsigned long) (nb + MINSIZE))
5013 {
5014 remainder_size = size - nb;
5015 remainder = chunk_at_offset (p, nb);
5016 set_head (remainder, remainder_size | PREV_INUSE |
5017 (av != &main_arena ? NON_MAIN_ARENA : 0));
5018 set_head_size (p, nb);
5019 _int_free (av, p: remainder, have_lock: 1);
5020 }
5021 }
5022
5023 check_inuse_chunk (av, p);
5024 return chunk2mem (p);
5025}
5026
5027
5028/*
5029 ------------------------------ malloc_trim ------------------------------
5030 */
5031
5032static int
5033mtrim (mstate av, size_t pad)
5034{
5035 /* Ensure all blocks are consolidated. */
5036 malloc_consolidate (av);
5037
5038 const size_t ps = GLRO (dl_pagesize);
5039 int psindex = bin_index (ps);
5040 const size_t psm1 = ps - 1;
5041
5042 int result = 0;
5043 for (int i = 1; i < NBINS; ++i)
5044 if (i == 1 || i >= psindex)
5045 {
5046 mbinptr bin = bin_at (av, i);
5047
5048 for (mchunkptr p = last (bin); p != bin; p = p->bk)
5049 {
5050 INTERNAL_SIZE_T size = chunksize (p);
5051
5052 if (size > psm1 + sizeof (struct malloc_chunk))
5053 {
5054 /* See whether the chunk contains at least one unused page. */
5055 char *paligned_mem = (char *) (((uintptr_t) p
5056 + sizeof (struct malloc_chunk)
5057 + psm1) & ~psm1);
5058
5059 assert ((char *) chunk2mem (p) + 2 * CHUNK_HDR_SZ
5060 <= paligned_mem);
5061 assert ((char *) p + size > paligned_mem);
5062
5063 /* This is the size we could potentially free. */
5064 size -= paligned_mem - (char *) p;
5065
5066 if (size > psm1)
5067 {
5068#if MALLOC_DEBUG
5069 /* When debugging we simulate destroying the memory
5070 content. */
5071 memset (paligned_mem, 0x89, size & ~psm1);
5072#endif
5073 __madvise (addr: paligned_mem, len: size & ~psm1, MADV_DONTNEED);
5074
5075 result = 1;
5076 }
5077 }
5078 }
5079 }
5080
5081#ifndef MORECORE_CANNOT_TRIM
5082 return result | (av == &main_arena ? systrim (pad, av) : 0);
5083
5084#else
5085 return result;
5086#endif
5087}
5088
5089
5090int
5091__malloc_trim (size_t s)
5092{
5093 int result = 0;
5094
5095 if (!__malloc_initialized)
5096 ptmalloc_init ();
5097
5098 mstate ar_ptr = &main_arena;
5099 do
5100 {
5101 __libc_lock_lock (ar_ptr->mutex);
5102 result |= mtrim (av: ar_ptr, pad: s);
5103 __libc_lock_unlock (ar_ptr->mutex);
5104
5105 ar_ptr = ar_ptr->next;
5106 }
5107 while (ar_ptr != &main_arena);
5108
5109 return result;
5110}
5111
5112
5113/*
5114 ------------------------- malloc_usable_size -------------------------
5115 */
5116
5117static size_t
5118musable (void *mem)
5119{
5120 mchunkptr p = mem2chunk (mem);
5121
5122 if (chunk_is_mmapped (p))
5123 return chunksize (p) - CHUNK_HDR_SZ;
5124 else if (inuse (p))
5125 return memsize (p);
5126
5127 return 0;
5128}
5129
5130#if IS_IN (libc)
5131size_t
5132__malloc_usable_size (void *m)
5133{
5134 if (m == NULL)
5135 return 0;
5136 return musable (m);
5137}
5138#endif
5139
5140/*
5141 ------------------------------ mallinfo ------------------------------
5142 Accumulate malloc statistics for arena AV into M.
5143 */
5144static void
5145int_mallinfo (mstate av, struct mallinfo2 *m)
5146{
5147 size_t i;
5148 mbinptr b;
5149 mchunkptr p;
5150 INTERNAL_SIZE_T avail;
5151 INTERNAL_SIZE_T fastavail;
5152 int nblocks;
5153 int nfastblocks;
5154
5155 check_malloc_state (av);
5156
5157 /* Account for top */
5158 avail = chunksize (av->top);
5159 nblocks = 1; /* top always exists */
5160
5161 /* traverse fastbins */
5162 nfastblocks = 0;
5163 fastavail = 0;
5164
5165 for (i = 0; i < NFASTBINS; ++i)
5166 {
5167 for (p = fastbin (av, i);
5168 p != 0;
5169 p = REVEAL_PTR (p->fd))
5170 {
5171 if (__glibc_unlikely (misaligned_chunk (p)))
5172 malloc_printerr (str: "int_mallinfo(): "
5173 "unaligned fastbin chunk detected");
5174 ++nfastblocks;
5175 fastavail += chunksize (p);
5176 }
5177 }
5178
5179 avail += fastavail;
5180
5181 /* traverse regular bins */
5182 for (i = 1; i < NBINS; ++i)
5183 {
5184 b = bin_at (av, i);
5185 for (p = last (b); p != b; p = p->bk)
5186 {
5187 ++nblocks;
5188 avail += chunksize (p);
5189 }
5190 }
5191
5192 m->smblks += nfastblocks;
5193 m->ordblks += nblocks;
5194 m->fordblks += avail;
5195 m->uordblks += av->system_mem - avail;
5196 m->arena += av->system_mem;
5197 m->fsmblks += fastavail;
5198 if (av == &main_arena)
5199 {
5200 m->hblks = mp_.n_mmaps;
5201 m->hblkhd = mp_.mmapped_mem;
5202 m->usmblks = 0;
5203 m->keepcost = chunksize (av->top);
5204 }
5205}
5206
5207
5208struct mallinfo2
5209__libc_mallinfo2 (void)
5210{
5211 struct mallinfo2 m;
5212 mstate ar_ptr;
5213
5214 if (!__malloc_initialized)
5215 ptmalloc_init ();
5216
5217 memset (s: &m, c: 0, n: sizeof (m));
5218 ar_ptr = &main_arena;
5219 do
5220 {
5221 __libc_lock_lock (ar_ptr->mutex);
5222 int_mallinfo (av: ar_ptr, m: &m);
5223 __libc_lock_unlock (ar_ptr->mutex);
5224
5225 ar_ptr = ar_ptr->next;
5226 }
5227 while (ar_ptr != &main_arena);
5228
5229 return m;
5230}
5231libc_hidden_def (__libc_mallinfo2)
5232
5233struct mallinfo
5234__libc_mallinfo (void)
5235{
5236 struct mallinfo m;
5237 struct mallinfo2 m2 = __libc_mallinfo2 ();
5238
5239 m.arena = m2.arena;
5240 m.ordblks = m2.ordblks;
5241 m.smblks = m2.smblks;
5242 m.hblks = m2.hblks;
5243 m.hblkhd = m2.hblkhd;
5244 m.usmblks = m2.usmblks;
5245 m.fsmblks = m2.fsmblks;
5246 m.uordblks = m2.uordblks;
5247 m.fordblks = m2.fordblks;
5248 m.keepcost = m2.keepcost;
5249
5250 return m;
5251}
5252
5253
5254/*
5255 ------------------------------ malloc_stats ------------------------------
5256 */
5257
5258void
5259__malloc_stats (void)
5260{
5261 int i;
5262 mstate ar_ptr;
5263 unsigned int in_use_b = mp_.mmapped_mem, system_b = in_use_b;
5264
5265 if (!__malloc_initialized)
5266 ptmalloc_init ();
5267 _IO_flockfile (stderr);
5268 int old_flags2 = stderr->_flags2;
5269 stderr->_flags2 |= _IO_FLAGS2_NOTCANCEL;
5270 for (i = 0, ar_ptr = &main_arena;; i++)
5271 {
5272 struct mallinfo2 mi;
5273
5274 memset (s: &mi, c: 0, n: sizeof (mi));
5275 __libc_lock_lock (ar_ptr->mutex);
5276 int_mallinfo (av: ar_ptr, m: &mi);
5277 fprintf (stderr, format: "Arena %d:\n", i);
5278 fprintf (stderr, format: "system bytes = %10u\n", (unsigned int) mi.arena);
5279 fprintf (stderr, format: "in use bytes = %10u\n", (unsigned int) mi.uordblks);
5280#if MALLOC_DEBUG > 1
5281 if (i > 0)
5282 dump_heap (heap_for_ptr (top (ar_ptr)));
5283#endif
5284 system_b += mi.arena;
5285 in_use_b += mi.uordblks;
5286 __libc_lock_unlock (ar_ptr->mutex);
5287 ar_ptr = ar_ptr->next;
5288 if (ar_ptr == &main_arena)
5289 break;
5290 }
5291 fprintf (stderr, format: "Total (incl. mmap):\n");
5292 fprintf (stderr, format: "system bytes = %10u\n", system_b);
5293 fprintf (stderr, format: "in use bytes = %10u\n", in_use_b);
5294 fprintf (stderr, format: "max mmap regions = %10u\n", (unsigned int) mp_.max_n_mmaps);
5295 fprintf (stderr, format: "max mmap bytes = %10lu\n",
5296 (unsigned long) mp_.max_mmapped_mem);
5297 stderr->_flags2 = old_flags2;
5298 _IO_funlockfile (stderr);
5299}
5300
5301
5302/*
5303 ------------------------------ mallopt ------------------------------
5304 */
5305static __always_inline int
5306do_set_trim_threshold (size_t value)
5307{
5308 LIBC_PROBE (memory_mallopt_trim_threshold, 3, value, mp_.trim_threshold,
5309 mp_.no_dyn_threshold);
5310 mp_.trim_threshold = value;
5311 mp_.no_dyn_threshold = 1;
5312 return 1;
5313}
5314
5315static __always_inline int
5316do_set_top_pad (size_t value)
5317{
5318 LIBC_PROBE (memory_mallopt_top_pad, 3, value, mp_.top_pad,
5319 mp_.no_dyn_threshold);
5320 mp_.top_pad = value;
5321 mp_.no_dyn_threshold = 1;
5322 return 1;
5323}
5324
5325static __always_inline int
5326do_set_mmap_threshold (size_t value)
5327{
5328 LIBC_PROBE (memory_mallopt_mmap_threshold, 3, value, mp_.mmap_threshold,
5329 mp_.no_dyn_threshold);
5330 mp_.mmap_threshold = value;
5331 mp_.no_dyn_threshold = 1;
5332 return 1;
5333}
5334
5335static __always_inline int
5336do_set_mmaps_max (int32_t value)
5337{
5338 LIBC_PROBE (memory_mallopt_mmap_max, 3, value, mp_.n_mmaps_max,
5339 mp_.no_dyn_threshold);
5340 mp_.n_mmaps_max = value;
5341 mp_.no_dyn_threshold = 1;
5342 return 1;
5343}
5344
5345static __always_inline int
5346do_set_mallopt_check (int32_t value)
5347{
5348 return 1;
5349}
5350
5351static __always_inline int
5352do_set_perturb_byte (int32_t value)
5353{
5354 LIBC_PROBE (memory_mallopt_perturb, 2, value, perturb_byte);
5355 perturb_byte = value;
5356 return 1;
5357}
5358
5359static __always_inline int
5360do_set_arena_test (size_t value)
5361{
5362 LIBC_PROBE (memory_mallopt_arena_test, 2, value, mp_.arena_test);
5363 mp_.arena_test = value;
5364 return 1;
5365}
5366
5367static __always_inline int
5368do_set_arena_max (size_t value)
5369{
5370 LIBC_PROBE (memory_mallopt_arena_max, 2, value, mp_.arena_max);
5371 mp_.arena_max = value;
5372 return 1;
5373}
5374
5375#if USE_TCACHE
5376static __always_inline int
5377do_set_tcache_max (size_t value)
5378{
5379 if (value <= MAX_TCACHE_SIZE)
5380 {
5381 LIBC_PROBE (memory_tunable_tcache_max_bytes, 2, value, mp_.tcache_max_bytes);
5382 mp_.tcache_max_bytes = value;
5383 mp_.tcache_bins = csize2tidx (request2size(value)) + 1;
5384 return 1;
5385 }
5386 return 0;
5387}
5388
5389static __always_inline int
5390do_set_tcache_count (size_t value)
5391{
5392 if (value <= MAX_TCACHE_COUNT)
5393 {
5394 LIBC_PROBE (memory_tunable_tcache_count, 2, value, mp_.tcache_count);
5395 mp_.tcache_count = value;
5396 return 1;
5397 }
5398 return 0;
5399}
5400
5401static __always_inline int
5402do_set_tcache_unsorted_limit (size_t value)
5403{
5404 LIBC_PROBE (memory_tunable_tcache_unsorted_limit, 2, value, mp_.tcache_unsorted_limit);
5405 mp_.tcache_unsorted_limit = value;
5406 return 1;
5407}
5408#endif
5409
5410static inline int
5411__always_inline
5412do_set_mxfast (size_t value)
5413{
5414 if (value <= MAX_FAST_SIZE)
5415 {
5416 LIBC_PROBE (memory_mallopt_mxfast, 2, value, get_max_fast ());
5417 set_max_fast (value);
5418 return 1;
5419 }
5420 return 0;
5421}
5422
5423#if HAVE_TUNABLES
5424static __always_inline int
5425do_set_hugetlb (size_t value)
5426{
5427 if (value == 1)
5428 {
5429 enum malloc_thp_mode_t thp_mode = __malloc_thp_mode ();
5430 /*
5431 Only enable THP madvise usage if system does support it and
5432 has 'madvise' mode. Otherwise the madvise() call is wasteful.
5433 */
5434 if (thp_mode == malloc_thp_mode_madvise)
5435 mp_.thp_pagesize = __malloc_default_thp_pagesize ();
5436 }
5437 else if (value >= 2)
5438 __malloc_hugepage_config (requested: value == 2 ? 0 : value, pagesize: &mp_.hp_pagesize,
5439 flags: &mp_.hp_flags);
5440 return 0;
5441}
5442#endif
5443
5444int
5445__libc_mallopt (int param_number, int value)
5446{
5447 mstate av = &main_arena;
5448 int res = 1;
5449
5450 if (!__malloc_initialized)
5451 ptmalloc_init ();
5452 __libc_lock_lock (av->mutex);
5453
5454 LIBC_PROBE (memory_mallopt, 2, param_number, value);
5455
5456 /* We must consolidate main arena before changing max_fast
5457 (see definition of set_max_fast). */
5458 malloc_consolidate (av);
5459
5460 /* Many of these helper functions take a size_t. We do not worry
5461 about overflow here, because negative int values will wrap to
5462 very large size_t values and the helpers have sufficient range
5463 checking for such conversions. Many of these helpers are also
5464 used by the tunables macros in arena.c. */
5465
5466 switch (param_number)
5467 {
5468 case M_MXFAST:
5469 res = do_set_mxfast (value);
5470 break;
5471
5472 case M_TRIM_THRESHOLD:
5473 res = do_set_trim_threshold (value);
5474 break;
5475
5476 case M_TOP_PAD:
5477 res = do_set_top_pad (value);
5478 break;
5479
5480 case M_MMAP_THRESHOLD:
5481 res = do_set_mmap_threshold (value);
5482 break;
5483
5484 case M_MMAP_MAX:
5485 res = do_set_mmaps_max (value);
5486 break;
5487
5488 case M_CHECK_ACTION:
5489 res = do_set_mallopt_check (value);
5490 break;
5491
5492 case M_PERTURB:
5493 res = do_set_perturb_byte (value);
5494 break;
5495
5496 case M_ARENA_TEST:
5497 if (value > 0)
5498 res = do_set_arena_test (value);
5499 break;
5500
5501 case M_ARENA_MAX:
5502 if (value > 0)
5503 res = do_set_arena_max (value);
5504 break;
5505 }
5506 __libc_lock_unlock (av->mutex);
5507 return res;
5508}
5509libc_hidden_def (__libc_mallopt)
5510
5511
5512/*
5513 -------------------- Alternative MORECORE functions --------------------
5514 */
5515
5516
5517/*
5518 General Requirements for MORECORE.
5519
5520 The MORECORE function must have the following properties:
5521
5522 If MORECORE_CONTIGUOUS is false:
5523
5524 * MORECORE must allocate in multiples of pagesize. It will
5525 only be called with arguments that are multiples of pagesize.
5526
5527 * MORECORE(0) must return an address that is at least
5528 MALLOC_ALIGNMENT aligned. (Page-aligning always suffices.)
5529
5530 else (i.e. If MORECORE_CONTIGUOUS is true):
5531
5532 * Consecutive calls to MORECORE with positive arguments
5533 return increasing addresses, indicating that space has been
5534 contiguously extended.
5535
5536 * MORECORE need not allocate in multiples of pagesize.
5537 Calls to MORECORE need not have args of multiples of pagesize.
5538
5539 * MORECORE need not page-align.
5540
5541 In either case:
5542
5543 * MORECORE may allocate more memory than requested. (Or even less,
5544 but this will generally result in a malloc failure.)
5545
5546 * MORECORE must not allocate memory when given argument zero, but
5547 instead return one past the end address of memory from previous
5548 nonzero call. This malloc does NOT call MORECORE(0)
5549 until at least one call with positive arguments is made, so
5550 the initial value returned is not important.
5551
5552 * Even though consecutive calls to MORECORE need not return contiguous
5553 addresses, it must be OK for malloc'ed chunks to span multiple
5554 regions in those cases where they do happen to be contiguous.
5555
5556 * MORECORE need not handle negative arguments -- it may instead
5557 just return MORECORE_FAILURE when given negative arguments.
5558 Negative arguments are always multiples of pagesize. MORECORE
5559 must not misinterpret negative args as large positive unsigned
5560 args. You can suppress all such calls from even occurring by defining
5561 MORECORE_CANNOT_TRIM,
5562
5563 There is some variation across systems about the type of the
5564 argument to sbrk/MORECORE. If size_t is unsigned, then it cannot
5565 actually be size_t, because sbrk supports negative args, so it is
5566 normally the signed type of the same width as size_t (sometimes
5567 declared as "intptr_t", and sometimes "ptrdiff_t"). It doesn't much
5568 matter though. Internally, we use "long" as arguments, which should
5569 work across all reasonable possibilities.
5570
5571 Additionally, if MORECORE ever returns failure for a positive
5572 request, then mmap is used as a noncontiguous system allocator. This
5573 is a useful backup strategy for systems with holes in address spaces
5574 -- in this case sbrk cannot contiguously expand the heap, but mmap
5575 may be able to map noncontiguous space.
5576
5577 If you'd like mmap to ALWAYS be used, you can define MORECORE to be
5578 a function that always returns MORECORE_FAILURE.
5579
5580 If you are using this malloc with something other than sbrk (or its
5581 emulation) to supply memory regions, you probably want to set
5582 MORECORE_CONTIGUOUS as false. As an example, here is a custom
5583 allocator kindly contributed for pre-OSX macOS. It uses virtually
5584 but not necessarily physically contiguous non-paged memory (locked
5585 in, present and won't get swapped out). You can use it by
5586 uncommenting this section, adding some #includes, and setting up the
5587 appropriate defines above:
5588
5589 *#define MORECORE osMoreCore
5590 *#define MORECORE_CONTIGUOUS 0
5591
5592 There is also a shutdown routine that should somehow be called for
5593 cleanup upon program exit.
5594
5595 *#define MAX_POOL_ENTRIES 100
5596 *#define MINIMUM_MORECORE_SIZE (64 * 1024)
5597 static int next_os_pool;
5598 void *our_os_pools[MAX_POOL_ENTRIES];
5599
5600 void *osMoreCore(int size)
5601 {
5602 void *ptr = 0;
5603 static void *sbrk_top = 0;
5604
5605 if (size > 0)
5606 {
5607 if (size < MINIMUM_MORECORE_SIZE)
5608 size = MINIMUM_MORECORE_SIZE;
5609 if (CurrentExecutionLevel() == kTaskLevel)
5610 ptr = PoolAllocateResident(size + RM_PAGE_SIZE, 0);
5611 if (ptr == 0)
5612 {
5613 return (void *) MORECORE_FAILURE;
5614 }
5615 // save ptrs so they can be freed during cleanup
5616 our_os_pools[next_os_pool] = ptr;
5617 next_os_pool++;
5618 ptr = (void *) ((((unsigned long) ptr) + RM_PAGE_MASK) & ~RM_PAGE_MASK);
5619 sbrk_top = (char *) ptr + size;
5620 return ptr;
5621 }
5622 else if (size < 0)
5623 {
5624 // we don't currently support shrink behavior
5625 return (void *) MORECORE_FAILURE;
5626 }
5627 else
5628 {
5629 return sbrk_top;
5630 }
5631 }
5632
5633 // cleanup any allocated memory pools
5634 // called as last thing before shutting down driver
5635
5636 void osCleanupMem(void)
5637 {
5638 void **ptr;
5639
5640 for (ptr = our_os_pools; ptr < &our_os_pools[MAX_POOL_ENTRIES]; ptr++)
5641 if (*ptr)
5642 {
5643 PoolDeallocate(*ptr);
5644 * ptr = 0;
5645 }
5646 }
5647
5648 */
5649
5650
5651/* Helper code. */
5652
5653extern char **__libc_argv attribute_hidden;
5654
5655static void
5656malloc_printerr (const char *str)
5657{
5658#if IS_IN (libc)
5659 __libc_message (do_abort, "%s\n", str);
5660#else
5661 __libc_fatal (message: str);
5662#endif
5663 __builtin_unreachable ();
5664}
5665
5666#if IS_IN (libc)
5667/* We need a wrapper function for one of the additions of POSIX. */
5668int
5669__posix_memalign (void **memptr, size_t alignment, size_t size)
5670{
5671 void *mem;
5672
5673 if (!__malloc_initialized)
5674 ptmalloc_init ();
5675
5676 /* Test whether the SIZE argument is valid. It must be a power of
5677 two multiple of sizeof (void *). */
5678 if (alignment % sizeof (void *) != 0
5679 || !powerof2 (alignment / sizeof (void *))
5680 || alignment == 0)
5681 return EINVAL;
5682
5683
5684 void *address = RETURN_ADDRESS (0);
5685 mem = _mid_memalign (alignment, size, address);
5686
5687 if (mem != NULL)
5688 {
5689 *memptr = mem;
5690 return 0;
5691 }
5692
5693 return ENOMEM;
5694}
5695weak_alias (__posix_memalign, posix_memalign)
5696#endif
5697
5698
5699int
5700__malloc_info (int options, FILE *fp)
5701{
5702 /* For now, at least. */
5703 if (options != 0)
5704 return EINVAL;
5705
5706 int n = 0;
5707 size_t total_nblocks = 0;
5708 size_t total_nfastblocks = 0;
5709 size_t total_avail = 0;
5710 size_t total_fastavail = 0;
5711 size_t total_system = 0;
5712 size_t total_max_system = 0;
5713 size_t total_aspace = 0;
5714 size_t total_aspace_mprotect = 0;
5715
5716
5717
5718 if (!__malloc_initialized)
5719 ptmalloc_init ();
5720
5721 fputs (s: "<malloc version=\"1\">\n", stream: fp);
5722
5723 /* Iterate over all arenas currently in use. */
5724 mstate ar_ptr = &main_arena;
5725 do
5726 {
5727 fprintf (stream: fp, format: "<heap nr=\"%d\">\n<sizes>\n", n++);
5728
5729 size_t nblocks = 0;
5730 size_t nfastblocks = 0;
5731 size_t avail = 0;
5732 size_t fastavail = 0;
5733 struct
5734 {
5735 size_t from;
5736 size_t to;
5737 size_t total;
5738 size_t count;
5739 } sizes[NFASTBINS + NBINS - 1];
5740#define nsizes (sizeof (sizes) / sizeof (sizes[0]))
5741
5742 __libc_lock_lock (ar_ptr->mutex);
5743
5744 /* Account for top chunk. The top-most available chunk is
5745 treated specially and is never in any bin. See "initial_top"
5746 comments. */
5747 avail = chunksize (ar_ptr->top);
5748 nblocks = 1; /* Top always exists. */
5749
5750 for (size_t i = 0; i < NFASTBINS; ++i)
5751 {
5752 mchunkptr p = fastbin (ar_ptr, i);
5753 if (p != NULL)
5754 {
5755 size_t nthissize = 0;
5756 size_t thissize = chunksize (p);
5757
5758 while (p != NULL)
5759 {
5760 if (__glibc_unlikely (misaligned_chunk (p)))
5761 malloc_printerr (str: "__malloc_info(): "
5762 "unaligned fastbin chunk detected");
5763 ++nthissize;
5764 p = REVEAL_PTR (p->fd);
5765 }
5766
5767 fastavail += nthissize * thissize;
5768 nfastblocks += nthissize;
5769 sizes[i].from = thissize - (MALLOC_ALIGNMENT - 1);
5770 sizes[i].to = thissize;
5771 sizes[i].count = nthissize;
5772 }
5773 else
5774 sizes[i].from = sizes[i].to = sizes[i].count = 0;
5775
5776 sizes[i].total = sizes[i].count * sizes[i].to;
5777 }
5778
5779
5780 mbinptr bin;
5781 struct malloc_chunk *r;
5782
5783 for (size_t i = 1; i < NBINS; ++i)
5784 {
5785 bin = bin_at (ar_ptr, i);
5786 r = bin->fd;
5787 sizes[NFASTBINS - 1 + i].from = ~((size_t) 0);
5788 sizes[NFASTBINS - 1 + i].to = sizes[NFASTBINS - 1 + i].total
5789 = sizes[NFASTBINS - 1 + i].count = 0;
5790
5791 if (r != NULL)
5792 while (r != bin)
5793 {
5794 size_t r_size = chunksize_nomask (r);
5795 ++sizes[NFASTBINS - 1 + i].count;
5796 sizes[NFASTBINS - 1 + i].total += r_size;
5797 sizes[NFASTBINS - 1 + i].from
5798 = MIN (sizes[NFASTBINS - 1 + i].from, r_size);
5799 sizes[NFASTBINS - 1 + i].to = MAX (sizes[NFASTBINS - 1 + i].to,
5800 r_size);
5801
5802 r = r->fd;
5803 }
5804
5805 if (sizes[NFASTBINS - 1 + i].count == 0)
5806 sizes[NFASTBINS - 1 + i].from = 0;
5807 nblocks += sizes[NFASTBINS - 1 + i].count;
5808 avail += sizes[NFASTBINS - 1 + i].total;
5809 }
5810
5811 size_t heap_size = 0;
5812 size_t heap_mprotect_size = 0;
5813 size_t heap_count = 0;
5814 if (ar_ptr != &main_arena)
5815 {
5816 /* Iterate over the arena heaps from back to front. */
5817 heap_info *heap = heap_for_ptr (top (ar_ptr));
5818 do
5819 {
5820 heap_size += heap->size;
5821 heap_mprotect_size += heap->mprotect_size;
5822 heap = heap->prev;
5823 ++heap_count;
5824 }
5825 while (heap != NULL);
5826 }
5827
5828 __libc_lock_unlock (ar_ptr->mutex);
5829
5830 total_nfastblocks += nfastblocks;
5831 total_fastavail += fastavail;
5832
5833 total_nblocks += nblocks;
5834 total_avail += avail;
5835
5836 for (size_t i = 0; i < nsizes; ++i)
5837 if (sizes[i].count != 0 && i != NFASTBINS)
5838 fprintf (stream: fp, format: "\
5839 <size from=\"%zu\" to=\"%zu\" total=\"%zu\" count=\"%zu\"/>\n",
5840 sizes[i].from, sizes[i].to, sizes[i].total, sizes[i].count);
5841
5842 if (sizes[NFASTBINS].count != 0)
5843 fprintf (stream: fp, format: "\
5844 <unsorted from=\"%zu\" to=\"%zu\" total=\"%zu\" count=\"%zu\"/>\n",
5845 sizes[NFASTBINS].from, sizes[NFASTBINS].to,
5846 sizes[NFASTBINS].total, sizes[NFASTBINS].count);
5847
5848 total_system += ar_ptr->system_mem;
5849 total_max_system += ar_ptr->max_system_mem;
5850
5851 fprintf (stream: fp,
5852 format: "</sizes>\n<total type=\"fast\" count=\"%zu\" size=\"%zu\"/>\n"
5853 "<total type=\"rest\" count=\"%zu\" size=\"%zu\"/>\n"
5854 "<system type=\"current\" size=\"%zu\"/>\n"
5855 "<system type=\"max\" size=\"%zu\"/>\n",
5856 nfastblocks, fastavail, nblocks, avail,
5857 ar_ptr->system_mem, ar_ptr->max_system_mem);
5858
5859 if (ar_ptr != &main_arena)
5860 {
5861 fprintf (stream: fp,
5862 format: "<aspace type=\"total\" size=\"%zu\"/>\n"
5863 "<aspace type=\"mprotect\" size=\"%zu\"/>\n"
5864 "<aspace type=\"subheaps\" size=\"%zu\"/>\n",
5865 heap_size, heap_mprotect_size, heap_count);
5866 total_aspace += heap_size;
5867 total_aspace_mprotect += heap_mprotect_size;
5868 }
5869 else
5870 {
5871 fprintf (stream: fp,
5872 format: "<aspace type=\"total\" size=\"%zu\"/>\n"
5873 "<aspace type=\"mprotect\" size=\"%zu\"/>\n",
5874 ar_ptr->system_mem, ar_ptr->system_mem);
5875 total_aspace += ar_ptr->system_mem;
5876 total_aspace_mprotect += ar_ptr->system_mem;
5877 }
5878
5879 fputs (s: "</heap>\n", stream: fp);
5880 ar_ptr = ar_ptr->next;
5881 }
5882 while (ar_ptr != &main_arena);
5883
5884 fprintf (stream: fp,
5885 format: "<total type=\"fast\" count=\"%zu\" size=\"%zu\"/>\n"
5886 "<total type=\"rest\" count=\"%zu\" size=\"%zu\"/>\n"
5887 "<total type=\"mmap\" count=\"%d\" size=\"%zu\"/>\n"
5888 "<system type=\"current\" size=\"%zu\"/>\n"
5889 "<system type=\"max\" size=\"%zu\"/>\n"
5890 "<aspace type=\"total\" size=\"%zu\"/>\n"
5891 "<aspace type=\"mprotect\" size=\"%zu\"/>\n"
5892 "</malloc>\n",
5893 total_nfastblocks, total_fastavail, total_nblocks, total_avail,
5894 mp_.n_mmaps, mp_.mmapped_mem,
5895 total_system, total_max_system,
5896 total_aspace, total_aspace_mprotect);
5897
5898 return 0;
5899}
5900#if IS_IN (libc)
5901weak_alias (__malloc_info, malloc_info)
5902
5903strong_alias (__libc_calloc, __calloc) weak_alias (__libc_calloc, calloc)
5904strong_alias (__libc_free, __free) strong_alias (__libc_free, free)
5905strong_alias (__libc_malloc, __malloc) strong_alias (__libc_malloc, malloc)
5906strong_alias (__libc_memalign, __memalign)
5907weak_alias (__libc_memalign, memalign)
5908strong_alias (__libc_realloc, __realloc) strong_alias (__libc_realloc, realloc)
5909strong_alias (__libc_valloc, __valloc) weak_alias (__libc_valloc, valloc)
5910strong_alias (__libc_pvalloc, __pvalloc) weak_alias (__libc_pvalloc, pvalloc)
5911strong_alias (__libc_mallinfo, __mallinfo)
5912weak_alias (__libc_mallinfo, mallinfo)
5913strong_alias (__libc_mallinfo2, __mallinfo2)
5914weak_alias (__libc_mallinfo2, mallinfo2)
5915strong_alias (__libc_mallopt, __mallopt) weak_alias (__libc_mallopt, mallopt)
5916
5917weak_alias (__malloc_stats, malloc_stats)
5918weak_alias (__malloc_usable_size, malloc_usable_size)
5919weak_alias (__malloc_trim, malloc_trim)
5920#endif
5921
5922#if SHLIB_COMPAT (libc, GLIBC_2_0, GLIBC_2_26)
5923compat_symbol (libc, __libc_free, cfree, GLIBC_2_0);
5924#endif
5925
5926/* ------------------------------------------------------------
5927 History:
5928
5929 [see ftp://g.oswego.edu/pub/misc/malloc.c for the history of dlmalloc]
5930
5931 */
5932/*
5933 * Local variables:
5934 * c-basic-offset: 2
5935 * End:
5936 */
5937

source code of glibc/malloc/malloc.c