OpenSolaris_b135/lib/libmtmalloc/common/mtmalloc.c
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#include <mtmalloc.h>
#include "mtmalloc_impl.h"
#include <unistd.h>
#include <synch.h>
#include <thread.h>
#include <pthread.h>
#include <stdio.h>
#include <limits.h>
#include <errno.h>
#include <string.h>
#include <strings.h>
#include <sys/param.h>
#include <sys/sysmacros.h>
/*
* To turn on the asserts just compile -DDEBUG
*/
#ifndef DEBUG
#define NDEBUG
#endif
#include <assert.h>
/*
* The MT hot malloc implementation contained herein is designed to be
* plug-compatible with the libc version of malloc. It is not intended
* to replace that implementation until we decide that it is ok to break
* customer apps (Solaris 3.0).
*
* For requests up to 2^^16, the allocator initializes itself into NCPUS
* worth of chains of caches. When a memory request is made, the calling thread
* is vectored into one of NCPUS worth of caches. The LWP id gives us a cheap,
* contention-reducing index to use, eventually, this should be replaced with
* the actual CPU sequence number, when an interface to get it is available.
*
* Once the thread is vectored into one of the list of caches the real
* allocation of the memory begins. The size is determined to figure out which
* bucket the allocation should be satisfied from. The management of free
* buckets is done via a bitmask. A free bucket is represented by a 1. The
* first free bit represents the first free bucket. The position of the bit,
* represents the position of the bucket in the arena.
*
* When the memory from the arena is handed out, the address of the cache
* control structure is written in the word preceeding the returned memory.
* This cache control address is used during free() to mark the buffer free
* in the cache control structure.
*
* When all available memory in a cache has been depleted, a new chunk of memory
* is allocated via sbrk(). The new cache is allocated from this chunk of memory
* and initialized in the function create_cache(). New caches are installed at
* the front of a singly linked list of the same size memory pools. This helps
* to ensure that there will tend to be available memory in the beginning of the
* list.
*
* Long linked lists hurt performance. To decrease this effect, there is a
* tunable, requestsize, that bumps up the sbrk allocation size and thus
* increases the number of available blocks within an arena. We also keep
* a "hint" for each cache list, which is the last cache in the list allocated
* from. This lowers the cost of searching if there are a lot of fully
* allocated blocks at the front of the list.
*
* For requests greater than 2^^16 (oversize allocations), there are two pieces
* of overhead. There is the OVERHEAD used to hold the cache addr
* (&oversize_list), plus an oversize_t structure to further describe the block.
*
* The oversize list is kept as defragmented as possible by coalescing
* freed oversized allocations with adjacent neighbors.
*
* Addresses handed out are stored in a hash table, and are aligned on
* MTMALLOC_MIN_ALIGN-byte boundaries at both ends. Request sizes are rounded-up
* where necessary in order to achieve this. This eases the implementation of
* MTDEBUGPATTERN and MTINITPATTERN, particularly where coalescing occurs.
*
* A memalign allocation takes memalign header overhead. There's two
* types of memalign headers distinguished by MTMALLOC_MEMALIGN_MAGIC
* and MTMALLOC_MEMALIGN_MIN_MAGIC. When the size of memory taken to
* get to the aligned address from malloc'ed address is the minimum size
* OVERHEAD, we create a header taking only one OVERHEAD space with magic
* number MTMALLOC_MEMALIGN_MIN_MAGIC, and we know by subtracting OVERHEAD
* from memaligned address, we can get to the malloc'ed address. Otherwise,
* we create a memalign header taking two OVERHEAD space, one stores
* MTMALLOC_MEMALIGN_MAGIC magic number, the other one points back to the
* malloc'ed address.
*/
#if defined(__i386) || defined(__amd64)
#include <arpa/inet.h> /* for htonl() */
#endif
static void * morecore(size_t);
static void create_cache(cache_t *, size_t bufsize, uint_t hunks);
static void * malloc_internal(size_t, percpu_t *);
static void * oversize(size_t);
static oversize_t *find_oversize(size_t);
static void add_oversize(oversize_t *);
static void copy_pattern(uint32_t, void *, size_t);
static void * verify_pattern(uint32_t, void *, size_t);
static void reinit_cpu_list(void);
static void reinit_cache(cache_t *);
static void free_oversize(oversize_t *);
static oversize_t *oversize_header_alloc(uintptr_t, size_t);
/*
* oversize hash table stuff
*/
#define NUM_BUCKETS 67 /* must be prime */
#define HASH_OVERSIZE(caddr) ((uintptr_t)(caddr) % NUM_BUCKETS)
oversize_t *ovsz_hashtab[NUM_BUCKETS];
#define ALIGN(x, a) ((((uintptr_t)(x) + ((uintptr_t)(a) - 1)) \
& ~((uintptr_t)(a) - 1)))
/* need this to deal with little endianess of x86 */
#if defined(__i386) || defined(__amd64)
#define FLIP_EM(x) htonl((x))
#else
#define FLIP_EM(x) (x)
#endif
#define INSERT_ONLY 0
#define COALESCE_LEFT 0x00000001
#define COALESCE_RIGHT 0x00000002
#define COALESCE_WITH_BOTH_SIDES (COALESCE_LEFT | COALESCE_RIGHT)
#define OVERHEAD 8 /* size needed to write cache addr */
#define HUNKSIZE 8192 /* just a multiplier */
#define MAX_CACHED_SHIFT 16 /* 64K is the max cached size */
#define MAX_CACHED (1 << MAX_CACHED_SHIFT)
#define MIN_CACHED_SHIFT 4 /* smaller requests rounded up */
#define MTMALLOC_MIN_ALIGN 8 /* min guaranteed alignment */
/* maximum size before overflow */
#define MAX_MTMALLOC (SIZE_MAX - (SIZE_MAX % MTMALLOC_MIN_ALIGN) \
- OVSZ_HEADER_SIZE)
#define NUM_CACHES (MAX_CACHED_SHIFT - MIN_CACHED_SHIFT + 1)
#define CACHELIST_SIZE ALIGN(NUM_CACHES * sizeof (cache_head_t), \
CACHE_COHERENCY_UNIT)
#define MINSIZE 9 /* for requestsize, tunable */
#define MAXSIZE 256 /* arbitrary, big enough, for requestsize */
#define FREEPATTERN 0xdeadbeef /* debug fill pattern for free buf */
#define INITPATTERN 0xbaddcafe /* debug fill pattern for new buf */
#define misaligned(p) ((unsigned)(p) & (sizeof (int) - 1))
#define IS_OVERSIZE(x, y) (((x) < (y)) && (((x) > MAX_CACHED)? 1 : 0))
static long requestsize = MINSIZE; /* 9 pages per cache; tunable; 9 is min */
static uint_t cpu_mask;
static curcpu_func curcpu;
static int32_t debugopt;
static int32_t reinit;
static percpu_t *cpu_list;
static oversize_t oversize_list;
static mutex_t oversize_lock = DEFAULTMUTEX;
static int ncpus = 0;
#define MTMALLOC_OVERSIZE_MAGIC ((uintptr_t)&oversize_list)
#define MTMALLOC_MEMALIGN_MAGIC ((uintptr_t)&oversize_list + 1)
#define MTMALLOC_MEMALIGN_MIN_MAGIC ((uintptr_t)&oversize_list + 2)
/*
* We require allocations handed out to be aligned on MTMALLOC_MIN_ALIGN-byte
* boundaries. We round up sizeof (oversize_t) (when necessary) to ensure that
* this is achieved.
*/
#define OVSZ_SIZE (ALIGN(sizeof (oversize_t), MTMALLOC_MIN_ALIGN))
#define OVSZ_HEADER_SIZE (OVSZ_SIZE + OVERHEAD)
/*
* memalign header takes 2 OVERHEAD space. One for memalign magic, and the
* other one points back to the start address of originally allocated space.
*/
#define MEMALIGN_HEADER_SIZE 2 * OVERHEAD
#define MEMALIGN_HEADER_ALLOC(x, shift, malloc_addr)\
if (shift == OVERHEAD)\
*((uintptr_t *)((caddr_t)x - OVERHEAD)) = \
MTMALLOC_MEMALIGN_MIN_MAGIC; \
else {\
*((uintptr_t *)((caddr_t)x - OVERHEAD)) = \
MTMALLOC_MEMALIGN_MAGIC; \
*((uintptr_t *)((caddr_t)x - 2 * OVERHEAD)) = \
(uintptr_t)malloc_addr; \
}
/*
* Add big to the oversize hash table at the head of the relevant bucket.
*/
static void
insert_hash(oversize_t *big)
{
caddr_t ret = big->addr;
int bucket = HASH_OVERSIZE(ret);
assert(MUTEX_HELD(&oversize_lock));
big->hash_next = ovsz_hashtab[bucket];
ovsz_hashtab[bucket] = big;
}
void *
malloc(size_t bytes)
{
percpu_t *list_rotor;
uint_t list_index;
if (bytes > MAX_CACHED)
return (oversize(bytes));
list_index = (curcpu() & cpu_mask);
list_rotor = &cpu_list[list_index];
return (malloc_internal(bytes, list_rotor));
}
void *
realloc(void * ptr, size_t bytes)
{
void *new, *data_ptr;
cache_t *cacheptr;
caddr_t mem;
size_t shift = 0;
if (ptr == NULL)
return (malloc(bytes));
if (bytes == 0) {
free(ptr);
return (NULL);
}
data_ptr = ptr;
mem = (caddr_t)ptr - OVERHEAD;
/*
* Optimization possibility :
* p = malloc(64);
* q = realloc(p, 64);
* q can be same as p.
* Apply this optimization for the normal
* sized caches for now.
*/
if (*(uintptr_t *)mem < MTMALLOC_OVERSIZE_MAGIC ||
*(uintptr_t *)mem > MTMALLOC_MEMALIGN_MIN_MAGIC) {
cacheptr = (cache_t *)*(uintptr_t *)mem;
if (bytes <= (cacheptr->mt_size - OVERHEAD))
return (ptr);
}
new = malloc(bytes);
if (new == NULL)
return (NULL);
/*
* If new == ptr, ptr has previously been freed. Passing a freed pointer
* to realloc() is not allowed - unless the caller specifically states
* otherwise, in which case we must avoid freeing ptr (ie new) before we
* return new. There is (obviously) no requirement to memcpy() ptr to
* new before we return.
*/
if (new == ptr) {
if (!(debugopt & MTDOUBLEFREE))
abort();
return (new);
}
if (*(uintptr_t *)mem == MTMALLOC_MEMALIGN_MAGIC) {
mem -= OVERHEAD;
ptr = (void *)*(uintptr_t *)mem;
mem = (caddr_t)ptr - OVERHEAD;
shift = (size_t)((uintptr_t)data_ptr - (uintptr_t)ptr);
} else if (*(uintptr_t *)mem == MTMALLOC_MEMALIGN_MIN_MAGIC) {
ptr = (void *) mem;
mem -= OVERHEAD;
shift = OVERHEAD;
}
if (*(uintptr_t *)mem == MTMALLOC_OVERSIZE_MAGIC) {
oversize_t *old;
old = (oversize_t *)(mem - OVSZ_SIZE);
(void) memcpy(new, data_ptr, MIN(bytes, old->size - shift));
free(ptr);
return (new);
}
cacheptr = (cache_t *)*(uintptr_t *)mem;
(void) memcpy(new, data_ptr,
MIN(cacheptr->mt_size - OVERHEAD - shift, bytes));
free(ptr);
return (new);
}
void *
calloc(size_t nelem, size_t bytes)
{
void * ptr;
size_t size = nelem * bytes;
ptr = malloc(size);
if (ptr == NULL)
return (NULL);
(void) memset(ptr, 0, size);
return (ptr);
}
void
free(void * ptr)
{
cache_t *cacheptr;
caddr_t mem;
int32_t i;
caddr_t freeblocks;
uintptr_t offset;
uchar_t mask;
int32_t which_bit, num_bytes;
if (ptr == NULL)
return;
mem = (caddr_t)ptr - OVERHEAD;
if (*(uintptr_t *)mem == MTMALLOC_MEMALIGN_MAGIC) {
mem -= OVERHEAD;
ptr = (void *)*(uintptr_t *)mem;
mem = (caddr_t)ptr - OVERHEAD;
} else if (*(uintptr_t *)mem == MTMALLOC_MEMALIGN_MIN_MAGIC) {
ptr = (void *) mem;
mem -= OVERHEAD;
}
if (*(uintptr_t *)mem == MTMALLOC_OVERSIZE_MAGIC) {
oversize_t *big, **opp;
int bucket;
big = (oversize_t *)(mem - OVSZ_SIZE);
(void) mutex_lock(&oversize_lock);
bucket = HASH_OVERSIZE(big->addr);
for (opp = &ovsz_hashtab[bucket]; *opp != NULL;
opp = &(*opp)->hash_next)
if (*opp == big)
break;
if (*opp == NULL) {
if (!(debugopt & MTDOUBLEFREE))
abort();
(void) mutex_unlock(&oversize_lock);
return;
}
*opp = big->hash_next; /* remove big from the hash table */
big->hash_next = NULL;
if (debugopt & MTDEBUGPATTERN)
copy_pattern(FREEPATTERN, ptr, big->size);
add_oversize(big);
(void) mutex_unlock(&oversize_lock);
return;
}
cacheptr = (cache_t *)*(uintptr_t *)mem;
freeblocks = cacheptr->mt_freelist;
/*
* This is the distance measured in bits into the arena.
* The value of offset is in bytes but there is a 1-1 correlation
* between distance into the arena and distance into the
* freelist bitmask.
*/
offset = mem - cacheptr->mt_arena;
/*
* i is total number of bits to offset into freelist bitmask.
*/
i = offset / cacheptr->mt_size;
num_bytes = i >> 3;
/*
* which_bit is the bit offset into the byte in the freelist.
* if our freelist bitmask looks like 0xf3 and we are freeing
* block 5 (ie: the 6th block) our mask will be 0xf7 after
* the free. Things go left to right that's why the mask is 0x80
* and not 0x01.
*/
which_bit = i - (num_bytes << 3);
mask = 0x80 >> which_bit;
freeblocks += num_bytes;
if (debugopt & MTDEBUGPATTERN)
copy_pattern(FREEPATTERN, ptr, cacheptr->mt_size - OVERHEAD);
(void) mutex_lock(&cacheptr->mt_cache_lock);
if (*freeblocks & mask) {
if (!(debugopt & MTDOUBLEFREE))
abort();
} else {
*freeblocks |= mask;
cacheptr->mt_nfree++;
}
(void) mutex_unlock(&cacheptr->mt_cache_lock);
}
void *
memalign(size_t alignment, size_t size)
{
size_t alloc_size;
uintptr_t offset;
void *alloc_buf;
void *ret_buf;
if (size == 0 || alignment == 0 || misaligned(alignment) ||
(alignment & (alignment - 1)) != 0) {
errno = EINVAL;
return (NULL);
}
/* <= MTMALLOC_MIN_ALIGN, malloc can provide directly */
if (alignment <= MTMALLOC_MIN_ALIGN)
return (malloc(size));
alloc_size = size + alignment - MTMALLOC_MIN_ALIGN;
if (alloc_size < size) { /* overflow */
errno = ENOMEM;
return (NULL);
}
alloc_buf = malloc(alloc_size);
if (alloc_buf == NULL)
/* malloc sets errno */
return (NULL);
/*
* If alloc_size > MAX_CACHED, malloc() will have returned a multiple of
* MTMALLOC_MIN_ALIGN, having rounded-up alloc_size if necessary. Since
* we will use alloc_size to return the excess fragments to the free
* list, we also round-up alloc_size if necessary.
*/
if ((alloc_size > MAX_CACHED) &&
(alloc_size & (MTMALLOC_MIN_ALIGN - 1)))
alloc_size = ALIGN(alloc_size, MTMALLOC_MIN_ALIGN);
if ((offset = (uintptr_t)alloc_buf & (alignment - 1)) == 0) {
/* aligned correctly */
size_t frag_size = alloc_size -
(size + MTMALLOC_MIN_ALIGN + OVSZ_HEADER_SIZE);
/*
* If the leftover piece of the memory > MAX_CACHED,
* split off the piece and return it back to the freelist.
*/
if (IS_OVERSIZE(frag_size, alloc_size)) {
oversize_t *orig, *tail;
uintptr_t taddr;
size_t data_size;
taddr = ALIGN((uintptr_t)alloc_buf + size,
MTMALLOC_MIN_ALIGN);
data_size = taddr - (uintptr_t)alloc_buf;
orig = (oversize_t *)((uintptr_t)alloc_buf -
OVSZ_HEADER_SIZE);
frag_size = orig->size - data_size -
OVSZ_HEADER_SIZE;
orig->size = data_size;
tail = oversize_header_alloc(taddr, frag_size);
free_oversize(tail);
}
ret_buf = alloc_buf;
} else {
uchar_t oversize_bits = 0;
size_t head_sz, data_sz, tail_sz;
uintptr_t ret_addr, taddr, shift, tshift;
oversize_t *orig, *tail, *big;
size_t tsize;
/* needs to be aligned */
shift = alignment - offset;
assert(shift >= MTMALLOC_MIN_ALIGN);
ret_addr = ((uintptr_t)alloc_buf + shift);
ret_buf = (void *)ret_addr;
if (alloc_size <= MAX_CACHED) {
MEMALIGN_HEADER_ALLOC(ret_addr, shift, alloc_buf);
return (ret_buf);
}
/*
* Only check for the fragments when the memory is allocted
* from oversize_list. Split off a fragment and return it
* to the oversize freelist when it's > MAX_CACHED.
*/
head_sz = shift - MAX(MEMALIGN_HEADER_SIZE, OVSZ_HEADER_SIZE);
tail_sz = alloc_size -
(shift + size + MTMALLOC_MIN_ALIGN + OVSZ_HEADER_SIZE);
oversize_bits |= IS_OVERSIZE(head_sz, alloc_size) |
IS_OVERSIZE(size, alloc_size) << DATA_SHIFT |
IS_OVERSIZE(tail_sz, alloc_size) << TAIL_SHIFT;
switch (oversize_bits) {
case NONE_OVERSIZE:
case DATA_OVERSIZE:
MEMALIGN_HEADER_ALLOC(ret_addr, shift,
alloc_buf);
break;
case HEAD_OVERSIZE:
/*
* If we can extend data > MAX_CACHED and have
* head still > MAX_CACHED, we split head-end
* as the case of head-end and data oversized,
* otherwise just create memalign header.
*/
tsize = (shift + size) - (MAX_CACHED + 8 +
MTMALLOC_MIN_ALIGN + OVSZ_HEADER_SIZE);
if (!IS_OVERSIZE(tsize, alloc_size)) {
MEMALIGN_HEADER_ALLOC(ret_addr, shift,
alloc_buf);
break;
} else {
tsize += OVSZ_HEADER_SIZE;
taddr = ALIGN((uintptr_t)alloc_buf +
tsize, MTMALLOC_MIN_ALIGN);
tshift = ret_addr - taddr;
MEMALIGN_HEADER_ALLOC(ret_addr, tshift,
taddr);
ret_addr = taddr;
shift = ret_addr - (uintptr_t)alloc_buf;
}
/* FALLTHROUGH */
case HEAD_AND_DATA_OVERSIZE:
/*
* Split off the head fragment and
* return it back to oversize freelist.
* Create oversize header for the piece
* of (data + tail fragment).
*/
orig = (oversize_t *)((uintptr_t)alloc_buf -
OVSZ_HEADER_SIZE);
big = oversize_header_alloc(ret_addr -
OVSZ_HEADER_SIZE, (orig->size - shift));
(void) mutex_lock(&oversize_lock);
insert_hash(big);
(void) mutex_unlock(&oversize_lock);
orig->size = shift - OVSZ_HEADER_SIZE;
/* free up the head fragment */
free_oversize(orig);
break;
case TAIL_OVERSIZE:
/*
* If we can extend data > MAX_CACHED and have
* tail-end still > MAX_CACHED, we split tail
* end, otherwise just create memalign header.
*/
orig = (oversize_t *)((uintptr_t)alloc_buf -
OVSZ_HEADER_SIZE);
tsize = orig->size - (MAX_CACHED + 8 +
shift + OVSZ_HEADER_SIZE +
MTMALLOC_MIN_ALIGN);
if (!IS_OVERSIZE(tsize, alloc_size)) {
MEMALIGN_HEADER_ALLOC(ret_addr, shift,
alloc_buf);
break;
} else {
size = MAX_CACHED + 8;
}
/* FALLTHROUGH */
case DATA_AND_TAIL_OVERSIZE:
/*
* Split off the tail fragment and
* return it back to oversize freelist.
* Create memalign header and adjust
* the size for the piece of
* (head fragment + data).
*/
taddr = ALIGN(ret_addr + size,
MTMALLOC_MIN_ALIGN);
data_sz = (size_t)(taddr -
(uintptr_t)alloc_buf);
orig = (oversize_t *)((uintptr_t)alloc_buf -
OVSZ_HEADER_SIZE);
tsize = orig->size - data_sz;
orig->size = data_sz;
MEMALIGN_HEADER_ALLOC(ret_buf, shift,
alloc_buf);
tsize -= OVSZ_HEADER_SIZE;
tail = oversize_header_alloc(taddr, tsize);
free_oversize(tail);
break;
case HEAD_AND_TAIL_OVERSIZE:
/*
* Split off the head fragment.
* We try to free up tail-end when we can
* extend data size to (MAX_CACHED + 8)
* and remain tail-end oversized.
* The bottom line is all split pieces
* should be oversize in size.
*/
orig = (oversize_t *)((uintptr_t)alloc_buf -
OVSZ_HEADER_SIZE);
tsize = orig->size - (MAX_CACHED + 8 +
OVSZ_HEADER_SIZE + shift +
MTMALLOC_MIN_ALIGN);
if (!IS_OVERSIZE(tsize, alloc_size)) {
/*
* If the chunk is not big enough
* to make both data and tail oversize
* we just keep them as one piece.
*/
big = oversize_header_alloc(ret_addr -
OVSZ_HEADER_SIZE,
orig->size - shift);
(void) mutex_lock(&oversize_lock);
insert_hash(big);
(void) mutex_unlock(&oversize_lock);
orig->size = shift - OVSZ_HEADER_SIZE;
free_oversize(orig);
break;
} else {
/*
* extend data size > MAX_CACHED
* and handle it as head, data, tail
* are all oversized.
*/
size = MAX_CACHED + 8;
}
/* FALLTHROUGH */
case ALL_OVERSIZE:
/*
* split off the head and tail fragments,
* return them back to the oversize freelist.
* Alloc oversize header for data seg.
*/
orig = (oversize_t *)((uintptr_t)alloc_buf -
OVSZ_HEADER_SIZE);
tsize = orig->size;
orig->size = shift - OVSZ_HEADER_SIZE;
free_oversize(orig);
taddr = ALIGN(ret_addr + size,
MTMALLOC_MIN_ALIGN);
data_sz = taddr - ret_addr;
assert(tsize > (shift + data_sz +
OVSZ_HEADER_SIZE));
tail_sz = tsize -
(shift + data_sz + OVSZ_HEADER_SIZE);
/* create oversize header for data seg */
big = oversize_header_alloc(ret_addr -
OVSZ_HEADER_SIZE, data_sz);
(void) mutex_lock(&oversize_lock);
insert_hash(big);
(void) mutex_unlock(&oversize_lock);
/* create oversize header for tail fragment */
tail = oversize_header_alloc(taddr, tail_sz);
free_oversize(tail);
break;
default:
/* should not reach here */
assert(0);
}
}
return (ret_buf);
}
void *
valloc(size_t size)
{
static unsigned pagesize;
if (size == 0)
return (NULL);
if (!pagesize)
pagesize = sysconf(_SC_PAGESIZE);
return (memalign(pagesize, size));
}
void
mallocctl(int cmd, long value)
{
switch (cmd) {
case MTDEBUGPATTERN:
/*
* Reinitialize free blocks in case malloc() is called prior
* to mallocctl().
*/
if (value && !(debugopt & cmd)) {
reinit++;
debugopt |= cmd;
reinit_cpu_list();
}
/*FALLTHRU*/
case MTDOUBLEFREE:
case MTINITBUFFER:
if (value)
debugopt |= cmd;
else
debugopt &= ~cmd;
break;
case MTCHUNKSIZE:
if (value >= MINSIZE && value <= MAXSIZE)
requestsize = value;
break;
default:
break;
}
}
/*
* Initialization function, called from the init section of the library.
* No locking is required here because we are single-threaded during
* library initialization.
*/
static void
setup_caches(void)
{
uintptr_t oldbrk;
uintptr_t newbrk;
size_t cache_space_needed;
size_t padding;
curcpu_func new_curcpu;
uint_t new_cpu_mask;
percpu_t *new_cpu_list;
uint_t i, j;
uintptr_t list_addr;
/*
* Get a decent "current cpu identifier", to be used to reduce
* contention. Eventually, this should be replaced by an interface
* to get the actual CPU sequence number in libthread/liblwp.
*/
new_curcpu = (curcpu_func)thr_self;
if ((ncpus = 2 * sysconf(_SC_NPROCESSORS_CONF)) <= 0)
ncpus = 4; /* decent default value */
/* round ncpus up to a power of 2 */
while (ncpus & (ncpus - 1))
ncpus++;
new_cpu_mask = ncpus - 1; /* create the cpu mask */
/*
* We now do some magic with the brk. What we want to get in the
* end is a bunch of well-aligned stuff in a big initial allocation.
* Along the way, we do sanity checks to make sure no one else has
* touched the brk (which shouldn't happen, but it's always good to
* check)
*
* First, make sure sbrk is sane, and store the current brk in oldbrk.
*/
oldbrk = (uintptr_t)sbrk(0);
if ((void *)oldbrk == (void *)-1)
abort(); /* sbrk is broken -- we're doomed. */
/*
* Now, align the brk to a multiple of CACHE_COHERENCY_UNIT, so that
* the percpu structures and cache lists will be properly aligned.
*
* 2. All hunks will be page-aligned, assuming HUNKSIZE >= PAGESIZE,
* so they can be paged out individually.
*/
newbrk = ALIGN(oldbrk, CACHE_COHERENCY_UNIT);
if (newbrk != oldbrk && (uintptr_t)sbrk(newbrk - oldbrk) != oldbrk)
abort(); /* sbrk is broken -- we're doomed. */
/*
* For each cpu, there is one percpu_t and a list of caches
*/
cache_space_needed = ncpus * (sizeof (percpu_t) + CACHELIST_SIZE);
new_cpu_list = (percpu_t *)sbrk(cache_space_needed);
if (new_cpu_list == (percpu_t *)-1 ||
(uintptr_t)new_cpu_list != newbrk)
abort(); /* sbrk is broken -- we're doomed. */
/*
* Finally, align the brk to HUNKSIZE so that all hunks are
* page-aligned, to avoid edge-effects.
*/
newbrk = (uintptr_t)new_cpu_list + cache_space_needed;
padding = ALIGN(newbrk, HUNKSIZE) - newbrk;
if (padding > 0 && (uintptr_t)sbrk(padding) != newbrk)
abort(); /* sbrk is broken -- we're doomed. */
list_addr = ((uintptr_t)new_cpu_list + (sizeof (percpu_t) * ncpus));
/* initialize the percpu list */
for (i = 0; i < ncpus; i++) {
new_cpu_list[i].mt_caches = (cache_head_t *)list_addr;
for (j = 0; j < NUM_CACHES; j++) {
new_cpu_list[i].mt_caches[j].mt_cache = NULL;
new_cpu_list[i].mt_caches[j].mt_hint = NULL;
}
(void) mutex_init(&new_cpu_list[i].mt_parent_lock,
USYNC_THREAD, NULL);
/* get the correct cache list alignment */
list_addr += CACHELIST_SIZE;
}
/*
* Initialize oversize listhead
*/
oversize_list.next_bysize = &oversize_list;
oversize_list.prev_bysize = &oversize_list;
oversize_list.next_byaddr = &oversize_list;
oversize_list.prev_byaddr = &oversize_list;
oversize_list.addr = NULL;
oversize_list.size = 0; /* sentinal */
/*
* Now install the global variables.
*/
curcpu = new_curcpu;
cpu_mask = new_cpu_mask;
cpu_list = new_cpu_list;
}
static void
create_cache(cache_t *cp, size_t size, uint_t chunksize)
{
long nblocks;
(void) mutex_init(&cp->mt_cache_lock, USYNC_THREAD, NULL);
cp->mt_size = size;
cp->mt_freelist = ((caddr_t)cp + sizeof (cache_t));
cp->mt_span = chunksize * HUNKSIZE - sizeof (cache_t);
cp->mt_hunks = chunksize;
/*
* rough calculation. We will need to adjust later.
*/
nblocks = cp->mt_span / cp->mt_size;
nblocks >>= 3;
if (nblocks == 0) { /* less than 8 free blocks in this pool */
int32_t numblocks = 0;
long i = cp->mt_span;
size_t sub = cp->mt_size;
uchar_t mask = 0;
while (i > sub) {
numblocks++;
i -= sub;
}
nblocks = numblocks;
cp->mt_arena = (caddr_t)ALIGN(cp->mt_freelist + 8, 8);
cp->mt_nfree = numblocks;
while (numblocks--) {
mask |= 0x80 >> numblocks;
}
*(cp->mt_freelist) = mask;
} else {
cp->mt_arena = (caddr_t)ALIGN((caddr_t)cp->mt_freelist +
nblocks, 32);
/* recompute nblocks */
nblocks = (uintptr_t)((caddr_t)cp->mt_freelist +
cp->mt_span - cp->mt_arena) / cp->mt_size;
cp->mt_nfree = ((nblocks >> 3) << 3);
/* Set everything to free */
(void) memset(cp->mt_freelist, 0xff, nblocks >> 3);
}
if (debugopt & MTDEBUGPATTERN)
copy_pattern(FREEPATTERN, cp->mt_arena, cp->mt_size * nblocks);
cp->mt_next = NULL;
}
static void
reinit_cpu_list(void)
{
oversize_t *wp = oversize_list.next_bysize;
percpu_t *cpuptr;
cache_t *thiscache;
cache_head_t *cachehead;
/* Reinitialize free oversize blocks. */
(void) mutex_lock(&oversize_lock);
if (debugopt & MTDEBUGPATTERN)
for (; wp != &oversize_list; wp = wp->next_bysize)
copy_pattern(FREEPATTERN, wp->addr, wp->size);
(void) mutex_unlock(&oversize_lock);
/* Reinitialize free blocks. */
for (cpuptr = &cpu_list[0]; cpuptr < &cpu_list[ncpus]; cpuptr++) {
(void) mutex_lock(&cpuptr->mt_parent_lock);
for (cachehead = &cpuptr->mt_caches[0]; cachehead <
&cpuptr->mt_caches[NUM_CACHES]; cachehead++) {
for (thiscache = cachehead->mt_cache; thiscache != NULL;
thiscache = thiscache->mt_next) {
(void) mutex_lock(&thiscache->mt_cache_lock);
if (thiscache->mt_nfree == 0) {
(void) mutex_unlock(
&thiscache->mt_cache_lock);
continue;
}
if (thiscache != NULL)
reinit_cache(thiscache);
(void) mutex_unlock(&thiscache->mt_cache_lock);
}
}
(void) mutex_unlock(&cpuptr->mt_parent_lock);
}
reinit = 0;
}
static void
reinit_cache(cache_t *thiscache)
{
uint32_t *freeblocks; /* not a uintptr_t on purpose */
int32_t i, n;
caddr_t ret;
freeblocks = (uint32_t *)thiscache->mt_freelist;
while (freeblocks < (uint32_t *)thiscache->mt_arena) {
if (*freeblocks & 0xffffffff) {
for (i = 0; i < 32; i++) {
if (FLIP_EM(*freeblocks) & (0x80000000 >> i)) {
n = (uintptr_t)(((freeblocks -
(uint32_t *)thiscache->mt_freelist)
<< 5) + i) * thiscache->mt_size;
ret = thiscache->mt_arena + n;
ret += OVERHEAD;
copy_pattern(FREEPATTERN, ret,
thiscache->mt_size);
}
}
}
freeblocks++;
}
}
static void *
malloc_internal(size_t size, percpu_t *cpuptr)
{
cache_head_t *cachehead;
cache_t *thiscache, *hintcache;
int32_t i, n, logsz, bucket;
uint32_t index;
uint32_t *freeblocks; /* not a uintptr_t on purpose */
caddr_t ret;
logsz = MIN_CACHED_SHIFT;
while (size > (1 << logsz))
logsz++;
bucket = logsz - MIN_CACHED_SHIFT;
(void) mutex_lock(&cpuptr->mt_parent_lock);
/*
* Find a cache of the appropriate size with free buffers.
*
* We don't need to lock each cache as we check their mt_nfree count,
* since:
* 1. We are only looking for caches with mt_nfree > 0. If a
* free happens during our search, it will increment mt_nfree,
* which will not effect the test.
* 2. Allocations can decrement mt_nfree, but they can't happen
* as long as we hold mt_parent_lock.
*/
cachehead = &cpuptr->mt_caches[bucket];
/* Search through the list, starting at the mt_hint */
thiscache = cachehead->mt_hint;
while (thiscache != NULL && thiscache->mt_nfree == 0)
thiscache = thiscache->mt_next;
if (thiscache == NULL) {
/* wrap around -- search up to the hint */
thiscache = cachehead->mt_cache;
hintcache = cachehead->mt_hint;
while (thiscache != NULL && thiscache != hintcache &&
thiscache->mt_nfree == 0)
thiscache = thiscache->mt_next;
if (thiscache == hintcache)
thiscache = NULL;
}
if (thiscache == NULL) { /* there are no free caches */
int32_t thisrequest = requestsize;
int32_t buffer_size = (1 << logsz) + OVERHEAD;
thiscache = (cache_t *)morecore(thisrequest * HUNKSIZE);
if (thiscache == (cache_t *)-1) {
(void) mutex_unlock(&cpuptr->mt_parent_lock);
errno = EAGAIN;
return (NULL);
}
create_cache(thiscache, buffer_size, thisrequest);
/* link in the new block at the beginning of the list */
thiscache->mt_next = cachehead->mt_cache;
cachehead->mt_cache = thiscache;
}
/* update the hint to the cache we found or created */
cachehead->mt_hint = thiscache;
/* thiscache now points to a cache with available space */
(void) mutex_lock(&thiscache->mt_cache_lock);
freeblocks = (uint32_t *)thiscache->mt_freelist;
while (freeblocks < (uint32_t *)thiscache->mt_arena) {
if (*freeblocks & 0xffffffff)
break;
freeblocks++;
if (freeblocks < (uint32_t *)thiscache->mt_arena &&
*freeblocks & 0xffffffff)
break;
freeblocks++;
if (freeblocks < (uint32_t *)thiscache->mt_arena &&
*freeblocks & 0xffffffff)
break;
freeblocks++;
if (freeblocks < (uint32_t *)thiscache->mt_arena &&
*freeblocks & 0xffffffff)
break;
freeblocks++;
}
/*
* the offset from mt_freelist to freeblocks is the offset into
* the arena. Be sure to include the offset into freeblocks
* of the bitmask. n is the offset.
*/
for (i = 0; i < 32; ) {
if (FLIP_EM(*freeblocks) & (0x80000000 >> i++))
break;
if (FLIP_EM(*freeblocks) & (0x80000000 >> i++))
break;
if (FLIP_EM(*freeblocks) & (0x80000000 >> i++))
break;
if (FLIP_EM(*freeblocks) & (0x80000000 >> i++))
break;
}
index = 0x80000000 >> --i;
*freeblocks &= FLIP_EM(~index);
thiscache->mt_nfree--;
(void) mutex_unlock(&thiscache->mt_cache_lock);
(void) mutex_unlock(&cpuptr->mt_parent_lock);
n = (uintptr_t)(((freeblocks - (uint32_t *)thiscache->mt_freelist) << 5)
+ i) * thiscache->mt_size;
/*
* Now you have the offset in n, you've changed the free mask
* in the freelist. Nothing left to do but find the block
* in the arena and put the value of thiscache in the word
* ahead of the handed out address and return the memory
* back to the user.
*/
ret = thiscache->mt_arena + n;
/* Store the cache addr for this buf. Makes free go fast. */
*(uintptr_t *)ret = (uintptr_t)thiscache;
/*
* This assert makes sure we don't hand out memory that is not
* owned by this cache.
*/
assert(ret + thiscache->mt_size <= thiscache->mt_freelist +
thiscache->mt_span);
ret += OVERHEAD;
assert(((uintptr_t)ret & 7) == 0); /* are we 8 byte aligned */
if (reinit == 0 && (debugopt & MTDEBUGPATTERN))
if (verify_pattern(FREEPATTERN, ret, size))
abort(); /* reference after free */
if (debugopt & MTINITBUFFER)
copy_pattern(INITPATTERN, ret, size);
return ((void *)ret);
}
static void *
morecore(size_t bytes)
{
void * ret;
if (bytes > LONG_MAX) {
intptr_t wad;
/*
* The request size is too big. We need to do this in
* chunks. Sbrk only takes an int for an arg.
*/
if (bytes == ULONG_MAX)
return ((void *)-1);
ret = sbrk(0);
wad = LONG_MAX;
while (wad > 0) {
if (sbrk(wad) == (void *)-1) {
if (ret != sbrk(0))
(void) sbrk(-LONG_MAX);
return ((void *)-1);
}
bytes -= LONG_MAX;
wad = bytes;
}
} else
ret = sbrk(bytes);
return (ret);
}
static void *
oversize(size_t size)
{
caddr_t ret;
oversize_t *big;
/* make sure we will not overflow */
if (size > MAX_MTMALLOC) {
errno = ENOMEM;
return (NULL);
}
/*
* Since we ensure every address we hand back is
* MTMALLOC_MIN_ALIGN-byte aligned, ALIGNing size ensures that the
* memory handed out is MTMALLOC_MIN_ALIGN-byte aligned at both ends.
* This eases the implementation of MTDEBUGPATTERN and MTINITPATTERN,
* particularly where coalescing occurs.
*/
size = ALIGN(size, MTMALLOC_MIN_ALIGN);
/*
* The idea with the global lock is that we are sure to
* block in the kernel anyway since given an oversize alloc
* we are sure to have to call morecore();
*/
(void) mutex_lock(&oversize_lock);
if ((big = find_oversize(size)) != NULL) {
if (reinit == 0 && (debugopt & MTDEBUGPATTERN))
if (verify_pattern(FREEPATTERN, big->addr, size))
abort(); /* reference after free */
} else {
/* Get more 8-byte aligned memory from heap */
ret = morecore(size + OVSZ_HEADER_SIZE);
if (ret == (caddr_t)-1) {
(void) mutex_unlock(&oversize_lock);
errno = ENOMEM;
return (NULL);
}
big = oversize_header_alloc((uintptr_t)ret, size);
}
ret = big->addr;
insert_hash(big);
if (debugopt & MTINITBUFFER)
copy_pattern(INITPATTERN, ret, size);
(void) mutex_unlock(&oversize_lock);
assert(((uintptr_t)ret & 7) == 0); /* are we 8 byte aligned */
return ((void *)ret);
}
static void
insert_oversize(oversize_t *op, oversize_t *nx)
{
oversize_t *sp;
/* locate correct insertion point in size-ordered list */
for (sp = oversize_list.next_bysize;
sp != &oversize_list && (op->size > sp->size);
sp = sp->next_bysize)
;
/* link into size-ordered list */
op->next_bysize = sp;
op->prev_bysize = sp->prev_bysize;
op->prev_bysize->next_bysize = op;
op->next_bysize->prev_bysize = op;
/*
* link item into address-ordered list
* (caller provides insertion point as an optimization)
*/
op->next_byaddr = nx;
op->prev_byaddr = nx->prev_byaddr;
op->prev_byaddr->next_byaddr = op;
op->next_byaddr->prev_byaddr = op;
}
static void
unlink_oversize(oversize_t *lp)
{
/* unlink from address list */
lp->prev_byaddr->next_byaddr = lp->next_byaddr;
lp->next_byaddr->prev_byaddr = lp->prev_byaddr;
/* unlink from size list */
lp->prev_bysize->next_bysize = lp->next_bysize;
lp->next_bysize->prev_bysize = lp->prev_bysize;
}
static void
position_oversize_by_size(oversize_t *op)
{
oversize_t *sp;
if (op->size > op->next_bysize->size ||
op->size < op->prev_bysize->size) {
/* unlink from size list */
op->prev_bysize->next_bysize = op->next_bysize;
op->next_bysize->prev_bysize = op->prev_bysize;
/* locate correct insertion point in size-ordered list */
for (sp = oversize_list.next_bysize;
sp != &oversize_list && (op->size > sp->size);
sp = sp->next_bysize)
;
/* link into size-ordered list */
op->next_bysize = sp;
op->prev_bysize = sp->prev_bysize;
op->prev_bysize->next_bysize = op;
op->next_bysize->prev_bysize = op;
}
}
static void
add_oversize(oversize_t *lp)
{
int merge_flags = INSERT_ONLY;
oversize_t *nx; /* ptr to item right of insertion point */
oversize_t *pv; /* ptr to item left of insertion point */
uint_t size_lp, size_pv, size_nx;
uintptr_t endp_lp, endp_pv, endp_nx;
/*
* Locate insertion point in address-ordered list
*/
for (nx = oversize_list.next_byaddr;
nx != &oversize_list && (lp->addr > nx->addr);
nx = nx->next_byaddr)
;
/*
* Determine how to add chunk to oversize freelist
*/
size_lp = OVSZ_HEADER_SIZE + lp->size;
endp_lp = ALIGN((uintptr_t)lp + size_lp, MTMALLOC_MIN_ALIGN);
size_lp = endp_lp - (uintptr_t)lp;
pv = nx->prev_byaddr;
if (pv->size) {
size_pv = OVSZ_HEADER_SIZE + pv->size;
endp_pv = ALIGN((uintptr_t)pv + size_pv,
MTMALLOC_MIN_ALIGN);
size_pv = endp_pv - (uintptr_t)pv;
/* Check for adjacency with left chunk */
if ((uintptr_t)lp == endp_pv)
merge_flags |= COALESCE_LEFT;
}
if (nx->size) {
/* Check for adjacency with right chunk */
if ((uintptr_t)nx == endp_lp) {
size_nx = OVSZ_HEADER_SIZE + nx->size;
endp_nx = ALIGN((uintptr_t)nx + size_nx,
MTMALLOC_MIN_ALIGN);
size_nx = endp_nx - (uintptr_t)nx;
merge_flags |= COALESCE_RIGHT;
}
}
/*
* If MTDEBUGPATTERN==1, lp->addr will have been overwritten with
* FREEPATTERN for lp->size bytes. If we can merge, the oversize
* header(s) that will also become part of the memory available for
* reallocation (ie lp and/or nx) must also be overwritten with
* FREEPATTERN or we will SIGABRT when this memory is next reallocated.
*/
switch (merge_flags) {
case INSERT_ONLY: /* Coalescing not possible */
insert_oversize(lp, nx);
break;
case COALESCE_LEFT:
pv->size += size_lp;
position_oversize_by_size(pv);
if (debugopt & MTDEBUGPATTERN)
copy_pattern(FREEPATTERN, lp, OVSZ_HEADER_SIZE);
break;
case COALESCE_RIGHT:
unlink_oversize(nx);
lp->size += size_nx;
insert_oversize(lp, pv->next_byaddr);
if (debugopt & MTDEBUGPATTERN)
copy_pattern(FREEPATTERN, nx, OVSZ_HEADER_SIZE);
break;
case COALESCE_WITH_BOTH_SIDES: /* Merge (with right) to the left */
pv->size += size_lp + size_nx;
unlink_oversize(nx);
position_oversize_by_size(pv);
if (debugopt & MTDEBUGPATTERN) {
copy_pattern(FREEPATTERN, lp, OVSZ_HEADER_SIZE);
copy_pattern(FREEPATTERN, nx, OVSZ_HEADER_SIZE);
}
break;
}
}
/*
* Find memory on our list that is at least size big. If we find a block that is
* big enough, we break it up and return the associated oversize_t struct back
* to the calling client. Any leftover piece of that block is returned to the
* freelist.
*/
static oversize_t *
find_oversize(size_t size)
{
oversize_t *wp = oversize_list.next_bysize;
while (wp != &oversize_list && size > wp->size)
wp = wp->next_bysize;
if (wp == &oversize_list) /* empty list or nothing big enough */
return (NULL);
/* breaking up a chunk of memory */
if ((long)((wp->size - (size + OVSZ_HEADER_SIZE + MTMALLOC_MIN_ALIGN)))
> MAX_CACHED) {
caddr_t off;
oversize_t *np;
size_t osize;
off = (caddr_t)ALIGN(wp->addr + size,
MTMALLOC_MIN_ALIGN);
osize = wp->size;
wp->size = (size_t)(off - wp->addr);
np = oversize_header_alloc((uintptr_t)off,
osize - (wp->size + OVSZ_HEADER_SIZE));
if ((long)np->size < 0)
abort();
unlink_oversize(wp);
add_oversize(np);
} else {
unlink_oversize(wp);
}
return (wp);
}
static void
copy_pattern(uint32_t pattern, void *buf_arg, size_t size)
{
uint32_t *bufend = (uint32_t *)((char *)buf_arg + size);
uint32_t *buf = buf_arg;
while (buf < bufend - 3) {
buf[3] = buf[2] = buf[1] = buf[0] = pattern;
buf += 4;
}
while (buf < bufend)
*buf++ = pattern;
}
static void *
verify_pattern(uint32_t pattern, void *buf_arg, size_t size)
{
uint32_t *bufend = (uint32_t *)((char *)buf_arg + size);
uint32_t *buf;
for (buf = buf_arg; buf < bufend; buf++)
if (*buf != pattern)
return (buf);
return (NULL);
}
static void
free_oversize(oversize_t *ovp)
{
assert(((uintptr_t)ovp->addr & 7) == 0); /* are we 8 byte aligned */
assert(ovp->size > MAX_CACHED);
ovp->next_bysize = ovp->prev_bysize = NULL;
ovp->next_byaddr = ovp->prev_byaddr = NULL;
(void) mutex_lock(&oversize_lock);
add_oversize(ovp);
(void) mutex_unlock(&oversize_lock);
}
static oversize_t *
oversize_header_alloc(uintptr_t mem, size_t size)
{
oversize_t *ovsz_hdr;
assert(size > MAX_CACHED);
ovsz_hdr = (oversize_t *)mem;
ovsz_hdr->prev_bysize = NULL;
ovsz_hdr->next_bysize = NULL;
ovsz_hdr->prev_byaddr = NULL;
ovsz_hdr->next_byaddr = NULL;
ovsz_hdr->hash_next = NULL;
ovsz_hdr->size = size;
mem += OVSZ_SIZE;
*(uintptr_t *)mem = MTMALLOC_OVERSIZE_MAGIC;
mem += OVERHEAD;
assert(((uintptr_t)mem & 7) == 0); /* are we 8 byte aligned */
ovsz_hdr->addr = (caddr_t)mem;
return (ovsz_hdr);
}
static void
malloc_prepare()
{
percpu_t *cpuptr;
cache_head_t *cachehead;
cache_t *thiscache;
(void) mutex_lock(&oversize_lock);
for (cpuptr = &cpu_list[0]; cpuptr < &cpu_list[ncpus]; cpuptr++) {
(void) mutex_lock(&cpuptr->mt_parent_lock);
for (cachehead = &cpuptr->mt_caches[0];
cachehead < &cpuptr->mt_caches[NUM_CACHES];
cachehead++) {
for (thiscache = cachehead->mt_cache;
thiscache != NULL;
thiscache = thiscache->mt_next) {
(void) mutex_lock(
&thiscache->mt_cache_lock);
}
}
}
}
static void
malloc_release()
{
percpu_t *cpuptr;
cache_head_t *cachehead;
cache_t *thiscache;
for (cpuptr = &cpu_list[ncpus - 1]; cpuptr >= &cpu_list[0]; cpuptr--) {
for (cachehead = &cpuptr->mt_caches[NUM_CACHES - 1];
cachehead >= &cpuptr->mt_caches[0];
cachehead--) {
for (thiscache = cachehead->mt_cache;
thiscache != NULL;
thiscache = thiscache->mt_next) {
(void) mutex_unlock(
&thiscache->mt_cache_lock);
}
}
(void) mutex_unlock(&cpuptr->mt_parent_lock);
}
(void) mutex_unlock(&oversize_lock);
}
#pragma init(malloc_init)
static void
malloc_init(void)
{
/*
* This works in the init section for this library
* because setup_caches() doesn't call anything in libc
* that calls malloc(). If it did, disaster would ensue.
*
* For this to work properly, this library must be the first
* one to have its init section called (after libc) by the
* dynamic linker. If some other library's init section
* ran first and called malloc(), disaster would ensue.
* Because this is an interposer library for malloc(), the
* dynamic linker arranges for its init section to run first.
*/
(void) setup_caches();
(void) pthread_atfork(malloc_prepare, malloc_release, malloc_release);
}