2002-01-29 08:54:51 +01:00
|
|
|
/* Malloc implementation for multiple threads without lock contention.
|
2017-01-01 01:14:16 +01:00
|
|
|
Copyright (C) 2001-2017 Free Software Foundation, Inc.
|
2002-01-29 08:54:51 +01:00
|
|
|
This file is part of the GNU C Library.
|
|
|
|
Contributed by Wolfram Gloger <wg@malloc.de>, 2001.
|
|
|
|
|
|
|
|
The GNU C Library is free software; you can redistribute it and/or
|
2002-08-27 00:40:48 +02:00
|
|
|
modify it under the terms of the GNU Lesser General Public License as
|
|
|
|
published by the Free Software Foundation; either version 2.1 of the
|
2002-01-29 08:54:51 +01:00
|
|
|
License, or (at your option) any later version.
|
|
|
|
|
|
|
|
The GNU C Library is distributed in the hope that it will be useful,
|
|
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
2002-08-27 00:40:48 +02:00
|
|
|
Lesser General Public License for more details.
|
2002-01-29 08:54:51 +01:00
|
|
|
|
2002-08-27 00:40:48 +02:00
|
|
|
You should have received a copy of the GNU Lesser General Public
|
2012-02-10 00:18:22 +01:00
|
|
|
License along with the GNU C Library; see the file COPYING.LIB. If
|
|
|
|
not, see <http://www.gnu.org/licenses/>. */
|
2002-01-29 08:54:51 +01:00
|
|
|
|
2005-03-01 21:30:29 +01:00
|
|
|
#include <stdbool.h>
|
|
|
|
|
2016-12-31 19:02:17 +01:00
|
|
|
#if HAVE_TUNABLES
|
|
|
|
# define TUNABLE_NAMESPACE malloc
|
|
|
|
#endif
|
|
|
|
#include <elf/dl-tunables.h>
|
|
|
|
|
2002-01-29 08:54:51 +01:00
|
|
|
/* Compile-time constants. */
|
|
|
|
|
2014-01-02 09:38:18 +01:00
|
|
|
#define HEAP_MIN_SIZE (32 * 1024)
|
2002-01-29 08:54:51 +01:00
|
|
|
#ifndef HEAP_MAX_SIZE
|
2006-08-22 08:19:12 +02:00
|
|
|
# ifdef DEFAULT_MMAP_THRESHOLD_MAX
|
2006-08-22 08:42:35 +02:00
|
|
|
# define HEAP_MAX_SIZE (2 * DEFAULT_MMAP_THRESHOLD_MAX)
|
2006-08-22 08:19:12 +02:00
|
|
|
# else
|
2014-01-02 09:38:18 +01:00
|
|
|
# define HEAP_MAX_SIZE (1024 * 1024) /* must be a power of two */
|
2006-08-22 08:19:12 +02:00
|
|
|
# endif
|
2002-01-29 08:54:51 +01:00
|
|
|
#endif
|
|
|
|
|
|
|
|
/* HEAP_MIN_SIZE and HEAP_MAX_SIZE limit the size of mmap()ed heaps
|
|
|
|
that are dynamically created for multi-threaded programs. The
|
|
|
|
maximum size must be a power of two, for fast determination of
|
|
|
|
which heap belongs to a chunk. It should be much larger than the
|
|
|
|
mmap threshold, so that requests with a size just below that
|
|
|
|
threshold can be fulfilled without creating too many heaps. */
|
|
|
|
|
|
|
|
/***************************************************************************/
|
|
|
|
|
|
|
|
#define top(ar_ptr) ((ar_ptr)->top)
|
|
|
|
|
|
|
|
/* A heap is a single contiguous memory region holding (coalesceable)
|
|
|
|
malloc_chunks. It is allocated with mmap() and always starts at an
|
2011-09-11 00:10:17 +02:00
|
|
|
address aligned to HEAP_MAX_SIZE. */
|
2002-01-29 08:54:51 +01:00
|
|
|
|
2014-01-02 09:38:18 +01:00
|
|
|
typedef struct _heap_info
|
|
|
|
{
|
2002-01-29 08:54:51 +01:00
|
|
|
mstate ar_ptr; /* Arena for this heap. */
|
|
|
|
struct _heap_info *prev; /* Previous heap. */
|
|
|
|
size_t size; /* Current size in bytes. */
|
2014-01-02 09:38:18 +01:00
|
|
|
size_t mprotect_size; /* Size in bytes that has been mprotected
|
|
|
|
PROT_READ|PROT_WRITE. */
|
2006-03-06 07:18:43 +01:00
|
|
|
/* Make sure the following data is properly aligned, particularly
|
|
|
|
that sizeof (heap_info) + 2 * SIZE_SZ is a multiple of
|
2007-05-07 17:30:57 +02:00
|
|
|
MALLOC_ALIGNMENT. */
|
|
|
|
char pad[-6 * SIZE_SZ & MALLOC_ALIGN_MASK];
|
2002-01-29 08:54:51 +01:00
|
|
|
} heap_info;
|
|
|
|
|
2006-03-06 07:18:43 +01:00
|
|
|
/* Get a compile-time error if the heap_info padding is not correct
|
|
|
|
to make alignment work as expected in sYSMALLOc. */
|
|
|
|
extern int sanity_check_heap_info_alignment[(sizeof (heap_info)
|
2014-01-02 09:38:18 +01:00
|
|
|
+ 2 * SIZE_SZ) % MALLOC_ALIGNMENT
|
|
|
|
? -1 : 1];
|
2006-03-06 07:18:43 +01:00
|
|
|
|
2015-10-17 12:06:48 +02:00
|
|
|
/* Thread specific data. */
|
|
|
|
|
|
|
|
static __thread mstate thread_arena attribute_tls_model_ie;
|
|
|
|
|
2015-12-21 16:42:46 +01:00
|
|
|
/* Arena free list. free_list_lock synchronizes access to the
|
|
|
|
free_list variable below, and the next_free and attached_threads
|
|
|
|
members of struct malloc_state objects. No other locks must be
|
|
|
|
acquired after free_list_lock has been acquired. */
|
2002-01-29 08:54:51 +01:00
|
|
|
|
2016-09-21 16:28:08 +02:00
|
|
|
__libc_lock_define_initialized (static, free_list_lock);
|
2011-09-11 03:47:36 +02:00
|
|
|
static size_t narenas = 1;
|
2009-03-14 00:53:18 +01:00
|
|
|
static mstate free_list;
|
2002-01-29 08:54:51 +01:00
|
|
|
|
2015-12-21 16:42:46 +01:00
|
|
|
/* list_lock prevents concurrent writes to the next member of struct
|
|
|
|
malloc_state objects.
|
|
|
|
|
|
|
|
Read access to the next member is supposed to synchronize with the
|
|
|
|
atomic_write_barrier and the write to the next member in
|
|
|
|
_int_new_arena. This suffers from data races; see the FIXME
|
|
|
|
comments in _int_new_arena and reused_arena.
|
|
|
|
|
2015-12-23 17:23:33 +01:00
|
|
|
list_lock also prevents concurrent forks. At the time list_lock is
|
|
|
|
acquired, no arena lock must have been acquired, but it is
|
|
|
|
permitted to acquire arena locks subsequently, while list_lock is
|
|
|
|
acquired. */
|
2016-09-21 16:28:08 +02:00
|
|
|
__libc_lock_define_initialized (static, list_lock);
|
2015-12-21 16:42:46 +01:00
|
|
|
|
2002-11-20 10:40:55 +01:00
|
|
|
/* Already initialized? */
|
|
|
|
int __malloc_initialized = -1;
|
|
|
|
|
2002-01-29 08:54:51 +01:00
|
|
|
/**************************************************************************/
|
|
|
|
|
|
|
|
|
|
|
|
/* arena_get() acquires an arena and locks the corresponding mutex.
|
|
|
|
First, try the one last locked successfully by this thread. (This
|
|
|
|
is the common case and handled with a macro for speed.) Then, loop
|
|
|
|
once over the circularly linked list of arenas. If no arena is
|
|
|
|
readily available, create a new one. In this latter case, `size'
|
|
|
|
is just a hint as to how much memory will be required immediately
|
|
|
|
in the new arena. */
|
|
|
|
|
|
|
|
#define arena_get(ptr, size) do { \
|
2015-10-17 12:06:48 +02:00
|
|
|
ptr = thread_arena; \
|
2014-01-02 09:38:18 +01:00
|
|
|
arena_lock (ptr, size); \
|
|
|
|
} while (0)
|
2009-03-14 00:53:18 +01:00
|
|
|
|
2014-01-02 09:38:18 +01:00
|
|
|
#define arena_lock(ptr, size) do { \
|
2017-08-30 17:16:08 +02:00
|
|
|
if (ptr) \
|
2016-09-21 16:28:08 +02:00
|
|
|
__libc_lock_lock (ptr->mutex); \
|
2014-01-02 09:38:18 +01:00
|
|
|
else \
|
2015-08-24 11:02:07 +02:00
|
|
|
ptr = arena_get2 ((size), NULL); \
|
2014-01-02 09:38:18 +01:00
|
|
|
} while (0)
|
2002-01-29 08:54:51 +01:00
|
|
|
|
|
|
|
/* find the heap and corresponding arena for a given ptr */
|
|
|
|
|
|
|
|
#define heap_for_ptr(ptr) \
|
2014-01-02 09:38:18 +01:00
|
|
|
((heap_info *) ((unsigned long) (ptr) & ~(HEAP_MAX_SIZE - 1)))
|
2002-01-29 08:54:51 +01:00
|
|
|
#define arena_for_chunk(ptr) \
|
2016-10-28 16:26:57 +02:00
|
|
|
(chunk_main_arena (ptr) ? &main_arena : heap_for_ptr (ptr)->ar_ptr)
|
2002-01-29 08:54:51 +01:00
|
|
|
|
|
|
|
|
|
|
|
/**************************************************************************/
|
|
|
|
|
|
|
|
/* atfork support. */
|
|
|
|
|
2016-04-14 09:17:02 +02:00
|
|
|
/* The following three functions are called around fork from a
|
|
|
|
multi-threaded process. We do not use the general fork handler
|
|
|
|
mechanism to make sure that our handlers are the last ones being
|
|
|
|
called, so that other fork handlers can use the malloc
|
|
|
|
subsystem. */
|
2002-01-29 08:54:51 +01:00
|
|
|
|
2016-04-14 09:17:02 +02:00
|
|
|
void
|
|
|
|
__malloc_fork_lock_parent (void)
|
2002-01-29 08:54:51 +01:00
|
|
|
{
|
2014-01-02 09:38:18 +01:00
|
|
|
if (__malloc_initialized < 1)
|
2002-11-20 10:40:55 +01:00
|
|
|
return;
|
2014-01-02 09:38:18 +01:00
|
|
|
|
2015-12-21 16:42:46 +01:00
|
|
|
/* We do not acquire free_list_lock here because we completely
|
2016-04-14 09:17:02 +02:00
|
|
|
reconstruct free_list in __malloc_fork_unlock_child. */
|
2015-12-21 16:42:46 +01:00
|
|
|
|
2016-09-06 12:49:54 +02:00
|
|
|
__libc_lock_lock (list_lock);
|
2005-09-27 07:45:26 +02:00
|
|
|
|
2016-04-14 09:18:30 +02:00
|
|
|
for (mstate ar_ptr = &main_arena;; )
|
2014-01-02 09:38:18 +01:00
|
|
|
{
|
2016-09-06 12:49:54 +02:00
|
|
|
__libc_lock_lock (ar_ptr->mutex);
|
2014-01-02 09:38:18 +01:00
|
|
|
ar_ptr = ar_ptr->next;
|
|
|
|
if (ar_ptr == &main_arena)
|
|
|
|
break;
|
2005-09-27 07:45:26 +02:00
|
|
|
}
|
2002-01-29 08:54:51 +01:00
|
|
|
}
|
|
|
|
|
2016-04-14 09:17:02 +02:00
|
|
|
void
|
|
|
|
__malloc_fork_unlock_parent (void)
|
2002-01-29 08:54:51 +01:00
|
|
|
{
|
2014-01-02 09:38:18 +01:00
|
|
|
if (__malloc_initialized < 1)
|
2002-11-20 10:40:55 +01:00
|
|
|
return;
|
2014-01-02 09:38:18 +01:00
|
|
|
|
2016-04-14 09:18:30 +02:00
|
|
|
for (mstate ar_ptr = &main_arena;; )
|
2014-01-02 09:38:18 +01:00
|
|
|
{
|
2016-09-06 12:49:54 +02:00
|
|
|
__libc_lock_unlock (ar_ptr->mutex);
|
2014-01-02 09:38:18 +01:00
|
|
|
ar_ptr = ar_ptr->next;
|
|
|
|
if (ar_ptr == &main_arena)
|
|
|
|
break;
|
|
|
|
}
|
2016-09-06 12:49:54 +02:00
|
|
|
__libc_lock_unlock (list_lock);
|
2002-01-29 08:54:51 +01:00
|
|
|
}
|
|
|
|
|
2016-04-14 09:17:02 +02:00
|
|
|
void
|
|
|
|
__malloc_fork_unlock_child (void)
|
2002-01-29 08:54:51 +01:00
|
|
|
{
|
2014-01-02 09:38:18 +01:00
|
|
|
if (__malloc_initialized < 1)
|
2002-11-20 10:40:55 +01:00
|
|
|
return;
|
2014-01-02 09:38:18 +01:00
|
|
|
|
2016-04-14 09:18:30 +02:00
|
|
|
/* Push all arenas to the free list, except thread_arena, which is
|
2015-10-28 19:32:46 +01:00
|
|
|
attached to the current thread. */
|
2016-09-06 12:49:54 +02:00
|
|
|
__libc_lock_init (free_list_lock);
|
2016-04-14 09:18:30 +02:00
|
|
|
if (thread_arena != NULL)
|
|
|
|
thread_arena->attached_threads = 1;
|
2009-03-14 00:53:18 +01:00
|
|
|
free_list = NULL;
|
2016-04-14 09:18:30 +02:00
|
|
|
for (mstate ar_ptr = &main_arena;; )
|
2014-01-02 09:38:18 +01:00
|
|
|
{
|
2016-09-06 12:49:54 +02:00
|
|
|
__libc_lock_init (ar_ptr->mutex);
|
2016-04-14 09:18:30 +02:00
|
|
|
if (ar_ptr != thread_arena)
|
2014-01-02 09:38:18 +01:00
|
|
|
{
|
2015-10-28 19:32:46 +01:00
|
|
|
/* This arena is no longer attached to any thread. */
|
|
|
|
ar_ptr->attached_threads = 0;
|
2014-01-02 09:38:18 +01:00
|
|
|
ar_ptr->next_free = free_list;
|
|
|
|
free_list = ar_ptr;
|
|
|
|
}
|
|
|
|
ar_ptr = ar_ptr->next;
|
|
|
|
if (ar_ptr == &main_arena)
|
|
|
|
break;
|
2009-03-14 00:53:18 +01:00
|
|
|
}
|
2015-12-21 16:42:46 +01:00
|
|
|
|
2016-09-06 12:49:54 +02:00
|
|
|
__libc_lock_init (list_lock);
|
2002-01-29 08:54:51 +01:00
|
|
|
}
|
|
|
|
|
2016-12-31 19:02:17 +01:00
|
|
|
#if HAVE_TUNABLES
|
|
|
|
static inline int do_set_mallopt_check (int32_t value);
|
|
|
|
void
|
tunables: Clean up hooks to get and set tunables
The TUNABLE_SET_VALUE and family of macros (and my later attempt to
add a TUNABLE_GET) never quite went together very well because the
overall interface was not clearly defined. This patch is an attempt
to do just that.
This patch consolidates the API to two simple sets of macros,
TUNABLE_GET* and TUNABLE_SET*. If TUNABLE_NAMESPACE is defined,
TUNABLE_GET takes just the tunable name, type and a (optionally NULL)
callback function to get the value of the tunable. The callback
function, if non-NULL, is called if the tunable was externally set
(i.e. via GLIBC_TUNABLES or any future mechanism). For example:
val = TUNABLE_GET (check, int32_t, check_callback)
returns the value of the glibc.malloc.check tunable (assuming
TUNABLE_NAMESPACE is set to malloc) as an int32_t into VAL after
calling check_callback.
Likewise, TUNABLE_SET can be used to set the value of the tunable,
although this is currently possible only in the dynamic linker before
it relocates itself. For example:
TUNABLE_SET (check, int32_t, 2)
will set glibc.malloc.check to 2. Of course, this is not possible
since we set (or read) glibc.malloc.check long after it is relocated.
To access or set a tunable outside of TUNABLE_NAMESPACE, use the
TUNABLE_GET_FULL and TUNABLE_SET_FULL macros, which have the following
prototype:
TUNABLE_GET_FULL (glibc, tune, hwcap_mask, uint64_t, NULL)
TUNABLE_SET_FULL (glibc, tune, hwcap_mask, uint64_t, 0xffff)
In future the tunable list may get split into mutable and immutable
tunables where mutable tunables can be modified by the library and
userspace after relocation as well and TUNABLE_SET will be more useful
than it currently is. However whenever we actually do that split, we
will have to ensure that the mutable tunables are protected with
locks.
* elf/Versions (__tunable_set_val): Rename to __tunable_get_val.
* elf/dl-tunables.c: Likewise.
(do_tunable_update_val): New function.
(__tunable_set_val): New function.
(__tunable_get_val): Call CB only if the tunable was externally
initialized.
(tunables_strtoul): Replace strval with initialized.
* elf/dl-tunables.h (strval): Replace with a bool initialized.
(TUNABLE_ENUM_NAME, TUNABLE_ENUM_NAME1): Adjust names to
prevent collision.
(__tunable_set_val): New function.
(TUNABLE_GET, TUNABLE_GET_FULL): New macros.
(TUNABLE_SET, TUNABLE_SET_FULL): Likewise.
(TUNABLE_SET_VAL): Remove.
(TUNABLE_SET_VAL_WITH_CALLBACK): Likewise.
* README.tunables: Document the new macros.
* malloc/arena.c (ptmalloc_init): Adjust.
2017-06-01 16:54:46 +02:00
|
|
|
TUNABLE_CALLBACK (set_mallopt_check) (tunable_val_t *valp)
|
2016-12-31 19:02:17 +01:00
|
|
|
{
|
2017-02-08 06:48:23 +01:00
|
|
|
int32_t value = (int32_t) valp->numval;
|
2017-08-30 19:29:38 +02:00
|
|
|
if (value != 0)
|
2016-12-31 19:02:17 +01:00
|
|
|
__malloc_check_init ();
|
|
|
|
}
|
|
|
|
|
tunables: Clean up hooks to get and set tunables
The TUNABLE_SET_VALUE and family of macros (and my later attempt to
add a TUNABLE_GET) never quite went together very well because the
overall interface was not clearly defined. This patch is an attempt
to do just that.
This patch consolidates the API to two simple sets of macros,
TUNABLE_GET* and TUNABLE_SET*. If TUNABLE_NAMESPACE is defined,
TUNABLE_GET takes just the tunable name, type and a (optionally NULL)
callback function to get the value of the tunable. The callback
function, if non-NULL, is called if the tunable was externally set
(i.e. via GLIBC_TUNABLES or any future mechanism). For example:
val = TUNABLE_GET (check, int32_t, check_callback)
returns the value of the glibc.malloc.check tunable (assuming
TUNABLE_NAMESPACE is set to malloc) as an int32_t into VAL after
calling check_callback.
Likewise, TUNABLE_SET can be used to set the value of the tunable,
although this is currently possible only in the dynamic linker before
it relocates itself. For example:
TUNABLE_SET (check, int32_t, 2)
will set glibc.malloc.check to 2. Of course, this is not possible
since we set (or read) glibc.malloc.check long after it is relocated.
To access or set a tunable outside of TUNABLE_NAMESPACE, use the
TUNABLE_GET_FULL and TUNABLE_SET_FULL macros, which have the following
prototype:
TUNABLE_GET_FULL (glibc, tune, hwcap_mask, uint64_t, NULL)
TUNABLE_SET_FULL (glibc, tune, hwcap_mask, uint64_t, 0xffff)
In future the tunable list may get split into mutable and immutable
tunables where mutable tunables can be modified by the library and
userspace after relocation as well and TUNABLE_SET will be more useful
than it currently is. However whenever we actually do that split, we
will have to ensure that the mutable tunables are protected with
locks.
* elf/Versions (__tunable_set_val): Rename to __tunable_get_val.
* elf/dl-tunables.c: Likewise.
(do_tunable_update_val): New function.
(__tunable_set_val): New function.
(__tunable_get_val): Call CB only if the tunable was externally
initialized.
(tunables_strtoul): Replace strval with initialized.
* elf/dl-tunables.h (strval): Replace with a bool initialized.
(TUNABLE_ENUM_NAME, TUNABLE_ENUM_NAME1): Adjust names to
prevent collision.
(__tunable_set_val): New function.
(TUNABLE_GET, TUNABLE_GET_FULL): New macros.
(TUNABLE_SET, TUNABLE_SET_FULL): Likewise.
(TUNABLE_SET_VAL): Remove.
(TUNABLE_SET_VAL_WITH_CALLBACK): Likewise.
* README.tunables: Document the new macros.
* malloc/arena.c (ptmalloc_init): Adjust.
2017-06-01 16:54:46 +02:00
|
|
|
# define TUNABLE_CALLBACK_FNDECL(__name, __type) \
|
2016-12-31 19:02:17 +01:00
|
|
|
static inline int do_ ## __name (__type value); \
|
|
|
|
void \
|
tunables: Clean up hooks to get and set tunables
The TUNABLE_SET_VALUE and family of macros (and my later attempt to
add a TUNABLE_GET) never quite went together very well because the
overall interface was not clearly defined. This patch is an attempt
to do just that.
This patch consolidates the API to two simple sets of macros,
TUNABLE_GET* and TUNABLE_SET*. If TUNABLE_NAMESPACE is defined,
TUNABLE_GET takes just the tunable name, type and a (optionally NULL)
callback function to get the value of the tunable. The callback
function, if non-NULL, is called if the tunable was externally set
(i.e. via GLIBC_TUNABLES or any future mechanism). For example:
val = TUNABLE_GET (check, int32_t, check_callback)
returns the value of the glibc.malloc.check tunable (assuming
TUNABLE_NAMESPACE is set to malloc) as an int32_t into VAL after
calling check_callback.
Likewise, TUNABLE_SET can be used to set the value of the tunable,
although this is currently possible only in the dynamic linker before
it relocates itself. For example:
TUNABLE_SET (check, int32_t, 2)
will set glibc.malloc.check to 2. Of course, this is not possible
since we set (or read) glibc.malloc.check long after it is relocated.
To access or set a tunable outside of TUNABLE_NAMESPACE, use the
TUNABLE_GET_FULL and TUNABLE_SET_FULL macros, which have the following
prototype:
TUNABLE_GET_FULL (glibc, tune, hwcap_mask, uint64_t, NULL)
TUNABLE_SET_FULL (glibc, tune, hwcap_mask, uint64_t, 0xffff)
In future the tunable list may get split into mutable and immutable
tunables where mutable tunables can be modified by the library and
userspace after relocation as well and TUNABLE_SET will be more useful
than it currently is. However whenever we actually do that split, we
will have to ensure that the mutable tunables are protected with
locks.
* elf/Versions (__tunable_set_val): Rename to __tunable_get_val.
* elf/dl-tunables.c: Likewise.
(do_tunable_update_val): New function.
(__tunable_set_val): New function.
(__tunable_get_val): Call CB only if the tunable was externally
initialized.
(tunables_strtoul): Replace strval with initialized.
* elf/dl-tunables.h (strval): Replace with a bool initialized.
(TUNABLE_ENUM_NAME, TUNABLE_ENUM_NAME1): Adjust names to
prevent collision.
(__tunable_set_val): New function.
(TUNABLE_GET, TUNABLE_GET_FULL): New macros.
(TUNABLE_SET, TUNABLE_SET_FULL): Likewise.
(TUNABLE_SET_VAL): Remove.
(TUNABLE_SET_VAL_WITH_CALLBACK): Likewise.
* README.tunables: Document the new macros.
* malloc/arena.c (ptmalloc_init): Adjust.
2017-06-01 16:54:46 +02:00
|
|
|
TUNABLE_CALLBACK (__name) (tunable_val_t *valp) \
|
2016-12-31 19:02:17 +01:00
|
|
|
{ \
|
2017-02-08 06:48:23 +01:00
|
|
|
__type value = (__type) (valp)->numval; \
|
2016-12-31 19:02:17 +01:00
|
|
|
do_ ## __name (value); \
|
|
|
|
}
|
|
|
|
|
tunables: Clean up hooks to get and set tunables
The TUNABLE_SET_VALUE and family of macros (and my later attempt to
add a TUNABLE_GET) never quite went together very well because the
overall interface was not clearly defined. This patch is an attempt
to do just that.
This patch consolidates the API to two simple sets of macros,
TUNABLE_GET* and TUNABLE_SET*. If TUNABLE_NAMESPACE is defined,
TUNABLE_GET takes just the tunable name, type and a (optionally NULL)
callback function to get the value of the tunable. The callback
function, if non-NULL, is called if the tunable was externally set
(i.e. via GLIBC_TUNABLES or any future mechanism). For example:
val = TUNABLE_GET (check, int32_t, check_callback)
returns the value of the glibc.malloc.check tunable (assuming
TUNABLE_NAMESPACE is set to malloc) as an int32_t into VAL after
calling check_callback.
Likewise, TUNABLE_SET can be used to set the value of the tunable,
although this is currently possible only in the dynamic linker before
it relocates itself. For example:
TUNABLE_SET (check, int32_t, 2)
will set glibc.malloc.check to 2. Of course, this is not possible
since we set (or read) glibc.malloc.check long after it is relocated.
To access or set a tunable outside of TUNABLE_NAMESPACE, use the
TUNABLE_GET_FULL and TUNABLE_SET_FULL macros, which have the following
prototype:
TUNABLE_GET_FULL (glibc, tune, hwcap_mask, uint64_t, NULL)
TUNABLE_SET_FULL (glibc, tune, hwcap_mask, uint64_t, 0xffff)
In future the tunable list may get split into mutable and immutable
tunables where mutable tunables can be modified by the library and
userspace after relocation as well and TUNABLE_SET will be more useful
than it currently is. However whenever we actually do that split, we
will have to ensure that the mutable tunables are protected with
locks.
* elf/Versions (__tunable_set_val): Rename to __tunable_get_val.
* elf/dl-tunables.c: Likewise.
(do_tunable_update_val): New function.
(__tunable_set_val): New function.
(__tunable_get_val): Call CB only if the tunable was externally
initialized.
(tunables_strtoul): Replace strval with initialized.
* elf/dl-tunables.h (strval): Replace with a bool initialized.
(TUNABLE_ENUM_NAME, TUNABLE_ENUM_NAME1): Adjust names to
prevent collision.
(__tunable_set_val): New function.
(TUNABLE_GET, TUNABLE_GET_FULL): New macros.
(TUNABLE_SET, TUNABLE_SET_FULL): Likewise.
(TUNABLE_SET_VAL): Remove.
(TUNABLE_SET_VAL_WITH_CALLBACK): Likewise.
* README.tunables: Document the new macros.
* malloc/arena.c (ptmalloc_init): Adjust.
2017-06-01 16:54:46 +02:00
|
|
|
TUNABLE_CALLBACK_FNDECL (set_mmap_threshold, size_t)
|
|
|
|
TUNABLE_CALLBACK_FNDECL (set_mmaps_max, int32_t)
|
|
|
|
TUNABLE_CALLBACK_FNDECL (set_top_pad, size_t)
|
|
|
|
TUNABLE_CALLBACK_FNDECL (set_perturb_byte, int32_t)
|
|
|
|
TUNABLE_CALLBACK_FNDECL (set_trim_threshold, size_t)
|
|
|
|
TUNABLE_CALLBACK_FNDECL (set_arena_max, size_t)
|
|
|
|
TUNABLE_CALLBACK_FNDECL (set_arena_test, size_t)
|
2017-07-06 19:37:30 +02:00
|
|
|
#if USE_TCACHE
|
|
|
|
TUNABLE_CALLBACK_FNDECL (set_tcache_max, size_t)
|
|
|
|
TUNABLE_CALLBACK_FNDECL (set_tcache_count, size_t)
|
|
|
|
TUNABLE_CALLBACK_FNDECL (set_tcache_unsorted_limit, size_t)
|
|
|
|
#endif
|
2016-12-31 19:02:17 +01:00
|
|
|
#else
|
2002-01-29 08:54:51 +01:00
|
|
|
/* Initialization routine. */
|
|
|
|
#include <string.h>
|
|
|
|
extern char **_environ;
|
|
|
|
|
|
|
|
static char *
|
|
|
|
next_env_entry (char ***position)
|
|
|
|
{
|
|
|
|
char **current = *position;
|
|
|
|
char *result = NULL;
|
|
|
|
|
|
|
|
while (*current != NULL)
|
|
|
|
{
|
|
|
|
if (__builtin_expect ((*current)[0] == 'M', 0)
|
2014-01-02 09:38:18 +01:00
|
|
|
&& (*current)[1] == 'A'
|
|
|
|
&& (*current)[2] == 'L'
|
|
|
|
&& (*current)[3] == 'L'
|
|
|
|
&& (*current)[4] == 'O'
|
|
|
|
&& (*current)[5] == 'C'
|
|
|
|
&& (*current)[6] == '_')
|
|
|
|
{
|
|
|
|
result = &(*current)[7];
|
2002-01-29 08:54:51 +01:00
|
|
|
|
2014-01-02 09:38:18 +01:00
|
|
|
/* Save current position for next visit. */
|
|
|
|
*position = ++current;
|
2002-01-29 08:54:51 +01:00
|
|
|
|
2014-01-02 09:38:18 +01:00
|
|
|
break;
|
|
|
|
}
|
2002-01-29 08:54:51 +01:00
|
|
|
|
|
|
|
++current;
|
|
|
|
}
|
|
|
|
|
|
|
|
return result;
|
|
|
|
}
|
2016-12-31 19:02:17 +01:00
|
|
|
#endif
|
2002-01-29 08:54:51 +01:00
|
|
|
|
[BZ #77]
Update.
Add support for namespaces in the dynamic linker.
* dlfcn/Makefile (libdl-routines): Add dlmopen.
* dlfcn/Versions [libdl, GLIBC_2.3.4]: Add dlmopen.
* dlfcn/dlfcn.h: Define Lmid_t, LM_ID_BASE, and LM_ID_NEWLM.
Declare dlmopen. Document RTLD_DI_LMID.
* dlfcn/dlinfo.c: Handle RTLD_DI_LMID.
* dlfcn/dlmopen.c: New file.
* dlfcn/dlopen.c: Pass new parameter to _dl_open.
* dlfcn/dlopenold.c: Likewise.
* elf/dl-addr.c: Adjust for removal of GL(dl_loaded).
* elf/dl-caller.c: Likewise.
* elf/dl-close.c: Likewise.
* elf/dl-conflict.c: Likewise.
* elf/dl-debug.c: Likewise.
* elf/dl-lookup.c: Likewise.
* elf/dl-sym.c: Likewise.
* elf/dl-version.c: Likewise.
* elf/do-lookup.h: Likewise.
* elf/rtld.c: Likewise.
* sysdeps/unix/sysv/linux/i386/dl-librecon.h: Likewise.
* elf/dl-depsc: Likewise. Add new parameter to _dl_map_object.
* elf/dl-fini.c: Call destructors in all namespaces.
* elf/dl-iteratephdr.c: Compute total nloaded. Adjust for removal of
GL(dl_loaded).
* elf/dl-libc.c: Pass new parameter to _dl_open. Adjust for removal
of GL(dl_loaded).
* elf/dl-load.c (_dl_map_object_from_fd): Don't load ld.so a second
time. Reuse the one from the main namespace in all others.
Pass new parameter to _dl_new_object.
Adjust for removal of GL(dl_loaded).
* elf/dl-object.c: Take new parameter. Use it to initialize l_ns.
Adjust for removal of GL(dl_loaded).
* elf/dl-open.c (_dl_open): Take new parameter.
Adjust for removal of GL(dl_loaded).
* elf/dl-support.c: Replace global _dl_loaded etc variables with
_dl_ns variable.
* include/dlfcn.h: Adjust prototype of _dl_open.
Define __LM_ID_CALLER.
* include/link.h: Add l_real, l_ns, and l_direct_opencount elements.
* sysdeps/generic/dl-tls.c: Bump TLS_STATIC_SURPLUS. Since libc is
using TLS we need memory appropriate to the number of namespaces.
* sysdeps/generic/ldsodefs.h (struct rtld_global): Replace _dl_loaded,
_dl_nloaded, _dl_global_scope, _dl_main_searchlist, and
_dl_global_scope_alloc with _dl_ns element. Define DL_NNS.
Adjust prototypes of _dl_map_object and member in rtld_global_ro.
* malloc/malloc.c: Include <dlfcn.h>.
* malloc/arena.c (ptmalloc_init): If libc is not in primary namespace,
never use brk.
* elf/Makefile: Add rules to build and run tst-dlmopen1 and
tst-dlmopen2.
* elf/tst-dlmopen1.c: New file.
* elf/tst-dlmopen1mod.c: New file.
* elf/tst-dlmopen2.c: New file.
* elf/dl-close.c: Improve reference counting by tracking direct loads.
* elf/dl-lookup.c (add_dependency): Likewise.
* elf/dl-open.c (dl_open_worker): Likewise.
* elf/rtld.c (dl_main): Likewise.
2004-09-09 GOTO Masanori <gotom@debian.or.jp>
[BZ #77]
* elf/dl-close.c: Count down l_opencount to check not only for
l_reldeps, but also l_initfini.
2004-10-13 Ulrich Drepper <drepper@redhat.com>
2004-10-14 04:08:23 +02:00
|
|
|
|
2011-09-11 00:10:17 +02:00
|
|
|
#ifdef SHARED
|
[BZ #77]
Update.
Add support for namespaces in the dynamic linker.
* dlfcn/Makefile (libdl-routines): Add dlmopen.
* dlfcn/Versions [libdl, GLIBC_2.3.4]: Add dlmopen.
* dlfcn/dlfcn.h: Define Lmid_t, LM_ID_BASE, and LM_ID_NEWLM.
Declare dlmopen. Document RTLD_DI_LMID.
* dlfcn/dlinfo.c: Handle RTLD_DI_LMID.
* dlfcn/dlmopen.c: New file.
* dlfcn/dlopen.c: Pass new parameter to _dl_open.
* dlfcn/dlopenold.c: Likewise.
* elf/dl-addr.c: Adjust for removal of GL(dl_loaded).
* elf/dl-caller.c: Likewise.
* elf/dl-close.c: Likewise.
* elf/dl-conflict.c: Likewise.
* elf/dl-debug.c: Likewise.
* elf/dl-lookup.c: Likewise.
* elf/dl-sym.c: Likewise.
* elf/dl-version.c: Likewise.
* elf/do-lookup.h: Likewise.
* elf/rtld.c: Likewise.
* sysdeps/unix/sysv/linux/i386/dl-librecon.h: Likewise.
* elf/dl-depsc: Likewise. Add new parameter to _dl_map_object.
* elf/dl-fini.c: Call destructors in all namespaces.
* elf/dl-iteratephdr.c: Compute total nloaded. Adjust for removal of
GL(dl_loaded).
* elf/dl-libc.c: Pass new parameter to _dl_open. Adjust for removal
of GL(dl_loaded).
* elf/dl-load.c (_dl_map_object_from_fd): Don't load ld.so a second
time. Reuse the one from the main namespace in all others.
Pass new parameter to _dl_new_object.
Adjust for removal of GL(dl_loaded).
* elf/dl-object.c: Take new parameter. Use it to initialize l_ns.
Adjust for removal of GL(dl_loaded).
* elf/dl-open.c (_dl_open): Take new parameter.
Adjust for removal of GL(dl_loaded).
* elf/dl-support.c: Replace global _dl_loaded etc variables with
_dl_ns variable.
* include/dlfcn.h: Adjust prototype of _dl_open.
Define __LM_ID_CALLER.
* include/link.h: Add l_real, l_ns, and l_direct_opencount elements.
* sysdeps/generic/dl-tls.c: Bump TLS_STATIC_SURPLUS. Since libc is
using TLS we need memory appropriate to the number of namespaces.
* sysdeps/generic/ldsodefs.h (struct rtld_global): Replace _dl_loaded,
_dl_nloaded, _dl_global_scope, _dl_main_searchlist, and
_dl_global_scope_alloc with _dl_ns element. Define DL_NNS.
Adjust prototypes of _dl_map_object and member in rtld_global_ro.
* malloc/malloc.c: Include <dlfcn.h>.
* malloc/arena.c (ptmalloc_init): If libc is not in primary namespace,
never use brk.
* elf/Makefile: Add rules to build and run tst-dlmopen1 and
tst-dlmopen2.
* elf/tst-dlmopen1.c: New file.
* elf/tst-dlmopen1mod.c: New file.
* elf/tst-dlmopen2.c: New file.
* elf/dl-close.c: Improve reference counting by tracking direct loads.
* elf/dl-lookup.c (add_dependency): Likewise.
* elf/dl-open.c (dl_open_worker): Likewise.
* elf/rtld.c (dl_main): Likewise.
2004-09-09 GOTO Masanori <gotom@debian.or.jp>
[BZ #77]
* elf/dl-close.c: Count down l_opencount to check not only for
l_reldeps, but also l_initfini.
2004-10-13 Ulrich Drepper <drepper@redhat.com>
2004-10-14 04:08:23 +02:00
|
|
|
static void *
|
|
|
|
__failing_morecore (ptrdiff_t d)
|
|
|
|
{
|
|
|
|
return (void *) MORECORE_FAILURE;
|
|
|
|
}
|
Update.
2004-10-18 Jakub Jelinek <jakub@redhat.com>
* elf/dl-libc.c (__libc_dlsym_private, __libc_register_dl_open_hook):
New functions.
(__libc_dlopen_mode): Call __libc_register_dl_open_hook and
__libc_register_dlfcn_hook.
* dlfcn/Makefile (routines, elide-routines.os): Set.
Add rules to build and test tststatic2.
* dlfcn/tststatic2.c: New test.
* dlfcn/modstatic2.c: New test module.
* dlfcn/dladdr.c: Call _dlfcn_hook from libdl.so if not NULL.
Define __ prefixed routine in libc.a and in libdl.a just call it.
* dlfcn/dladdr1.c: Likewise.
* dlfcn/dlclose.c: Likewise.
* dlfcn/dlerror.c: Likewise.
* dlfcn/dlinfo.c: Likewise.
* dlfcn/dlmopen.c: Likewise.
* dlfcn/dlopen.c: Likewise.
* dlfcn/dlopenold.c: Likewise.
* dlfcn/dlsym.c: Likewise.
* dlfcn/dlvsym.c: Likewise.
* dlfcn/sdladdr.c: New file.
* dlfcn/sdladdr1.c: New file.
* dlfcn/sdlclose.c: New file.
* dlfcn/sdlerror.c: New file.
* dlfcn/sdlinfo.c: New file.
* dlfcn/sdlopen.c: New file.
* dlfcn/sdlsym.c: New file.
* dlfcn/sdlvsym.c: New file.
* dlfcn/Versions (libdl): Export _dlfcn_hook@GLIBC_PRIVATE.
* include/dlfcn.h (DL_CALLER_DECL, DL_CALLER RETURN_ADDRESS): Define.
(struct dlfcn_hook): New type.
(_dlfcn_hook): New extern decl.
(__dlopen, __dlclose, __dlsym, __dlerror, __dladdr, __dladdr1,
__dlinfo, __dlmopen, __libc_dlsym_private,
__libc_register_dl_open_hook, __libc_register_dlfcn_hook): New
prototypes.
(__dlvsym): Use DL_CALLER_DECL.
* include/libc-symbols.h: Define libdl_hidden_proto and friends.
* malloc/arena.c (_dl_open_hook): Extern decl.
(ptmalloc_init): Don't call _dl_addr when dlopened from statically
linked programs but don't use brk for them either.
2004-10-19 01:17:46 +02:00
|
|
|
|
|
|
|
extern struct dl_open_hook *_dl_open_hook;
|
|
|
|
libc_hidden_proto (_dl_open_hook);
|
2002-12-06 12:15:07 +01:00
|
|
|
#endif
|
|
|
|
|
2002-01-29 08:54:51 +01:00
|
|
|
static void
|
2004-09-08 10:12:39 +02:00
|
|
|
ptmalloc_init (void)
|
2002-01-29 08:54:51 +01:00
|
|
|
{
|
2014-01-02 09:38:18 +01:00
|
|
|
if (__malloc_initialized >= 0)
|
|
|
|
return;
|
|
|
|
|
2002-01-29 08:54:51 +01:00
|
|
|
__malloc_initialized = 0;
|
|
|
|
|
2011-09-11 00:10:17 +02:00
|
|
|
#ifdef SHARED
|
Update.
2004-10-18 Jakub Jelinek <jakub@redhat.com>
* elf/dl-libc.c (__libc_dlsym_private, __libc_register_dl_open_hook):
New functions.
(__libc_dlopen_mode): Call __libc_register_dl_open_hook and
__libc_register_dlfcn_hook.
* dlfcn/Makefile (routines, elide-routines.os): Set.
Add rules to build and test tststatic2.
* dlfcn/tststatic2.c: New test.
* dlfcn/modstatic2.c: New test module.
* dlfcn/dladdr.c: Call _dlfcn_hook from libdl.so if not NULL.
Define __ prefixed routine in libc.a and in libdl.a just call it.
* dlfcn/dladdr1.c: Likewise.
* dlfcn/dlclose.c: Likewise.
* dlfcn/dlerror.c: Likewise.
* dlfcn/dlinfo.c: Likewise.
* dlfcn/dlmopen.c: Likewise.
* dlfcn/dlopen.c: Likewise.
* dlfcn/dlopenold.c: Likewise.
* dlfcn/dlsym.c: Likewise.
* dlfcn/dlvsym.c: Likewise.
* dlfcn/sdladdr.c: New file.
* dlfcn/sdladdr1.c: New file.
* dlfcn/sdlclose.c: New file.
* dlfcn/sdlerror.c: New file.
* dlfcn/sdlinfo.c: New file.
* dlfcn/sdlopen.c: New file.
* dlfcn/sdlsym.c: New file.
* dlfcn/sdlvsym.c: New file.
* dlfcn/Versions (libdl): Export _dlfcn_hook@GLIBC_PRIVATE.
* include/dlfcn.h (DL_CALLER_DECL, DL_CALLER RETURN_ADDRESS): Define.
(struct dlfcn_hook): New type.
(_dlfcn_hook): New extern decl.
(__dlopen, __dlclose, __dlsym, __dlerror, __dladdr, __dladdr1,
__dlinfo, __dlmopen, __libc_dlsym_private,
__libc_register_dl_open_hook, __libc_register_dlfcn_hook): New
prototypes.
(__dlvsym): Use DL_CALLER_DECL.
* include/libc-symbols.h: Define libdl_hidden_proto and friends.
* malloc/arena.c (_dl_open_hook): Extern decl.
(ptmalloc_init): Don't call _dl_addr when dlopened from statically
linked programs but don't use brk for them either.
2004-10-19 01:17:46 +02:00
|
|
|
/* In case this libc copy is in a non-default namespace, never use brk.
|
|
|
|
Likewise if dlopened from statically linked program. */
|
[BZ #77]
Update.
Add support for namespaces in the dynamic linker.
* dlfcn/Makefile (libdl-routines): Add dlmopen.
* dlfcn/Versions [libdl, GLIBC_2.3.4]: Add dlmopen.
* dlfcn/dlfcn.h: Define Lmid_t, LM_ID_BASE, and LM_ID_NEWLM.
Declare dlmopen. Document RTLD_DI_LMID.
* dlfcn/dlinfo.c: Handle RTLD_DI_LMID.
* dlfcn/dlmopen.c: New file.
* dlfcn/dlopen.c: Pass new parameter to _dl_open.
* dlfcn/dlopenold.c: Likewise.
* elf/dl-addr.c: Adjust for removal of GL(dl_loaded).
* elf/dl-caller.c: Likewise.
* elf/dl-close.c: Likewise.
* elf/dl-conflict.c: Likewise.
* elf/dl-debug.c: Likewise.
* elf/dl-lookup.c: Likewise.
* elf/dl-sym.c: Likewise.
* elf/dl-version.c: Likewise.
* elf/do-lookup.h: Likewise.
* elf/rtld.c: Likewise.
* sysdeps/unix/sysv/linux/i386/dl-librecon.h: Likewise.
* elf/dl-depsc: Likewise. Add new parameter to _dl_map_object.
* elf/dl-fini.c: Call destructors in all namespaces.
* elf/dl-iteratephdr.c: Compute total nloaded. Adjust for removal of
GL(dl_loaded).
* elf/dl-libc.c: Pass new parameter to _dl_open. Adjust for removal
of GL(dl_loaded).
* elf/dl-load.c (_dl_map_object_from_fd): Don't load ld.so a second
time. Reuse the one from the main namespace in all others.
Pass new parameter to _dl_new_object.
Adjust for removal of GL(dl_loaded).
* elf/dl-object.c: Take new parameter. Use it to initialize l_ns.
Adjust for removal of GL(dl_loaded).
* elf/dl-open.c (_dl_open): Take new parameter.
Adjust for removal of GL(dl_loaded).
* elf/dl-support.c: Replace global _dl_loaded etc variables with
_dl_ns variable.
* include/dlfcn.h: Adjust prototype of _dl_open.
Define __LM_ID_CALLER.
* include/link.h: Add l_real, l_ns, and l_direct_opencount elements.
* sysdeps/generic/dl-tls.c: Bump TLS_STATIC_SURPLUS. Since libc is
using TLS we need memory appropriate to the number of namespaces.
* sysdeps/generic/ldsodefs.h (struct rtld_global): Replace _dl_loaded,
_dl_nloaded, _dl_global_scope, _dl_main_searchlist, and
_dl_global_scope_alloc with _dl_ns element. Define DL_NNS.
Adjust prototypes of _dl_map_object and member in rtld_global_ro.
* malloc/malloc.c: Include <dlfcn.h>.
* malloc/arena.c (ptmalloc_init): If libc is not in primary namespace,
never use brk.
* elf/Makefile: Add rules to build and run tst-dlmopen1 and
tst-dlmopen2.
* elf/tst-dlmopen1.c: New file.
* elf/tst-dlmopen1mod.c: New file.
* elf/tst-dlmopen2.c: New file.
* elf/dl-close.c: Improve reference counting by tracking direct loads.
* elf/dl-lookup.c (add_dependency): Likewise.
* elf/dl-open.c (dl_open_worker): Likewise.
* elf/rtld.c (dl_main): Likewise.
2004-09-09 GOTO Masanori <gotom@debian.or.jp>
[BZ #77]
* elf/dl-close.c: Count down l_opencount to check not only for
l_reldeps, but also l_initfini.
2004-10-13 Ulrich Drepper <drepper@redhat.com>
2004-10-14 04:08:23 +02:00
|
|
|
Dl_info di;
|
|
|
|
struct link_map *l;
|
Update.
2004-10-18 Jakub Jelinek <jakub@redhat.com>
* elf/dl-libc.c (__libc_dlsym_private, __libc_register_dl_open_hook):
New functions.
(__libc_dlopen_mode): Call __libc_register_dl_open_hook and
__libc_register_dlfcn_hook.
* dlfcn/Makefile (routines, elide-routines.os): Set.
Add rules to build and test tststatic2.
* dlfcn/tststatic2.c: New test.
* dlfcn/modstatic2.c: New test module.
* dlfcn/dladdr.c: Call _dlfcn_hook from libdl.so if not NULL.
Define __ prefixed routine in libc.a and in libdl.a just call it.
* dlfcn/dladdr1.c: Likewise.
* dlfcn/dlclose.c: Likewise.
* dlfcn/dlerror.c: Likewise.
* dlfcn/dlinfo.c: Likewise.
* dlfcn/dlmopen.c: Likewise.
* dlfcn/dlopen.c: Likewise.
* dlfcn/dlopenold.c: Likewise.
* dlfcn/dlsym.c: Likewise.
* dlfcn/dlvsym.c: Likewise.
* dlfcn/sdladdr.c: New file.
* dlfcn/sdladdr1.c: New file.
* dlfcn/sdlclose.c: New file.
* dlfcn/sdlerror.c: New file.
* dlfcn/sdlinfo.c: New file.
* dlfcn/sdlopen.c: New file.
* dlfcn/sdlsym.c: New file.
* dlfcn/sdlvsym.c: New file.
* dlfcn/Versions (libdl): Export _dlfcn_hook@GLIBC_PRIVATE.
* include/dlfcn.h (DL_CALLER_DECL, DL_CALLER RETURN_ADDRESS): Define.
(struct dlfcn_hook): New type.
(_dlfcn_hook): New extern decl.
(__dlopen, __dlclose, __dlsym, __dlerror, __dladdr, __dladdr1,
__dlinfo, __dlmopen, __libc_dlsym_private,
__libc_register_dl_open_hook, __libc_register_dlfcn_hook): New
prototypes.
(__dlvsym): Use DL_CALLER_DECL.
* include/libc-symbols.h: Define libdl_hidden_proto and friends.
* malloc/arena.c (_dl_open_hook): Extern decl.
(ptmalloc_init): Don't call _dl_addr when dlopened from statically
linked programs but don't use brk for them either.
2004-10-19 01:17:46 +02:00
|
|
|
|
|
|
|
if (_dl_open_hook != NULL
|
|
|
|
|| (_dl_addr (ptmalloc_init, &di, &l, NULL) != 0
|
2014-01-02 09:38:18 +01:00
|
|
|
&& l->l_ns != LM_ID_BASE))
|
[BZ #77]
Update.
Add support for namespaces in the dynamic linker.
* dlfcn/Makefile (libdl-routines): Add dlmopen.
* dlfcn/Versions [libdl, GLIBC_2.3.4]: Add dlmopen.
* dlfcn/dlfcn.h: Define Lmid_t, LM_ID_BASE, and LM_ID_NEWLM.
Declare dlmopen. Document RTLD_DI_LMID.
* dlfcn/dlinfo.c: Handle RTLD_DI_LMID.
* dlfcn/dlmopen.c: New file.
* dlfcn/dlopen.c: Pass new parameter to _dl_open.
* dlfcn/dlopenold.c: Likewise.
* elf/dl-addr.c: Adjust for removal of GL(dl_loaded).
* elf/dl-caller.c: Likewise.
* elf/dl-close.c: Likewise.
* elf/dl-conflict.c: Likewise.
* elf/dl-debug.c: Likewise.
* elf/dl-lookup.c: Likewise.
* elf/dl-sym.c: Likewise.
* elf/dl-version.c: Likewise.
* elf/do-lookup.h: Likewise.
* elf/rtld.c: Likewise.
* sysdeps/unix/sysv/linux/i386/dl-librecon.h: Likewise.
* elf/dl-depsc: Likewise. Add new parameter to _dl_map_object.
* elf/dl-fini.c: Call destructors in all namespaces.
* elf/dl-iteratephdr.c: Compute total nloaded. Adjust for removal of
GL(dl_loaded).
* elf/dl-libc.c: Pass new parameter to _dl_open. Adjust for removal
of GL(dl_loaded).
* elf/dl-load.c (_dl_map_object_from_fd): Don't load ld.so a second
time. Reuse the one from the main namespace in all others.
Pass new parameter to _dl_new_object.
Adjust for removal of GL(dl_loaded).
* elf/dl-object.c: Take new parameter. Use it to initialize l_ns.
Adjust for removal of GL(dl_loaded).
* elf/dl-open.c (_dl_open): Take new parameter.
Adjust for removal of GL(dl_loaded).
* elf/dl-support.c: Replace global _dl_loaded etc variables with
_dl_ns variable.
* include/dlfcn.h: Adjust prototype of _dl_open.
Define __LM_ID_CALLER.
* include/link.h: Add l_real, l_ns, and l_direct_opencount elements.
* sysdeps/generic/dl-tls.c: Bump TLS_STATIC_SURPLUS. Since libc is
using TLS we need memory appropriate to the number of namespaces.
* sysdeps/generic/ldsodefs.h (struct rtld_global): Replace _dl_loaded,
_dl_nloaded, _dl_global_scope, _dl_main_searchlist, and
_dl_global_scope_alloc with _dl_ns element. Define DL_NNS.
Adjust prototypes of _dl_map_object and member in rtld_global_ro.
* malloc/malloc.c: Include <dlfcn.h>.
* malloc/arena.c (ptmalloc_init): If libc is not in primary namespace,
never use brk.
* elf/Makefile: Add rules to build and run tst-dlmopen1 and
tst-dlmopen2.
* elf/tst-dlmopen1.c: New file.
* elf/tst-dlmopen1mod.c: New file.
* elf/tst-dlmopen2.c: New file.
* elf/dl-close.c: Improve reference counting by tracking direct loads.
* elf/dl-lookup.c (add_dependency): Likewise.
* elf/dl-open.c (dl_open_worker): Likewise.
* elf/rtld.c (dl_main): Likewise.
2004-09-09 GOTO Masanori <gotom@debian.or.jp>
[BZ #77]
* elf/dl-close.c: Count down l_opencount to check not only for
l_reldeps, but also l_initfini.
2004-10-13 Ulrich Drepper <drepper@redhat.com>
2004-10-14 04:08:23 +02:00
|
|
|
__morecore = __failing_morecore;
|
|
|
|
#endif
|
|
|
|
|
2015-10-17 12:06:48 +02:00
|
|
|
thread_arena = &main_arena;
|
2016-12-31 19:02:17 +01:00
|
|
|
|
2017-10-17 19:55:16 +02:00
|
|
|
malloc_init_state (&main_arena);
|
2016-12-31 19:02:17 +01:00
|
|
|
|
2017-10-17 19:55:16 +02:00
|
|
|
#if HAVE_TUNABLES
|
tunables: Clean up hooks to get and set tunables
The TUNABLE_SET_VALUE and family of macros (and my later attempt to
add a TUNABLE_GET) never quite went together very well because the
overall interface was not clearly defined. This patch is an attempt
to do just that.
This patch consolidates the API to two simple sets of macros,
TUNABLE_GET* and TUNABLE_SET*. If TUNABLE_NAMESPACE is defined,
TUNABLE_GET takes just the tunable name, type and a (optionally NULL)
callback function to get the value of the tunable. The callback
function, if non-NULL, is called if the tunable was externally set
(i.e. via GLIBC_TUNABLES or any future mechanism). For example:
val = TUNABLE_GET (check, int32_t, check_callback)
returns the value of the glibc.malloc.check tunable (assuming
TUNABLE_NAMESPACE is set to malloc) as an int32_t into VAL after
calling check_callback.
Likewise, TUNABLE_SET can be used to set the value of the tunable,
although this is currently possible only in the dynamic linker before
it relocates itself. For example:
TUNABLE_SET (check, int32_t, 2)
will set glibc.malloc.check to 2. Of course, this is not possible
since we set (or read) glibc.malloc.check long after it is relocated.
To access or set a tunable outside of TUNABLE_NAMESPACE, use the
TUNABLE_GET_FULL and TUNABLE_SET_FULL macros, which have the following
prototype:
TUNABLE_GET_FULL (glibc, tune, hwcap_mask, uint64_t, NULL)
TUNABLE_SET_FULL (glibc, tune, hwcap_mask, uint64_t, 0xffff)
In future the tunable list may get split into mutable and immutable
tunables where mutable tunables can be modified by the library and
userspace after relocation as well and TUNABLE_SET will be more useful
than it currently is. However whenever we actually do that split, we
will have to ensure that the mutable tunables are protected with
locks.
* elf/Versions (__tunable_set_val): Rename to __tunable_get_val.
* elf/dl-tunables.c: Likewise.
(do_tunable_update_val): New function.
(__tunable_set_val): New function.
(__tunable_get_val): Call CB only if the tunable was externally
initialized.
(tunables_strtoul): Replace strval with initialized.
* elf/dl-tunables.h (strval): Replace with a bool initialized.
(TUNABLE_ENUM_NAME, TUNABLE_ENUM_NAME1): Adjust names to
prevent collision.
(__tunable_set_val): New function.
(TUNABLE_GET, TUNABLE_GET_FULL): New macros.
(TUNABLE_SET, TUNABLE_SET_FULL): Likewise.
(TUNABLE_SET_VAL): Remove.
(TUNABLE_SET_VAL_WITH_CALLBACK): Likewise.
* README.tunables: Document the new macros.
* malloc/arena.c (ptmalloc_init): Adjust.
2017-06-01 16:54:46 +02:00
|
|
|
TUNABLE_GET (check, int32_t, TUNABLE_CALLBACK (set_mallopt_check));
|
|
|
|
TUNABLE_GET (top_pad, size_t, TUNABLE_CALLBACK (set_top_pad));
|
|
|
|
TUNABLE_GET (perturb, int32_t, TUNABLE_CALLBACK (set_perturb_byte));
|
|
|
|
TUNABLE_GET (mmap_threshold, size_t, TUNABLE_CALLBACK (set_mmap_threshold));
|
|
|
|
TUNABLE_GET (trim_threshold, size_t, TUNABLE_CALLBACK (set_trim_threshold));
|
|
|
|
TUNABLE_GET (mmap_max, int32_t, TUNABLE_CALLBACK (set_mmaps_max));
|
|
|
|
TUNABLE_GET (arena_max, size_t, TUNABLE_CALLBACK (set_arena_max));
|
|
|
|
TUNABLE_GET (arena_test, size_t, TUNABLE_CALLBACK (set_arena_test));
|
2017-10-17 19:55:16 +02:00
|
|
|
# if USE_TCACHE
|
2017-07-06 19:37:30 +02:00
|
|
|
TUNABLE_GET (tcache_max, size_t, TUNABLE_CALLBACK (set_tcache_max));
|
|
|
|
TUNABLE_GET (tcache_count, size_t, TUNABLE_CALLBACK (set_tcache_count));
|
|
|
|
TUNABLE_GET (tcache_unsorted_limit, size_t,
|
|
|
|
TUNABLE_CALLBACK (set_tcache_unsorted_limit));
|
2017-10-17 19:55:16 +02:00
|
|
|
# endif
|
2016-12-31 19:02:17 +01:00
|
|
|
#else
|
2011-09-11 03:47:36 +02:00
|
|
|
const char *s = NULL;
|
2014-02-10 14:45:42 +01:00
|
|
|
if (__glibc_likely (_environ != NULL))
|
2002-12-27 01:37:06 +01:00
|
|
|
{
|
|
|
|
char **runp = _environ;
|
|
|
|
char *envline;
|
|
|
|
|
|
|
|
while (__builtin_expect ((envline = next_env_entry (&runp)) != NULL,
|
2014-01-02 09:38:18 +01:00
|
|
|
0))
|
|
|
|
{
|
|
|
|
size_t len = strcspn (envline, "=");
|
|
|
|
|
|
|
|
if (envline[len] != '=')
|
|
|
|
/* This is a "MALLOC_" variable at the end of the string
|
|
|
|
without a '=' character. Ignore it since otherwise we
|
|
|
|
will access invalid memory below. */
|
|
|
|
continue;
|
|
|
|
|
|
|
|
switch (len)
|
|
|
|
{
|
|
|
|
case 6:
|
|
|
|
if (memcmp (envline, "CHECK_", 6) == 0)
|
|
|
|
s = &envline[7];
|
|
|
|
break;
|
|
|
|
case 8:
|
|
|
|
if (!__builtin_expect (__libc_enable_secure, 0))
|
|
|
|
{
|
|
|
|
if (memcmp (envline, "TOP_PAD_", 8) == 0)
|
|
|
|
__libc_mallopt (M_TOP_PAD, atoi (&envline[9]));
|
|
|
|
else if (memcmp (envline, "PERTURB_", 8) == 0)
|
|
|
|
__libc_mallopt (M_PERTURB, atoi (&envline[9]));
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case 9:
|
|
|
|
if (!__builtin_expect (__libc_enable_secure, 0))
|
|
|
|
{
|
|
|
|
if (memcmp (envline, "MMAP_MAX_", 9) == 0)
|
|
|
|
__libc_mallopt (M_MMAP_MAX, atoi (&envline[10]));
|
|
|
|
else if (memcmp (envline, "ARENA_MAX", 9) == 0)
|
|
|
|
__libc_mallopt (M_ARENA_MAX, atoi (&envline[10]));
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case 10:
|
|
|
|
if (!__builtin_expect (__libc_enable_secure, 0))
|
|
|
|
{
|
|
|
|
if (memcmp (envline, "ARENA_TEST", 10) == 0)
|
|
|
|
__libc_mallopt (M_ARENA_TEST, atoi (&envline[11]));
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case 15:
|
|
|
|
if (!__builtin_expect (__libc_enable_secure, 0))
|
|
|
|
{
|
|
|
|
if (memcmp (envline, "TRIM_THRESHOLD_", 15) == 0)
|
|
|
|
__libc_mallopt (M_TRIM_THRESHOLD, atoi (&envline[16]));
|
|
|
|
else if (memcmp (envline, "MMAP_THRESHOLD_", 15) == 0)
|
|
|
|
__libc_mallopt (M_MMAP_THRESHOLD, atoi (&envline[16]));
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2017-08-30 19:29:38 +02:00
|
|
|
if (s && s[0] != '\0' && s[0] != '0')
|
|
|
|
__malloc_check_init ();
|
2016-12-31 19:02:17 +01:00
|
|
|
#endif
|
|
|
|
|
2016-06-10 10:46:05 +02:00
|
|
|
#if HAVE_MALLOC_INIT_HOOK
|
2016-06-20 11:11:29 +02:00
|
|
|
void (*hook) (void) = atomic_forced_read (__malloc_initialize_hook);
|
2009-04-16 23:22:16 +02:00
|
|
|
if (hook != NULL)
|
|
|
|
(*hook)();
|
2016-06-10 10:46:05 +02:00
|
|
|
#endif
|
2002-01-29 08:54:51 +01:00
|
|
|
__malloc_initialized = 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Managing heaps and arenas (for concurrent threads) */
|
|
|
|
|
|
|
|
#if MALLOC_DEBUG > 1
|
|
|
|
|
|
|
|
/* Print the complete contents of a single heap to stderr. */
|
|
|
|
|
|
|
|
static void
|
2014-01-02 09:38:18 +01:00
|
|
|
dump_heap (heap_info *heap)
|
2002-01-29 08:54:51 +01:00
|
|
|
{
|
|
|
|
char *ptr;
|
|
|
|
mchunkptr p;
|
|
|
|
|
2014-01-02 09:38:18 +01:00
|
|
|
fprintf (stderr, "Heap %p, size %10lx:\n", heap, (long) heap->size);
|
|
|
|
ptr = (heap->ar_ptr != (mstate) (heap + 1)) ?
|
|
|
|
(char *) (heap + 1) : (char *) (heap + 1) + sizeof (struct malloc_state);
|
|
|
|
p = (mchunkptr) (((unsigned long) ptr + MALLOC_ALIGN_MASK) &
|
|
|
|
~MALLOC_ALIGN_MASK);
|
|
|
|
for (;; )
|
|
|
|
{
|
|
|
|
fprintf (stderr, "chunk %p size %10lx", p, (long) p->size);
|
|
|
|
if (p == top (heap->ar_ptr))
|
|
|
|
{
|
|
|
|
fprintf (stderr, " (top)\n");
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
else if (p->size == (0 | PREV_INUSE))
|
|
|
|
{
|
|
|
|
fprintf (stderr, " (fence)\n");
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
fprintf (stderr, "\n");
|
|
|
|
p = next_chunk (p);
|
2002-01-29 08:54:51 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif /* MALLOC_DEBUG > 1 */
|
|
|
|
|
2004-10-04 04:27:39 +02:00
|
|
|
/* If consecutive mmap (0, HEAP_MAX_SIZE << 1, ...) calls return decreasing
|
|
|
|
addresses as opposed to increasing, new_heap would badly fragment the
|
|
|
|
address space. In that case remember the second HEAP_MAX_SIZE part
|
|
|
|
aligned to HEAP_MAX_SIZE from last mmap (0, HEAP_MAX_SIZE << 1, ...)
|
|
|
|
call (if it is already aligned) and try to reuse it next time. We need
|
|
|
|
no locking for it, as kernel ensures the atomicity for us - worst case
|
|
|
|
we'll call mmap (addr, HEAP_MAX_SIZE, ...) for some value of addr in
|
|
|
|
multiple threads, but only one will succeed. */
|
|
|
|
static char *aligned_heap_area;
|
|
|
|
|
2002-01-29 08:54:51 +01:00
|
|
|
/* Create a new heap. size is automatically rounded up to a multiple
|
|
|
|
of the page size. */
|
|
|
|
|
|
|
|
static heap_info *
|
2014-01-02 09:38:18 +01:00
|
|
|
new_heap (size_t size, size_t top_pad)
|
2002-01-29 08:54:51 +01:00
|
|
|
{
|
2015-02-18 01:25:01 +01:00
|
|
|
size_t pagesize = GLRO (dl_pagesize);
|
2002-01-29 08:54:51 +01:00
|
|
|
char *p1, *p2;
|
|
|
|
unsigned long ul;
|
|
|
|
heap_info *h;
|
|
|
|
|
2014-01-02 09:38:18 +01:00
|
|
|
if (size + top_pad < HEAP_MIN_SIZE)
|
2002-01-29 08:54:51 +01:00
|
|
|
size = HEAP_MIN_SIZE;
|
2014-01-02 09:38:18 +01:00
|
|
|
else if (size + top_pad <= HEAP_MAX_SIZE)
|
2002-01-29 08:54:51 +01:00
|
|
|
size += top_pad;
|
2014-01-02 09:38:18 +01:00
|
|
|
else if (size > HEAP_MAX_SIZE)
|
2002-01-29 08:54:51 +01:00
|
|
|
return 0;
|
|
|
|
else
|
|
|
|
size = HEAP_MAX_SIZE;
|
2015-02-18 01:25:01 +01:00
|
|
|
size = ALIGN_UP (size, pagesize);
|
2002-01-29 08:54:51 +01:00
|
|
|
|
|
|
|
/* A memory region aligned to a multiple of HEAP_MAX_SIZE is needed.
|
|
|
|
No swap space needs to be reserved for the following large
|
|
|
|
mapping (on Linux, this is the case for all non-writable mappings
|
|
|
|
anyway). */
|
2004-10-04 04:27:39 +02:00
|
|
|
p2 = MAP_FAILED;
|
2014-01-02 09:38:18 +01:00
|
|
|
if (aligned_heap_area)
|
|
|
|
{
|
|
|
|
p2 = (char *) MMAP (aligned_heap_area, HEAP_MAX_SIZE, PROT_NONE,
|
|
|
|
MAP_NORESERVE);
|
|
|
|
aligned_heap_area = NULL;
|
|
|
|
if (p2 != MAP_FAILED && ((unsigned long) p2 & (HEAP_MAX_SIZE - 1)))
|
|
|
|
{
|
|
|
|
__munmap (p2, HEAP_MAX_SIZE);
|
|
|
|
p2 = MAP_FAILED;
|
|
|
|
}
|
2004-10-04 04:27:39 +02:00
|
|
|
}
|
2014-01-02 09:38:18 +01:00
|
|
|
if (p2 == MAP_FAILED)
|
|
|
|
{
|
|
|
|
p1 = (char *) MMAP (0, HEAP_MAX_SIZE << 1, PROT_NONE, MAP_NORESERVE);
|
|
|
|
if (p1 != MAP_FAILED)
|
|
|
|
{
|
|
|
|
p2 = (char *) (((unsigned long) p1 + (HEAP_MAX_SIZE - 1))
|
|
|
|
& ~(HEAP_MAX_SIZE - 1));
|
|
|
|
ul = p2 - p1;
|
|
|
|
if (ul)
|
|
|
|
__munmap (p1, ul);
|
|
|
|
else
|
|
|
|
aligned_heap_area = p2 + HEAP_MAX_SIZE;
|
|
|
|
__munmap (p2 + HEAP_MAX_SIZE, HEAP_MAX_SIZE - ul);
|
|
|
|
}
|
2004-10-04 04:27:39 +02:00
|
|
|
else
|
2014-01-02 09:38:18 +01:00
|
|
|
{
|
|
|
|
/* Try to take the chance that an allocation of only HEAP_MAX_SIZE
|
|
|
|
is already aligned. */
|
|
|
|
p2 = (char *) MMAP (0, HEAP_MAX_SIZE, PROT_NONE, MAP_NORESERVE);
|
|
|
|
if (p2 == MAP_FAILED)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
if ((unsigned long) p2 & (HEAP_MAX_SIZE - 1))
|
|
|
|
{
|
|
|
|
__munmap (p2, HEAP_MAX_SIZE);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
}
|
2002-01-29 08:54:51 +01:00
|
|
|
}
|
2014-01-02 09:38:18 +01:00
|
|
|
if (__mprotect (p2, size, PROT_READ | PROT_WRITE) != 0)
|
|
|
|
{
|
|
|
|
__munmap (p2, HEAP_MAX_SIZE);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
h = (heap_info *) p2;
|
2002-01-29 08:54:51 +01:00
|
|
|
h->size = size;
|
2007-05-07 17:30:57 +02:00
|
|
|
h->mprotect_size = size;
|
2013-09-20 16:10:56 +02:00
|
|
|
LIBC_PROBE (memory_heap_new, 2, h, h->size);
|
2002-01-29 08:54:51 +01:00
|
|
|
return h;
|
|
|
|
}
|
|
|
|
|
2007-12-12 01:11:49 +01:00
|
|
|
/* Grow a heap. size is automatically rounded up to a
|
|
|
|
multiple of the page size. */
|
2002-01-29 08:54:51 +01:00
|
|
|
|
|
|
|
static int
|
2014-01-02 09:38:18 +01:00
|
|
|
grow_heap (heap_info *h, long diff)
|
2002-01-29 08:54:51 +01:00
|
|
|
{
|
2015-02-18 01:25:01 +01:00
|
|
|
size_t pagesize = GLRO (dl_pagesize);
|
2002-01-29 08:54:51 +01:00
|
|
|
long new_size;
|
|
|
|
|
2015-02-18 01:25:01 +01:00
|
|
|
diff = ALIGN_UP (diff, pagesize);
|
2014-01-02 09:38:18 +01:00
|
|
|
new_size = (long) h->size + diff;
|
|
|
|
if ((unsigned long) new_size > (unsigned long) HEAP_MAX_SIZE)
|
2007-12-12 01:11:49 +01:00
|
|
|
return -1;
|
2014-01-02 09:38:18 +01:00
|
|
|
|
|
|
|
if ((unsigned long) new_size > h->mprotect_size)
|
|
|
|
{
|
|
|
|
if (__mprotect ((char *) h + h->mprotect_size,
|
|
|
|
(unsigned long) new_size - h->mprotect_size,
|
|
|
|
PROT_READ | PROT_WRITE) != 0)
|
|
|
|
return -2;
|
|
|
|
|
|
|
|
h->mprotect_size = new_size;
|
|
|
|
}
|
2007-12-12 01:11:49 +01:00
|
|
|
|
|
|
|
h->size = new_size;
|
2013-09-20 16:10:56 +02:00
|
|
|
LIBC_PROBE (memory_heap_more, 2, h, h->size);
|
2007-12-12 01:11:49 +01:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Shrink a heap. */
|
|
|
|
|
|
|
|
static int
|
2014-01-02 09:38:18 +01:00
|
|
|
shrink_heap (heap_info *h, long diff)
|
2007-12-12 01:11:49 +01:00
|
|
|
{
|
|
|
|
long new_size;
|
|
|
|
|
2014-01-02 09:38:18 +01:00
|
|
|
new_size = (long) h->size - diff;
|
|
|
|
if (new_size < (long) sizeof (*h))
|
2007-12-12 01:11:49 +01:00
|
|
|
return -1;
|
2014-01-02 09:38:18 +01:00
|
|
|
|
2012-09-25 10:40:29 +02:00
|
|
|
/* Try to re-map the extra heap space freshly to save memory, and make it
|
|
|
|
inaccessible. See malloc-sysdep.h to know when this is true. */
|
2014-02-10 14:45:42 +01:00
|
|
|
if (__glibc_unlikely (check_may_shrink_heap ()))
|
2007-12-12 01:11:49 +01:00
|
|
|
{
|
2014-01-02 09:38:18 +01:00
|
|
|
if ((char *) MMAP ((char *) h + new_size, diff, PROT_NONE,
|
|
|
|
MAP_FIXED) == (char *) MAP_FAILED)
|
|
|
|
return -2;
|
|
|
|
|
2007-12-12 01:11:49 +01:00
|
|
|
h->mprotect_size = new_size;
|
|
|
|
}
|
|
|
|
else
|
2014-01-02 09:38:18 +01:00
|
|
|
__madvise ((char *) h + new_size, diff, MADV_DONTNEED);
|
2007-12-12 01:11:49 +01:00
|
|
|
/*fprintf(stderr, "shrink %p %08lx\n", h, new_size);*/
|
|
|
|
|
2002-01-29 08:54:51 +01:00
|
|
|
h->size = new_size;
|
2013-09-20 16:10:56 +02:00
|
|
|
LIBC_PROBE (memory_heap_less, 2, h, h->size);
|
2002-01-29 08:54:51 +01:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Delete a heap. */
|
|
|
|
|
2004-10-04 04:27:39 +02:00
|
|
|
#define delete_heap(heap) \
|
2014-01-02 09:38:18 +01:00
|
|
|
do { \
|
|
|
|
if ((char *) (heap) + HEAP_MAX_SIZE == aligned_heap_area) \
|
|
|
|
aligned_heap_area = NULL; \
|
|
|
|
__munmap ((char *) (heap), HEAP_MAX_SIZE); \
|
|
|
|
} while (0)
|
2002-01-29 08:54:51 +01:00
|
|
|
|
|
|
|
static int
|
2014-01-02 09:38:18 +01:00
|
|
|
heap_trim (heap_info *heap, size_t pad)
|
2002-01-29 08:54:51 +01:00
|
|
|
{
|
|
|
|
mstate ar_ptr = heap->ar_ptr;
|
2014-01-02 09:38:18 +01:00
|
|
|
unsigned long pagesz = GLRO (dl_pagesize);
|
|
|
|
mchunkptr top_chunk = top (ar_ptr), p, bck, fwd;
|
2002-01-29 08:54:51 +01:00
|
|
|
heap_info *prev_heap;
|
malloc: Consistently apply trim_threshold to all heaps [BZ #17195]
Trimming heaps is a balance between saving memory and the system overhead
required to update page tables and discard allocated pages. The malloc
option M_TRIM_THRESHOLD is a tunable that users are meant to use to decide
where this balance point is but it is only applied to the main arena.
For scalability reasons, glibc malloc has per-thread heaps but these are
shrunk with madvise() if there is one page free at the top of the heap.
In some circumstances this can lead to high system overhead if a thread
has a control flow like
while (data_to_process) {
buf = malloc(large_size);
do_stuff();
free(buf);
}
For a large size, the free() will call madvise (pagetable teardown, page
free and TLB flush) every time followed immediately by a malloc (fault,
kernel page alloc, zeroing and charge accounting). The kernel overhead
can dominate such a workload.
This patch allows the user to tune when madvise gets called by applying
the trim threshold to the per-thread heaps and using similar logic to the
main arena when deciding whether to shrink. Alternatively if the dynamic
brk/mmap threshold gets adjusted then the new values will be obeyed by
the per-thread heaps.
Bug 17195 was a test case motivated by a problem encountered in scientific
applications written in python that performance badly due to high page fault
overhead. The basic operation of such a program was posted by Julian Taylor
https://sourceware.org/ml/libc-alpha/2015-02/msg00373.html
With this patch applied, the overhead is eliminated. All numbers in this
report are in seconds and were recorded by running Julian's program 30
times.
pyarray
glibc madvise
2.21 v2
System min 1.81 ( 0.00%) 0.00 (100.00%)
System mean 1.93 ( 0.00%) 0.02 ( 99.20%)
System stddev 0.06 ( 0.00%) 0.01 ( 88.99%)
System max 2.06 ( 0.00%) 0.03 ( 98.54%)
Elapsed min 3.26 ( 0.00%) 2.37 ( 27.30%)
Elapsed mean 3.39 ( 0.00%) 2.41 ( 28.84%)
Elapsed stddev 0.14 ( 0.00%) 0.02 ( 82.73%)
Elapsed max 4.05 ( 0.00%) 2.47 ( 39.01%)
glibc madvise
2.21 v2
User 141.86 142.28
System 57.94 0.60
Elapsed 102.02 72.66
Note that almost a minutes worth of system time is eliminted and the
program completes 28% faster on average.
To illustrate the problem without python this is a basic test-case for
the worst case scenario where every free is a madvise followed by a an alloc
/* gcc bench-free.c -lpthread -o bench-free */
static int num = 1024;
void __attribute__((noinline,noclone)) dostuff (void *p)
{
}
void *worker (void *data)
{
int i;
for (i = num; i--;)
{
void *m = malloc (48*4096);
dostuff (m);
free (m);
}
return NULL;
}
int main()
{
int i;
pthread_t t;
void *ret;
if (pthread_create (&t, NULL, worker, NULL))
exit (2);
if (pthread_join (t, &ret))
exit (3);
return 0;
}
Before the patch, this resulted in 1024 calls to madvise. With the patch applied,
madvise is called twice because the default trim threshold is high enough to avoid
this.
This a more complex case where there is a mix of frees. It's simply a different worker
function for the test case above
void *worker (void *data)
{
int i;
int j = 0;
void *free_index[num];
for (i = num; i--;)
{
void *m = malloc ((i % 58) *4096);
dostuff (m);
if (i % 2 == 0) {
free (m);
} else {
free_index[j++] = m;
}
}
for (; j >= 0; j--)
{
free(free_index[j]);
}
return NULL;
}
glibc 2.21 calls malloc 90305 times but with the patch applied, it's
called 13438. Increasing the trim threshold will decrease the number of
times it's called with the option of eliminating the overhead.
ebizzy is meant to generate a workload resembling common web application
server workloads. It is threaded with a large working set that at its core
has an allocation, do_stuff, free loop that also hits this case. The primary
metric of the benchmark is records processed per second. This is running on
my desktop which is a single socket machine with an I7-4770 and 8 cores.
Each thread count was run for 30 seconds. It was only run once as the
performance difference is so high that the variation is insignificant.
glibc 2.21 patch
threads 1 10230 44114
threads 2 19153 84925
threads 4 34295 134569
threads 8 51007 183387
Note that the saving happens to be a concidence as the size allocated
by ebizzy was less than the default threshold. If a different number of
chunks were specified then it may also be necessary to tune the threshold
to compensate
This is roughly quadrupling the performance of this benchmark. The difference in
system CPU usage illustrates why.
ebizzy running 1 thread with glibc 2.21
10230 records/s 306904
real 30.00 s
user 7.47 s
sys 22.49 s
22.49 seconds was spent in the kernel for a workload runinng 30 seconds. With the
patch applied
ebizzy running 1 thread with patch applied
44126 records/s 1323792
real 30.00 s
user 29.97 s
sys 0.00 s
system CPU usage was zero with the patch applied. strace shows that glibc
running this workload calls madvise approximately 9000 times a second. With
the patch applied madvise was called twice during the workload (or 0.06
times per second).
2015-02-10 Mel Gorman <mgorman@suse.de>
[BZ #17195]
* malloc/arena.c (free): Apply trim threshold to per-thread heaps
as well as the main arena.
2015-04-02 08:44:14 +02:00
|
|
|
long new_size, top_size, top_area, extra, prev_size, misalign;
|
2002-01-29 08:54:51 +01:00
|
|
|
|
|
|
|
/* Can this heap go away completely? */
|
2014-01-02 09:38:18 +01:00
|
|
|
while (top_chunk == chunk_at_offset (heap, sizeof (*heap)))
|
|
|
|
{
|
|
|
|
prev_heap = heap->prev;
|
|
|
|
prev_size = prev_heap->size - (MINSIZE - 2 * SIZE_SZ);
|
|
|
|
p = chunk_at_offset (prev_heap, prev_size);
|
|
|
|
/* fencepost must be properly aligned. */
|
|
|
|
misalign = ((long) p) & MALLOC_ALIGN_MASK;
|
|
|
|
p = chunk_at_offset (prev_heap, prev_size - misalign);
|
2016-10-28 16:26:57 +02:00
|
|
|
assert (chunksize_nomask (p) == (0 | PREV_INUSE)); /* must be fencepost */
|
2014-01-02 09:38:18 +01:00
|
|
|
p = prev_chunk (p);
|
|
|
|
new_size = chunksize (p) + (MINSIZE - 2 * SIZE_SZ) + misalign;
|
|
|
|
assert (new_size > 0 && new_size < (long) (2 * MINSIZE));
|
|
|
|
if (!prev_inuse (p))
|
2016-10-28 16:26:57 +02:00
|
|
|
new_size += prev_size (p);
|
2014-01-02 09:38:18 +01:00
|
|
|
assert (new_size > 0 && new_size < HEAP_MAX_SIZE);
|
|
|
|
if (new_size + (HEAP_MAX_SIZE - prev_heap->size) < pad + MINSIZE + pagesz)
|
|
|
|
break;
|
|
|
|
ar_ptr->system_mem -= heap->size;
|
|
|
|
LIBC_PROBE (memory_heap_free, 2, heap, heap->size);
|
|
|
|
delete_heap (heap);
|
|
|
|
heap = prev_heap;
|
|
|
|
if (!prev_inuse (p)) /* consolidate backward */
|
|
|
|
{
|
|
|
|
p = prev_chunk (p);
|
Avoid deadlock in malloc on backtrace (BZ #16159)
When the malloc subsystem detects some kind of memory corruption,
depending on the configuration it prints the error, a backtrace, a
memory map and then aborts the process. In this process, the
backtrace() call may result in a call to malloc, resulting in
various kinds of problematic behavior.
In one case, the malloc it calls may detect a corruption and call
backtrace again, and a stack overflow may result due to the infinite
recursion. In another case, the malloc it calls may deadlock on an
arena lock with the malloc (or free, realloc, etc.) that detected the
corruption. In yet another case, if the program is linked with
pthreads, backtrace may do a pthread_once initialization, which
deadlocks on itself.
In all these cases, the program exit is not as intended. This is
avoidable by marking the arena that malloc detected a corruption on,
as unusable. The following patch does that. Features of this patch
are as follows:
- A flag is added to the mstate struct of the arena to indicate if the
arena is corrupt.
- The flag is checked whenever malloc functions try to get a lock on
an arena. If the arena is unusable, a NULL is returned, causing the
malloc to use mmap or try the next arena.
- malloc_printerr sets the corrupt flag on the arena when it detects a
corruption
- free does not concern itself with the flag at all. It is not
important since the backtrace workflow does not need free. A free
in a parallel thread may cause another corruption, but that's not
new
- The flag check and set are not atomic and may race. This is fine
since we don't care about contention during the flag check. We want
to make sure that the malloc call in the backtrace does not trip on
itself and all that action happens in the same thread and not across
threads.
I verified that the test case does not show any regressions due to
this patch. I also ran the malloc benchmarks and found an
insignificant difference in timings (< 2%).
* malloc/Makefile (tests): New test case tst-malloc-backtrace.
* malloc/arena.c (arena_lock): Check if arena is corrupt.
(reused_arena): Find a non-corrupt arena.
(heap_trim): Pass arena to unlink.
* malloc/hooks.c (malloc_check_get_size): Pass arena to
malloc_printerr.
(top_check): Likewise.
(free_check): Likewise.
(realloc_check): Likewise.
* malloc/malloc.c (malloc_printerr): Add arena argument.
(unlink): Likewise.
(munmap_chunk): Adjust.
(ARENA_CORRUPTION_BIT): New macro.
(arena_is_corrupt): Likewise.
(set_arena_corrupt): Likewise.
(sysmalloc): Use mmap if there are no usable arenas.
(_int_malloc): Likewise.
(__libc_malloc): Don't fail if arena_get returns NULL.
(_mid_memalign): Likewise.
(__libc_calloc): Likewise.
(__libc_realloc): Adjust for additional argument to
malloc_printerr.
(_int_free): Likewise.
(malloc_consolidate): Likewise.
(_int_realloc): Likewise.
(_int_memalign): Don't touch corrupt arenas.
* malloc/tst-malloc-backtrace.c: New test case.
2015-05-19 03:10:37 +02:00
|
|
|
unlink (ar_ptr, p, bck, fwd);
|
2014-01-02 09:38:18 +01:00
|
|
|
}
|
|
|
|
assert (((unsigned long) ((char *) p + new_size) & (pagesz - 1)) == 0);
|
|
|
|
assert (((char *) p + new_size) == ((char *) heap + heap->size));
|
|
|
|
top (ar_ptr) = top_chunk = p;
|
|
|
|
set_head (top_chunk, new_size | PREV_INUSE);
|
|
|
|
/*check_chunk(ar_ptr, top_chunk);*/
|
2002-01-29 08:54:51 +01:00
|
|
|
}
|
malloc: Consistently apply trim_threshold to all heaps [BZ #17195]
Trimming heaps is a balance between saving memory and the system overhead
required to update page tables and discard allocated pages. The malloc
option M_TRIM_THRESHOLD is a tunable that users are meant to use to decide
where this balance point is but it is only applied to the main arena.
For scalability reasons, glibc malloc has per-thread heaps but these are
shrunk with madvise() if there is one page free at the top of the heap.
In some circumstances this can lead to high system overhead if a thread
has a control flow like
while (data_to_process) {
buf = malloc(large_size);
do_stuff();
free(buf);
}
For a large size, the free() will call madvise (pagetable teardown, page
free and TLB flush) every time followed immediately by a malloc (fault,
kernel page alloc, zeroing and charge accounting). The kernel overhead
can dominate such a workload.
This patch allows the user to tune when madvise gets called by applying
the trim threshold to the per-thread heaps and using similar logic to the
main arena when deciding whether to shrink. Alternatively if the dynamic
brk/mmap threshold gets adjusted then the new values will be obeyed by
the per-thread heaps.
Bug 17195 was a test case motivated by a problem encountered in scientific
applications written in python that performance badly due to high page fault
overhead. The basic operation of such a program was posted by Julian Taylor
https://sourceware.org/ml/libc-alpha/2015-02/msg00373.html
With this patch applied, the overhead is eliminated. All numbers in this
report are in seconds and were recorded by running Julian's program 30
times.
pyarray
glibc madvise
2.21 v2
System min 1.81 ( 0.00%) 0.00 (100.00%)
System mean 1.93 ( 0.00%) 0.02 ( 99.20%)
System stddev 0.06 ( 0.00%) 0.01 ( 88.99%)
System max 2.06 ( 0.00%) 0.03 ( 98.54%)
Elapsed min 3.26 ( 0.00%) 2.37 ( 27.30%)
Elapsed mean 3.39 ( 0.00%) 2.41 ( 28.84%)
Elapsed stddev 0.14 ( 0.00%) 0.02 ( 82.73%)
Elapsed max 4.05 ( 0.00%) 2.47 ( 39.01%)
glibc madvise
2.21 v2
User 141.86 142.28
System 57.94 0.60
Elapsed 102.02 72.66
Note that almost a minutes worth of system time is eliminted and the
program completes 28% faster on average.
To illustrate the problem without python this is a basic test-case for
the worst case scenario where every free is a madvise followed by a an alloc
/* gcc bench-free.c -lpthread -o bench-free */
static int num = 1024;
void __attribute__((noinline,noclone)) dostuff (void *p)
{
}
void *worker (void *data)
{
int i;
for (i = num; i--;)
{
void *m = malloc (48*4096);
dostuff (m);
free (m);
}
return NULL;
}
int main()
{
int i;
pthread_t t;
void *ret;
if (pthread_create (&t, NULL, worker, NULL))
exit (2);
if (pthread_join (t, &ret))
exit (3);
return 0;
}
Before the patch, this resulted in 1024 calls to madvise. With the patch applied,
madvise is called twice because the default trim threshold is high enough to avoid
this.
This a more complex case where there is a mix of frees. It's simply a different worker
function for the test case above
void *worker (void *data)
{
int i;
int j = 0;
void *free_index[num];
for (i = num; i--;)
{
void *m = malloc ((i % 58) *4096);
dostuff (m);
if (i % 2 == 0) {
free (m);
} else {
free_index[j++] = m;
}
}
for (; j >= 0; j--)
{
free(free_index[j]);
}
return NULL;
}
glibc 2.21 calls malloc 90305 times but with the patch applied, it's
called 13438. Increasing the trim threshold will decrease the number of
times it's called with the option of eliminating the overhead.
ebizzy is meant to generate a workload resembling common web application
server workloads. It is threaded with a large working set that at its core
has an allocation, do_stuff, free loop that also hits this case. The primary
metric of the benchmark is records processed per second. This is running on
my desktop which is a single socket machine with an I7-4770 and 8 cores.
Each thread count was run for 30 seconds. It was only run once as the
performance difference is so high that the variation is insignificant.
glibc 2.21 patch
threads 1 10230 44114
threads 2 19153 84925
threads 4 34295 134569
threads 8 51007 183387
Note that the saving happens to be a concidence as the size allocated
by ebizzy was less than the default threshold. If a different number of
chunks were specified then it may also be necessary to tune the threshold
to compensate
This is roughly quadrupling the performance of this benchmark. The difference in
system CPU usage illustrates why.
ebizzy running 1 thread with glibc 2.21
10230 records/s 306904
real 30.00 s
user 7.47 s
sys 22.49 s
22.49 seconds was spent in the kernel for a workload runinng 30 seconds. With the
patch applied
ebizzy running 1 thread with patch applied
44126 records/s 1323792
real 30.00 s
user 29.97 s
sys 0.00 s
system CPU usage was zero with the patch applied. strace shows that glibc
running this workload calls madvise approximately 9000 times a second. With
the patch applied madvise was called twice during the workload (or 0.06
times per second).
2015-02-10 Mel Gorman <mgorman@suse.de>
[BZ #17195]
* malloc/arena.c (free): Apply trim threshold to per-thread heaps
as well as the main arena.
2015-04-02 08:44:14 +02:00
|
|
|
|
|
|
|
/* Uses similar logic for per-thread arenas as the main arena with systrim
|
2015-10-08 04:21:36 +02:00
|
|
|
and _int_free by preserving the top pad and rounding down to the nearest
|
|
|
|
page. */
|
2014-01-02 09:38:18 +01:00
|
|
|
top_size = chunksize (top_chunk);
|
2015-10-08 04:21:36 +02:00
|
|
|
if ((unsigned long)(top_size) <
|
|
|
|
(unsigned long)(mp_.trim_threshold))
|
|
|
|
return 0;
|
|
|
|
|
malloc: Consistently apply trim_threshold to all heaps [BZ #17195]
Trimming heaps is a balance between saving memory and the system overhead
required to update page tables and discard allocated pages. The malloc
option M_TRIM_THRESHOLD is a tunable that users are meant to use to decide
where this balance point is but it is only applied to the main arena.
For scalability reasons, glibc malloc has per-thread heaps but these are
shrunk with madvise() if there is one page free at the top of the heap.
In some circumstances this can lead to high system overhead if a thread
has a control flow like
while (data_to_process) {
buf = malloc(large_size);
do_stuff();
free(buf);
}
For a large size, the free() will call madvise (pagetable teardown, page
free and TLB flush) every time followed immediately by a malloc (fault,
kernel page alloc, zeroing and charge accounting). The kernel overhead
can dominate such a workload.
This patch allows the user to tune when madvise gets called by applying
the trim threshold to the per-thread heaps and using similar logic to the
main arena when deciding whether to shrink. Alternatively if the dynamic
brk/mmap threshold gets adjusted then the new values will be obeyed by
the per-thread heaps.
Bug 17195 was a test case motivated by a problem encountered in scientific
applications written in python that performance badly due to high page fault
overhead. The basic operation of such a program was posted by Julian Taylor
https://sourceware.org/ml/libc-alpha/2015-02/msg00373.html
With this patch applied, the overhead is eliminated. All numbers in this
report are in seconds and were recorded by running Julian's program 30
times.
pyarray
glibc madvise
2.21 v2
System min 1.81 ( 0.00%) 0.00 (100.00%)
System mean 1.93 ( 0.00%) 0.02 ( 99.20%)
System stddev 0.06 ( 0.00%) 0.01 ( 88.99%)
System max 2.06 ( 0.00%) 0.03 ( 98.54%)
Elapsed min 3.26 ( 0.00%) 2.37 ( 27.30%)
Elapsed mean 3.39 ( 0.00%) 2.41 ( 28.84%)
Elapsed stddev 0.14 ( 0.00%) 0.02 ( 82.73%)
Elapsed max 4.05 ( 0.00%) 2.47 ( 39.01%)
glibc madvise
2.21 v2
User 141.86 142.28
System 57.94 0.60
Elapsed 102.02 72.66
Note that almost a minutes worth of system time is eliminted and the
program completes 28% faster on average.
To illustrate the problem without python this is a basic test-case for
the worst case scenario where every free is a madvise followed by a an alloc
/* gcc bench-free.c -lpthread -o bench-free */
static int num = 1024;
void __attribute__((noinline,noclone)) dostuff (void *p)
{
}
void *worker (void *data)
{
int i;
for (i = num; i--;)
{
void *m = malloc (48*4096);
dostuff (m);
free (m);
}
return NULL;
}
int main()
{
int i;
pthread_t t;
void *ret;
if (pthread_create (&t, NULL, worker, NULL))
exit (2);
if (pthread_join (t, &ret))
exit (3);
return 0;
}
Before the patch, this resulted in 1024 calls to madvise. With the patch applied,
madvise is called twice because the default trim threshold is high enough to avoid
this.
This a more complex case where there is a mix of frees. It's simply a different worker
function for the test case above
void *worker (void *data)
{
int i;
int j = 0;
void *free_index[num];
for (i = num; i--;)
{
void *m = malloc ((i % 58) *4096);
dostuff (m);
if (i % 2 == 0) {
free (m);
} else {
free_index[j++] = m;
}
}
for (; j >= 0; j--)
{
free(free_index[j]);
}
return NULL;
}
glibc 2.21 calls malloc 90305 times but with the patch applied, it's
called 13438. Increasing the trim threshold will decrease the number of
times it's called with the option of eliminating the overhead.
ebizzy is meant to generate a workload resembling common web application
server workloads. It is threaded with a large working set that at its core
has an allocation, do_stuff, free loop that also hits this case. The primary
metric of the benchmark is records processed per second. This is running on
my desktop which is a single socket machine with an I7-4770 and 8 cores.
Each thread count was run for 30 seconds. It was only run once as the
performance difference is so high that the variation is insignificant.
glibc 2.21 patch
threads 1 10230 44114
threads 2 19153 84925
threads 4 34295 134569
threads 8 51007 183387
Note that the saving happens to be a concidence as the size allocated
by ebizzy was less than the default threshold. If a different number of
chunks were specified then it may also be necessary to tune the threshold
to compensate
This is roughly quadrupling the performance of this benchmark. The difference in
system CPU usage illustrates why.
ebizzy running 1 thread with glibc 2.21
10230 records/s 306904
real 30.00 s
user 7.47 s
sys 22.49 s
22.49 seconds was spent in the kernel for a workload runinng 30 seconds. With the
patch applied
ebizzy running 1 thread with patch applied
44126 records/s 1323792
real 30.00 s
user 29.97 s
sys 0.00 s
system CPU usage was zero with the patch applied. strace shows that glibc
running this workload calls madvise approximately 9000 times a second. With
the patch applied madvise was called twice during the workload (or 0.06
times per second).
2015-02-10 Mel Gorman <mgorman@suse.de>
[BZ #17195]
* malloc/arena.c (free): Apply trim threshold to per-thread heaps
as well as the main arena.
2015-04-02 08:44:14 +02:00
|
|
|
top_area = top_size - MINSIZE - 1;
|
2015-06-08 14:36:13 +02:00
|
|
|
if (top_area < 0 || (size_t) top_area <= pad)
|
malloc: Consistently apply trim_threshold to all heaps [BZ #17195]
Trimming heaps is a balance between saving memory and the system overhead
required to update page tables and discard allocated pages. The malloc
option M_TRIM_THRESHOLD is a tunable that users are meant to use to decide
where this balance point is but it is only applied to the main arena.
For scalability reasons, glibc malloc has per-thread heaps but these are
shrunk with madvise() if there is one page free at the top of the heap.
In some circumstances this can lead to high system overhead if a thread
has a control flow like
while (data_to_process) {
buf = malloc(large_size);
do_stuff();
free(buf);
}
For a large size, the free() will call madvise (pagetable teardown, page
free and TLB flush) every time followed immediately by a malloc (fault,
kernel page alloc, zeroing and charge accounting). The kernel overhead
can dominate such a workload.
This patch allows the user to tune when madvise gets called by applying
the trim threshold to the per-thread heaps and using similar logic to the
main arena when deciding whether to shrink. Alternatively if the dynamic
brk/mmap threshold gets adjusted then the new values will be obeyed by
the per-thread heaps.
Bug 17195 was a test case motivated by a problem encountered in scientific
applications written in python that performance badly due to high page fault
overhead. The basic operation of such a program was posted by Julian Taylor
https://sourceware.org/ml/libc-alpha/2015-02/msg00373.html
With this patch applied, the overhead is eliminated. All numbers in this
report are in seconds and were recorded by running Julian's program 30
times.
pyarray
glibc madvise
2.21 v2
System min 1.81 ( 0.00%) 0.00 (100.00%)
System mean 1.93 ( 0.00%) 0.02 ( 99.20%)
System stddev 0.06 ( 0.00%) 0.01 ( 88.99%)
System max 2.06 ( 0.00%) 0.03 ( 98.54%)
Elapsed min 3.26 ( 0.00%) 2.37 ( 27.30%)
Elapsed mean 3.39 ( 0.00%) 2.41 ( 28.84%)
Elapsed stddev 0.14 ( 0.00%) 0.02 ( 82.73%)
Elapsed max 4.05 ( 0.00%) 2.47 ( 39.01%)
glibc madvise
2.21 v2
User 141.86 142.28
System 57.94 0.60
Elapsed 102.02 72.66
Note that almost a minutes worth of system time is eliminted and the
program completes 28% faster on average.
To illustrate the problem without python this is a basic test-case for
the worst case scenario where every free is a madvise followed by a an alloc
/* gcc bench-free.c -lpthread -o bench-free */
static int num = 1024;
void __attribute__((noinline,noclone)) dostuff (void *p)
{
}
void *worker (void *data)
{
int i;
for (i = num; i--;)
{
void *m = malloc (48*4096);
dostuff (m);
free (m);
}
return NULL;
}
int main()
{
int i;
pthread_t t;
void *ret;
if (pthread_create (&t, NULL, worker, NULL))
exit (2);
if (pthread_join (t, &ret))
exit (3);
return 0;
}
Before the patch, this resulted in 1024 calls to madvise. With the patch applied,
madvise is called twice because the default trim threshold is high enough to avoid
this.
This a more complex case where there is a mix of frees. It's simply a different worker
function for the test case above
void *worker (void *data)
{
int i;
int j = 0;
void *free_index[num];
for (i = num; i--;)
{
void *m = malloc ((i % 58) *4096);
dostuff (m);
if (i % 2 == 0) {
free (m);
} else {
free_index[j++] = m;
}
}
for (; j >= 0; j--)
{
free(free_index[j]);
}
return NULL;
}
glibc 2.21 calls malloc 90305 times but with the patch applied, it's
called 13438. Increasing the trim threshold will decrease the number of
times it's called with the option of eliminating the overhead.
ebizzy is meant to generate a workload resembling common web application
server workloads. It is threaded with a large working set that at its core
has an allocation, do_stuff, free loop that also hits this case. The primary
metric of the benchmark is records processed per second. This is running on
my desktop which is a single socket machine with an I7-4770 and 8 cores.
Each thread count was run for 30 seconds. It was only run once as the
performance difference is so high that the variation is insignificant.
glibc 2.21 patch
threads 1 10230 44114
threads 2 19153 84925
threads 4 34295 134569
threads 8 51007 183387
Note that the saving happens to be a concidence as the size allocated
by ebizzy was less than the default threshold. If a different number of
chunks were specified then it may also be necessary to tune the threshold
to compensate
This is roughly quadrupling the performance of this benchmark. The difference in
system CPU usage illustrates why.
ebizzy running 1 thread with glibc 2.21
10230 records/s 306904
real 30.00 s
user 7.47 s
sys 22.49 s
22.49 seconds was spent in the kernel for a workload runinng 30 seconds. With the
patch applied
ebizzy running 1 thread with patch applied
44126 records/s 1323792
real 30.00 s
user 29.97 s
sys 0.00 s
system CPU usage was zero with the patch applied. strace shows that glibc
running this workload calls madvise approximately 9000 times a second. With
the patch applied madvise was called twice during the workload (or 0.06
times per second).
2015-02-10 Mel Gorman <mgorman@suse.de>
[BZ #17195]
* malloc/arena.c (free): Apply trim threshold to per-thread heaps
as well as the main arena.
2015-04-02 08:44:14 +02:00
|
|
|
return 0;
|
|
|
|
|
2015-10-08 04:21:36 +02:00
|
|
|
/* Release in pagesize units and round down to the nearest page. */
|
malloc: Consistently apply trim_threshold to all heaps [BZ #17195]
Trimming heaps is a balance between saving memory and the system overhead
required to update page tables and discard allocated pages. The malloc
option M_TRIM_THRESHOLD is a tunable that users are meant to use to decide
where this balance point is but it is only applied to the main arena.
For scalability reasons, glibc malloc has per-thread heaps but these are
shrunk with madvise() if there is one page free at the top of the heap.
In some circumstances this can lead to high system overhead if a thread
has a control flow like
while (data_to_process) {
buf = malloc(large_size);
do_stuff();
free(buf);
}
For a large size, the free() will call madvise (pagetable teardown, page
free and TLB flush) every time followed immediately by a malloc (fault,
kernel page alloc, zeroing and charge accounting). The kernel overhead
can dominate such a workload.
This patch allows the user to tune when madvise gets called by applying
the trim threshold to the per-thread heaps and using similar logic to the
main arena when deciding whether to shrink. Alternatively if the dynamic
brk/mmap threshold gets adjusted then the new values will be obeyed by
the per-thread heaps.
Bug 17195 was a test case motivated by a problem encountered in scientific
applications written in python that performance badly due to high page fault
overhead. The basic operation of such a program was posted by Julian Taylor
https://sourceware.org/ml/libc-alpha/2015-02/msg00373.html
With this patch applied, the overhead is eliminated. All numbers in this
report are in seconds and were recorded by running Julian's program 30
times.
pyarray
glibc madvise
2.21 v2
System min 1.81 ( 0.00%) 0.00 (100.00%)
System mean 1.93 ( 0.00%) 0.02 ( 99.20%)
System stddev 0.06 ( 0.00%) 0.01 ( 88.99%)
System max 2.06 ( 0.00%) 0.03 ( 98.54%)
Elapsed min 3.26 ( 0.00%) 2.37 ( 27.30%)
Elapsed mean 3.39 ( 0.00%) 2.41 ( 28.84%)
Elapsed stddev 0.14 ( 0.00%) 0.02 ( 82.73%)
Elapsed max 4.05 ( 0.00%) 2.47 ( 39.01%)
glibc madvise
2.21 v2
User 141.86 142.28
System 57.94 0.60
Elapsed 102.02 72.66
Note that almost a minutes worth of system time is eliminted and the
program completes 28% faster on average.
To illustrate the problem without python this is a basic test-case for
the worst case scenario where every free is a madvise followed by a an alloc
/* gcc bench-free.c -lpthread -o bench-free */
static int num = 1024;
void __attribute__((noinline,noclone)) dostuff (void *p)
{
}
void *worker (void *data)
{
int i;
for (i = num; i--;)
{
void *m = malloc (48*4096);
dostuff (m);
free (m);
}
return NULL;
}
int main()
{
int i;
pthread_t t;
void *ret;
if (pthread_create (&t, NULL, worker, NULL))
exit (2);
if (pthread_join (t, &ret))
exit (3);
return 0;
}
Before the patch, this resulted in 1024 calls to madvise. With the patch applied,
madvise is called twice because the default trim threshold is high enough to avoid
this.
This a more complex case where there is a mix of frees. It's simply a different worker
function for the test case above
void *worker (void *data)
{
int i;
int j = 0;
void *free_index[num];
for (i = num; i--;)
{
void *m = malloc ((i % 58) *4096);
dostuff (m);
if (i % 2 == 0) {
free (m);
} else {
free_index[j++] = m;
}
}
for (; j >= 0; j--)
{
free(free_index[j]);
}
return NULL;
}
glibc 2.21 calls malloc 90305 times but with the patch applied, it's
called 13438. Increasing the trim threshold will decrease the number of
times it's called with the option of eliminating the overhead.
ebizzy is meant to generate a workload resembling common web application
server workloads. It is threaded with a large working set that at its core
has an allocation, do_stuff, free loop that also hits this case. The primary
metric of the benchmark is records processed per second. This is running on
my desktop which is a single socket machine with an I7-4770 and 8 cores.
Each thread count was run for 30 seconds. It was only run once as the
performance difference is so high that the variation is insignificant.
glibc 2.21 patch
threads 1 10230 44114
threads 2 19153 84925
threads 4 34295 134569
threads 8 51007 183387
Note that the saving happens to be a concidence as the size allocated
by ebizzy was less than the default threshold. If a different number of
chunks were specified then it may also be necessary to tune the threshold
to compensate
This is roughly quadrupling the performance of this benchmark. The difference in
system CPU usage illustrates why.
ebizzy running 1 thread with glibc 2.21
10230 records/s 306904
real 30.00 s
user 7.47 s
sys 22.49 s
22.49 seconds was spent in the kernel for a workload runinng 30 seconds. With the
patch applied
ebizzy running 1 thread with patch applied
44126 records/s 1323792
real 30.00 s
user 29.97 s
sys 0.00 s
system CPU usage was zero with the patch applied. strace shows that glibc
running this workload calls madvise approximately 9000 times a second. With
the patch applied madvise was called twice during the workload (or 0.06
times per second).
2015-02-10 Mel Gorman <mgorman@suse.de>
[BZ #17195]
* malloc/arena.c (free): Apply trim threshold to per-thread heaps
as well as the main arena.
2015-04-02 08:44:14 +02:00
|
|
|
extra = ALIGN_DOWN(top_area - pad, pagesz);
|
2015-10-08 04:21:36 +02:00
|
|
|
if (extra == 0)
|
2002-01-29 08:54:51 +01:00
|
|
|
return 0;
|
2014-01-02 09:38:18 +01:00
|
|
|
|
2002-01-29 08:54:51 +01:00
|
|
|
/* Try to shrink. */
|
2014-01-02 09:38:18 +01:00
|
|
|
if (shrink_heap (heap, extra) != 0)
|
2002-01-29 08:54:51 +01:00
|
|
|
return 0;
|
2014-01-02 09:38:18 +01:00
|
|
|
|
2002-01-29 08:54:51 +01:00
|
|
|
ar_ptr->system_mem -= extra;
|
|
|
|
|
|
|
|
/* Success. Adjust top accordingly. */
|
2014-01-02 09:38:18 +01:00
|
|
|
set_head (top_chunk, (top_size - extra) | PREV_INUSE);
|
2002-01-29 08:54:51 +01:00
|
|
|
/*check_chunk(ar_ptr, top_chunk);*/
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2005-10-12 22:49:06 +02:00
|
|
|
/* Create a new arena with initial size "size". */
|
|
|
|
|
2015-10-28 19:32:46 +01:00
|
|
|
/* If REPLACED_ARENA is not NULL, detach it from this thread. Must be
|
2015-12-21 16:42:46 +01:00
|
|
|
called while free_list_lock is held. */
|
2015-10-28 19:32:46 +01:00
|
|
|
static void
|
|
|
|
detach_arena (mstate replaced_arena)
|
|
|
|
{
|
|
|
|
if (replaced_arena != NULL)
|
|
|
|
{
|
|
|
|
assert (replaced_arena->attached_threads > 0);
|
|
|
|
/* The current implementation only detaches from main_arena in
|
|
|
|
case of allocation failure. This means that it is likely not
|
|
|
|
beneficial to put the arena on free_list even if the
|
|
|
|
reference count reaches zero. */
|
|
|
|
--replaced_arena->attached_threads;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2005-10-12 22:49:06 +02:00
|
|
|
static mstate
|
2014-01-02 09:38:18 +01:00
|
|
|
_int_new_arena (size_t size)
|
2005-10-12 22:49:06 +02:00
|
|
|
{
|
|
|
|
mstate a;
|
|
|
|
heap_info *h;
|
|
|
|
char *ptr;
|
|
|
|
unsigned long misalign;
|
|
|
|
|
2014-01-02 09:38:18 +01:00
|
|
|
h = new_heap (size + (sizeof (*h) + sizeof (*a) + MALLOC_ALIGNMENT),
|
|
|
|
mp_.top_pad);
|
|
|
|
if (!h)
|
|
|
|
{
|
|
|
|
/* Maybe size is too large to fit in a single heap. So, just try
|
|
|
|
to create a minimally-sized arena and let _int_malloc() attempt
|
|
|
|
to deal with the large request via mmap_chunk(). */
|
|
|
|
h = new_heap (sizeof (*h) + sizeof (*a) + MALLOC_ALIGNMENT, mp_.top_pad);
|
|
|
|
if (!h)
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
a = h->ar_ptr = (mstate) (h + 1);
|
|
|
|
malloc_init_state (a);
|
2015-10-28 19:32:46 +01:00
|
|
|
a->attached_threads = 1;
|
2005-10-12 22:49:06 +02:00
|
|
|
/*a->next = NULL;*/
|
|
|
|
a->system_mem = a->max_system_mem = h->size;
|
|
|
|
|
|
|
|
/* Set up the top chunk, with proper alignment. */
|
2014-01-02 09:38:18 +01:00
|
|
|
ptr = (char *) (a + 1);
|
|
|
|
misalign = (unsigned long) chunk2mem (ptr) & MALLOC_ALIGN_MASK;
|
2005-10-12 22:49:06 +02:00
|
|
|
if (misalign > 0)
|
|
|
|
ptr += MALLOC_ALIGNMENT - misalign;
|
2014-01-02 09:38:18 +01:00
|
|
|
top (a) = (mchunkptr) ptr;
|
|
|
|
set_head (top (a), (((char *) h + h->size) - ptr) | PREV_INUSE);
|
2005-10-12 22:49:06 +02:00
|
|
|
|
2013-09-20 16:10:54 +02:00
|
|
|
LIBC_PROBE (memory_arena_new, 2, a, size);
|
2015-10-28 19:32:46 +01:00
|
|
|
mstate replaced_arena = thread_arena;
|
2015-10-17 12:06:48 +02:00
|
|
|
thread_arena = a;
|
2016-09-06 12:49:54 +02:00
|
|
|
__libc_lock_init (a->mutex);
|
2009-03-14 00:53:18 +01:00
|
|
|
|
2016-09-06 12:49:54 +02:00
|
|
|
__libc_lock_lock (list_lock);
|
2009-03-14 00:53:18 +01:00
|
|
|
|
|
|
|
/* Add the new arena to the global list. */
|
|
|
|
a->next = main_arena.next;
|
2015-12-21 16:42:46 +01:00
|
|
|
/* FIXME: The barrier is an attempt to synchronize with read access
|
|
|
|
in reused_arena, which does not acquire list_lock while
|
|
|
|
traversing the list. */
|
2009-03-14 00:53:18 +01:00
|
|
|
atomic_write_barrier ();
|
|
|
|
main_arena.next = a;
|
|
|
|
|
2016-09-06 12:49:54 +02:00
|
|
|
__libc_lock_unlock (list_lock);
|
2009-03-14 00:53:18 +01:00
|
|
|
|
2016-09-06 12:49:54 +02:00
|
|
|
__libc_lock_lock (free_list_lock);
|
2015-12-21 16:42:46 +01:00
|
|
|
detach_arena (replaced_arena);
|
2016-09-06 12:49:54 +02:00
|
|
|
__libc_lock_unlock (free_list_lock);
|
2015-12-21 16:42:46 +01:00
|
|
|
|
|
|
|
/* Lock this arena. NB: Another thread may have been attached to
|
|
|
|
this arena because the arena is now accessible from the
|
|
|
|
main_arena.next list and could have been picked by reused_arena.
|
|
|
|
This can only happen for the last arena created (before the arena
|
|
|
|
limit is reached). At this point, some arena has to be attached
|
|
|
|
to two threads. We could acquire the arena lock before list_lock
|
|
|
|
to make it less likely that reused_arena picks this new arena,
|
2016-04-14 09:17:02 +02:00
|
|
|
but this could result in a deadlock with
|
|
|
|
__malloc_fork_lock_parent. */
|
2015-12-21 16:42:46 +01:00
|
|
|
|
2016-09-06 12:49:54 +02:00
|
|
|
__libc_lock_lock (a->mutex);
|
2015-12-21 16:42:46 +01:00
|
|
|
|
2005-10-12 22:49:06 +02:00
|
|
|
return a;
|
|
|
|
}
|
|
|
|
|
2009-03-14 00:53:18 +01:00
|
|
|
|
malloc: Preserve arena free list/thread count invariant [BZ #20370]
It is necessary to preserve the invariant that if an arena is
on the free list, it has thread attach count zero. Otherwise,
when arena_thread_freeres sees the zero attach count, it will
add it, and without the invariant, an arena could get pushed
to the list twice, resulting in a cycle.
One possible execution trace looks like this:
Thread 1 examines free list and observes it as empty.
Thread 2 exits and adds its arena to the free list,
with attached_threads == 0).
Thread 1 selects this arena in reused_arena (not from the free list).
Thread 1 increments attached_threads and attaches itself.
(The arena remains on the free list.)
Thread 1 exits, decrements attached_threads,
and adds the arena to the free list.
The final step creates a cycle in the usual way (by overwriting the
next_free member with the former list head, while there is another
list item pointing to the arena structure).
tst-malloc-thread-exit exhibits this issue, but it was only visible
with a debugger because the incorrect fix in bug 19243 removed
the assert from get_free_list.
2016-08-02 12:24:50 +02:00
|
|
|
/* Remove an arena from free_list. */
|
2009-03-14 00:53:18 +01:00
|
|
|
static mstate
|
|
|
|
get_free_list (void)
|
|
|
|
{
|
2015-10-28 19:32:46 +01:00
|
|
|
mstate replaced_arena = thread_arena;
|
2009-03-14 00:53:18 +01:00
|
|
|
mstate result = free_list;
|
|
|
|
if (result != NULL)
|
|
|
|
{
|
2016-09-06 12:49:54 +02:00
|
|
|
__libc_lock_lock (free_list_lock);
|
2009-03-14 00:53:18 +01:00
|
|
|
result = free_list;
|
|
|
|
if (result != NULL)
|
2015-10-28 19:32:46 +01:00
|
|
|
{
|
|
|
|
free_list = result->next_free;
|
|
|
|
|
2015-12-16 12:39:48 +01:00
|
|
|
/* The arena will be attached to this thread. */
|
malloc: Preserve arena free list/thread count invariant [BZ #20370]
It is necessary to preserve the invariant that if an arena is
on the free list, it has thread attach count zero. Otherwise,
when arena_thread_freeres sees the zero attach count, it will
add it, and without the invariant, an arena could get pushed
to the list twice, resulting in a cycle.
One possible execution trace looks like this:
Thread 1 examines free list and observes it as empty.
Thread 2 exits and adds its arena to the free list,
with attached_threads == 0).
Thread 1 selects this arena in reused_arena (not from the free list).
Thread 1 increments attached_threads and attaches itself.
(The arena remains on the free list.)
Thread 1 exits, decrements attached_threads,
and adds the arena to the free list.
The final step creates a cycle in the usual way (by overwriting the
next_free member with the former list head, while there is another
list item pointing to the arena structure).
tst-malloc-thread-exit exhibits this issue, but it was only visible
with a debugger because the incorrect fix in bug 19243 removed
the assert from get_free_list.
2016-08-02 12:24:50 +02:00
|
|
|
assert (result->attached_threads == 0);
|
|
|
|
result->attached_threads = 1;
|
2015-10-28 19:32:46 +01:00
|
|
|
|
|
|
|
detach_arena (replaced_arena);
|
|
|
|
}
|
2016-09-06 12:49:54 +02:00
|
|
|
__libc_lock_unlock (free_list_lock);
|
2009-03-14 00:53:18 +01:00
|
|
|
|
|
|
|
if (result != NULL)
|
2014-01-02 09:38:18 +01:00
|
|
|
{
|
|
|
|
LIBC_PROBE (memory_arena_reuse_free_list, 1, result);
|
2016-09-06 12:49:54 +02:00
|
|
|
__libc_lock_lock (result->mutex);
|
2015-10-17 12:06:48 +02:00
|
|
|
thread_arena = result;
|
2014-01-02 09:38:18 +01:00
|
|
|
}
|
2009-03-14 00:53:18 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
malloc: Preserve arena free list/thread count invariant [BZ #20370]
It is necessary to preserve the invariant that if an arena is
on the free list, it has thread attach count zero. Otherwise,
when arena_thread_freeres sees the zero attach count, it will
add it, and without the invariant, an arena could get pushed
to the list twice, resulting in a cycle.
One possible execution trace looks like this:
Thread 1 examines free list and observes it as empty.
Thread 2 exits and adds its arena to the free list,
with attached_threads == 0).
Thread 1 selects this arena in reused_arena (not from the free list).
Thread 1 increments attached_threads and attaches itself.
(The arena remains on the free list.)
Thread 1 exits, decrements attached_threads,
and adds the arena to the free list.
The final step creates a cycle in the usual way (by overwriting the
next_free member with the former list head, while there is another
list item pointing to the arena structure).
tst-malloc-thread-exit exhibits this issue, but it was only visible
with a debugger because the incorrect fix in bug 19243 removed
the assert from get_free_list.
2016-08-02 12:24:50 +02:00
|
|
|
/* Remove the arena from the free list (if it is present).
|
|
|
|
free_list_lock must have been acquired by the caller. */
|
|
|
|
static void
|
|
|
|
remove_from_free_list (mstate arena)
|
|
|
|
{
|
|
|
|
mstate *previous = &free_list;
|
|
|
|
for (mstate p = free_list; p != NULL; p = p->next_free)
|
|
|
|
{
|
|
|
|
assert (p->attached_threads == 0);
|
|
|
|
if (p == arena)
|
|
|
|
{
|
|
|
|
/* Remove the requested arena from the list. */
|
|
|
|
*previous = p->next_free;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
previous = &p->next_free;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-08-10 17:39:38 +02:00
|
|
|
/* Lock and return an arena that can be reused for memory allocation.
|
2012-08-10 17:37:04 +02:00
|
|
|
Avoid AVOID_ARENA as we have already failed to allocate memory in
|
|
|
|
it and it is currently locked. */
|
2009-03-14 00:53:18 +01:00
|
|
|
static mstate
|
2012-08-10 17:37:04 +02:00
|
|
|
reused_arena (mstate avoid_arena)
|
2009-03-14 00:53:18 +01:00
|
|
|
{
|
|
|
|
mstate result;
|
2015-12-21 16:42:46 +01:00
|
|
|
/* FIXME: Access to next_to_use suffers from data races. */
|
2009-03-14 00:53:18 +01:00
|
|
|
static mstate next_to_use;
|
|
|
|
if (next_to_use == NULL)
|
|
|
|
next_to_use = &main_arena;
|
|
|
|
|
2015-12-16 12:39:48 +01:00
|
|
|
/* Iterate over all arenas (including those linked from
|
|
|
|
free_list). */
|
2009-03-14 00:53:18 +01:00
|
|
|
result = next_to_use;
|
|
|
|
do
|
|
|
|
{
|
2017-08-30 17:16:08 +02:00
|
|
|
if (!__libc_lock_trylock (result->mutex))
|
2014-01-02 09:38:18 +01:00
|
|
|
goto out;
|
2009-03-14 00:53:18 +01:00
|
|
|
|
2015-12-21 16:42:46 +01:00
|
|
|
/* FIXME: This is a data race, see _int_new_arena. */
|
2009-03-14 00:53:18 +01:00
|
|
|
result = result->next;
|
|
|
|
}
|
|
|
|
while (result != next_to_use);
|
|
|
|
|
2012-08-10 17:37:04 +02:00
|
|
|
/* Avoid AVOID_ARENA as we have already failed to allocate memory
|
|
|
|
in that arena and it is currently locked. */
|
|
|
|
if (result == avoid_arena)
|
|
|
|
result = result->next;
|
|
|
|
|
Avoid deadlock in malloc on backtrace (BZ #16159)
When the malloc subsystem detects some kind of memory corruption,
depending on the configuration it prints the error, a backtrace, a
memory map and then aborts the process. In this process, the
backtrace() call may result in a call to malloc, resulting in
various kinds of problematic behavior.
In one case, the malloc it calls may detect a corruption and call
backtrace again, and a stack overflow may result due to the infinite
recursion. In another case, the malloc it calls may deadlock on an
arena lock with the malloc (or free, realloc, etc.) that detected the
corruption. In yet another case, if the program is linked with
pthreads, backtrace may do a pthread_once initialization, which
deadlocks on itself.
In all these cases, the program exit is not as intended. This is
avoidable by marking the arena that malloc detected a corruption on,
as unusable. The following patch does that. Features of this patch
are as follows:
- A flag is added to the mstate struct of the arena to indicate if the
arena is corrupt.
- The flag is checked whenever malloc functions try to get a lock on
an arena. If the arena is unusable, a NULL is returned, causing the
malloc to use mmap or try the next arena.
- malloc_printerr sets the corrupt flag on the arena when it detects a
corruption
- free does not concern itself with the flag at all. It is not
important since the backtrace workflow does not need free. A free
in a parallel thread may cause another corruption, but that's not
new
- The flag check and set are not atomic and may race. This is fine
since we don't care about contention during the flag check. We want
to make sure that the malloc call in the backtrace does not trip on
itself and all that action happens in the same thread and not across
threads.
I verified that the test case does not show any regressions due to
this patch. I also ran the malloc benchmarks and found an
insignificant difference in timings (< 2%).
* malloc/Makefile (tests): New test case tst-malloc-backtrace.
* malloc/arena.c (arena_lock): Check if arena is corrupt.
(reused_arena): Find a non-corrupt arena.
(heap_trim): Pass arena to unlink.
* malloc/hooks.c (malloc_check_get_size): Pass arena to
malloc_printerr.
(top_check): Likewise.
(free_check): Likewise.
(realloc_check): Likewise.
* malloc/malloc.c (malloc_printerr): Add arena argument.
(unlink): Likewise.
(munmap_chunk): Adjust.
(ARENA_CORRUPTION_BIT): New macro.
(arena_is_corrupt): Likewise.
(set_arena_corrupt): Likewise.
(sysmalloc): Use mmap if there are no usable arenas.
(_int_malloc): Likewise.
(__libc_malloc): Don't fail if arena_get returns NULL.
(_mid_memalign): Likewise.
(__libc_calloc): Likewise.
(__libc_realloc): Adjust for additional argument to
malloc_printerr.
(_int_free): Likewise.
(malloc_consolidate): Likewise.
(_int_realloc): Likewise.
(_int_memalign): Don't touch corrupt arenas.
* malloc/tst-malloc-backtrace.c: New test case.
2015-05-19 03:10:37 +02:00
|
|
|
/* No arena available without contention. Wait for the next in line. */
|
2013-09-20 16:10:55 +02:00
|
|
|
LIBC_PROBE (memory_arena_reuse_wait, 3, &result->mutex, result, avoid_arena);
|
2016-09-06 12:49:54 +02:00
|
|
|
__libc_lock_lock (result->mutex);
|
2009-03-14 00:53:18 +01:00
|
|
|
|
2014-01-02 09:38:18 +01:00
|
|
|
out:
|
malloc: Preserve arena free list/thread count invariant [BZ #20370]
It is necessary to preserve the invariant that if an arena is
on the free list, it has thread attach count zero. Otherwise,
when arena_thread_freeres sees the zero attach count, it will
add it, and without the invariant, an arena could get pushed
to the list twice, resulting in a cycle.
One possible execution trace looks like this:
Thread 1 examines free list and observes it as empty.
Thread 2 exits and adds its arena to the free list,
with attached_threads == 0).
Thread 1 selects this arena in reused_arena (not from the free list).
Thread 1 increments attached_threads and attaches itself.
(The arena remains on the free list.)
Thread 1 exits, decrements attached_threads,
and adds the arena to the free list.
The final step creates a cycle in the usual way (by overwriting the
next_free member with the former list head, while there is another
list item pointing to the arena structure).
tst-malloc-thread-exit exhibits this issue, but it was only visible
with a debugger because the incorrect fix in bug 19243 removed
the assert from get_free_list.
2016-08-02 12:24:50 +02:00
|
|
|
/* Attach the arena to the current thread. */
|
2015-10-28 19:32:46 +01:00
|
|
|
{
|
2015-12-21 16:42:46 +01:00
|
|
|
/* Update the arena thread attachment counters. */
|
2015-10-28 19:32:46 +01:00
|
|
|
mstate replaced_arena = thread_arena;
|
2016-09-06 12:49:54 +02:00
|
|
|
__libc_lock_lock (free_list_lock);
|
2015-10-28 19:32:46 +01:00
|
|
|
detach_arena (replaced_arena);
|
malloc: Preserve arena free list/thread count invariant [BZ #20370]
It is necessary to preserve the invariant that if an arena is
on the free list, it has thread attach count zero. Otherwise,
when arena_thread_freeres sees the zero attach count, it will
add it, and without the invariant, an arena could get pushed
to the list twice, resulting in a cycle.
One possible execution trace looks like this:
Thread 1 examines free list and observes it as empty.
Thread 2 exits and adds its arena to the free list,
with attached_threads == 0).
Thread 1 selects this arena in reused_arena (not from the free list).
Thread 1 increments attached_threads and attaches itself.
(The arena remains on the free list.)
Thread 1 exits, decrements attached_threads,
and adds the arena to the free list.
The final step creates a cycle in the usual way (by overwriting the
next_free member with the former list head, while there is another
list item pointing to the arena structure).
tst-malloc-thread-exit exhibits this issue, but it was only visible
with a debugger because the incorrect fix in bug 19243 removed
the assert from get_free_list.
2016-08-02 12:24:50 +02:00
|
|
|
|
|
|
|
/* We may have picked up an arena on the free list. We need to
|
|
|
|
preserve the invariant that no arena on the free list has a
|
|
|
|
positive attached_threads counter (otherwise,
|
|
|
|
arena_thread_freeres cannot use the counter to determine if the
|
|
|
|
arena needs to be put on the free list). We unconditionally
|
|
|
|
remove the selected arena from the free list. The caller of
|
|
|
|
reused_arena checked the free list and observed it to be empty,
|
|
|
|
so the list is very short. */
|
|
|
|
remove_from_free_list (result);
|
|
|
|
|
2015-10-28 19:32:46 +01:00
|
|
|
++result->attached_threads;
|
malloc: Preserve arena free list/thread count invariant [BZ #20370]
It is necessary to preserve the invariant that if an arena is
on the free list, it has thread attach count zero. Otherwise,
when arena_thread_freeres sees the zero attach count, it will
add it, and without the invariant, an arena could get pushed
to the list twice, resulting in a cycle.
One possible execution trace looks like this:
Thread 1 examines free list and observes it as empty.
Thread 2 exits and adds its arena to the free list,
with attached_threads == 0).
Thread 1 selects this arena in reused_arena (not from the free list).
Thread 1 increments attached_threads and attaches itself.
(The arena remains on the free list.)
Thread 1 exits, decrements attached_threads,
and adds the arena to the free list.
The final step creates a cycle in the usual way (by overwriting the
next_free member with the former list head, while there is another
list item pointing to the arena structure).
tst-malloc-thread-exit exhibits this issue, but it was only visible
with a debugger because the incorrect fix in bug 19243 removed
the assert from get_free_list.
2016-08-02 12:24:50 +02:00
|
|
|
|
2016-09-06 12:49:54 +02:00
|
|
|
__libc_lock_unlock (free_list_lock);
|
2015-10-28 19:32:46 +01:00
|
|
|
}
|
|
|
|
|
2013-09-20 16:10:55 +02:00
|
|
|
LIBC_PROBE (memory_arena_reuse, 2, result, avoid_arena);
|
2015-10-17 12:06:48 +02:00
|
|
|
thread_arena = result;
|
2009-03-14 00:53:18 +01:00
|
|
|
next_to_use = result->next;
|
|
|
|
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2002-01-29 08:54:51 +01:00
|
|
|
static mstate
|
2015-08-24 11:02:07 +02:00
|
|
|
arena_get2 (size_t size, mstate avoid_arena)
|
2002-01-29 08:54:51 +01:00
|
|
|
{
|
|
|
|
mstate a;
|
|
|
|
|
2011-11-09 17:14:39 +01:00
|
|
|
static size_t narenas_limit;
|
|
|
|
|
|
|
|
a = get_free_list ();
|
|
|
|
if (a == NULL)
|
|
|
|
{
|
|
|
|
/* Nothing immediately available, so generate a new arena. */
|
|
|
|
if (narenas_limit == 0)
|
2014-01-02 09:38:18 +01:00
|
|
|
{
|
|
|
|
if (mp_.arena_max != 0)
|
|
|
|
narenas_limit = mp_.arena_max;
|
|
|
|
else if (narenas > mp_.arena_test)
|
|
|
|
{
|
|
|
|
int n = __get_nprocs ();
|
|
|
|
|
|
|
|
if (n >= 1)
|
|
|
|
narenas_limit = NARENAS_FROM_NCORES (n);
|
|
|
|
else
|
|
|
|
/* We have no information about the system. Assume two
|
|
|
|
cores. */
|
|
|
|
narenas_limit = NARENAS_FROM_NCORES (2);
|
|
|
|
}
|
|
|
|
}
|
2011-11-09 17:14:39 +01:00
|
|
|
repeat:;
|
|
|
|
size_t n = narenas;
|
2012-01-31 20:42:34 +01:00
|
|
|
/* NB: the following depends on the fact that (size_t)0 - 1 is a
|
2014-01-02 09:38:18 +01:00
|
|
|
very large number and that the underflow is OK. If arena_max
|
|
|
|
is set the value of arena_test is irrelevant. If arena_test
|
|
|
|
is set but narenas is not yet larger or equal to arena_test
|
|
|
|
narenas_limit is 0. There is no possibility for narenas to
|
|
|
|
be too big for the test to always fail since there is not
|
|
|
|
enough address space to create that many arenas. */
|
2014-02-10 14:45:42 +01:00
|
|
|
if (__glibc_unlikely (n <= narenas_limit - 1))
|
2014-01-02 09:38:18 +01:00
|
|
|
{
|
|
|
|
if (catomic_compare_and_exchange_bool_acq (&narenas, n + 1, n))
|
|
|
|
goto repeat;
|
|
|
|
a = _int_new_arena (size);
|
2014-02-10 14:45:42 +01:00
|
|
|
if (__glibc_unlikely (a == NULL))
|
2014-01-02 09:38:18 +01:00
|
|
|
catomic_decrement (&narenas);
|
|
|
|
}
|
2011-11-14 11:41:52 +01:00
|
|
|
else
|
2014-01-02 09:38:18 +01:00
|
|
|
a = reused_arena (avoid_arena);
|
2011-11-09 17:14:39 +01:00
|
|
|
}
|
2002-01-29 08:54:51 +01:00
|
|
|
return a;
|
|
|
|
}
|
|
|
|
|
2012-09-07 11:09:52 +02:00
|
|
|
/* If we don't have the main arena, then maybe the failure is due to running
|
|
|
|
out of mmapped areas, so we can try allocating on the main arena.
|
|
|
|
Otherwise, it is likely that sbrk() has failed and there is still a chance
|
|
|
|
to mmap(), so try one of the other arenas. */
|
|
|
|
static mstate
|
|
|
|
arena_get_retry (mstate ar_ptr, size_t bytes)
|
|
|
|
{
|
2013-09-20 16:10:56 +02:00
|
|
|
LIBC_PROBE (memory_arena_retry, 2, bytes, ar_ptr);
|
2014-01-02 09:38:18 +01:00
|
|
|
if (ar_ptr != &main_arena)
|
|
|
|
{
|
2016-09-06 12:49:54 +02:00
|
|
|
__libc_lock_unlock (ar_ptr->mutex);
|
2014-01-02 09:38:18 +01:00
|
|
|
ar_ptr = &main_arena;
|
2016-09-06 12:49:54 +02:00
|
|
|
__libc_lock_lock (ar_ptr->mutex);
|
2014-01-02 09:38:18 +01:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2016-09-06 12:49:54 +02:00
|
|
|
__libc_lock_unlock (ar_ptr->mutex);
|
2015-08-24 11:02:07 +02:00
|
|
|
ar_ptr = arena_get2 (bytes, ar_ptr);
|
2014-01-02 09:38:18 +01:00
|
|
|
}
|
2012-09-07 11:09:52 +02:00
|
|
|
|
|
|
|
return ar_ptr;
|
|
|
|
}
|
|
|
|
|
2009-03-14 00:53:18 +01:00
|
|
|
static void __attribute__ ((section ("__libc_thread_freeres_fn")))
|
|
|
|
arena_thread_freeres (void)
|
|
|
|
{
|
2015-10-17 12:06:48 +02:00
|
|
|
mstate a = thread_arena;
|
|
|
|
thread_arena = NULL;
|
2009-03-14 00:53:18 +01:00
|
|
|
|
|
|
|
if (a != NULL)
|
|
|
|
{
|
2016-09-06 12:49:54 +02:00
|
|
|
__libc_lock_lock (free_list_lock);
|
2015-10-28 19:32:46 +01:00
|
|
|
/* If this was the last attached thread for this arena, put the
|
|
|
|
arena on the free list. */
|
|
|
|
assert (a->attached_threads > 0);
|
|
|
|
if (--a->attached_threads == 0)
|
|
|
|
{
|
|
|
|
a->next_free = free_list;
|
|
|
|
free_list = a;
|
|
|
|
}
|
2016-09-06 12:49:54 +02:00
|
|
|
__libc_lock_unlock (free_list_lock);
|
2009-03-14 00:53:18 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
text_set_element (__libc_thread_subfreeres, arena_thread_freeres);
|
|
|
|
|
2002-01-29 08:54:51 +01:00
|
|
|
/*
|
|
|
|
* Local variables:
|
|
|
|
* c-basic-offset: 2
|
|
|
|
* End:
|
|
|
|
*/
|