OpenE2K
/
glibc
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

* config.h.in (USE_MULTIARCH): Define. 

* configure.in: Handle --enable-multi-arch.
	* elf/dl-runtime.c (_dl_fixup): Handle STT_GNU_IFUNC.
	(_dl_fixup_profile): Likewise.
	* elf/do-lookup.c (dl_lookup_x): Likewise.
	* sysdeps/x86_64/dl-machine.h: Handle STT_GNU_IFUNC.
	* elf/elf.h (STT_GNU_IFUNC): Define.
	* include/libc-symbols.h (libc_ifunc): Define.
	* sysdeps/x86_64/cacheinfo.c: If USE_MULTIARCH is defined, use the
	framework in init-arch.h to get CPUID values.
	* sysdeps/x86_64/multiarch/Makefile: New file.
	* sysdeps/x86_64/multiarch/init-arch.c: New file.
	* sysdeps/x86_64/multiarch/init-arch.h: New file.
	* sysdeps/x86_64/multiarch/sched_cpucount.c: New file.

	* config.make.in (experimental-malloc): Define.
	* configure.in: Handle --enable-experimental-malloc.
	* malloc/Makefile: Handle experimental-malloc flag.
	* malloc/malloc.c: Implement PER_THREAD and ATOMIC_FASTBINS features.
	* malloc/arena.c: Likewise.
	* malloc/hooks.c: Likewise.
	* malloc/malloc.h: Define M_ARENA_TEST and M_ARENA_MAX.
This commit is contained in:
Ulrich Drepper 2009-03-13 23:53:18 +00:00
parent e7f110cdbd
commit 425ce2edb9
20 changed files with 813 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
ChangeLog
View File

@ -1,3 +1,28 @@
2009-03-13 Ulrich Drepper <drepper@redhat.com>
* config.h.in (USE_MULTIARCH): Define.
* configure.in: Handle --enable-multi-arch.
* elf/dl-runtime.c (_dl_fixup): Handle STT_GNU_IFUNC.
(_dl_fixup_profile): Likewise.
* elf/do-lookup.c (dl_lookup_x): Likewise.
* sysdeps/x86_64/dl-machine.h: Handle STT_GNU_IFUNC.
* elf/elf.h (STT_GNU_IFUNC): Define.
* include/libc-symbols.h (libc_ifunc): Define.
* sysdeps/x86_64/cacheinfo.c: If USE_MULTIARCH is defined, use the
framework in init-arch.h to get CPUID values.
* sysdeps/x86_64/multiarch/Makefile: New file.
* sysdeps/x86_64/multiarch/init-arch.c: New file.
* sysdeps/x86_64/multiarch/init-arch.h: New file.
* sysdeps/x86_64/multiarch/sched_cpucount.c: New file.
* config.make.in (experimental-malloc): Define.
* configure.in: Handle --enable-experimental-malloc.
* malloc/Makefile: Handle experimental-malloc flag.
* malloc/malloc.c: Implement PER_THREAD and ATOMIC_FASTBINS features.
* malloc/arena.c: Likewise.
* malloc/hooks.c: Likewise.
* malloc/malloc.h: Define M_ARENA_TEST and M_ARENA_MAX.
2009-03-11 Ulrich Drepper <drepper@redhat.com>
* sysdeps/x86_64/dl-machine.h (elf_machine_rela): Add branch

4
NEWS
View File

@ -1,4 +1,4 @@
GNU C Library NEWS -- history of user-visible changes. 2009-3-8
GNU C Library NEWS -- history of user-visible changes. 2009-3-12
Copyright (C) 1992-2008, 2009 Free Software Foundation, Inc.
See the end for copying conditions.
@ -7,7 +7,7 @@ using `glibc' in the "product" field.
Version 2.10
* New Linux interface: accept4
* New Linux interfaces: accept4, fallocate, fallocate64.
* Correct declarations of string function when used in C++ code. This
could lead to compile error for invalid C++ code.

3
config.h.in
View File

@ -189,6 +189,9 @@
/* Define if __stack_chk_guard canary should be randomized at program startup. */
#undef ENABLE_STACKGUARD_RANDOMIZE
/* Define if multi-arch DSOs should be generated. */
#undef USE_MULTIARCH
/*
*/

3
config.make.in
View File

@ -70,6 +70,7 @@ versioning = @VERSIONING@
oldest-abi = @oldest_abi@
no-whole-archive = @no_whole_archive@
exceptions = @exceptions@
multi-arch = @multi_arch@
mach-interface-list = @mach_interface_list@
@ -78,6 +79,8 @@ have-ksh = @libc_cv_have_ksh@
sizeof-long-double = @sizeof_long_double@
experimental-malloc = @experimental_malloc@
# Configuration options.
build-static = @static@
build-shared = @shared@

70
configure vendored
View File

@ -660,6 +660,8 @@ oldest_abi
bindnow
force_install
all_warnings
multi_arch
experimental_malloc
build
build_cpu
build_vendor
@ -1380,6 +1382,10 @@ Optional Features:
--enable-kernel=VERSION compile for compatibility with kernel not older than
VERSION
--enable-all-warnings enable all useful warnings gcc can issue
--enable-multi-arch enable single DSO with optimizations for multiple
architectures
--enable-experimental-malloc
enable experimental malloc features
Optional Packages:
--with-PACKAGE[=ARG] use PACKAGE [ARG=yes]
@ -2173,6 +2179,29 @@ fi
# Check whether --enable-multi-arch was given.
if test "${enable_multi_arch+set}" = set; then
enableval=$enable_multi_arch; multi_arch=$enableval
else
multi_arch=no
fi
if test x"$multi_arch" = xyes; then
cat >>confdefs.h <<\_ACEOF
#define USE_MULTIARCH 1
_ACEOF
multi_arch_d=/multiarch
fi
# Check whether --enable-experimental-malloc was given.
if test "${enable_experimental_malloc+set}" = set; then
enableval=$enable_experimental_malloc; experimental_malloc=$enableval
fi
# Make sure we can run config.sub.
$SHELL "$ac_aux_dir/config.sub" sun4 >/dev/null 2>&1 ||
{ { echo "$as_me:$LINENO: error: cannot run $SHELL $ac_aux_dir/config.sub" >&5
@ -2627,7 +2656,7 @@ for b in $base ''; do
test "$v" = / && continue
for o in /$ostry ''; do
test "$o" = / && continue
for m in $mach ''; do
for m in $multi_arch_d $mach ''; do
for d in $add_ons_pfx ''; do
for a in $add_ons_sfx ''; do
if test -n "$m0$m0sub$b$v$o$m$msub"; then
@ -5684,6 +5713,37 @@ _ACEOF
fi
# For the multi-arch option we need support in the assembler.
if test "$multi_arch" = yes; then
if test "x$libc_cv_asm_type_prefix" != xno; then
{ echo "$as_me:$LINENO: checking for assembler gnu_indirect_function symbol type support" >&5
echo $ECHO_N "checking for assembler gnu_indirect_function symbol type support... $ECHO_C" >&6; }
if test "${libc_cv_asm_gnu_indirect_function+set}" = set; then
echo $ECHO_N "(cached) $ECHO_C" >&6
else
cat > conftest.s <<EOF
.type foo,%gnu_indirect_function
EOF
if ${CC-cc} -c $ASFLAGS conftest.s 1>&5 2>&5;
then
libc_cv_asm_gnu_indirect_function=yes
else
libc_cv_asm_gnu_indirect_function=no
fi
rm -f conftest*
fi
{ echo "$as_me:$LINENO: result: $libc_cv_asm_gnu_indirect_function" >&5
echo "${ECHO_T}$libc_cv_asm_gnu_indirect_function" >&6; }
else
libc_cv_asm_gnu_indirect_function=no
fi
if test x"$libc_cv_asm_gnu_indirect_function" != xyes; then
{ { echo "$as_me:$LINENO: error: --enable-multi-arch support requires assembler and linker support" >&5
echo "$as_me: error: --enable-multi-arch support requires assembler and linker support" >&2;}
{ (exit 1); exit 1; }; }
fi
fi
{ echo "$as_me:$LINENO: checking for .symver assembler directive" >&5
echo $ECHO_N "checking for .symver assembler directive... $ECHO_C" >&6; }
if test "${libc_cv_asm_symver_directive+set}" = set; then
@ -9184,6 +9244,8 @@ oldest_abi!$oldest_abi$ac_delim
bindnow!$bindnow$ac_delim
force_install!$force_install$ac_delim
all_warnings!$all_warnings$ac_delim
multi_arch!$multi_arch$ac_delim
experimental_malloc!$experimental_malloc$ac_delim
build!$build$ac_delim
build_cpu!$build_cpu$ac_delim
build_vendor!$build_vendor$ac_delim
@ -9235,8 +9297,6 @@ libc_cv_have_bash2!$libc_cv_have_bash2$ac_delim
KSH!$KSH$ac_delim
libc_cv_have_ksh!$libc_cv_have_ksh$ac_delim
AWK!$AWK$ac_delim
PERL!$PERL$ac_delim
INSTALL_INFO!$INSTALL_INFO$ac_delim
_ACEOF
if test `sed -n "s/.*$ac_delim\$/X/p" conf$$subs.sed | grep -c X` = 97; then
@ -9278,6 +9338,8 @@ _ACEOF
ac_delim='%!_!# '
for ac_last_try in false false false false false :; do
cat >conf$$subs.sed <<_ACEOF
PERL!$PERL$ac_delim
INSTALL_INFO!$INSTALL_INFO$ac_delim
BISON!$BISON$ac_delim
VERSIONING!$VERSIONING$ac_delim
libc_cv_cc_with_libunwind!$libc_cv_cc_with_libunwind$ac_delim
@ -9334,7 +9396,7 @@ LIBOBJS!$LIBOBJS$ac_delim
LTLIBOBJS!$LTLIBOBJS$ac_delim
_ACEOF
if test `sed -n "s/.*$ac_delim\$/X/p" conf$$subs.sed | grep -c X` = 54; then
if test `sed -n "s/.*$ac_delim\$/X/p" conf$$subs.sed | grep -c X` = 56; then
break
elif $ac_last_try; then
{ { echo "$as_me:$LINENO: error: could not make $CONFIG_STATUS" >&5

43
configure.in
View File

@ -254,6 +254,24 @@ AC_ARG_ENABLE([all-warnings],
[])
AC_SUBST(all_warnings)
AC_ARG_ENABLE([multi-arch],
AC_HELP_STRING([--enable-multi-arch],
[enable single DSO with optimizations for multiple architectures]),
[multi_arch=$enableval],
[multi_arch=no])
if test x"$multi_arch" = xyes; then
AC_DEFINE(USE_MULTIARCH)
multi_arch_d=/multiarch
fi
AC_SUBST(multi_arch)
AC_ARG_ENABLE([experimental-malloc],
AC_HELP_STRING([--enable-experimental-malloc],
[enable experimental malloc features]),
[experimental_malloc=$enableval],
[])
AC_SUBST(experimental_malloc)
AC_CANONICAL_HOST
# The way shlib-versions is used to generate soversions.mk uses a
@ -608,7 +626,7 @@ for b in $base ''; do
test "$v" = / && continue
for o in /$ostry ''; do
test "$o" = / && continue
for m in $mach ''; do
for m in $multi_arch_d $mach ''; do
for d in $add_ons_pfx ''; do
for a in $add_ons_sfx ''; do
if test -n "$m0$m0sub$b$v$o$m$msub"; then
@ -1157,6 +1175,29 @@ if test "x$libc_cv_asm_type_prefix" != xno; then
AC_DEFINE_UNQUOTED(ASM_TYPE_DIRECTIVE_PREFIX, ${libc_cv_asm_type_prefix})
fi
# For the multi-arch option we need support in the assembler.
if test "$multi_arch" = yes; then
if test "x$libc_cv_asm_type_prefix" != xno; then
AC_CACHE_CHECK([for assembler gnu_indirect_function symbol type support],
libc_cv_asm_gnu_indirect_function, [dnl
cat > conftest.s <<EOF
.type foo,%gnu_indirect_function
EOF
if ${CC-cc} -c $ASFLAGS conftest.s 1>&AS_MESSAGE_LOG_FD 2>&AS_MESSAGE_LOG_FD;
then
libc_cv_asm_gnu_indirect_function=yes
else
libc_cv_asm_gnu_indirect_function=no
fi
rm -f conftest*])
else
libc_cv_asm_gnu_indirect_function=no
fi
if test x"$libc_cv_asm_gnu_indirect_function" != xyes; then
AC_MSG_ERROR([--enable-multi-arch support requires assembler and linker support])
fi
fi
AC_CACHE_CHECK(for .symver assembler directive, libc_cv_asm_symver_directive,
[cat > conftest.s <<EOF
${libc_cv_dot_text}

14
elf/dl-runtime.c
View File

@ -1,5 +1,5 @@
/* On-demand PLT fixup for shared objects.
Copyright (C) 1995-2006, 2007 Free Software Foundation, Inc.
Copyright (C) 1995-2006, 2007, 2008 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
@ -130,6 +130,9 @@ _dl_fixup (
/* And now perhaps the relocation addend. */
value = elf_machine_plt_value (l, reloc, value);
if (__builtin_expect (ELFW(ST_TYPE) (sym->st_info) == STT_GNU_IFUNC, 0))
value = ((DL_FIXUP_VALUE_TYPE (*) (void)) value) ();
/* Finally, fix up the plt itself. */
if (__builtin_expect (GLRO(dl_bind_not), 0))
return value;
@ -215,12 +218,21 @@ _dl_profile_fixup (
defsym != NULL
? LOOKUP_VALUE_ADDRESS (result)
+ defsym->st_value : 0);
if (__builtin_expect (ELFW(ST_TYPE) (defsym->st_info)
== STT_GNU_IFUNC, 0))
value = ((DL_FIXUP_VALUE_TYPE (*) (void)) value) ();
}
else
{
/* We already found the symbol. The module (and therefore its load
address) is also known. */
value = DL_FIXUP_MAKE_VALUE (l, l->l_addr + refsym->st_value);
if (__builtin_expect (ELFW(ST_TYPE) (refsym->st_info)
== STT_GNU_IFUNC, 0))
value = ((DL_FIXUP_VALUE_TYPE (*) (void)) value) ();
result = l;
}
/* And now perhaps the relocation addend. */

12
elf/do-lookup.h
View File

@ -1,5 +1,5 @@
/* Look up a symbol in the loaded objects.
Copyright (C) 1995-2004, 2005, 2006, 2007 Free Software Foundation, Inc.
Copyright (C) 1995-2007, 2008 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
@ -88,10 +88,12 @@ do_lookup_x (const char *undef_name, uint_fast32_t new_hash,
if (__builtin_expect (ELFW(ST_TYPE) (sym->st_info) > STT_FUNC
&& ELFW(ST_TYPE) (sym->st_info) != STT_COMMON
&& ELFW(ST_TYPE) (sym->st_info) != STT_TLS, 0))
/* Ignore all but STT_NOTYPE, STT_OBJECT, STT_FUNC, and STT_COMMON
entries (and STT_TLS if TLS is supported) since these
are no code/data definitions. */
&& ELFW(ST_TYPE) (sym->st_info) != STT_TLS
&& ELFW(ST_TYPE) (sym->st_info) != STT_GNU_IFUNC,
0))
/* Ignore all but STT_NOTYPE, STT_OBJECT, STT_FUNC, STT_COMMON,
STT_TLS, and STT_GNU_IFUNC since these are no code/data
definitions. */
return NULL;
if (sym != ref && strcmp (strtab + sym->st_name, undef_name))

1
elf/elf.h
View File

@ -459,6 +459,7 @@ typedef struct
#define STT_TLS 6 /* Symbol is thread-local data object*/
#define STT_NUM 7 /* Number of defined types. */
#define STT_LOOS 10 /* Start of OS-specific */
#define STT_GNU_IFUNC 10 /* Symbol is indirect code object */
#define STT_HIOS 12 /* End of OS-specific */
#define STT_LOPROC 13 /* Start of processor-specific */
#define STT_HIPROC 15 /* End of processor-specific */

14
include/libc-symbols.h
View File

@ -1,7 +1,6 @@
/* Support macros for making weak and strong aliases for symbols,
and for using symbol sets and linker warnings with GNU ld.
Copyright (C) 1995-1998,2000-2003,2004,2005,2006
Free Software Foundation, Inc.
Copyright (C) 1995-1998, 2000-2006, 2008 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
@ -835,4 +834,15 @@ for linking")
# define compat_data_section .section ".data.compat", "aw";
#endif
/* Marker used for indirection function symbols. */
#define libc_ifunc(name, expr) \
extern void *name##_ifunc (void) __asm__ (#name); \
void *name##_ifunc (void) \
{ \
INIT_ARCH (); \
__typeof (name) *res = expr; \
return res; \
} \
__asm__ (".type " #name ", %gnu_indirect_function");
#endif /* libc-symbols.h */

5
malloc/Makefile
View File

@ -1,4 +1,4 @@
# Copyright (C) 1991-1999, 2000, 2001, 2002, 2003, 2005, 2006, 2007
# Copyright (C) 1991-1999, 2000, 2001, 2002, 2003, 2005, 2006, 2007, 2009
# Free Software Foundation, Inc.
# This file is part of the GNU C Library.
@ -124,6 +124,9 @@ endif
tst-mcheck-ENV = MALLOC_CHECK_=3
ifeq ($(experimental-malloc),yes)
CPPFLAGS-malloc.c += -DPER_THREAD -DATOMIC_FASTBINS
endif
# Uncomment this for test releases. For public releases it is too expensive.
#CPPFLAGS-malloc.o += -DMALLOC_DEBUG=1

203
malloc/arena.c
View File

@ -1,5 +1,5 @@
/* Malloc implementation for multiple threads without lock contention.
Copyright (C) 2001,2002,2003,2004,2005,2006,2007
Copyright (C) 2001,2002,2003,2004,2005,2006,2007,2009
Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Wolfram Gloger <wg@malloc.de>, 2001.
@ -78,6 +78,10 @@ extern int sanity_check_heap_info_alignment[(sizeof (heap_info)
static tsd_key_t arena_key;
static mutex_t list_lock;
#ifdef PER_THREAD
static size_t narenas;
static mstate free_list;
#endif
#if THREAD_STATS
static int stat_n_heaps;
@ -105,13 +109,30 @@ int __malloc_initialized = -1;
in the new arena. */
#define arena_get(ptr, size) do { \
arena_lookup(ptr); \
arena_lock(ptr, size); \
} while(0)
#define arena_lookup(ptr) do { \
Void_t *vptr = NULL; \
ptr = (mstate)tsd_getspecific(arena_key, vptr); \
} while(0)
#ifdef PER_THREAD
#define arena_lock(ptr, size) do { \
if(ptr) \
(void)mutex_lock(&ptr->mutex); \
else \
ptr = arena_get2(ptr, (size)); \
} while(0)
#else
#define arena_lock(ptr, size) do { \
if(ptr && !mutex_trylock(&ptr->mutex)) { \
THREAD_STAT(++(ptr->stat_lock_direct)); \
} else \
ptr = arena_get2(ptr, (size)); \
} while(0)
#endif
/* find the heap and corresponding arena for a given ptr */
@ -219,6 +240,11 @@ free_atfork(Void_t* mem, const Void_t *caller)
}
#endif
#ifdef ATOMIC_FASTBINS
ar_ptr = arena_for_chunk(p);
tsd_getspecific(arena_key, vptr);
_int_free(ar_ptr, p, vptr == ATFORK_ARENA_PTR);
#else
ar_ptr = arena_for_chunk(p);
tsd_getspecific(arena_key, vptr);
if(vptr != ATFORK_ARENA_PTR)
@ -226,6 +252,7 @@ free_atfork(Void_t* mem, const Void_t *caller)
_int_free(ar_ptr, p);
if(vptr != ATFORK_ARENA_PTR)
(void)mutex_unlock(&ar_ptr->mutex);
#endif
}
@ -311,9 +338,18 @@ ptmalloc_unlock_all2 (void)
tsd_setspecific(arena_key, save_arena);
__malloc_hook = save_malloc_hook;
__free_hook = save_free_hook;
#endif
#ifdef PER_THREAD
free_list = NULL;
#endif
for(ar_ptr = &main_arena;;) {
mutex_init(&ar_ptr->mutex);
#ifdef PER_THREAD
if (ar_ptr != save_arena) {
ar_ptr->next_free = free_list;
free_list = ar_ptr;
}
#endif
ar_ptr = ar_ptr->next;
if(ar_ptr == &main_arena) break;
}
@ -377,6 +413,11 @@ ptmalloc_init_minimal (void)
mp_.mmap_threshold = DEFAULT_MMAP_THRESHOLD;
mp_.trim_threshold = DEFAULT_TRIM_THRESHOLD;
mp_.pagesize = malloc_getpagesize;
#ifdef PER_THREAD
# define NARENAS_FROM_NCORES(n) ((n) * (sizeof(long) == 4 ? 2 : 8))
mp_.arena_test = NARENAS_FROM_NCORES (1);
narenas = 1;
#endif
}
@ -529,9 +570,25 @@ ptmalloc_init (void)
}
break;
case 9:
if (! secure && memcmp (envline, "MMAP_MAX_", 9) == 0)
mALLOPt(M_MMAP_MAX, atoi(&envline[10]));
if (! secure)
{
if (memcmp (envline, "MMAP_MAX_", 9) == 0)
mALLOPt(M_MMAP_MAX, atoi(&envline[10]));
#ifdef PER_THREAD
else if (memcmp (envline, "ARENA_MAX", 9) == 0)
mALLOPt(M_ARENA_MAX, atoi(&envline[10]));
#endif
}
break;
#ifdef PER_THREAD
case 10:
if (! secure)
{
if (memcmp (envline, "ARENA_TEST", 10) == 0)
mALLOPt(M_ARENA_TEST, atoi(&envline[11]));
}
break;
#endif
case 15:
if (! secure)
{
@ -875,9 +932,110 @@ _int_new_arena(size_t size)
top(a) = (mchunkptr)ptr;
set_head(top(a), (((char*)h + h->size) - ptr) | PREV_INUSE);
tsd_setspecific(arena_key, (Void_t *)a);
mutex_init(&a->mutex);
(void)mutex_lock(&a->mutex);
#ifdef PER_THREAD
(void)mutex_lock(&list_lock);
#endif
/* Add the new arena to the global list. */
a->next = main_arena.next;
atomic_write_barrier ();
main_arena.next = a;
#ifdef PER_THREAD
++narenas;
(void)mutex_unlock(&list_lock);
#endif
THREAD_STAT(++(a->stat_lock_loop));
return a;
}
#ifdef PER_THREAD
static mstate
get_free_list (void)
{
mstate result = free_list;
if (result != NULL)
{
(void)mutex_lock(&list_lock);
result = free_list;
if (result != NULL)
free_list = result->next_free;
(void)mutex_unlock(&list_lock);
if (result != NULL)
{
(void)mutex_lock(&result->mutex);
tsd_setspecific(arena_key, (Void_t *)result);
THREAD_STAT(++(result->stat_lock_loop));
}
}
return result;
}
static mstate
reused_arena (void)
{
if (narenas <= mp_.arena_test)
return NULL;
static int narenas_limit;
if (narenas_limit == 0)
{
if (mp_.arena_max != 0)
narenas_limit = mp_.arena_max;
else
{
int n = __get_nprocs ();
if (n >= 1)
narenas_limit = NARENAS_FROM_NCORES (n);
else
/* We have no information about the system. Assume two
cores. */
narenas_limit = NARENAS_FROM_NCORES (2);
}
}
if (narenas < narenas_limit)
return NULL;
mstate result;
static mstate next_to_use;
if (next_to_use == NULL)
next_to_use = &main_arena;
result = next_to_use;
do
{
if (!mutex_trylock(&result->mutex))
goto out;
result = result->next;
}
while (result != next_to_use);
/* No arena available. Wait for the next in line. */
(void)mutex_lock(&result->mutex);
out:
tsd_setspecific(arena_key, (Void_t *)result);
THREAD_STAT(++(result->stat_lock_loop));
next_to_use = result->next;
return result;
}
#endif
static mstate
internal_function
#if __STD_C
@ -888,6 +1046,12 @@ arena_get2(a_tsd, size) mstate a_tsd; size_t size;
{
mstate a;
#ifdef PER_THREAD
if ((a = get_free_list ()) == NULL
&& (a = reused_arena ()) == NULL)
/* Nothing immediately available, so generate a new arena. */
a = _int_new_arena(size);
#else
if(!a_tsd)
a = a_tsd = &main_arena;
else {
@ -930,24 +1094,31 @@ arena_get2(a_tsd, size) mstate a_tsd; size_t size;
/* Nothing immediately available, so generate a new arena. */
a = _int_new_arena(size);
if(a)
{
tsd_setspecific(arena_key, (Void_t *)a);
mutex_init(&a->mutex);
mutex_lock(&a->mutex); /* remember result */
/* Add the new arena to the global list. */
a->next = main_arena.next;
atomic_write_barrier ();
main_arena.next = a;
THREAD_STAT(++(a->stat_lock_loop));
}
(void)mutex_unlock(&list_lock);
#endif
return a;
}
#ifdef PER_THREAD
static void __attribute__ ((section ("__libc_thread_freeres_fn")))
arena_thread_freeres (void)
{
Void_t *vptr = NULL;
mstate a = tsd_getspecific(arena_key, vptr);
tsd_setspecific(arena_key, NULL);
if (a != NULL)
{
(void)mutex_lock(&list_lock);
a->next_free = free_list;
free_list = a;
(void)mutex_unlock(&list_lock);
}
}
text_set_element (__libc_thread_subfreeres, arena_thread_freeres);
#endif
#endif /* USE_ARENAS */
/*

15
malloc/hooks.c
View File

@ -275,17 +275,13 @@ free_check(mem, caller) Void_t* mem; const Void_t *caller;
mchunkptr p;
if(!mem) return;
(void)mutex_lock(&main_arena.mutex);
p = mem2chunk_check(mem, NULL);
if(!p) {
(void)mutex_unlock(&main_arena.mutex);
malloc_printerr(check_action, "free(): invalid pointer", mem);
return;
}
#if HAVE_MMAP
if (chunk_is_mmapped(p)) {
(void)mutex_unlock(&main_arena.mutex);
munmap_chunk(p);
return;
}
@ -293,8 +289,13 @@ free_check(mem, caller) Void_t* mem; const Void_t *caller;
#if 0 /* Erase freed memory. */
memset(mem, 0, chunksize(p) - (SIZE_SZ+1));
#endif
#ifdef ATOMIC_FASTBINS
_int_free(&main_arena, p, 0);
#else
(void)mutex_lock(&main_arena.mutex);
_int_free(&main_arena, p);
(void)mutex_unlock(&main_arena.mutex);
#endif
}
static Void_t*
@ -472,7 +473,11 @@ free_starter(mem, caller) Void_t* mem; const Void_t *caller;
return;
}
#endif
#ifdef ATOMIC_FASTBINS
_int_free(&main_arena, p, 1);
#else
_int_free(&main_arena, p);
#endif
}
# endif /* !defiend NO_STARTER */
@ -584,7 +589,7 @@ public_sET_STATe(Void_t* msptr)
clear_fastchunks(&main_arena);
set_max_fast(DEFAULT_MXFAST);
for (i=0; i<NFASTBINS; ++i)
main_arena.fastbins[i] = 0;
fastbin (&main_arena, i) = 0;
for (i=0; i<BINMAPSIZE; ++i)
main_arena.binmap[i] = 0;
top(&main_arena) = ms->av[2];

260
malloc/malloc.c
View File

@ -208,7 +208,7 @@
Tuning options that are also dynamically changeable via mallopt:
DEFAULT_MXFAST 64
DEFAULT_MXFAST 64 (for 32bit), 128 (for 64bit)
DEFAULT_TRIM_THRESHOLD 128 * 1024
DEFAULT_TOP_PAD 0
DEFAULT_MMAP_THRESHOLD 128 * 1024
@ -254,8 +254,12 @@
#include <malloc-machine.h>
#ifdef _LIBC
#ifdef ATOMIC_FASTBINS
#include <atomic.h>
#endif
#include <stdio-common/_itoa.h>
#include <bits/wordsize.h>
#include <sys/sysinfo.h>
#endif
#ifdef __cplusplus
@ -321,12 +325,7 @@ extern "C" {
or other mallocs available that do this.
*/
#if MALLOC_DEBUG
#include <assert.h>
#else
#undef assert
#define assert(x) ((void)0)
#endif
/*
@ -1308,7 +1307,7 @@ int __posix_memalign(void **, size_t, size_t);
#endif
#ifndef DEFAULT_MXFAST
#define DEFAULT_MXFAST 64
#define DEFAULT_MXFAST (64 * SIZE_SZ / 4)
#endif
@ -1582,7 +1581,11 @@ typedef struct malloc_chunk* mchunkptr;
#if __STD_C
static Void_t* _int_malloc(mstate, size_t);
#ifdef ATOMIC_FASTBINS
static void _int_free(mstate, mchunkptr, int);
#else
static void _int_free(mstate, mchunkptr);
#endif
static Void_t* _int_realloc(mstate, mchunkptr, INTERNAL_SIZE_T);
static Void_t* _int_memalign(mstate, size_t, size_t);
static Void_t* _int_valloc(mstate, size_t);
@ -2239,12 +2242,15 @@ typedef struct malloc_chunk* mbinptr;
*/
typedef struct malloc_chunk* mfastbinptr;
#define fastbin(ar_ptr, idx) ((ar_ptr)->fastbinsY[idx])
/* offset 2 to use otherwise unindexable first 2 bins */
#define fastbin_index(sz) ((((unsigned int)(sz)) >> 3) - 2)
#define fastbin_index(sz) \
((((unsigned int)(sz)) >> (SIZE_SZ == 8 ? 4 : 3)) - 2)
/* The maximum fastbin request size we support */
#define MAX_FAST_SIZE 80
#define MAX_FAST_SIZE (80 * SIZE_SZ / 4)
#define NFASTBINS (fastbin_index(request2size(MAX_FAST_SIZE))+1)
@ -2279,8 +2285,13 @@ typedef struct malloc_chunk* mfastbinptr;
#define FASTCHUNKS_BIT (1U)
#define have_fastchunks(M) (((M)->flags & FASTCHUNKS_BIT) == 0)
#ifdef ATOMIC_FASTBINS
#define clear_fastchunks(M) catomic_or (&(M)->flags, FASTCHUNKS_BIT)
#define set_fastchunks(M) catomic_and (&(M)->flags, ~FASTCHUNKS_BIT)
#else
#define clear_fastchunks(M) ((M)->flags |= FASTCHUNKS_BIT)
#define set_fastchunks(M) ((M)->flags &= ~FASTCHUNKS_BIT)
#endif
/*
NONCONTIGUOUS_BIT indicates that MORECORE does not return contiguous
@ -2327,7 +2338,7 @@ struct malloc_state {
#endif
/* Fastbins */
mfastbinptr fastbins[NFASTBINS];
mfastbinptr fastbinsY[NFASTBINS];
/* Base of the topmost chunk -- not otherwise kept in a bin */
mchunkptr top;
@ -2344,6 +2355,11 @@ struct malloc_state {
/* Linked list */
struct malloc_state *next;
#ifdef PER_THREAD
/* Linked list for free arenas. */
struct malloc_state *next_free;
#endif
/* Memory allocated from the system in this arena. */
INTERNAL_SIZE_T system_mem;
INTERNAL_SIZE_T max_system_mem;
@ -2354,6 +2370,10 @@ struct malloc_par {
unsigned long trim_threshold;
INTERNAL_SIZE_T top_pad;
INTERNAL_SIZE_T mmap_threshold;
#ifdef PER_THREAD
INTERNAL_SIZE_T arena_test;
INTERNAL_SIZE_T arena_max;
#endif
/* Memory map support */
int n_mmaps;
@ -2391,6 +2411,13 @@ static struct malloc_state main_arena;
static struct malloc_par mp_;
#ifdef PER_THREAD
/* Non public mallopt parameters. */
#define M_ARENA_TEST -7
#define M_ARENA_MAX -8
#endif
/* Maximum size of memory handled in fastbins. */
static INTERNAL_SIZE_T global_max_fast;
@ -3037,8 +3064,10 @@ static Void_t* sYSMALLOc(nb, av) INTERNAL_SIZE_T nb; mstate av;
/* Precondition: not enough current space to satisfy nb request */
assert((unsigned long)(old_size) < (unsigned long)(nb + MINSIZE));
#ifndef ATOMIC_FASTBINS
/* Precondition: all fastbins are consolidated */
assert(!have_fastchunks(av));
#endif
if (av != &main_arena) {
@ -3084,7 +3113,11 @@ static Void_t* sYSMALLOc(nb, av) INTERNAL_SIZE_T nb; mstate av;
set_head(chunk_at_offset(old_top, old_size), (2*SIZE_SZ)|PREV_INUSE);
set_foot(chunk_at_offset(old_top, old_size), (2*SIZE_SZ));
set_head(old_top, old_size|PREV_INUSE|NON_MAIN_ARENA);
#ifdef ATOMIC_FASTBINS
_int_free(av, old_top, 1);
#else
_int_free(av, old_top);
#endif
} else {
set_head(old_top, (old_size + 2*SIZE_SZ)|PREV_INUSE);
set_foot(old_top, (old_size + 2*SIZE_SZ));
@ -3323,7 +3356,11 @@ static Void_t* sYSMALLOc(nb, av) INTERNAL_SIZE_T nb; mstate av;
/* If possible, release the rest. */
if (old_size >= MINSIZE) {
#ifdef ATOMIC_FASTBINS
_int_free(av, old_top, 1);
#else
_int_free(av, old_top);
#endif
}
}
@ -3545,7 +3582,40 @@ public_mALLOc(size_t bytes)
if (__builtin_expect (hook != NULL, 0))
return (*hook)(bytes, RETURN_ADDRESS (0));
arena_get(ar_ptr, bytes);
arena_lookup(ar_ptr);
#if 0
// XXX We need double-word CAS and fastbins must be extended to also
// XXX hold a generation counter for each entry.
if (ar_ptr) {
INTERNAL_SIZE_T nb; /* normalized request size */
checked_request2size(bytes, nb);
if (nb <= get_max_fast ()) {
long int idx = fastbin_index(nb);
mfastbinptr* fb = &fastbin (ar_ptr, idx);
mchunkptr pp = *fb;
mchunkptr v;
do
{
v = pp;
if (v == NULL)
break;
}
while ((pp = catomic_compare_and_exchange_val_acq (fb, v->fd, v)) != v);
if (v != 0) {
if (__builtin_expect (fastbin_index (chunksize (v)) != idx, 0))
malloc_printerr (check_action, "malloc(): memory corruption (fast)",
chunk2mem (v));
check_remalloced_chunk(ar_ptr, v, nb);
void *p = chunk2mem(v);
if (__builtin_expect (perturb_byte, 0))
alloc_perturb (p, bytes);
return p;
}
}
}
#endif
arena_lock(ar_ptr, bytes);
if(!ar_ptr)
return 0;
victim = _int_malloc(ar_ptr, bytes);
@ -3612,18 +3682,22 @@ public_fREe(Void_t* mem)
#endif
ar_ptr = arena_for_chunk(p);
#if THREAD_STATS
#ifdef ATOMIC_FASTBINS
_int_free(ar_ptr, p, 0);
#else
# if THREAD_STATS
if(!mutex_trylock(&ar_ptr->mutex))
++(ar_ptr->stat_lock_direct);
else {
(void)mutex_lock(&ar_ptr->mutex);
++(ar_ptr->stat_lock_wait);
}
#else
# else
(void)mutex_lock(&ar_ptr->mutex);
#endif
# endif
_int_free(ar_ptr, p);
(void)mutex_unlock(&ar_ptr->mutex);
#endif
}
#ifdef libc_hidden_def
libc_hidden_def (public_fREe)
@ -3699,7 +3773,7 @@ public_rEALLOc(Void_t* oldmem, size_t bytes)
(void)mutex_lock(&ar_ptr->mutex);
#endif
#ifndef NO_THREADS
#if !defined NO_THREADS && !defined PER_THREAD
/* As in malloc(), remember this arena for the next allocation. */
tsd_setspecific(arena_key, (Void_t *)ar_ptr);
#endif
@ -3717,18 +3791,22 @@ public_rEALLOc(Void_t* oldmem, size_t bytes)
if (newp != NULL)
{
MALLOC_COPY (newp, oldmem, oldsize - SIZE_SZ);
#if THREAD_STATS
#ifdef ATOMIC_FASTBINS
_int_free(ar_ptr, oldp, 0);
#else
# if THREAD_STATS
if(!mutex_trylock(&ar_ptr->mutex))
++(ar_ptr->stat_lock_direct);
else {
(void)mutex_lock(&ar_ptr->mutex);
++(ar_ptr->stat_lock_wait);
}
#else
# else
(void)mutex_lock(&ar_ptr->mutex);
#endif
# endif
_int_free(ar_ptr, oldp);
(void)mutex_unlock(&ar_ptr->mutex);
#endif
}
}
@ -4130,7 +4208,6 @@ _int_malloc(mstate av, size_t bytes)
INTERNAL_SIZE_T nb; /* normalized request size */
unsigned int idx; /* associated bin index */
mbinptr bin; /* associated bin */
mfastbinptr* fb; /* associated fastbin */
mchunkptr victim; /* inspected/selected chunk */
INTERNAL_SIZE_T size; /* its size */
@ -4164,13 +4241,28 @@ _int_malloc(mstate av, size_t bytes)
*/
if ((unsigned long)(nb) <= (unsigned long)(get_max_fast ())) {
long int idx = fastbin_index(nb);
fb = &(av->fastbins[idx]);
if ( (victim = *fb) != 0) {
idx = fastbin_index(nb);
mfastbinptr* fb = &fastbin (av, idx);
#ifdef ATOMIC_FASTBINS
mchunkptr pp = *fb;
do
{
victim = pp;
if (victim == NULL)
break;
}
while ((pp = catomic_compare_and_exchange_val_acq (fb, victim->fd, victim))
!= victim);
#else
victim = *fb;
#endif
if (victim != 0) {
if (__builtin_expect (fastbin_index (chunksize (victim)) != idx, 0))
malloc_printerr (check_action, "malloc(): memory corruption (fast)",
chunk2mem (victim));
#ifndef ATOMIC_FASTBINS
*fb = victim->fd;
#endif
check_remalloced_chunk(av, victim, nb);
void *p = chunk2mem(victim);
if (__builtin_expect (perturb_byte, 0))
@ -4560,6 +4652,18 @@ _int_malloc(mstate av, size_t bytes)
return p;
}
#ifdef ATOMIC_FASTBINS
/* When we are using atomic ops to free fast chunks we can get
here for all block sizes. */
else if (have_fastchunks(av)) {
malloc_consolidate(av);
/* restore original bin index */
if (in_smallbin_range(nb))
idx = smallbin_index(nb);
else
idx = largebin_index(nb);
}
#else
/*
If there is space available in fastbins, consolidate and retry,
to possibly avoid expanding memory. This can occur only if nb is
@ -4571,6 +4675,7 @@ _int_malloc(mstate av, size_t bytes)
malloc_consolidate(av);
idx = smallbin_index(nb); /* restore original bin index */
}
#endif
/*
Otherwise, relay to handle system-dependent cases
@ -4589,7 +4694,11 @@ _int_malloc(mstate av, size_t bytes)
*/
static void
#ifdef ATOMIC_FASTBINS
_int_free(mstate av, mchunkptr p, int have_lock)
#else
_int_free(mstate av, mchunkptr p)
#endif
{
INTERNAL_SIZE_T size; /* its size */
mfastbinptr* fb; /* associated fastbin */
@ -4601,6 +4710,9 @@ _int_free(mstate av, mchunkptr p)
mchunkptr fwd; /* misc temp for linking */
const char *errstr = NULL;
#ifdef ATOMIC_FASTBINS
int locked = 0;
#endif
size = chunksize(p);
@ -4613,6 +4725,10 @@ _int_free(mstate av, mchunkptr p)
{
errstr = "free(): invalid pointer";
errout:
#ifdef ATOMIC_FASTBINS
if (! have_lock && locked)
(void)mutex_unlock(&av->mutex);
#endif
malloc_printerr (check_action, errstr, chunk2mem(p));
return;
}
@ -4649,8 +4765,28 @@ _int_free(mstate av, mchunkptr p)
goto errout;
}
if (__builtin_expect (perturb_byte, 0))
free_perturb (chunk2mem(p), size - SIZE_SZ);
set_fastchunks(av);
fb = &(av->fastbins[fastbin_index(size)]);
fb = &fastbin (av, fastbin_index(size));
#ifdef ATOMIC_FASTBINS
mchunkptr fd;
mchunkptr old = *fb;
do
{
/* Another simple check: make sure the top of the bin is not the
record we are going to add (i.e., double free). */
if (__builtin_expect (old == p, 0))
{
errstr = "double free or corruption (fasttop)";
goto errout;
}
p->fd = fd = old;
}
while ((old = catomic_compare_and_exchange_val_acq (fb, p, fd)) != fd);
#else
/* Another simple check: make sure the top of the bin is not the
record we are going to add (i.e., double free). */
if (__builtin_expect (*fb == p, 0))
@ -4659,11 +4795,9 @@ _int_free(mstate av, mchunkptr p)
goto errout;
}
if (__builtin_expect (perturb_byte, 0))
free_perturb (chunk2mem(p), size - SIZE_SZ);
p->fd = *fb;
*fb = p;
#endif
}
/*
@ -4671,6 +4805,22 @@ _int_free(mstate av, mchunkptr p)
*/
else if (!chunk_is_mmapped(p)) {
#ifdef ATOMIC_FASTBINS
if (! have_lock) {
# if THREAD_STATS
if(!mutex_trylock(&av->mutex))
++(av->stat_lock_direct);
else {
(void)mutex_lock(&av->mutex);
++(av->stat_lock_wait);
}
# else
(void)mutex_lock(&av->mutex);
# endif
locked = 1;
}
#endif
nextchunk = chunk_at_offset(p, size);
/* Lightweight tests: check whether the block is already the
@ -4794,6 +4944,12 @@ _int_free(mstate av, mchunkptr p)
}
}
#ifdef ATOMIC_FASTBINS
if (! have_lock) {
assert (locked);
(void)mutex_unlock(&av->mutex);
}
#endif
}
/*
If the chunk was allocated via mmap, release via munmap(). Note
@ -4869,15 +5025,21 @@ static void malloc_consolidate(av) mstate av;
because, except for the main arena, all the others might have
blocks in the high fast bins. It's not worth it anyway, just
search all bins all the time. */
maxfb = &(av->fastbins[fastbin_index(get_max_fast ())]);
maxfb = &fastbin (av, fastbin_index(get_max_fast ()));
#else
maxfb = &(av->fastbins[NFASTBINS - 1]);
maxfb = &fastbin (av, NFASTBINS - 1);
#endif
fb = &(av->fastbins[0]);
fb = &fastbin (av, 0);
do {
if ( (p = *fb) != 0) {
*fb = 0;
#ifdef ATOMIC_FASTBINS
p = atomic_exchange_acq (fb, 0);
#else
p = *fb;
#endif
if (p != 0) {
#ifndef ATOMIC_FASTBINS
*fb = 0;
#endif
do {
check_inuse_chunk(av, p);
nextp = p->fd;
@ -5070,7 +5232,11 @@ _int_realloc(mstate av, mchunkptr oldp, INTERNAL_SIZE_T nb)
}
}
#ifdef ATOMIC_FASTBINS
_int_free(av, oldp, 1);
#else
_int_free(av, oldp);
#endif
check_inuse_chunk(av, newp);
return chunk2mem(newp);
}
@ -5094,7 +5260,11 @@ _int_realloc(mstate av, mchunkptr oldp, INTERNAL_SIZE_T nb)
(av != &main_arena ? NON_MAIN_ARENA : 0));
/* Mark remainder as inuse so free() won't complain */
set_inuse_bit_at_offset(remainder, remainder_size);
#ifdef ATOMIC_FASTBINS
_int_free(av, remainder, 1);
#else
_int_free(av, remainder);
#endif
}
check_inuse_chunk(av, newp);
@ -5153,7 +5323,11 @@ _int_realloc(mstate av, mchunkptr oldp, INTERNAL_SIZE_T nb)
newmem = _int_malloc(av, nb - MALLOC_ALIGN_MASK);
if (newmem != 0) {
MALLOC_COPY(newmem, chunk2mem(oldp), oldsize - 2*SIZE_SZ);
#ifdef ATOMIC_FASTBINS
_int_free(av, oldp, 1);
#else
_int_free(av, oldp);
#endif
}
}
return newmem;
@ -5247,7 +5421,11 @@ _int_memalign(mstate av, size_t alignment, size_t bytes)
(av != &main_arena ? NON_MAIN_ARENA : 0));
set_inuse_bit_at_offset(newp, newsize);
set_head_size(p, leadsize | (av != &main_arena ? NON_MAIN_ARENA : 0));
#ifdef ATOMIC_FASTBINS
_int_free(av, p, 1);
#else
_int_free(av, p);
#endif
p = newp;
assert (newsize >= nb &&
@ -5263,7 +5441,11 @@ _int_memalign(mstate av, size_t alignment, size_t bytes)
set_head(remainder, remainder_size | PREV_INUSE |
(av != &main_arena ? NON_MAIN_ARENA : 0));
set_head_size(p, nb);
#ifdef ATOMIC_FASTBINS
_int_free(av, remainder, 1);
#else
_int_free(av, remainder);
#endif
}
}
@ -5650,7 +5832,7 @@ struct mallinfo mALLINFo(mstate av)
fastavail = 0;
for (i = 0; i < NFASTBINS; ++i) {
for (p = av->fastbins[i]; p != 0; p = p->fd) {
for (p = fastbin (av, i); p != 0; p = p->fd) {
++nfastblocks;
fastavail += chunksize(p);
}
@ -5818,6 +6000,18 @@ int mALLOPt(param_number, value) int param_number; int value;
case M_PERTURB:
perturb_byte = value;
break;
#ifdef PER_THREAD
case M_ARENA_TEST:
if (value > 0)
mp_.arena_test = value;
break;
case M_ARENA_MAX:
if (value > 0)
mp_.arena_max = value;
break;
#endif
}
(void)mutex_unlock(&av->mutex);
return res;

4
malloc/malloc.h
View File

@ -1,5 +1,5 @@
/* Prototypes and definition for malloc implementation.
Copyright (C) 1996, 1997, 1999, 2000, 2002-2004, 2005, 2007
Copyright (C) 1996, 1997, 1999, 2000, 2002-2004, 2005, 2007, 2009
Free Software Foundation, Inc.
This file is part of the GNU C Library.
@ -127,6 +127,8 @@ extern struct mallinfo mallinfo __MALLOC_P ((void));
#define M_MMAP_MAX -4
#define M_CHECK_ACTION -5
#define M_PERTURB -6
#define M_ARENA_TEST -7
#define M_ARENA_MAX -8
/* General SVID/XPG interface to tunable parameters. */
extern int mallopt __MALLOC_P ((int __param, int __val));

32
sysdeps/x86_64/cacheinfo.c
View File

@ -23,6 +23,10 @@
#include <stdlib.h>
#include <unistd.h>
#ifdef USE_MULTIARCH
# include "multiarch/init-arch.h"
#endif
static const struct intel_02_cache_info
{
unsigned int idx;
@ -443,19 +447,32 @@ init_cacheinfo (void)
unsigned int ebx;
unsigned int ecx;
unsigned int edx;
int max_cpuid;
int max_cpuid_ex;
long int data = -1;
long int shared = -1;
unsigned int level;
unsigned int threads = 0;
#ifdef USE_MULTIARCH
if (__cpu_features.kind == arch_kind_unknown)
__init_cpu_features ();
# define is_intel __cpu_features.kind == arch_kind_intel
# define is_amd __cpu_features.kind == arch_kind_amd
# define max_cpuid __cpu_features.max_cpuid
#else
int max_cpuid;
asm volatile ("cpuid"
: "=a" (max_cpuid), "=b" (ebx), "=c" (ecx), "=d" (edx)
: "0" (0));
/* This spells out "GenuineIntel". */
if (ebx == 0x756e6547 && ecx == 0x6c65746e && edx == 0x49656e69)
# define is_intel \
ebx == 0x756e6547 && ecx == 0x6c65746e && edx == 0x49656e69
/* This spells out "AuthenticAMD". */
# define is_amd \
ebx == 0x68747541 && ecx == 0x444d4163 && edx == 0x69746e65
#endif
if (is_intel)
{
data = handle_intel (_SC_LEVEL1_DCACHE_SIZE, max_cpuid);
@ -470,9 +487,16 @@ init_cacheinfo (void)
shared = handle_intel (_SC_LEVEL2_CACHE_SIZE, max_cpuid);
}
#ifdef USE_MULTIARCH
eax = __cpu_features.cpuid[INTEL_CPUID_INDEX_1].eax;
ebx = __cpu_features.cpuid[INTEL_CPUID_INDEX_1].ebx;
ecx = __cpu_features.cpuid[INTEL_CPUID_INDEX_1].ecx;
edx = __cpu_features.cpuid[INTEL_CPUID_INDEX_1].edx;
#else
asm volatile ("cpuid"
: "=a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx)
: "0" (1));
#endif
/* Intel prefers SSSE3 instructions for memory/string routines
if they are avaiable. */
@ -519,7 +543,7 @@ init_cacheinfo (void)
shared /= threads;
}
/* This spells out "AuthenticAMD". */
else if (ebx == 0x68747541 && ecx == 0x444d4163 && edx == 0x69746e65)
else if (is_amd)
{
data = handle_amd (_SC_LEVEL1_DCACHE_SIZE);
long int core = handle_amd (_SC_LEVEL2_CACHE_SIZE);

3
sysdeps/x86_64/multiarch/Makefile Normal file
View File

@ -0,0 +1,3 @@
ifeq ($(subdir),csu)
aux += init-arch
endif

65
sysdeps/x86_64/multiarch/init-arch.c Normal file
View File

@ -0,0 +1,65 @@
/* Initialize CPU feature data.
This file is part of the GNU C Library.
Copyright (C) 2008 Free Software Foundation, Inc.
Contributed by Ulrich Drepper <drepper@redhat.com>.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, write to the Free
Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
02111-1307 USA. */
#include "init-arch.h"
struct cpu_features __cpu_features attribute_hidden;
void
__init_cpu_features (void)
{
unsigned int ebx;
unsigned int ecx;
unsigned int edx;
asm volatile ("cpuid"
: "=a" (__cpu_features.max_cpuid), "=b" (ebx), "=c" (ecx),
"=d" (edx)
: "0" (0));
/* This spells out "GenuineIntel". */
if (ebx == 0x756e6547 && ecx == 0x6c65746e && edx == 0x49656e69)
{
__cpu_features.kind = arch_kind_intel;
asm volatile ("cpuid"
: "=a" (__cpu_features.cpuid[INTEL_CPUID_INDEX_1].eax),
"=b" (__cpu_features.cpuid[INTEL_CPUID_INDEX_1].ebx),
"=c" (__cpu_features.cpuid[INTEL_CPUID_INDEX_1].ecx),
"=d" (__cpu_features.cpuid[INTEL_CPUID_INDEX_1].edx)
: "0" (1));
}
/* This spells out "AuthenticAMD". */
else if (ebx == 0x68747541 && ecx == 0x444d4163 && edx == 0x69746e65)
{
__cpu_features.kind = arch_kind_amd;
asm volatile ("cpuid"
: "=a" (__cpu_features.cpuid[AMD_CPUID_INDEX_1].eax),
"=b" (__cpu_features.cpuid[AMD_CPUID_INDEX_1].ebx),
"=c" (__cpu_features.cpuid[AMD_CPUID_INDEX_1].ecx),
"=d" (__cpu_features.cpuid[AMD_CPUID_INDEX_1].edx)
: "0" (1));
}
else
__cpu_features.kind = arch_kind_other;
}

70
sysdeps/x86_64/multiarch/init-arch.h Normal file
View File

@ -0,0 +1,70 @@
/* This file is part of the GNU C Library.
Copyright (C) 2008 Free Software Foundation, Inc.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, write to the Free
Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
02111-1307 USA. */
#include <sys/param.h>
enum
{
INTEL_CPUID_INDEX_1 = 0,
/* Keep the following line at the end. */
INTEL_CPUID_INDEX_MAX
};
enum
{
AMD_CPUID_INDEX_1 = 0,
/* Keep the following line at the end. */
AMD_CPUID_INDEX_MAX
};
extern struct cpu_features
{
enum
{
arch_kind_unknown = 0,
arch_kind_intel,
arch_kind_amd,
arch_kind_other
} kind;
int max_cpuid;
struct
{
unsigned int eax;
unsigned int ebx;
unsigned int ecx;
unsigned int edx;
} cpuid[MAX (INTEL_CPUID_INDEX_MAX, AMD_CPUID_INDEX_MAX)];
} __cpu_features attribute_hidden;
extern void __init_cpu_features (void) attribute_hidden;
#define INIT_ARCH()\
do \
if (__cpu_features.kind == arch_kind_unknown) \
__init_cpu_features (); \
while (0)
/* Following are the feature tests used throughout libc. */
#define INTEL_HAS_POPCOUNT \
(__cpu_features.kind == arch_kind_intel \
&& (__cpu_features.cpuid[INTEL_CPUID_INDEX_1].ecx & (1 << 23)) != 0)
#define AMD_HAS_POPCOUNT \
(__cpu_features.kind == arch_kind_amd \
&& (__cpu_features.cpuid[AMD_CPUID_INDEX_1].ecx & (1 << 23)) != 0)

42
sysdeps/x86_64/multiarch/sched_cpucount.c Normal file
View File

@ -0,0 +1,42 @@
/* Count bits in CPU set. x86-64 multi-arch version.
This file is part of the GNU C Library.
Copyright (C) 2008 Free Software Foundation, Inc.
Contributed by Ulrich Drepper <drepper@redhat.com>.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, write to the Free
Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
02111-1307 USA. */
#ifdef SHARED
# include <sched.h>
# include "init-arch.h"
# define __sched_cpucount static generic_cpucount
# include <posix/sched_cpucount.c>
# undef __sched_cpucount
# define POPCNT(l) \
({ __cpu_mask r; \
asm ("popcntq %1, %0" : "=r" (r) : "0" (l));\
r; })
# define __sched_cpucount static popcount_cpucount
# include <posix/sched_cpucount.c>
# undef __sched_cpucount
libc_ifunc (__sched_cpucount,
INTEL_HAS_POPCOUNT || AMD_HAS_POPCOUNT
? popcount_cpucount : generic_cpucount);
#else
# include_next <sched_cpucount.c>
#endif