OpenE2K/glibc
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

x86-64: Implement memmove family IFUNC selectors in C

Browse Source 
Implement memmove family IFUNC selectors in C.

All internal calls within libc.so can use IFUNC on x86-64 since unlike
x86, x86-64 supports PC-relative addressing to access the GOT entry so
that it can call via PLT without using an extra register.  For libc.a,
we can't use IFUNC for functions which are called before IFUNC has been
initialized.  Use IFUNC internally reduces the icache footprint since
libc.so and other codes in the process use the same implementations.
This patch uses IFUNC for memmove family functions within libc.

	* sysdeps/x86_64/multiarch/Makefile (sysdep_routines): Add
	memmove-sse2-unaligned-erms, memcpy_chk-nonshared,
	mempcpy_chk-nonshared and memmove_chk-nonshared.
	* sysdeps/x86_64/multiarch/ifunc-impl-list.c
	(__libc_ifunc_impl_list): Add tests for __memmove_chk_erms,
	__memcpy_chk_erms and __mempcpy_chk_erms.  Update comments.
	* sysdeps/x86_64/multiarch/ifunc-memmove.h: New file.
	* sysdeps/x86_64/multiarch/memcpy.c: Likewise.
	* sysdeps/x86_64/multiarch/memcpy_chk-nonshared.S: Likewise.
	* sysdeps/x86_64/multiarch/memcpy_chk.c: Likewise.
	* sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S: Likewise.
	* sysdeps/x86_64/multiarch/memmove.c: Likewise.
	* sysdeps/x86_64/multiarch/memmove_chk-nonshared.S: Likewise.
	* sysdeps/x86_64/multiarch/memmove_chk.c: Likewise.
	* sysdeps/x86_64/multiarch/mempcpy.c: Likewise.
	* sysdeps/x86_64/multiarch/mempcpy_chk-nonshared.S: Likewise.
	* sysdeps/x86_64/multiarch/mempcpy_chk.c: Likewise.
	* sysdeps/x86_64/multiarch/memcpy.S: Removed.
	* sysdeps/x86_64/multiarch/memcpy_chk.S: Likewise.
	* sysdeps/x86_64/multiarch/memmove.S: Likewise.
	* sysdeps/x86_64/multiarch/memmove_chk.S: Likewise.
	* sysdeps/x86_64/multiarch/mempcpy.S: Likewise.
	* sysdeps/x86_64/multiarch/mempcpy_chk.S: Likewise.
	* sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
	(__mempcpy_chk_erms): New function.
	(__memmove_chk_erms): Likewise.
	(__memcpy_chk_erms): New alias.
This commit is contained in:
H.J. Lu 2017-06-14 12:10:57 -07:00
parent 69052a3a95
commit 5c3e322d3b
21 changed files with 450 additions and 474 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
ChangeLog
View File

@ -1,3 +1,35 @@
2017-06-14 H.J. Lu <hongjiu.lu@intel.com>
Erich Elsen <eriche@google.com>
Adhemerval Zanella <adhemerval.zanella@linaro.org>
* sysdeps/x86_64/multiarch/Makefile (sysdep_routines): Add
memmove-sse2-unaligned-erms, memcpy_chk-nonshared,
mempcpy_chk-nonshared and memmove_chk-nonshared.
* sysdeps/x86_64/multiarch/ifunc-impl-list.c
(__libc_ifunc_impl_list): Add tests for __memmove_chk_erms,
__memcpy_chk_erms and __mempcpy_chk_erms. Update comments.
* sysdeps/x86_64/multiarch/ifunc-memmove.h: New file.
* sysdeps/x86_64/multiarch/memcpy.c: Likewise.
* sysdeps/x86_64/multiarch/memcpy_chk-nonshared.S: Likewise.
* sysdeps/x86_64/multiarch/memcpy_chk.c: Likewise.
* sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S: Likewise.
* sysdeps/x86_64/multiarch/memmove.c: Likewise.
* sysdeps/x86_64/multiarch/memmove_chk-nonshared.S: Likewise.
* sysdeps/x86_64/multiarch/memmove_chk.c: Likewise.
* sysdeps/x86_64/multiarch/mempcpy.c: Likewise.
* sysdeps/x86_64/multiarch/mempcpy_chk-nonshared.S: Likewise.
* sysdeps/x86_64/multiarch/mempcpy_chk.c: Likewise.
* sysdeps/x86_64/multiarch/memcpy.S: Removed.
* sysdeps/x86_64/multiarch/memcpy_chk.S: Likewise.
* sysdeps/x86_64/multiarch/memmove.S: Likewise.
* sysdeps/x86_64/multiarch/memmove_chk.S: Likewise.
* sysdeps/x86_64/multiarch/mempcpy.S: Likewise.
* sysdeps/x86_64/multiarch/mempcpy_chk.S: Likewise.
* sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
(__mempcpy_chk_erms): New function.
(__memmove_chk_erms): Likewise.
(__memcpy_chk_erms): New alias.
2017-06-14 Florian Weimer <fweimer@redhat.com>
* sysdeps/i386/i686/multiarch/strcspn-c.c: Add IS_IN (libc) guard.

5
sysdeps/x86_64/multiarch/Makefile
View File

@ -26,6 +26,7 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c strcmp-ssse3 \
strchr-sse2-no-bsf memcmp-ssse3 strstr-sse2-unaligned \
strcspn-c strpbrk-c strspn-c varshift \
memset-avx512-no-vzeroupper \
memmove-sse2-unaligned-erms \
memmove-avx-unaligned-erms \
memmove-avx512-unaligned-erms \
memset-avx2-unaligned-erms \
@ -48,5 +49,7 @@ sysdep_routines += wmemcmp-sse4 wmemcmp-ssse3 wmemcmp-c \
endif
ifeq ($(subdir),debug)
sysdep_routines += wmemset_chk-nonshared
sysdep_routines += memcpy_chk-nonshared mempcpy_chk-nonshared \
memmove_chk-nonshared \
wmemset_chk-nonshared
endif

24
sysdeps/x86_64/multiarch/ifunc-impl-list.c
View File

@ -57,7 +57,7 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
__memcmp_ssse3)
IFUNC_IMPL_ADD (array, i, memcmp, 1, __memcmp_sse2))
/* Support sysdeps/x86_64/multiarch/memmove_chk.S. */
/* Support sysdeps/x86_64/multiarch/memmove_chk.c. */
IFUNC_IMPL (i, name, __memmove_chk,
IFUNC_IMPL_ADD (array, i, __memmove_chk,
HAS_ARCH_FEATURE (AVX512F_Usable),
@ -83,9 +83,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
IFUNC_IMPL_ADD (array, i, __memmove_chk, 1,
__memmove_chk_sse2_unaligned)
IFUNC_IMPL_ADD (array, i, __memmove_chk, 1,
__memmove_chk_sse2_unaligned_erms))
__memmove_chk_sse2_unaligned_erms)
IFUNC_IMPL_ADD (array, i, __memmove_chk, 1,
__memmove_chk_erms))
/* Support sysdeps/x86_64/multiarch/memmove.S. */
/* Support sysdeps/x86_64/multiarch/memmove.c. */
IFUNC_IMPL (i, name, memmove,
IFUNC_IMPL_ADD (array, i, memmove,
HAS_ARCH_FEATURE (AVX_Usable),
@ -410,7 +412,7 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
__wmemset_avx512_unaligned))
#ifdef SHARED
/* Support sysdeps/x86_64/multiarch/memcpy_chk.S. */
/* Support sysdeps/x86_64/multiarch/memcpy_chk.c. */
IFUNC_IMPL (i, name, __memcpy_chk,
IFUNC_IMPL_ADD (array, i, __memcpy_chk,
HAS_ARCH_FEATURE (AVX512F_Usable),
@ -436,9 +438,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
IFUNC_IMPL_ADD (array, i, __memcpy_chk, 1,
__memcpy_chk_sse2_unaligned)
IFUNC_IMPL_ADD (array, i, __memcpy_chk, 1,
__memcpy_chk_sse2_unaligned_erms))
__memcpy_chk_sse2_unaligned_erms)
IFUNC_IMPL_ADD (array, i, __memcpy_chk, 1,
__memcpy_chk_erms))
/* Support sysdeps/x86_64/multiarch/memcpy.S. */
/* Support sysdeps/x86_64/multiarch/memcpy.c. */
IFUNC_IMPL (i, name, memcpy,
IFUNC_IMPL_ADD (array, i, memcpy,
HAS_ARCH_FEATURE (AVX_Usable),
@ -464,7 +468,7 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
__memcpy_sse2_unaligned_erms)
IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_erms))
/* Support sysdeps/x86_64/multiarch/mempcpy_chk.S. */
/* Support sysdeps/x86_64/multiarch/mempcpy_chk.c. */
IFUNC_IMPL (i, name, __mempcpy_chk,
IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
HAS_ARCH_FEATURE (AVX512F_Usable),
@ -490,9 +494,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
IFUNC_IMPL_ADD (array, i, __mempcpy_chk, 1,
__mempcpy_chk_sse2_unaligned)
IFUNC_IMPL_ADD (array, i, __mempcpy_chk, 1,
__mempcpy_chk_sse2_unaligned_erms))
__mempcpy_chk_sse2_unaligned_erms)
IFUNC_IMPL_ADD (array, i, __mempcpy_chk, 1,
__mempcpy_chk_erms))
/* Support sysdeps/x86_64/multiarch/mempcpy.S. */
/* Support sysdeps/x86_64/multiarch/mempcpy.c. */
IFUNC_IMPL (i, name, mempcpy,
IFUNC_IMPL_ADD (array, i, mempcpy,
HAS_ARCH_FEATURE (AVX512F_Usable),

80
sysdeps/x86_64/multiarch/ifunc-memmove.h Normal file
View File

@ -0,0 +1,80 @@
/* Common definition for memcpy, mempcpy and memmove implementation.
All versions must be listed in ifunc-impl-list.c.
Copyright (C) 2017 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
#include <init-arch.h>
extern __typeof (REDIRECT_NAME) OPTIMIZE (erms) attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned)
attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned_erms)
attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3_back) attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned) attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_erms)
attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned)
attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned_erms)
attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_no_vzeroupper)
attribute_hidden;
static inline void *
IFUNC_SELECTOR (void)
{
const struct cpu_features* cpu_features = __get_cpu_features ();
if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_ERMS))
return OPTIMIZE (erms);
if (CPU_FEATURES_ARCH_P (cpu_features, AVX512F_Usable)
&& !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512))
{
if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER))
return OPTIMIZE (avx512_no_vzeroupper);
if (CPU_FEATURES_CPU_P (cpu_features, ERMS))
return OPTIMIZE (avx512_unaligned_erms);
return OPTIMIZE (avx512_unaligned);
}
if (CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
{
if (CPU_FEATURES_CPU_P (cpu_features, ERMS))
return OPTIMIZE (avx_unaligned_erms);
return OPTIMIZE (avx_unaligned);
}
if (!CPU_FEATURES_CPU_P (cpu_features, SSSE3)
|| CPU_FEATURES_ARCH_P (cpu_features, Fast_Unaligned_Copy))
{
if (CPU_FEATURES_CPU_P (cpu_features, ERMS))
return OPTIMIZE (sse2_unaligned_erms);
return OPTIMIZE (sse2_unaligned);
}
if (CPU_FEATURES_ARCH_P (cpu_features, Fast_Copy_Backward))
return OPTIMIZE (ssse3_back);
return OPTIMIZE (ssse3);
}

75
sysdeps/x86_64/multiarch/memcpy.S
View File

@ -1,75 +0,0 @@
/* Multiple versions of memcpy
All versions must be listed in ifunc-impl-list.c.
Copyright (C) 2010-2017 Free Software Foundation, Inc.
Contributed by Intel Corporation.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
#include <sysdep.h>
#include <init-arch.h>
/* Define multiple versions only for the definition in lib and for
DSO. In static binaries we need memcpy before the initialization
happened. */
#if defined SHARED && IS_IN (libc)
.text
ENTRY(__new_memcpy)
.type __new_memcpy, @gnu_indirect_function
LOAD_RTLD_GLOBAL_RO_RDX
lea __memcpy_erms(%rip), %RAX_LP
HAS_ARCH_FEATURE (Prefer_ERMS)
jnz 2f
HAS_ARCH_FEATURE (Prefer_No_AVX512)
jnz 1f
HAS_ARCH_FEATURE (AVX512F_Usable)
jz 1f
lea __memcpy_avx512_no_vzeroupper(%rip), %RAX_LP
HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER)
jnz 2f
lea __memcpy_avx512_unaligned_erms(%rip), %RAX_LP
HAS_CPU_FEATURE (ERMS)
jnz 2f
lea __memcpy_avx512_unaligned(%rip), %RAX_LP
ret
1: lea __memcpy_avx_unaligned(%rip), %RAX_LP
HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
jz L(Fast_Unaligned_Load)
HAS_CPU_FEATURE (ERMS)
jz 2f
lea __memcpy_avx_unaligned_erms(%rip), %RAX_LP
ret
L(Fast_Unaligned_Load):
lea __memcpy_sse2_unaligned(%rip), %RAX_LP
HAS_ARCH_FEATURE (Fast_Unaligned_Copy)
jz L(SSSE3)
HAS_CPU_FEATURE (ERMS)
jz 2f
lea __memcpy_sse2_unaligned_erms(%rip), %RAX_LP
ret
L(SSSE3):
HAS_CPU_FEATURE (SSSE3)
jz 2f
lea __memcpy_ssse3_back(%rip), %RAX_LP
HAS_ARCH_FEATURE (Fast_Copy_Backward)
jnz 2f
lea __memcpy_ssse3(%rip), %RAX_LP
2: ret
END(__new_memcpy)
# undef memcpy
# include <shlib-compat.h>
versioned_symbol (libc, __new_memcpy, memcpy, GLIBC_2_14);
#endif

39
sysdeps/x86_64/multiarch/memcpy.c Normal file
View File

@ -0,0 +1,39 @@
/* Multiple versions of memcpy.
All versions must be listed in ifunc-impl-list.c.
Copyright (C) 2017 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
/* Define multiple versions only for the definition in lib and for
DSO. In static binaries we need memcpy before the initialization
happened. */
#if defined SHARED && IS_IN (libc)
# define memcpy __redirect_memcpy
# include <string.h>
# undef memcpy
# define SYMBOL_NAME memcpy
# include "ifunc-memmove.h"
libc_ifunc_redirected (__redirect_memcpy, __new_memcpy,
IFUNC_SELECTOR ());
__hidden_ver1 (__new_memcpy, __GI_memcpy, __redirect_memcpy)
__attribute__ ((visibility ("hidden")));
# include <shlib-compat.h>
versioned_symbol (libc, __new_memcpy, memcpy, GLIBC_2_14);
#endif

21
sysdeps/x86_64/multiarch/memcpy_chk-nonshared.S Normal file
View File

@ -0,0 +1,21 @@
/* Non-shared version of memcpy_chk for x86-64.
Copyright (C) 2017 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
#if IS_IN (libc) && !defined SHARED
# include <sysdeps/x86_64/memcpy_chk.S>
#endif

72
sysdeps/x86_64/multiarch/memcpy_chk.S
View File

@ -1,72 +0,0 @@
/* Multiple versions of __memcpy_chk
All versions must be listed in ifunc-impl-list.c.
Copyright (C) 2010-2017 Free Software Foundation, Inc.
Contributed by Intel Corporation.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
#include <sysdep.h>
#include <init-arch.h>
/* Define multiple versions only for the definition in lib and for
DSO. There are no multiarch memcpy functions for static binaries.
*/
#if IS_IN (libc)
# ifdef SHARED
.text
ENTRY(__memcpy_chk)
.type __memcpy_chk, @gnu_indirect_function
LOAD_RTLD_GLOBAL_RO_RDX
HAS_ARCH_FEATURE (Prefer_No_AVX512)
jnz 1f
HAS_ARCH_FEATURE (AVX512F_Usable)
jz 1f
lea __memcpy_chk_avx512_no_vzeroupper(%rip), %RAX_LP
HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER)
jnz 2f
lea __memcpy_chk_avx512_unaligned_erms(%rip), %RAX_LP
HAS_CPU_FEATURE (ERMS)
jnz 2f
lea __memcpy_chk_avx512_unaligned(%rip), %RAX_LP
ret
1: lea __memcpy_chk_avx_unaligned(%rip), %RAX_LP
HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
jz L(Fast_Unaligned_Load)
HAS_CPU_FEATURE (ERMS)
jz 2f
lea __memcpy_chk_avx_unaligned_erms(%rip), %RAX_LP
ret
L(Fast_Unaligned_Load):
lea __memcpy_chk_sse2_unaligned(%rip), %RAX_LP
HAS_ARCH_FEATURE (Fast_Unaligned_Copy)
jz L(SSSE3)
HAS_CPU_FEATURE (ERMS)
jz 2f
lea __memcpy_chk_sse2_unaligned_erms(%rip), %RAX_LP
ret
L(SSSE3):
HAS_CPU_FEATURE (SSSE3)
jz 2f
lea __memcpy_chk_ssse3_back(%rip), %RAX_LP
HAS_ARCH_FEATURE (Fast_Copy_Backward)
jnz 2f
lea __memcpy_chk_ssse3(%rip), %RAX_LP
2: ret
END(__memcpy_chk)
# else
# include "../memcpy_chk.S"
# endif
#endif

31
sysdeps/x86_64/multiarch/memcpy_chk.c Normal file
View File

@ -0,0 +1,31 @@
/* Multiple versions of __memcpy_chk
All versions must be listed in ifunc-impl-list.c.
Copyright (C) 2017 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
/* Define multiple versions only for the definition in libc.so. */
#if IS_IN (libc) && defined SHARED
# define __memcpy_chk __redirect_memcpy_chk
# include <string.h>
# undef __memcpy_chk
# define SYMBOL_NAME memcpy_chk
# include "ifunc-memmove.h"
libc_ifunc_redirected (__redirect_memcpy_chk, __memcpy_chk,
IFUNC_SELECTOR ());
#endif

35
sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S Normal file
View File

@ -0,0 +1,35 @@
/* memmove with SSE2.
Copyright (C) 2017 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
#if IS_IN (libc)
# define MEMMOVE_SYMBOL(p,s) p##_sse2_##s
#endif
#if !defined SHARED || !IS_IN (libc)
weak_alias (__mempcpy, mempcpy)
#endif
#include <sysdeps/x86_64/memmove.S>
#if defined SHARED && IS_IN (libc)
# include <shlib-compat.h>
# if SHLIB_COMPAT (libc, GLIBC_2_2_5, GLIBC_2_14)
/* Use __memmove_sse2_unaligned to support overlapping addresses. */
compat_symbol (libc, __memmove_sse2_unaligned, memcpy, GLIBC_2_2_5);
# endif
#endif

11
sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
View File

@ -152,6 +152,11 @@ END (MEMMOVE_SYMBOL (__memmove, unaligned))
# if VEC_SIZE == 16
# if defined SHARED
ENTRY (__mempcpy_chk_erms)
cmpq %rdx, %rcx
jb HIDDEN_JUMPTARGET (__chk_fail)
END (__mempcpy_chk_erms)
/* Only used to measure performance of REP MOVSB. */
ENTRY (__mempcpy_erms)
movq %rdi, %rax
@ -160,6 +165,11 @@ ENTRY (__mempcpy_erms)
END (__mempcpy_erms)
# endif
ENTRY (__memmove_chk_erms)
cmpq %rdx, %rcx
jb HIDDEN_JUMPTARGET (__chk_fail)
END (__memmove_chk_erms)
ENTRY (__memmove_erms)
movq %rdi, %rax
L(start_movsb):
@ -185,6 +195,7 @@ L(movsb_backward):
END (__memmove_erms)
# if defined SHARED
strong_alias (__memmove_erms, __memcpy_erms)
strong_alias (__memmove_chk_erms, __memcpy_chk_erms)
# endif
# endif

101
sysdeps/x86_64/multiarch/memmove.S
View File

@ -1,101 +0,0 @@
/* Multiple versions of memmove
All versions must be listed in ifunc-impl-list.c.
Copyright (C) 2016-2017 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
#include <sysdep.h>
#include <init-arch.h>
/* Define multiple versions only for the definition in lib and for
DSO. */
#if IS_IN (libc)
.text
ENTRY(__libc_memmove)
.type __libc_memmove, @gnu_indirect_function
LOAD_RTLD_GLOBAL_RO_RDX
lea __memmove_erms(%rip), %RAX_LP
HAS_ARCH_FEATURE (Prefer_ERMS)
jnz 2f
HAS_ARCH_FEATURE (Prefer_No_AVX512)
jnz 1f
HAS_ARCH_FEATURE (AVX512F_Usable)
jz 1f
lea __memmove_avx512_no_vzeroupper(%rip), %RAX_LP
HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER)
jnz 2f
lea __memmove_avx512_unaligned_erms(%rip), %RAX_LP
HAS_CPU_FEATURE (ERMS)
jnz 2f
lea __memmove_avx512_unaligned(%rip), %RAX_LP
ret
1: lea __memmove_avx_unaligned(%rip), %RAX_LP
HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
jz L(Fast_Unaligned_Load)
HAS_CPU_FEATURE (ERMS)
jz 2f
lea __memmove_avx_unaligned_erms(%rip), %RAX_LP
ret
L(Fast_Unaligned_Load):
lea __memmove_sse2_unaligned(%rip), %RAX_LP
HAS_ARCH_FEATURE (Fast_Unaligned_Copy)
jz L(SSSE3)
HAS_CPU_FEATURE (ERMS)
jz 2f
lea __memmove_sse2_unaligned_erms(%rip), %RAX_LP
ret
L(SSSE3):
HAS_CPU_FEATURE (SSSE3)
jz 2f
lea __memmove_ssse3_back(%rip), %RAX_LP
HAS_ARCH_FEATURE (Fast_Copy_Backward)
jnz 2f
lea __memmove_ssse3(%rip), %RAX_LP
2: ret
END(__libc_memmove)
#endif
#if IS_IN (libc)
# define MEMMOVE_SYMBOL(p,s) p##_sse2_##s
# ifdef SHARED
libc_hidden_ver (__memmove_sse2_unaligned, memmove)
libc_hidden_ver (__memcpy_sse2_unaligned, memcpy)
libc_hidden_ver (__mempcpy_sse2_unaligned, mempcpy)
libc_hidden_ver (__mempcpy_sse2_unaligned, __mempcpy)
# undef libc_hidden_builtin_def
/* It doesn't make sense to send libc-internal memmove calls through a PLT.
The speedup we get from using SSE2 instructions is likely eaten away
by the indirect call in the PLT. */
# define libc_hidden_builtin_def
# endif
strong_alias (__libc_memmove, memmove)
#endif
#if !defined SHARED || !IS_IN (libc)
weak_alias (__mempcpy, mempcpy)
#endif
#include "../memmove.S"
#if defined SHARED && IS_IN (libc)
# include <shlib-compat.h>
# if SHLIB_COMPAT (libc, GLIBC_2_2_5, GLIBC_2_14)
/* Use __memmove_sse2_unaligned to support overlapping addresses. */
compat_symbol (libc, __memmove_sse2_unaligned, memcpy, GLIBC_2_2_5);
# endif
#endif

38
sysdeps/x86_64/multiarch/memmove.c Normal file
View File

@ -0,0 +1,38 @@
/* Multiple versions of memmmove.
All versions must be listed in ifunc-impl-list.c.
Copyright (C) 2016-2017 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
/* Define multiple versions only for the definition in lib and for
DSO. */
#if IS_IN (libc)
# define memmove __redirect_memmove
# include <string.h>
# undef memmove
# define SYMBOL_NAME memmove
# include "ifunc-memmove.h"
libc_ifunc_redirected (__redirect_memmove, __libc_memmove,
IFUNC_SELECTOR ());
strong_alias (__libc_memmove, memmove);
# ifdef SHARED
__hidden_ver1 (__libc_memmove, __GI_memmove, __redirect_memmove)
__attribute__ ((visibility ("hidden")));
# endif
#endif

21
sysdeps/x86_64/multiarch/memmove_chk-nonshared.S Normal file
View File

@ -0,0 +1,21 @@
/* Non-shared version of memmove_chk for x86-64.
Copyright (C) 2017 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
#if IS_IN (libc) && !defined SHARED
# include <sysdeps/x86_64/memmove_chk.S>
#endif

71
sysdeps/x86_64/multiarch/memmove_chk.S
View File

@ -1,71 +0,0 @@
/* Multiple versions of __memmove_chk
All versions must be listed in ifunc-impl-list.c.
Copyright (C) 2016-2017 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
#include <sysdep.h>
#include <init-arch.h>
/* Define multiple versions only for the definition in lib and for
DSO. There are no multiarch memmove functions for static binaries.
*/
#if IS_IN (libc)
# ifdef SHARED
.text
ENTRY(__memmove_chk)
.type __memmove_chk, @gnu_indirect_function
LOAD_RTLD_GLOBAL_RO_RDX
HAS_ARCH_FEATURE (Prefer_No_AVX512)
jnz 1f
HAS_ARCH_FEATURE (AVX512F_Usable)
jz 1f
lea __memmove_chk_avx512_no_vzeroupper(%rip), %RAX_LP
HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER)
jnz 2f
lea __memmove_chk_avx512_unaligned_erms(%rip), %RAX_LP
HAS_CPU_FEATURE (ERMS)
jnz 2f
lea __memmove_chk_avx512_unaligned(%rip), %RAX_LP
ret
1: lea __memmove_chk_avx_unaligned(%rip), %RAX_LP
HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
jz L(Fast_Unaligned_Load)
HAS_CPU_FEATURE (ERMS)
jz 2f
lea __memmove_chk_avx_unaligned_erms(%rip), %RAX_LP
ret
L(Fast_Unaligned_Load):
lea __memmove_chk_sse2_unaligned(%rip), %RAX_LP
HAS_ARCH_FEATURE (Fast_Unaligned_Copy)
jz L(SSSE3)
HAS_CPU_FEATURE (ERMS)
jz 2f
lea __memmove_chk_sse2_unaligned_erms(%rip), %RAX_LP
ret
L(SSSE3):
HAS_CPU_FEATURE (SSSE3)
jz 2f
lea __memmove_chk_ssse3_back(%rip), %RAX_LP
HAS_ARCH_FEATURE (Fast_Copy_Backward)
jnz 2f
lea __memmove_chk_ssse3(%rip), %RAX_LP
2: ret
END(__memmove_chk)
# else
# include "../memmove_chk.S"
# endif
#endif

31
sysdeps/x86_64/multiarch/memmove_chk.c Normal file
View File

@ -0,0 +1,31 @@
/* Multiple versions of __memmove_chk
All versions must be listed in ifunc-impl-list.c.
Copyright (C) 2017 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
/* Define multiple versions only for the definition in libc.so. */
#if IS_IN (libc) && defined SHARED
# define __memmove_chk __redirect_memmove_chk
# include <string.h>
# undef __memmove_chk
# define SYMBOL_NAME memmove_chk
# include "ifunc-memmove.h"
libc_ifunc_redirected (__redirect_memmove_chk, __memmove_chk,
IFUNC_SELECTOR ());
#endif

73
sysdeps/x86_64/multiarch/mempcpy.S
View File

@ -1,73 +0,0 @@
/* Multiple versions of mempcpy
All versions must be listed in ifunc-impl-list.c.
Copyright (C) 2010-2017 Free Software Foundation, Inc.
Contributed by Intel Corporation.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
#include <sysdep.h>
#include <init-arch.h>
/* Define multiple versions only for the definition in lib and for
DSO. In static binaries we need mempcpy before the initialization
happened. */
#if defined SHARED && IS_IN (libc)
.text
ENTRY(__mempcpy)
.type __mempcpy, @gnu_indirect_function
LOAD_RTLD_GLOBAL_RO_RDX
lea __mempcpy_erms(%rip), %RAX_LP
HAS_ARCH_FEATURE (Prefer_ERMS)
jnz 2f
HAS_ARCH_FEATURE (Prefer_No_AVX512)
jnz 1f
HAS_ARCH_FEATURE (AVX512F_Usable)
jz 1f
lea __mempcpy_avx512_no_vzeroupper(%rip), %RAX_LP
HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER)
jnz 2f
lea __mempcpy_avx512_unaligned_erms(%rip), %RAX_LP
HAS_CPU_FEATURE (ERMS)
jnz 2f
lea __mempcpy_avx512_unaligned(%rip), %RAX_LP
ret
1: lea __mempcpy_avx_unaligned(%rip), %RAX_LP
HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
jz L(Fast_Unaligned_Load)
HAS_CPU_FEATURE (ERMS)
jz 2f
lea __mempcpy_avx_unaligned_erms(%rip), %RAX_LP
ret
L(Fast_Unaligned_Load):
lea __mempcpy_sse2_unaligned(%rip), %RAX_LP
HAS_ARCH_FEATURE (Fast_Unaligned_Copy)
jz L(SSSE3)
HAS_CPU_FEATURE (ERMS)
jz 2f
lea __mempcpy_sse2_unaligned_erms(%rip), %RAX_LP
ret
L(SSSE3):
HAS_CPU_FEATURE (SSSE3)
jz 2f
lea __mempcpy_ssse3_back(%rip), %RAX_LP
HAS_ARCH_FEATURE (Fast_Copy_Backward)
jnz 2f
lea __mempcpy_ssse3(%rip), %RAX_LP
2: ret
END(__mempcpy)
weak_alias (__mempcpy, mempcpy)
#endif

40
sysdeps/x86_64/multiarch/mempcpy.c Normal file
View File

@ -0,0 +1,40 @@
/* Multiple versions of mempcpy.
All versions must be listed in ifunc-impl-list.c.
Copyright (C) 2017 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
/* Define multiple versions only for the definition in lib and for
DSO. In static binaries we need mempcpy before the initialization
happened. */
#if defined SHARED && IS_IN (libc)
# define mempcpy __redirect_mempcpy
# define __mempcpy __redirect___mempcpy
# include <string.h>
# undef mempcpy
# undef __mempcpy
# define SYMBOL_NAME mempcpy
# include "ifunc-memmove.h"
libc_ifunc_redirected (__redirect_mempcpy, __mempcpy, IFUNC_SELECTOR ());
weak_alias (__mempcpy, mempcpy)
__hidden_ver1 (__mempcpy, __GI___mempcpy, __redirect___mempcpy)
__attribute__ ((visibility ("hidden")));
__hidden_ver1 (mempcpy, __GI_mempcpy, __redirect_mempcpy)
__attribute__ ((visibility ("hidden")));
#endif

21
sysdeps/x86_64/multiarch/mempcpy_chk-nonshared.S Normal file
View File

@ -0,0 +1,21 @@
/* Non-shared version of mempcpy_chk for x86-64.
Copyright (C) 2017 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
#if IS_IN (libc) && !defined SHARED
# include <sysdeps/x86_64/mempcpy_chk.S>
#endif

72
sysdeps/x86_64/multiarch/mempcpy_chk.S
View File

@ -1,72 +0,0 @@
/* Multiple versions of __mempcpy_chk
All versions must be listed in ifunc-impl-list.c.
Copyright (C) 2010-2017 Free Software Foundation, Inc.
Contributed by Intel Corporation.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
#include <sysdep.h>
#include <init-arch.h>
/* Define multiple versions only for the definition in lib and for
DSO. There are no multiarch mempcpy functions for static binaries.
*/
#if IS_IN (libc)
# ifdef SHARED
.text
ENTRY(__mempcpy_chk)
.type __mempcpy_chk, @gnu_indirect_function
LOAD_RTLD_GLOBAL_RO_RDX
HAS_ARCH_FEATURE (Prefer_No_AVX512)
jnz 1f
HAS_ARCH_FEATURE (AVX512F_Usable)
jz 1f
lea __mempcpy_chk_avx512_no_vzeroupper(%rip), %RAX_LP
HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER)
jnz 2f
lea __mempcpy_chk_avx512_unaligned_erms(%rip), %RAX_LP
HAS_CPU_FEATURE (ERMS)
jnz 2f
lea __mempcpy_chk_avx512_unaligned(%rip), %RAX_LP
ret
1: lea __mempcpy_chk_avx_unaligned(%rip), %RAX_LP
HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
jz L(Fast_Unaligned_Load)
HAS_CPU_FEATURE (ERMS)
jz 2f
lea __mempcpy_chk_avx_unaligned_erms(%rip), %RAX_LP
ret
L(Fast_Unaligned_Load):
lea __mempcpy_chk_sse2_unaligned(%rip), %RAX_LP
HAS_ARCH_FEATURE (Fast_Unaligned_Copy)
jz L(SSSE3)
HAS_CPU_FEATURE (ERMS)
jz 2f
lea __mempcpy_chk_sse2_unaligned_erms(%rip), %RAX_LP
ret
L(SSSE3):
HAS_CPU_FEATURE (SSSE3)
jz 2f
lea __mempcpy_chk_ssse3_back(%rip), %RAX_LP
HAS_ARCH_FEATURE (Fast_Copy_Backward)
jnz 2f
lea __mempcpy_chk_ssse3(%rip), %RAX_LP
2: ret
END(__mempcpy_chk)
# else
# include "../mempcpy_chk.S"
# endif
#endif

31
sysdeps/x86_64/multiarch/mempcpy_chk.c Normal file
View File

@ -0,0 +1,31 @@
/* Multiple versions of __mempcpy_chk
All versions must be listed in ifunc-impl-list.c.
Copyright (C) 2017 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
/* Define multiple versions only for the definition in libc.so. */
#if IS_IN (libc) && defined SHARED
# define __mempcpy_chk __redirect_mempcpy_chk
# include <string.h>
# undef __mempcpy_chk
# define SYMBOL_NAME mempcpy_chk
# include "ifunc-memmove.h"
libc_ifunc_redirected (__redirect_mempcpy_chk, __mempcpy_chk,
IFUNC_SELECTOR ());
#endif