OpenE2K
/
glibc
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

x86-64: Implement strcpy family IFUNC selectors in C 

Implement strcpy family IFUNC selectors in C.

All internal calls within libc.so can use IFUNC on x86-64 since unlike
x86, x86-64 supports PC-relative addressing to access the GOT entry so
that it can call via PLT without using an extra register.  For libc.a,
we can't use IFUNC for functions which are called before IFUNC has been
initialized.  Use IFUNC internally reduces the icache footprint since
libc.so and other codes in the process use the same implementations.
This patch uses IFUNC for strcpy family functions within libc.

	* sysdeps/x86_64/multiarch/Makefile (sysdep_routines): Add
	strcpy-sse2 and stpcpy-sse2.
	* sysdeps/x86_64/multiarch/ifunc-unaligned-ssse3.h: New file.
	* sysdeps/x86_64/multiarch/stpcpy-sse2.S: Likewise.
	* sysdeps/x86_64/multiarch/stpcpy.c: Likewise.
	* sysdeps/x86_64/multiarch/stpncpy.c: Likewise.
	* sysdeps/x86_64/multiarch/strcpy-sse2.S: Likewise.
	* sysdeps/x86_64/multiarch/strcpy.c: Likewise.
	* sysdeps/x86_64/multiarch/strncpy.c: Likewise.
	* sysdeps/x86_64/multiarch/stpcpy.S: Removed.
	* sysdeps/x86_64/multiarch/stpncpy.S: Likewise.
	* sysdeps/x86_64/multiarch/strcpy.S: Likewise.
	* sysdeps/x86_64/multiarch/strncpy.S: Likewise.
	* sysdeps/x86_64/multiarch/stpncpy-c.c (weak_alias): New.
	(libc_hidden_def): Always defined as empty.
	* sysdeps/x86_64/multiarch/strncpy-c.c (libc_hidden_builtin_def):
	Always Defined as empty.
This commit is contained in:
H.J. Lu 2017-06-12 09:05:57 -07:00
parent 18b10de7ce
commit 5a103908c0
15 changed files with 278 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
ChangeLog
View File

@ -1,3 +1,23 @@
2017-06-12 H.J. Lu <hongjiu.lu@intel.com>
* sysdeps/x86_64/multiarch/Makefile (sysdep_routines): Add
strcpy-sse2 and stpcpy-sse2.
* sysdeps/x86_64/multiarch/ifunc-unaligned-ssse3.h: New file.
* sysdeps/x86_64/multiarch/stpcpy-sse2.S: Likewise.
* sysdeps/x86_64/multiarch/stpcpy.c: Likewise.
* sysdeps/x86_64/multiarch/stpncpy.c: Likewise.
* sysdeps/x86_64/multiarch/strcpy-sse2.S: Likewise.
* sysdeps/x86_64/multiarch/strcpy.c: Likewise.
* sysdeps/x86_64/multiarch/strncpy.c: Likewise.
* sysdeps/x86_64/multiarch/stpcpy.S: Removed.
* sysdeps/x86_64/multiarch/stpncpy.S: Likewise.
* sysdeps/x86_64/multiarch/strcpy.S: Likewise.
* sysdeps/x86_64/multiarch/strncpy.S: Likewise.
* sysdeps/x86_64/multiarch/stpncpy-c.c (weak_alias): New.
(libc_hidden_def): Always defined as empty.
* sysdeps/x86_64/multiarch/strncpy-c.c (libc_hidden_builtin_def):
Always Defined as empty.
2017-06-12 Wilco Dijkstra <wdijkstr@arm.com>
[BZ #15105]

1
sysdeps/x86_64/multiarch/Makefile
View File

@ -18,6 +18,7 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c strcmp-ssse3 \
strrchr-sse2 strrchr-avx2 \
strlen-sse2 strnlen-sse2 strlen-avx2 strnlen-avx2 \
strncase_l-ssse3 strcat-ssse3 strncat-ssse3\
strcpy-sse2 stpcpy-sse2 \
strcpy-ssse3 strncpy-ssse3 stpcpy-ssse3 stpncpy-ssse3 \
strcpy-sse2-unaligned strncpy-sse2-unaligned \
stpcpy-sse2-unaligned stpncpy-sse2-unaligned \

40
sysdeps/x86_64/multiarch/ifunc-unaligned-ssse3.h Normal file
View File

@ -0,0 +1,40 @@
/* Common definition for ifunc selections optimized with SSE2, unaligned
SSE2 and SSSE3.
All versions must be listed in ifunc-impl-list.c.
Copyright (C) 2017 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
#include <init-arch.h>
extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned)
attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden;
static inline void *
IFUNC_SELECTOR (void)
{
const struct cpu_features* cpu_features = __get_cpu_features ();
if (CPU_FEATURES_ARCH_P (cpu_features, Fast_Unaligned_Load))
return OPTIMIZE (sse2_unaligned);
if (CPU_FEATURES_CPU_P (cpu_features, SSSE3))
return OPTIMIZE (ssse3);
return OPTIMIZE (sse2);
}

33
sysdeps/x86_64/multiarch/stpcpy-sse2.S Normal file
View File

@ -0,0 +1,33 @@
/* stpcpy optimized with SSE2.
Copyright (C) 2017 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
#if IS_IN (libc)
# include <sysdep.h>
# define __stpcpy __stpcpy_sse2
# undef weak_alias
# define weak_alias(ignored1, ignored2)
# undef libc_hidden_def
# define libc_hidden_def(__stpcpy)
# undef libc_hidden_builtin_def
# define libc_hidden_builtin_def(stpcpy)
#endif
#define USE_AS_STPCPY
#include <sysdeps/x86_64/stpcpy.S>

9
sysdeps/x86_64/multiarch/stpcpy.S
View File

@ -1,9 +0,0 @@
/* Multiple versions of stpcpy
All versions must be listed in ifunc-impl-list.c. */
#define USE_AS_STPCPY
#define STRCPY __stpcpy
#include "strcpy.S"
weak_alias (__stpcpy, stpcpy)
libc_hidden_def (__stpcpy)
libc_hidden_builtin_def (stpcpy)

41
sysdeps/x86_64/multiarch/stpcpy.c Normal file
View File

@ -0,0 +1,41 @@
/* Multiple versions of stpcpy.
All versions must be listed in ifunc-impl-list.c.
Copyright (C) 2017 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
/* Define multiple versions only for the definition in libc. */
#if IS_IN (libc)
# define _HAVE_STRING_ARCH_stpcpy 1
# define stpcpy __redirect_stpcpy
# define __stpcpy __redirect___stpcpy
# include <string.h>
# undef stpcpy
# undef __stpcpy
# define SYMBOL_NAME stpcpy
# include "ifunc-unaligned-ssse3.h"
libc_ifunc_redirected (__redirect_stpcpy, __stpcpy, IFUNC_SELECTOR ());
weak_alias (__stpcpy, stpcpy)
# ifdef SHARED
__hidden_ver1 (__stpcpy, __GI___stpcpy, __redirect___stpcpy)
__attribute__ ((visibility ("hidden")));
__hidden_ver1 (stpcpy, __GI_stpcpy, __redirect_stpcpy)
__attribute__ ((visibility ("hidden")));
# endif
#endif

9
sysdeps/x86_64/multiarch/stpncpy-c.c
View File

@ -1,8 +1,7 @@
#define STPNCPY __stpncpy_sse2
#ifdef SHARED
#undef weak_alias
#define weak_alias(ignored1, ignored2)
#undef libc_hidden_def
#define libc_hidden_def(name) \
__hidden_ver1 (__stpncpy_sse2, __GI___stpncpy, __stpncpy_sse2);
#endif
#define libc_hidden_def(stpncpy)
#include "stpncpy.c"
#include <string/stpncpy.c>

8
sysdeps/x86_64/multiarch/stpncpy.S
View File

@ -1,8 +0,0 @@
/* Multiple versions of stpncpy
All versions must be listed in ifunc-impl-list.c. */
#define STRCPY __stpncpy
#define USE_AS_STPCPY
#define USE_AS_STRNCPY
#include "strcpy.S"
weak_alias (__stpncpy, stpncpy)

38
sysdeps/x86_64/multiarch/stpncpy.c Normal file
View File

@ -0,0 +1,38 @@
/* Multiple versions of stpncpy.
All versions must be listed in ifunc-impl-list.c.
Copyright (C) 2017 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
/* Define multiple versions only for the definition in libc. */
#if IS_IN (libc)
# define stpncpy __redirect_stpncpy
# define __stpncpy __redirect___stpncpy
# include <string.h>
# undef stpncpy
# undef __stpncpy
# define SYMBOL_NAME stpncpy
# include "ifunc-unaligned-ssse3.h"
libc_ifunc_redirected (__redirect_stpncpy, __stpncpy, IFUNC_SELECTOR ());
weak_alias (__stpncpy, stpncpy)
# ifdef SHARED
__hidden_ver1 (__stpncpy, __GI___stpncpy, __redirect___stpncpy)
__attribute__ ((visibility ("hidden")));
# endif
#endif

28
sysdeps/x86_64/multiarch/strcpy-sse2.S Normal file
View File

@ -0,0 +1,28 @@
/* strcpy optimized with SSE2.
Copyright (C) 2017 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
#if IS_IN (libc)
# include <sysdep.h>
# define strcpy __strcpy_sse2
# undef libc_hidden_builtin_def
# define libc_hidden_builtin_def(strcpy)
#endif
#include <sysdeps/x86_64/strcpy.S>

99
sysdeps/x86_64/multiarch/strcpy.S
View File

@ -1,99 +0,0 @@
/* Multiple versions of strcpy
All versions must be listed in ifunc-impl-list.c.
Copyright (C) 2009-2017 Free Software Foundation, Inc.
Contributed by Intel Corporation.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
#include <sysdep.h>
#include <init-arch.h>
#if !defined (USE_AS_STPCPY) && !defined (USE_AS_STRNCPY)
# ifndef STRCPY
# define STRCPY strcpy
# endif
#endif
#ifdef USE_AS_STPCPY
# ifdef USE_AS_STRNCPY
# define STRCPY_SSSE3 __stpncpy_ssse3
# define STRCPY_SSE2 __stpncpy_sse2
# define STRCPY_SSE2_UNALIGNED __stpncpy_sse2_unaligned
# define __GI_STRCPY __GI_stpncpy
# define __GI___STRCPY __GI___stpncpy
# else
# define STRCPY_SSSE3 __stpcpy_ssse3
# define STRCPY_SSE2 __stpcpy_sse2
# define STRCPY_SSE2_UNALIGNED __stpcpy_sse2_unaligned
# define __GI_STRCPY __GI_stpcpy
# define __GI___STRCPY __GI___stpcpy
# endif
#else
# ifdef USE_AS_STRNCPY
# define STRCPY_SSSE3 __strncpy_ssse3
# define STRCPY_SSE2 __strncpy_sse2
# define STRCPY_SSE2_UNALIGNED __strncpy_sse2_unaligned
# define __GI_STRCPY __GI_strncpy
# else
# define STRCPY_SSSE3 __strcpy_ssse3
# define STRCPY_SSE2 __strcpy_sse2
# define STRCPY_SSE2_UNALIGNED __strcpy_sse2_unaligned
# define __GI_STRCPY __GI_strcpy
# endif
#endif
/* Define multiple versions only for the definition in libc. */
#if IS_IN (libc)
.text
ENTRY(STRCPY)
.type STRCPY, @gnu_indirect_function
LOAD_RTLD_GLOBAL_RO_RDX
leaq STRCPY_SSE2_UNALIGNED(%rip), %rax
HAS_ARCH_FEATURE (Fast_Unaligned_Load)
jnz 2f
leaq STRCPY_SSE2(%rip), %rax
HAS_CPU_FEATURE (SSSE3)
jz 2f
leaq STRCPY_SSSE3(%rip), %rax
2: ret
END(STRCPY)
# undef ENTRY
# define ENTRY(name) \
.type STRCPY_SSE2, @function; \
.align 16; \
.globl STRCPY_SSE2; \
.hidden STRCPY_SSE2; \
STRCPY_SSE2: cfi_startproc; \
CALL_MCOUNT
# undef END
# define END(name) \
cfi_endproc; .size STRCPY_SSE2, .-STRCPY_SSE2
# undef libc_hidden_builtin_def
/* It doesn't make sense to send libc-internal strcpy calls through a PLT.
The speedup we get from using SSSE3 instruction is likely eaten away
by the indirect call in the PLT. */
# define libc_hidden_builtin_def(name) \
.globl __GI_STRCPY; __GI_STRCPY = STRCPY_SSE2
# undef libc_hidden_def
# define libc_hidden_def(name) \
.globl __GI___STRCPY; __GI___STRCPY = STRCPY_SSE2
#endif
#ifndef USE_AS_STRNCPY
#include "../strcpy.S"
#endif

35
sysdeps/x86_64/multiarch/strcpy.c Normal file
View File

@ -0,0 +1,35 @@
/* Multiple versions of strcpy.
All versions must be listed in ifunc-impl-list.c.
Copyright (C) 2017 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
/* Define multiple versions only for the definition in libc. */
#if IS_IN (libc)
# define strcpy __redirect_strcpy
# include <string.h>
# undef strcpy
# define SYMBOL_NAME strcpy
# include "ifunc-unaligned-ssse3.h"
libc_ifunc_redirected (__redirect_strcpy, strcpy, IFUNC_SELECTOR ());
# ifdef SHARED
__hidden_ver1 (strcpy, __GI_strcpy, __redirect_strcpy)
__attribute__ ((visibility ("hidden")));
# endif
#endif

7
sysdeps/x86_64/multiarch/strncpy-c.c
View File

@ -1,8 +1,5 @@
#define STRNCPY __strncpy_sse2
#ifdef SHARED
#undef libc_hidden_builtin_def
#define libc_hidden_builtin_def(name) \
__hidden_ver1 (__strncpy_sse2, __GI_strncpy, __strncpy_sse2);
#endif
#define libc_hidden_builtin_def(strncpy)
#include "strncpy.c"
#include <string/strncpy.c>

5
sysdeps/x86_64/multiarch/strncpy.S
View File

@ -1,5 +0,0 @@
/* Multiple versions of strncpy
All versions must be listed in ifunc-impl-list.c. */
#define STRCPY strncpy
#define USE_AS_STRNCPY
#include "strcpy.S"

36
sysdeps/x86_64/multiarch/strncpy.c Normal file
View File

@ -0,0 +1,36 @@
/* Multiple versions of strncpy.
All versions must be listed in ifunc-impl-list.c.
Copyright (C) 2017 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
/* Define multiple versions only for the definition in libc. */
#if IS_IN (libc)
# define _HAVE_STRING_ARCH_strncpy 1
# define strncpy __redirect_strncpy
# include <string.h>
# undef strncpy
# define SYMBOL_NAME strncpy
# include "ifunc-unaligned-ssse3.h"
libc_ifunc_redirected (__redirect_strncpy, strncpy, IFUNC_SELECTOR ());
# ifdef SHARED
__hidden_ver1 (strncpy, __GI_strncpy, __redirect_strncpy)
__attribute__ ((visibility ("hidden")));
# endif
#endif