S390: Add z13 memmove ifunc variant.
This patch introduces a z13 specific ifunc variant for memmove. As the common code implementation, it checks if we can copy from the beginning to the end - with z196 memcpy implementation - or if we have to copy from the end to the beginning. The latter case is done by using vector load/store instructions. If vector instructions are not available, the common-code is used as fallback. Therefore it is implemented in memmove-c with a different name. Furthermore the ifunc logic decides if we need the common-code implementation at all. If vector instructions are supported due to the minimum architecture level set we can skip the common-code ifunc variant. ChangeLog: * sysdeps/s390/Makefile (sysdep_routines): Add memmove-c. * sysdeps/s390/ifunc-memcpy.h (HAVE_MEMMOVE_IFUNC, HAVE_MEMMOVE_IFUNC_AND_VX_SUPPORT, MEMMOVE_DEFAULT, HAVE_MEMMOVE_C, MEMMOVE_C, HAVE_MEMMOVE_Z13, MEMMOVE_Z13): New defines. * sysdeps/s390/memcpy-z900.S: Add z13 memmove implementation. * sysdeps/s390/memmove-c.c: New file. * sysdeps/s390/memmove.c: Likewise. * sysdeps/s390/multiarch/ifunc-impl-list.c (__libc_ifunc_impl_list): Add ifunc variants for memmove.
This commit is contained in:
Stefan Liebler
parent
2ee1bc57ab
commit
cdd927d98c
13
ChangeLog
13
ChangeLog
|
|
@ -1,3 +1,16 @@
|
|||
2018-12-18 Stefan Liebler <stli@linux.ibm.com>
|
||||
|
||||
* sysdeps/s390/Makefile (sysdep_routines): Add memmove-c.
|
||||
* sysdeps/s390/ifunc-memcpy.h (HAVE_MEMMOVE_IFUNC,
|
||||
HAVE_MEMMOVE_IFUNC_AND_VX_SUPPORT, MEMMOVE_DEFAULT,
|
||||
HAVE_MEMMOVE_C, MEMMOVE_C, HAVE_MEMMOVE_Z13, MEMMOVE_Z13):
|
||||
New defines.
|
||||
* sysdeps/s390/memcpy-z900.S: Add z13 memmove implementation.
|
||||
* sysdeps/s390/memmove-c.c: New file.
|
||||
* sysdeps/s390/memmove.c: Likewise.
|
||||
* sysdeps/s390/multiarch/ifunc-impl-list.c
|
||||
(__libc_ifunc_impl_list): Add ifunc variants for memmove.
|
||||
|
||||
2018-12-18 Stefan Liebler <stli@linux.ibm.com>
|
||||
|
||||
* config.h.in (HAVE_S390_MIN_Z13_ZARCH_ASM_SUPPORT): New undefine.
|
||||
|
|
|
|||
|
|
@ -56,5 +56,6 @@ endif
|
|||
ifeq ($(subdir),string)
|
||||
sysdep_routines += bzero memset memset-z900 \
|
||||
memcmp memcmp-z900 \
|
||||
mempcpy memcpy memcpy-z900
|
||||
mempcpy memcpy memcpy-z900 \
|
||||
memmove memmove-c
|
||||
endif
|
||||
|
|
|
|||
|
|
@ -43,6 +43,29 @@
|
|||
# define HAVE_MEMCPY_Z196 HAVE_MEMCPY_IFUNC
|
||||
#endif
|
||||
|
||||
#if defined SHARED && defined USE_MULTIARCH && IS_IN (libc) \
|
||||
&& ! defined HAVE_S390_MIN_Z13_ZARCH_ASM_SUPPORT
|
||||
# define HAVE_MEMMOVE_IFUNC 1
|
||||
#else
|
||||
# define HAVE_MEMMOVE_IFUNC 0
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_S390_VX_ASM_SUPPORT
|
||||
# define HAVE_MEMMOVE_IFUNC_AND_VX_SUPPORT HAVE_MEMMOVE_IFUNC
|
||||
#else
|
||||
# define HAVE_MEMMOVE_IFUNC_AND_VX_SUPPORT 0
|
||||
#endif
|
||||
|
||||
#if defined HAVE_S390_MIN_Z13_ZARCH_ASM_SUPPORT
|
||||
# define MEMMOVE_DEFAULT MEMMOVE_Z13
|
||||
# define HAVE_MEMMOVE_C 0
|
||||
# define HAVE_MEMMOVE_Z13 1
|
||||
#else
|
||||
# define MEMMOVE_DEFAULT MEMMOVE_C
|
||||
# define HAVE_MEMMOVE_C 1
|
||||
# define HAVE_MEMMOVE_Z13 HAVE_MEMMOVE_IFUNC_AND_VX_SUPPORT
|
||||
#endif
|
||||
|
||||
#if HAVE_MEMCPY_Z900_G5
|
||||
# define MEMCPY_Z900_G5 __memcpy_default
|
||||
# define MEMPCPY_Z900_G5 __mempcpy_default
|
||||
|
|
@ -66,3 +89,15 @@
|
|||
# define MEMCPY_Z196 NULL
|
||||
# define MEMPCPY_Z196 NULL
|
||||
#endif
|
||||
|
||||
#if HAVE_MEMMOVE_C
|
||||
# define MEMMOVE_C __memmove_c
|
||||
#else
|
||||
# define MEMMOVE_C NULL
|
||||
#endif
|
||||
|
||||
#if HAVE_MEMMOVE_Z13
|
||||
# define MEMMOVE_Z13 __memmove_z13
|
||||
#else
|
||||
# define MEMMOVE_Z13 NULL
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -182,6 +182,7 @@ ENTRY(MEMCPY_Z196)
|
|||
# endif /* !defined __s390x__ */
|
||||
ltgr %r4,%r4
|
||||
je .L_Z196_4
|
||||
.L_Z196_start2:
|
||||
aghi %r4,-1
|
||||
srlg %r5,%r4,8
|
||||
ltgr %r5,%r5
|
||||
|
|
@ -207,6 +208,75 @@ ENTRY(MEMCPY_Z196)
|
|||
END(MEMCPY_Z196)
|
||||
#endif /* HAVE_MEMCPY_Z196 */
|
||||
|
||||
#if HAVE_MEMMOVE_Z13
|
||||
ENTRY(MEMMOVE_Z13)
|
||||
.machine "z13"
|
||||
.machinemode "zarch_nohighgprs"
|
||||
# if !defined __s390x__
|
||||
/* Note: The 31bit dst and src pointers are prefixed with zeroes. */
|
||||
llgfr %r4,%r4
|
||||
llgfr %r3,%r3
|
||||
llgfr %r2,%r2
|
||||
# endif /* !defined __s390x__ */
|
||||
sgrk %r0,%r2,%r3
|
||||
clgijh %r4,16,.L_MEMMOVE_Z13_LARGE
|
||||
aghik %r5,%r4,-1
|
||||
.L_MEMMOVE_Z13_SMALL:
|
||||
jl .L_MEMMOVE_Z13_END /* Jump away if len was zero. */
|
||||
/* Store up to 16 bytes with vll/vstl which needs the index
|
||||
instead of lengths. */
|
||||
vll %v16,%r5,0(%r3)
|
||||
vstl %v16,%r5,0(%r2)
|
||||
.L_MEMMOVE_Z13_END:
|
||||
br %r14
|
||||
.L_MEMMOVE_Z13_LARGE:
|
||||
lgr %r1,%r2 /* For memcpy: r1: Use as dest ;
|
||||
r2: Return dest */
|
||||
/* The unsigned comparison (dst - src >= len) determines if we can
|
||||
execute the forward case with memcpy. */
|
||||
#if ! HAVE_MEMCPY_Z196
|
||||
# error The z13 variant of memmove needs the z196 variant of memcpy!
|
||||
#endif
|
||||
clgrjhe %r0,%r4,.L_Z196_start2
|
||||
risbgn %r5,%r4,4,128+63,60 /* r5 = r4 / 16 */
|
||||
aghi %r4,-16
|
||||
clgijhe %r5,8,.L_MEMMOVE_Z13_LARGE_64B
|
||||
.L_MEMMOVE_Z13_LARGE_16B_LOOP:
|
||||
/* Store at least 16 bytes with vl/vst. The number of 16byte blocks
|
||||
is stored in r5. */
|
||||
vl %v16,0(%r4,%r3)
|
||||
vst %v16,0(%r4,%r2)
|
||||
aghi %r4,-16
|
||||
brctg %r5,.L_MEMMOVE_Z13_LARGE_16B_LOOP
|
||||
aghik %r5,%r4,15
|
||||
j .L_MEMMOVE_Z13_SMALL
|
||||
.L_MEMMOVE_Z13_LARGE_64B:
|
||||
/* Store at least 128 bytes with 4x vl/vst. The number of 64byte blocks
|
||||
will be stored in r0. */
|
||||
aghi %r4,-48
|
||||
srlg %r0,%r5,2 /* r5 = %r0 / 4
|
||||
=> Number of 64byte blocks. */
|
||||
.L_MEMMOVE_Z13_LARGE_64B_LOOP:
|
||||
vl %v20,48(%r4,%r3)
|
||||
vl %v19,32(%r4,%r3)
|
||||
vl %v18,16(%r4,%r3)
|
||||
vl %v17,0(%r4,%r3)
|
||||
vst %v20,48(%r4,%r2)
|
||||
vst %v19,32(%r4,%r2)
|
||||
vst %v18,16(%r4,%r2)
|
||||
vst %v17,0(%r4,%r2)
|
||||
aghi %r4,-64
|
||||
brctg %r0,.L_MEMMOVE_Z13_LARGE_64B_LOOP
|
||||
aghi %r4,48
|
||||
/* Recalculate the number of 16byte blocks. */
|
||||
risbg %r5,%r5,62,128+63,0 /* r5 = r5 & 3
|
||||
=> Remaining 16byte blocks. */
|
||||
jne .L_MEMMOVE_Z13_LARGE_16B_LOOP
|
||||
aghik %r5,%r4,15
|
||||
j .L_MEMMOVE_Z13_SMALL
|
||||
END(MEMMOVE_Z13)
|
||||
#endif /* HAVE_MEMMOVE_Z13 */
|
||||
|
||||
#if ! HAVE_MEMCPY_IFUNC
|
||||
/* If we don't use ifunc, define an alias for mem[p]cpy here.
|
||||
Otherwise see sysdeps/s390/mem[p]cpy.c. */
|
||||
|
|
@ -215,10 +285,27 @@ strong_alias (MEMPCPY_DEFAULT, __mempcpy)
|
|||
weak_alias (__mempcpy, mempcpy)
|
||||
#endif
|
||||
|
||||
#if ! HAVE_MEMMOVE_IFUNC
|
||||
/* If we don't use ifunc, define an alias for memmove here.
|
||||
Otherwise see sysdeps/s390/memmove.c. */
|
||||
# if ! HAVE_MEMMOVE_C
|
||||
/* If the c variant is needed, then sysdeps/s390/memmove-c.c
|
||||
defines memmove.
|
||||
Otherwise MEMMOVE_DEFAULT is implemented here and we have to define it. */
|
||||
strong_alias (MEMMOVE_DEFAULT, memmove)
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#if defined SHARED && IS_IN (libc)
|
||||
/* Defines the internal symbols.
|
||||
Compare to libc_hidden_[builtin_]def (mem[p]cpy) in string/mem[p]cpy.c. */
|
||||
strong_alias (MEMCPY_DEFAULT, __GI_memcpy)
|
||||
strong_alias (MEMPCPY_DEFAULT, __GI_mempcpy)
|
||||
strong_alias (MEMPCPY_DEFAULT, __GI___mempcpy)
|
||||
# if ! HAVE_MEMMOVE_C
|
||||
/* If the c variant is needed, then sysdeps/s390/memmove-c.c
|
||||
defines the internal symbol.
|
||||
Otherwise MEMMOVE_DEFAULT is implemented here and we have to define it. */
|
||||
strong_alias (MEMMOVE_DEFAULT, __GI_memmove)
|
||||
# endif
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -0,0 +1,37 @@
|
|||
/* Fallback C version of memmove.
|
||||
Copyright (C) 2018 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<http://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <ifunc-memcpy.h>
|
||||
|
||||
#if HAVE_MEMMOVE_C
|
||||
# if HAVE_MEMMOVE_IFUNC
|
||||
/* If we use ifunc, then the memmove symbol is defined
|
||||
in sysdeps/s390/memmove.c and we use a different name here.
|
||||
Otherwise, we have to define memmove here or in
|
||||
sysdeps/s390/memcpy.S depending on the used default implementation. */
|
||||
# define MEMMOVE MEMMOVE_C
|
||||
# if defined SHARED && IS_IN (libc)
|
||||
/* Define the internal symbol. */
|
||||
# undef libc_hidden_builtin_def
|
||||
# define libc_hidden_builtin_def(name) \
|
||||
__hidden_ver1 (__memmove_c, __GI_memmove, __memmove_c);
|
||||
# endif
|
||||
# endif
|
||||
|
||||
# include <string/memmove.c>
|
||||
#endif
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
/* Multiple versions of memmove.
|
||||
Copyright (C) 2016-2018 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<http://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <ifunc-memcpy.h>
|
||||
|
||||
#if HAVE_MEMMOVE_IFUNC
|
||||
/* If we don't use ifunc, an alias is defined for memmove
|
||||
in sysdeps/s390/memmove-c.c or sysdeps/s390/memcpy.S
|
||||
depending on the used default implementation. */
|
||||
# undef memmove
|
||||
# define memmove __redirect_memmove
|
||||
# include <string.h>
|
||||
# include <ifunc-resolve.h>
|
||||
# undef memmove
|
||||
|
||||
# if HAVE_MEMMOVE_C
|
||||
extern __typeof (__redirect_memmove) MEMMOVE_C attribute_hidden;
|
||||
# endif
|
||||
|
||||
# if HAVE_MEMMOVE_Z13
|
||||
extern __typeof (__redirect_memmove) MEMMOVE_Z13 attribute_hidden;
|
||||
# endif
|
||||
|
||||
s390_libc_ifunc_expr (__redirect_memmove, memmove,
|
||||
(HAVE_MEMMOVE_Z13 && (hwcap & HWCAP_S390_VX))
|
||||
? MEMMOVE_Z13
|
||||
: MEMMOVE_DEFAULT
|
||||
)
|
||||
#endif
|
||||
|
|
@ -126,6 +126,18 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
|||
)
|
||||
#endif /* HAVE_MEMCPY_IFUNC */
|
||||
|
||||
#if HAVE_MEMMOVE_IFUNC
|
||||
IFUNC_IMPL (i, name, memmove,
|
||||
# if HAVE_MEMMOVE_Z13
|
||||
IFUNC_IMPL_ADD (array, i, memmove,
|
||||
dl_hwcap & HWCAP_S390_VX, MEMMOVE_Z13)
|
||||
# endif
|
||||
# if HAVE_MEMMOVE_C
|
||||
IFUNC_IMPL_ADD (array, i, memmove, 1, MEMMOVE_C)
|
||||
# endif
|
||||
)
|
||||
#endif /* HAVE_MEMMOVE_IFUNC */
|
||||
|
||||
#ifdef HAVE_S390_VX_ASM_SUPPORT
|
||||
|
||||
# define IFUNC_VX_IMPL(FUNC) \
|
||||
|
|
|
|||
Loading…
Reference in New Issue