x86-64: Avoid rep movsb with short distance [BZ #27130]
When copying with "rep movsb", if the distance between source and destination is N*4GB + [1..63] with N >= 0, performance may be very slow. This patch updates memmove-vec-unaligned-erms.S for AVX and AVX512 versions with the distance in RCX: cmpl $63, %ecx // Don't use "rep movsb" if ECX <= 63 jbe L(Don't use rep movsb") Use "rep movsb" Benchtests data with bench-memcpy, bench-memcpy-large, bench-memcpy-random and bench-memcpy-walk on Skylake, Ice Lake and Tiger Lake show that its performance impact is within noise range as "rep movsb" is only used for data size >= 4KB. (cherry picked from commit 3ec5d83d2a237d39e7fd6ef7a0bc8ac4c171a4a5)
This commit is contained in:
parent
a3c78954ee
commit
8493ba72b1
1
NEWS
1
NEWS
|
@ -41,6 +41,7 @@ The following bugs are resolved with this release:
|
||||||
[25232] No const correctness for strchr et al. for Clang++
|
[25232] No const correctness for strchr et al. for Clang++
|
||||||
[25933] Off by one error in __strncmp_avx2
|
[25933] Off by one error in __strncmp_avx2
|
||||||
[25976] nss_compat: internal_end*ent may clobber errno, hiding ERANGE
|
[25976] nss_compat: internal_end*ent may clobber errno, hiding ERANGE
|
||||||
|
[27130] "rep movsb" performance issue
|
||||||
|
|
||||||
Security related changes:
|
Security related changes:
|
||||||
|
|
||||||
|
|
|
@ -67,6 +67,13 @@
|
||||||
# define REP_MOVSB_THRESHOLD (2048 * (VEC_SIZE / 16))
|
# define REP_MOVSB_THRESHOLD (2048 * (VEC_SIZE / 16))
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
/* Avoid short distance rep movsb only with non-SSE vector. */
|
||||||
|
#ifndef AVOID_SHORT_DISTANCE_REP_MOVSB
|
||||||
|
# define AVOID_SHORT_DISTANCE_REP_MOVSB (VEC_SIZE > 16)
|
||||||
|
#else
|
||||||
|
# define AVOID_SHORT_DISTANCE_REP_MOVSB 0
|
||||||
|
#endif
|
||||||
|
|
||||||
#ifndef PREFETCH
|
#ifndef PREFETCH
|
||||||
# define PREFETCH(addr) prefetcht0 addr
|
# define PREFETCH(addr) prefetcht0 addr
|
||||||
#endif
|
#endif
|
||||||
|
@ -257,7 +264,21 @@ L(movsb):
|
||||||
# error Unsupported REP_MOVSB_THRESHOLD and VEC_SIZE!
|
# error Unsupported REP_MOVSB_THRESHOLD and VEC_SIZE!
|
||||||
# endif
|
# endif
|
||||||
jb L(more_8x_vec_backward)
|
jb L(more_8x_vec_backward)
|
||||||
|
# if AVOID_SHORT_DISTANCE_REP_MOVSB
|
||||||
|
movq %rdi, %rcx
|
||||||
|
subq %rsi, %rcx
|
||||||
|
jmp 2f
|
||||||
|
# endif
|
||||||
1:
|
1:
|
||||||
|
# if AVOID_SHORT_DISTANCE_REP_MOVSB
|
||||||
|
movq %rsi, %rcx
|
||||||
|
subq %rdi, %rcx
|
||||||
|
2:
|
||||||
|
/* Avoid "rep movsb" if RCX, the distance between source and destination,
|
||||||
|
is N*4GB + [1..63] with N >= 0. */
|
||||||
|
cmpl $63, %ecx
|
||||||
|
jbe L(more_2x_vec) /* Avoid "rep movsb" if ECX <= 63. */
|
||||||
|
# endif
|
||||||
mov %RDX_LP, %RCX_LP
|
mov %RDX_LP, %RCX_LP
|
||||||
rep movsb
|
rep movsb
|
||||||
L(nop):
|
L(nop):
|
||||||
|
|
Loading…
Reference in New Issue