436 lines
11 KiB
ArmAsm
436 lines
11 KiB
ArmAsm
/* Optimized version of the standard memcpy() function.
|
|
This file is part of the GNU C Library.
|
|
Copyright (C) 2000-2014 Free Software Foundation, Inc.
|
|
Contributed by Dan Pop for Itanium <Dan.Pop@cern.ch>.
|
|
Rewritten for McKinley by Sverre Jarp, HP Labs/CERN <Sverre.Jarp@cern.ch>
|
|
|
|
The GNU C Library is free software; you can redistribute it and/or
|
|
modify it under the terms of the GNU Lesser General Public
|
|
License as published by the Free Software Foundation; either
|
|
version 2.1 of the License, or (at your option) any later version.
|
|
|
|
The GNU C Library is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
Lesser General Public License for more details.
|
|
|
|
You should have received a copy of the GNU Lesser General Public
|
|
License along with the GNU C Library; if not, see
|
|
<http://www.gnu.org/licenses/>. */
|
|
|
|
/* Return: dest
|
|
|
|
Inputs:
|
|
in0: dest
|
|
in1: src
|
|
in2: byte count
|
|
|
|
An assembly implementation of the algorithm used by the generic C
|
|
version from glibc. The case when source and sest are aligned is
|
|
treated separately, for extra performance.
|
|
|
|
In this form, memcpy assumes little endian mode. For big endian mode,
|
|
sh1 must be computed using an extra instruction: sub sh1 = 64, sh1
|
|
and the order of r[MEMLAT] and r[MEMLAT+1] must be reverted in the
|
|
shrp instruction. */
|
|
|
|
#define USE_LFETCH
|
|
#define USE_FLP
|
|
#include <sysdep.h>
|
|
#undef ret
|
|
|
|
#define LFETCH_DIST 500
|
|
|
|
#define ALIGN_UNROLL_no 4 // no. of elements
|
|
#define ALIGN_UNROLL_sh 2 // (shift amount)
|
|
|
|
#define MEMLAT 8
|
|
#define Nrot ((4*(MEMLAT+2) + 7) & ~7)
|
|
|
|
#define OP_T_THRES 16
|
|
#define OPSIZ 8
|
|
|
|
#define loopcnt r14
|
|
#define elemcnt r15
|
|
#define saved_pr r16
|
|
#define saved_lc r17
|
|
#define adest r18
|
|
#define dest r19
|
|
#define asrc r20
|
|
#define src r21
|
|
#define len r22
|
|
#define tmp2 r23
|
|
#define tmp3 r24
|
|
#define tmp4 r25
|
|
#define ptable r26
|
|
#define ploop56 r27
|
|
#define loopaddr r28
|
|
#define sh1 r29
|
|
#define ptr1 r30
|
|
#define ptr2 r31
|
|
|
|
#define movi0 mov
|
|
|
|
#define p_scr p6
|
|
#define p_xtr p7
|
|
#define p_nxtr p8
|
|
#define p_few p9
|
|
|
|
#if defined(USE_FLP)
|
|
#define load ldf8
|
|
#define store stf8
|
|
#define tempreg f6
|
|
#define the_r fr
|
|
#define the_s fs
|
|
#define the_t ft
|
|
#define the_q fq
|
|
#define the_w fw
|
|
#define the_x fx
|
|
#define the_y fy
|
|
#define the_z fz
|
|
#elif defined(USE_INT)
|
|
#define load ld8
|
|
#define store st8
|
|
#define tempreg tmp2
|
|
#define the_r r
|
|
#define the_s s
|
|
#define the_t t
|
|
#define the_q q
|
|
#define the_w w
|
|
#define the_x x
|
|
#define the_y y
|
|
#define the_z z
|
|
#endif
|
|
|
|
#ifdef GAS_ALIGN_BREAKS_UNWIND_INFO
|
|
/* Manually force proper loop-alignment. Note: be sure to
|
|
double-check the code-layout after making any changes to
|
|
this routine! */
|
|
# define ALIGN(n) { nop 0 }
|
|
#else
|
|
# define ALIGN(n) .align n
|
|
#endif
|
|
|
|
#if defined(USE_LFETCH)
|
|
#define LOOP(shift) \
|
|
ALIGN(32); \
|
|
.loop##shift##: \
|
|
{ .mmb \
|
|
(p[0]) ld8.nt1 r[0] = [asrc], 8 ; \
|
|
(p[0]) lfetch.nt1 [ptr1], 16 ; \
|
|
nop.b 0 ; \
|
|
} { .mib \
|
|
(p[MEMLAT+1]) st8 [dest] = tmp3, 8 ; \
|
|
(p[MEMLAT]) shrp tmp3 = r[MEMLAT], s[MEMLAT+1], shift ; \
|
|
nop.b 0 ;; \
|
|
} { .mmb \
|
|
(p[0]) ld8.nt1 s[0] = [asrc], 8 ; \
|
|
(p[0]) lfetch.nt1 [ptr2], 16 ; \
|
|
nop.b 0 ; \
|
|
} { .mib \
|
|
(p[MEMLAT+1]) st8 [dest] = tmp4, 8 ; \
|
|
(p[MEMLAT]) shrp tmp4 = s[MEMLAT], r[MEMLAT], shift ; \
|
|
br.ctop.sptk.many .loop##shift \
|
|
;; } \
|
|
{ .mib \
|
|
br.cond.sptk.many .copy_bytes ; /* deal with the remaining bytes */ \
|
|
}
|
|
#else
|
|
#define LOOP(shift) \
|
|
ALIGN(32); \
|
|
.loop##shift##: \
|
|
{ .mmb \
|
|
(p[0]) ld8.nt1 r[0] = [asrc], 8 ; \
|
|
nop.b 0 ; \
|
|
} { .mib \
|
|
(p[MEMLAT+1]) st8 [dest] = tmp3, 8 ; \
|
|
(p[MEMLAT]) shrp tmp3 = r[MEMLAT], s[MEMLAT+1], shift ; \
|
|
nop.b 0 ;; \
|
|
} { .mmb \
|
|
(p[0]) ld8.nt1 s[0] = [asrc], 8 ; \
|
|
nop.b 0 ; \
|
|
} { .mib \
|
|
(p[MEMLAT+1]) st8 [dest] = tmp4, 8 ; \
|
|
(p[MEMLAT]) shrp tmp4 = s[MEMLAT], r[MEMLAT], shift ; \
|
|
br.ctop.sptk.many .loop##shift \
|
|
;; } \
|
|
{ .mib \
|
|
br.cond.sptk.many .copy_bytes ; /* deal with the remaining bytes */ \
|
|
}
|
|
#endif
|
|
|
|
|
|
ENTRY(memcpy)
|
|
{ .mmi
|
|
.prologue
|
|
alloc r2 = ar.pfs, 3, Nrot - 3, 0, Nrot
|
|
.rotr r[MEMLAT+1], s[MEMLAT+2], q[MEMLAT+1], t[MEMLAT+1]
|
|
.rotp p[MEMLAT+2]
|
|
.rotf fr[MEMLAT+1], fq[MEMLAT+1], fs[MEMLAT+1], ft[MEMLAT+1]
|
|
mov ret0 = in0 // return tmp2 = dest
|
|
.save pr, saved_pr
|
|
movi0 saved_pr = pr // save the predicate registers
|
|
} { .mmi
|
|
and tmp4 = 7, in0 // check if destination is aligned
|
|
mov dest = in0 // dest
|
|
mov src = in1 // src
|
|
;; }
|
|
{ .mii
|
|
cmp.eq p_scr, p0 = in2, r0 // if (len == 0)
|
|
.save ar.lc, saved_lc
|
|
movi0 saved_lc = ar.lc // save the loop counter
|
|
.body
|
|
cmp.ge p_few, p0 = OP_T_THRES, in2 // is len <= OP_T_THRESH
|
|
} { .mbb
|
|
mov len = in2 // len
|
|
(p_scr) br.cond.dpnt.few .restore_and_exit // Branch no. 1: return dest
|
|
(p_few) br.cond.dpnt.many .copy_bytes // Branch no. 2: copy byte by byte
|
|
;; }
|
|
{ .mmi
|
|
#if defined(USE_LFETCH)
|
|
lfetch.nt1 [dest] //
|
|
lfetch.nt1 [src] //
|
|
#endif
|
|
shr.u elemcnt = len, 3 // elemcnt = len / 8
|
|
} { .mib
|
|
cmp.eq p_scr, p0 = tmp4, r0 // is destination aligned?
|
|
sub loopcnt = 7, tmp4 //
|
|
(p_scr) br.cond.dptk.many .dest_aligned
|
|
;; }
|
|
{ .mmi
|
|
ld1 tmp2 = [src], 1 //
|
|
sub len = len, loopcnt, 1 // reduce len
|
|
movi0 ar.lc = loopcnt //
|
|
} { .mib
|
|
cmp.ne p_scr, p0 = 0, loopcnt // avoid loading beyond end-point
|
|
;; }
|
|
|
|
.l0: // ---------------------------- // L0: Align src on 8-byte boundary
|
|
{ .mmi
|
|
st1 [dest] = tmp2, 1 //
|
|
(p_scr) ld1 tmp2 = [src], 1 //
|
|
} { .mib
|
|
cmp.lt p_scr, p0 = 1, loopcnt // avoid load beyond end-point
|
|
add loopcnt = -1, loopcnt
|
|
br.cloop.dptk.few .l0 //
|
|
;; }
|
|
|
|
.dest_aligned:
|
|
{ .mmi
|
|
and tmp4 = 7, src // ready for alignment check
|
|
shr.u elemcnt = len, 3 // elemcnt = len / 8
|
|
;; }
|
|
{ .mib
|
|
cmp.ne p_scr, p0 = tmp4, r0 // is source also aligned
|
|
tbit.nz p_xtr, p_nxtr = src, 3 // prepare a separate move if src
|
|
} { .mib // is not 16B aligned
|
|
add ptr2 = LFETCH_DIST, dest // prefetch address
|
|
add ptr1 = LFETCH_DIST, src
|
|
(p_scr) br.cond.dptk.many .src_not_aligned
|
|
;; }
|
|
|
|
// The optimal case, when dest, and src are aligned
|
|
|
|
.both_aligned:
|
|
{ .mmi
|
|
.pred.rel "mutex",p_xtr,p_nxtr
|
|
(p_xtr) cmp.gt p_scr, p0 = ALIGN_UNROLL_no+1, elemcnt // Need N + 1 to qualify
|
|
(p_nxtr) cmp.gt p_scr, p0 = ALIGN_UNROLL_no, elemcnt // Need only N to qualify
|
|
movi0 pr.rot = 1 << 16 // set rotating predicates
|
|
} { .mib
|
|
(p_scr) br.cond.dpnt.many .copy_full_words
|
|
;; }
|
|
|
|
{ .mmi
|
|
(p_xtr) load tempreg = [src], 8
|
|
(p_xtr) add elemcnt = -1, elemcnt
|
|
movi0 ar.ec = MEMLAT + 1 // set the epilog counter
|
|
;; }
|
|
{ .mmi
|
|
(p_xtr) add len = -8, len //
|
|
add asrc = 16, src // one bank apart (for USE_INT)
|
|
shr.u loopcnt = elemcnt, ALIGN_UNROLL_sh // cater for unrolling
|
|
;;}
|
|
{ .mmi
|
|
add loopcnt = -1, loopcnt
|
|
(p_xtr) store [dest] = tempreg, 8 // copy the "extra" word
|
|
nop.i 0
|
|
;; }
|
|
{ .mib
|
|
add adest = 16, dest
|
|
movi0 ar.lc = loopcnt // set the loop counter
|
|
;; }
|
|
|
|
#ifdef GAS_ALIGN_BREAKS_UNWIND_INFO
|
|
{ nop 0 }
|
|
#else
|
|
.align 32
|
|
#endif
|
|
#if defined(USE_FLP)
|
|
.l1: // ------------------------------- // L1: Everything a multiple of 8
|
|
{ .mmi
|
|
#if defined(USE_LFETCH)
|
|
(p[0]) lfetch.nt1 [ptr2],32
|
|
#endif
|
|
(p[0]) ldfp8 the_r[0],the_q[0] = [src], 16
|
|
(p[0]) add len = -32, len
|
|
} {.mmb
|
|
(p[MEMLAT]) store [dest] = the_r[MEMLAT], 8
|
|
(p[MEMLAT]) store [adest] = the_s[MEMLAT], 8
|
|
;; }
|
|
{ .mmi
|
|
#if defined(USE_LFETCH)
|
|
(p[0]) lfetch.nt1 [ptr1],32
|
|
#endif
|
|
(p[0]) ldfp8 the_s[0], the_t[0] = [src], 16
|
|
} {.mmb
|
|
(p[MEMLAT]) store [dest] = the_q[MEMLAT], 24
|
|
(p[MEMLAT]) store [adest] = the_t[MEMLAT], 24
|
|
br.ctop.dptk.many .l1
|
|
;; }
|
|
#elif defined(USE_INT)
|
|
.l1: // ------------------------------- // L1: Everything a multiple of 8
|
|
{ .mmi
|
|
(p[0]) load the_r[0] = [src], 8
|
|
(p[0]) load the_q[0] = [asrc], 8
|
|
(p[0]) add len = -32, len
|
|
} {.mmb
|
|
(p[MEMLAT]) store [dest] = the_r[MEMLAT], 8
|
|
(p[MEMLAT]) store [adest] = the_q[MEMLAT], 8
|
|
;; }
|
|
{ .mmi
|
|
(p[0]) load the_s[0] = [src], 24
|
|
(p[0]) load the_t[0] = [asrc], 24
|
|
} {.mmb
|
|
(p[MEMLAT]) store [dest] = the_s[MEMLAT], 24
|
|
(p[MEMLAT]) store [adest] = the_t[MEMLAT], 24
|
|
#if defined(USE_LFETCH)
|
|
;; }
|
|
{ .mmb
|
|
(p[0]) lfetch.nt1 [ptr2],32
|
|
(p[0]) lfetch.nt1 [ptr1],32
|
|
#endif
|
|
br.ctop.dptk.many .l1
|
|
;; }
|
|
#endif
|
|
|
|
.copy_full_words:
|
|
{ .mib
|
|
cmp.gt p_scr, p0 = 8, len //
|
|
shr.u elemcnt = len, 3 //
|
|
(p_scr) br.cond.dpnt.many .copy_bytes
|
|
;; }
|
|
{ .mii
|
|
load tempreg = [src], 8
|
|
add loopcnt = -1, elemcnt //
|
|
;; }
|
|
{ .mii
|
|
cmp.ne p_scr, p0 = 0, loopcnt //
|
|
mov ar.lc = loopcnt //
|
|
;; }
|
|
|
|
.l2: // ------------------------------- // L2: Max 4 words copied separately
|
|
{ .mmi
|
|
store [dest] = tempreg, 8
|
|
(p_scr) load tempreg = [src], 8 //
|
|
add len = -8, len
|
|
} { .mib
|
|
cmp.lt p_scr, p0 = 1, loopcnt // avoid load beyond end-point
|
|
add loopcnt = -1, loopcnt
|
|
br.cloop.dptk.few .l2
|
|
;; }
|
|
|
|
.copy_bytes:
|
|
{ .mib
|
|
cmp.eq p_scr, p0 = len, r0 // is len == 0 ?
|
|
add loopcnt = -1, len // len--;
|
|
(p_scr) br.cond.spnt .restore_and_exit
|
|
;; }
|
|
{ .mii
|
|
ld1 tmp2 = [src], 1
|
|
movi0 ar.lc = loopcnt
|
|
cmp.ne p_scr, p0 = 0, loopcnt // avoid load beyond end-point
|
|
;; }
|
|
|
|
.l3: // ------------------------------- // L3: Final byte move
|
|
{ .mmi
|
|
st1 [dest] = tmp2, 1
|
|
(p_scr) ld1 tmp2 = [src], 1
|
|
} { .mib
|
|
cmp.lt p_scr, p0 = 1, loopcnt // avoid load beyond end-point
|
|
add loopcnt = -1, loopcnt
|
|
br.cloop.dptk.few .l3
|
|
;; }
|
|
|
|
.restore_and_exit:
|
|
{ .mmi
|
|
movi0 pr = saved_pr, -1 // restore the predicate registers
|
|
;; }
|
|
{ .mib
|
|
movi0 ar.lc = saved_lc // restore the loop counter
|
|
br.ret.sptk.many b0
|
|
;; }
|
|
|
|
|
|
.src_not_aligned:
|
|
{ .mmi
|
|
cmp.gt p_scr, p0 = 16, len
|
|
and sh1 = 7, src // sh1 = src % 8
|
|
shr.u loopcnt = len, 4 // element-cnt = len / 16
|
|
} { .mib
|
|
add tmp4 = @ltoff(.table), gp
|
|
add tmp3 = @ltoff(.loop56), gp
|
|
(p_scr) br.cond.dpnt.many .copy_bytes // do byte by byte if too few
|
|
;; }
|
|
{ .mmi
|
|
and asrc = -8, src // asrc = (-8) -- align src for loop
|
|
add loopcnt = -1, loopcnt // loopcnt--
|
|
shl sh1 = sh1, 3 // sh1 = 8 * (src % 8)
|
|
} { .mmi
|
|
ld8 ptable = [tmp4] // ptable = &table
|
|
ld8 ploop56 = [tmp3] // ploop56 = &loop56
|
|
and tmp2 = -16, len // tmp2 = len & -OPSIZ
|
|
;; }
|
|
{ .mmi
|
|
add tmp3 = ptable, sh1 // tmp3 = &table + sh1
|
|
add src = src, tmp2 // src += len & (-16)
|
|
movi0 ar.lc = loopcnt // set LC
|
|
;; }
|
|
{ .mmi
|
|
ld8 tmp4 = [tmp3] // tmp4 = loop offset
|
|
sub len = len, tmp2 // len -= len & (-16)
|
|
movi0 ar.ec = MEMLAT + 2 // one more pass needed
|
|
;; }
|
|
{ .mmi
|
|
ld8 s[1] = [asrc], 8 // preload
|
|
sub loopaddr = ploop56,tmp4 // loopadd = &loop56 - loop offset
|
|
movi0 pr.rot = 1 << 16 // set rotating predicates
|
|
;; }
|
|
{ .mib
|
|
nop.m 0
|
|
movi0 b6 = loopaddr
|
|
br b6 // jump to the appropriate loop
|
|
;; }
|
|
|
|
LOOP(8)
|
|
LOOP(16)
|
|
LOOP(24)
|
|
LOOP(32)
|
|
LOOP(40)
|
|
LOOP(48)
|
|
LOOP(56)
|
|
END(memcpy)
|
|
libc_hidden_builtin_def (memcpy)
|
|
|
|
.rodata
|
|
.align 8
|
|
.table:
|
|
data8 0 // dummy entry
|
|
data8 .loop56 - .loop8
|
|
data8 .loop56 - .loop16
|
|
data8 .loop56 - .loop24
|
|
data8 .loop56 - .loop32
|
|
data8 .loop56 - .loop40
|
|
data8 .loop56 - .loop48
|
|
data8 .loop56 - .loop56
|