2013-12-17 20:42:32 +01:00
|
|
|
/*
|
|
|
|
* AArch64 specific helper definitions
|
|
|
|
*
|
|
|
|
* Copyright (c) 2013 Alexander Graf <agraf@suse.de>
|
|
|
|
*
|
|
|
|
* This library is free software; you can redistribute it and/or
|
|
|
|
* modify it under the terms of the GNU Lesser General Public
|
|
|
|
* License as published by the Free Software Foundation; either
|
2020-10-23 14:29:13 +02:00
|
|
|
* version 2.1 of the License, or (at your option) any later version.
|
2013-12-17 20:42:32 +01:00
|
|
|
*
|
|
|
|
* This library is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
* Lesser General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU Lesser General Public
|
|
|
|
* License along with this library; if not, see <http://www.gnu.org/licenses/>.
|
|
|
|
*/
|
2013-12-17 20:42:34 +01:00
|
|
|
DEF_HELPER_FLAGS_2(udiv64, TCG_CALL_NO_RWG_SE, i64, i64, i64)
|
|
|
|
DEF_HELPER_FLAGS_2(sdiv64, TCG_CALL_NO_RWG_SE, s64, s64, s64)
|
2013-12-17 20:42:35 +01:00
|
|
|
DEF_HELPER_FLAGS_1(rbit64, TCG_CALL_NO_RWG_SE, i64, i64)
|
2019-03-01 21:04:55 +01:00
|
|
|
DEF_HELPER_2(msr_i_spsel, void, env, i32)
|
|
|
|
DEF_HELPER_2(msr_i_daifset, void, env, i32)
|
|
|
|
DEF_HELPER_2(msr_i_daifclear, void, env, i32)
|
2018-05-15 15:58:43 +02:00
|
|
|
DEF_HELPER_3(vfp_cmph_a64, i64, f16, f16, ptr)
|
|
|
|
DEF_HELPER_3(vfp_cmpeh_a64, i64, f16, f16, ptr)
|
2014-01-04 23:15:50 +01:00
|
|
|
DEF_HELPER_3(vfp_cmps_a64, i64, f32, f32, ptr)
|
|
|
|
DEF_HELPER_3(vfp_cmpes_a64, i64, f32, f32, ptr)
|
|
|
|
DEF_HELPER_3(vfp_cmpd_a64, i64, f64, f64, ptr)
|
|
|
|
DEF_HELPER_3(vfp_cmped_a64, i64, f64, f64, ptr)
|
2021-02-25 00:05:32 +01:00
|
|
|
DEF_HELPER_FLAGS_4(simd_tblx, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
|
2014-02-20 11:35:48 +01:00
|
|
|
DEF_HELPER_FLAGS_3(vfp_mulxs, TCG_CALL_NO_RWG, f32, f32, f32, ptr)
|
|
|
|
DEF_HELPER_FLAGS_3(vfp_mulxd, TCG_CALL_NO_RWG, f64, f64, f64, ptr)
|
2014-02-20 11:35:49 +01:00
|
|
|
DEF_HELPER_FLAGS_3(neon_ceq_f64, TCG_CALL_NO_RWG, i64, i64, i64, ptr)
|
|
|
|
DEF_HELPER_FLAGS_3(neon_cge_f64, TCG_CALL_NO_RWG, i64, i64, i64, ptr)
|
|
|
|
DEF_HELPER_FLAGS_3(neon_cgt_f64, TCG_CALL_NO_RWG, i64, i64, i64, ptr)
|
2018-03-01 12:05:50 +01:00
|
|
|
DEF_HELPER_FLAGS_3(recpsf_f16, TCG_CALL_NO_RWG, f16, f16, f16, ptr)
|
2014-02-20 11:35:50 +01:00
|
|
|
DEF_HELPER_FLAGS_3(recpsf_f32, TCG_CALL_NO_RWG, f32, f32, f32, ptr)
|
|
|
|
DEF_HELPER_FLAGS_3(recpsf_f64, TCG_CALL_NO_RWG, f64, f64, f64, ptr)
|
2018-03-01 12:05:50 +01:00
|
|
|
DEF_HELPER_FLAGS_3(rsqrtsf_f16, TCG_CALL_NO_RWG, f16, f16, f16, ptr)
|
2014-02-20 11:35:50 +01:00
|
|
|
DEF_HELPER_FLAGS_3(rsqrtsf_f32, TCG_CALL_NO_RWG, f32, f32, f32, ptr)
|
|
|
|
DEF_HELPER_FLAGS_3(rsqrtsf_f64, TCG_CALL_NO_RWG, f64, f64, f64, ptr)
|
2014-03-17 17:31:48 +01:00
|
|
|
DEF_HELPER_FLAGS_1(neon_addlp_s8, TCG_CALL_NO_RWG_SE, i64, i64)
|
|
|
|
DEF_HELPER_FLAGS_1(neon_addlp_u8, TCG_CALL_NO_RWG_SE, i64, i64)
|
|
|
|
DEF_HELPER_FLAGS_1(neon_addlp_s16, TCG_CALL_NO_RWG_SE, i64, i64)
|
|
|
|
DEF_HELPER_FLAGS_1(neon_addlp_u16, TCG_CALL_NO_RWG_SE, i64, i64)
|
2014-03-17 17:31:50 +01:00
|
|
|
DEF_HELPER_FLAGS_2(frecpx_f64, TCG_CALL_NO_RWG, f64, f64, ptr)
|
|
|
|
DEF_HELPER_FLAGS_2(frecpx_f32, TCG_CALL_NO_RWG, f32, f32, ptr)
|
2018-03-01 12:05:55 +01:00
|
|
|
DEF_HELPER_FLAGS_2(frecpx_f16, TCG_CALL_NO_RWG, f16, f16, ptr)
|
2014-03-17 17:31:53 +01:00
|
|
|
DEF_HELPER_FLAGS_2(fcvtx_f64_to_f32, TCG_CALL_NO_RWG, f32, f64, env)
|
2014-06-09 16:43:25 +02:00
|
|
|
DEF_HELPER_FLAGS_3(crc32_64, TCG_CALL_NO_RWG_SE, i64, i64, i64, i32)
|
|
|
|
DEF_HELPER_FLAGS_3(crc32c_64, TCG_CALL_NO_RWG_SE, i64, i64, i64, i32)
|
target-arm: emulate aarch64's LL/SC using cmpxchg helpers
Emulating LL/SC with cmpxchg is not correct, since it can
suffer from the ABA problem. Portable parallel code, however,
is written assuming only cmpxchg--and not LL/SC--is available.
This means that in practice emulating LL/SC with cmpxchg is
a viable alternative.
The appended emulates LL/SC pairs in aarch64 with cmpxchg helpers.
This works in both user and system mode. In usermode, it avoids
pausing all other CPUs to perform the LL/SC pair. The subsequent
performance and scalability improvement is significant, as the
plots below show. They plot the throughput of atomic_add-bench
compiled for ARM and executed on a 64-core x86 machine.
Hi-res plots: http://imgur.com/a/JVc8Y
atomic_add-bench: 1000000 ops/thread, [0,1] range
18 ++---------+----------+---------+----------+----------+----------+---++
+cmpxchg +-E--+ + + + + + |
16 ++master +-H--+ ++
|| |
14 ++ ++
| | |
12 ++| ++
| | |
10 ++++ ++
8 ++E ++
|+++ |
6 ++ | ++
| | |
4 ++ | ++
| | |
2 +H++E+--- ++
+ | +E++----+E+---+--+E+----++E+------+E+------+E++----+E+---+--+E|
0 ++H-H----H-+-----H----+---------+----------+----------+----------+---++
0 10 20 30 40 50 60
Number of threads
atomic_add-bench: 1000000 ops/thread, [0,2] range
18 ++---------+----------+---------+----------+----------+----------+---++
+cmpxchg +-E--+ + + + + + |
16 ++master +-H--+ ++
| | |
14 ++E ++
| | |
12 ++| ++
|+++ |
10 ++ | ++
8 ++ | ++
| | |
6 ++ | ++
| | |
4 ++ | ++
| +E+--- |
2 +H+ +E+-----+++ +++ +++ ---+E+-----+E+------+++
+++ + +E+---+--+E+----++E+------+E+--- ++++ +++ + +E|
0 ++H-H----H-+-----H----+---------+----------+----------+----------+---++
0 10 20 30 40 50 60
Number of threads
atomic_add-bench: 1000000 ops/thread, [0,128] range
70 ++---------+----------+---------+----------+----------+----------+---++
+cmpxchg +-E--+ + + + + + |
60 ++master +-H--+ +++ ---+E+-----+E+------+E+
| +E+------E-------+E+--- |
| --- +++ |
50 ++ +++--- ++
| -+E+ |
40 ++ +++---- ++
| E- |
| --| |
30 ++ -- +++ ++
| +E+ |
20 ++E+ ++
|E+ |
| |
10 ++ ++
+ + + + + + + |
0 +HH-H----H-+-----H----+---------+----------+----------+----------+---++
0 10 20 30 40 50 60
Number of threads
atomic_add-bench: 1000000 ops/thread, [0,1024] range
160 ++---------+---------+----------+---------+----------+----------+---++
+cmpxchg +-E--+ + + + + + |
140 ++master +-H--+ +++ +++
| -+E+-----+E+-------E|
120 ++ +++ ---- +++
| +++ ----E-- |
100 ++ --E--- +++ ++
| +++ ---- +++ |
80 ++ --E-- ++
| ---- +++ |
| -+E+ |
60 ++ ---- +++ ++
| +E+- |
40 ++ -- ++
| +E+ |
20 +EE+ ++
+++ + + + + + + |
0 +HH-H---H--+-----H---+----------+---------+----------+----------+---++
0 10 20 30 40 50 60
Number of threads
[rth: Rearrange 128-bit cmpxchg helper. Enforce alignment on LL.]
Signed-off-by: Emilio G. Cota <cota@braap.org>
Message-Id: <1467054136-10430-28-git-send-email-cota@braap.org>
Signed-off-by: Richard Henderson <rth@twiddle.net>
2016-06-27 21:02:13 +02:00
|
|
|
DEF_HELPER_FLAGS_4(paired_cmpxchg64_le, TCG_CALL_NO_WG, i64, env, i64, i64, i64)
|
2017-07-15 00:20:49 +02:00
|
|
|
DEF_HELPER_FLAGS_4(paired_cmpxchg64_le_parallel, TCG_CALL_NO_WG,
|
|
|
|
i64, env, i64, i64, i64)
|
target-arm: emulate aarch64's LL/SC using cmpxchg helpers
Emulating LL/SC with cmpxchg is not correct, since it can
suffer from the ABA problem. Portable parallel code, however,
is written assuming only cmpxchg--and not LL/SC--is available.
This means that in practice emulating LL/SC with cmpxchg is
a viable alternative.
The appended emulates LL/SC pairs in aarch64 with cmpxchg helpers.
This works in both user and system mode. In usermode, it avoids
pausing all other CPUs to perform the LL/SC pair. The subsequent
performance and scalability improvement is significant, as the
plots below show. They plot the throughput of atomic_add-bench
compiled for ARM and executed on a 64-core x86 machine.
Hi-res plots: http://imgur.com/a/JVc8Y
atomic_add-bench: 1000000 ops/thread, [0,1] range
18 ++---------+----------+---------+----------+----------+----------+---++
+cmpxchg +-E--+ + + + + + |
16 ++master +-H--+ ++
|| |
14 ++ ++
| | |
12 ++| ++
| | |
10 ++++ ++
8 ++E ++
|+++ |
6 ++ | ++
| | |
4 ++ | ++
| | |
2 +H++E+--- ++
+ | +E++----+E+---+--+E+----++E+------+E+------+E++----+E+---+--+E|
0 ++H-H----H-+-----H----+---------+----------+----------+----------+---++
0 10 20 30 40 50 60
Number of threads
atomic_add-bench: 1000000 ops/thread, [0,2] range
18 ++---------+----------+---------+----------+----------+----------+---++
+cmpxchg +-E--+ + + + + + |
16 ++master +-H--+ ++
| | |
14 ++E ++
| | |
12 ++| ++
|+++ |
10 ++ | ++
8 ++ | ++
| | |
6 ++ | ++
| | |
4 ++ | ++
| +E+--- |
2 +H+ +E+-----+++ +++ +++ ---+E+-----+E+------+++
+++ + +E+---+--+E+----++E+------+E+--- ++++ +++ + +E|
0 ++H-H----H-+-----H----+---------+----------+----------+----------+---++
0 10 20 30 40 50 60
Number of threads
atomic_add-bench: 1000000 ops/thread, [0,128] range
70 ++---------+----------+---------+----------+----------+----------+---++
+cmpxchg +-E--+ + + + + + |
60 ++master +-H--+ +++ ---+E+-----+E+------+E+
| +E+------E-------+E+--- |
| --- +++ |
50 ++ +++--- ++
| -+E+ |
40 ++ +++---- ++
| E- |
| --| |
30 ++ -- +++ ++
| +E+ |
20 ++E+ ++
|E+ |
| |
10 ++ ++
+ + + + + + + |
0 +HH-H----H-+-----H----+---------+----------+----------+----------+---++
0 10 20 30 40 50 60
Number of threads
atomic_add-bench: 1000000 ops/thread, [0,1024] range
160 ++---------+---------+----------+---------+----------+----------+---++
+cmpxchg +-E--+ + + + + + |
140 ++master +-H--+ +++ +++
| -+E+-----+E+-------E|
120 ++ +++ ---- +++
| +++ ----E-- |
100 ++ --E--- +++ ++
| +++ ---- +++ |
80 ++ --E-- ++
| ---- +++ |
| -+E+ |
60 ++ ---- +++ ++
| +E+- |
40 ++ -- ++
| +E+ |
20 +EE+ ++
+++ + + + + + + |
0 +HH-H---H--+-----H---+----------+---------+----------+----------+---++
0 10 20 30 40 50 60
Number of threads
[rth: Rearrange 128-bit cmpxchg helper. Enforce alignment on LL.]
Signed-off-by: Emilio G. Cota <cota@braap.org>
Message-Id: <1467054136-10430-28-git-send-email-cota@braap.org>
Signed-off-by: Richard Henderson <rth@twiddle.net>
2016-06-27 21:02:13 +02:00
|
|
|
DEF_HELPER_FLAGS_4(paired_cmpxchg64_be, TCG_CALL_NO_WG, i64, env, i64, i64, i64)
|
2017-07-15 00:20:49 +02:00
|
|
|
DEF_HELPER_FLAGS_4(paired_cmpxchg64_be_parallel, TCG_CALL_NO_WG,
|
|
|
|
i64, env, i64, i64, i64)
|
2018-05-10 19:10:57 +02:00
|
|
|
DEF_HELPER_5(casp_le_parallel, void, env, i32, i64, i64, i64)
|
|
|
|
DEF_HELPER_5(casp_be_parallel, void, env, i32, i64, i64, i64)
|
2018-03-01 12:05:48 +01:00
|
|
|
DEF_HELPER_FLAGS_3(advsimd_maxh, TCG_CALL_NO_RWG, f16, f16, f16, ptr)
|
|
|
|
DEF_HELPER_FLAGS_3(advsimd_minh, TCG_CALL_NO_RWG, f16, f16, f16, ptr)
|
|
|
|
DEF_HELPER_FLAGS_3(advsimd_maxnumh, TCG_CALL_NO_RWG, f16, f16, f16, ptr)
|
|
|
|
DEF_HELPER_FLAGS_3(advsimd_minnumh, TCG_CALL_NO_RWG, f16, f16, f16, ptr)
|
2018-03-01 12:05:49 +01:00
|
|
|
DEF_HELPER_3(advsimd_addh, f16, f16, f16, ptr)
|
|
|
|
DEF_HELPER_3(advsimd_subh, f16, f16, f16, ptr)
|
|
|
|
DEF_HELPER_3(advsimd_mulh, f16, f16, f16, ptr)
|
|
|
|
DEF_HELPER_3(advsimd_divh, f16, f16, f16, ptr)
|
2018-03-01 12:05:49 +01:00
|
|
|
DEF_HELPER_3(advsimd_ceq_f16, i32, f16, f16, ptr)
|
|
|
|
DEF_HELPER_3(advsimd_cge_f16, i32, f16, f16, ptr)
|
|
|
|
DEF_HELPER_3(advsimd_cgt_f16, i32, f16, f16, ptr)
|
|
|
|
DEF_HELPER_3(advsimd_acge_f16, i32, f16, f16, ptr)
|
|
|
|
DEF_HELPER_3(advsimd_acgt_f16, i32, f16, f16, ptr)
|
2018-03-01 12:05:50 +01:00
|
|
|
DEF_HELPER_3(advsimd_mulxh, f16, f16, f16, ptr)
|
|
|
|
DEF_HELPER_4(advsimd_muladdh, f16, f16, f16, f16, ptr)
|
2018-03-01 12:05:52 +01:00
|
|
|
DEF_HELPER_3(advsimd_add2h, i32, i32, i32, ptr)
|
|
|
|
DEF_HELPER_3(advsimd_sub2h, i32, i32, i32, ptr)
|
|
|
|
DEF_HELPER_3(advsimd_mul2h, i32, i32, i32, ptr)
|
|
|
|
DEF_HELPER_3(advsimd_div2h, i32, i32, i32, ptr)
|
|
|
|
DEF_HELPER_3(advsimd_max2h, i32, i32, i32, ptr)
|
|
|
|
DEF_HELPER_3(advsimd_min2h, i32, i32, i32, ptr)
|
|
|
|
DEF_HELPER_3(advsimd_maxnum2h, i32, i32, i32, ptr)
|
|
|
|
DEF_HELPER_3(advsimd_minnum2h, i32, i32, i32, ptr)
|
|
|
|
DEF_HELPER_3(advsimd_mulx2h, i32, i32, i32, ptr)
|
|
|
|
DEF_HELPER_4(advsimd_muladd2h, i32, i32, i32, i32, ptr)
|
2018-03-01 12:05:53 +01:00
|
|
|
DEF_HELPER_2(advsimd_rinth_exact, f16, f16, ptr)
|
|
|
|
DEF_HELPER_2(advsimd_rinth, f16, f16, ptr)
|
2018-03-01 12:05:53 +01:00
|
|
|
DEF_HELPER_2(advsimd_f16tosinth, i32, f16, ptr)
|
|
|
|
DEF_HELPER_2(advsimd_f16touinth, i32, f16, ptr)
|
2018-03-01 12:05:55 +01:00
|
|
|
DEF_HELPER_2(sqrt_f16, f16, f16, ptr)
|
2019-01-21 11:23:11 +01:00
|
|
|
|
2019-01-21 11:23:12 +01:00
|
|
|
DEF_HELPER_2(exception_return, void, env, i64)
|
2020-03-05 17:09:21 +01:00
|
|
|
DEF_HELPER_FLAGS_2(dc_zva, TCG_CALL_NO_WG, void, env, i64)
|
2019-01-21 11:23:12 +01:00
|
|
|
|
2019-01-21 11:23:11 +01:00
|
|
|
DEF_HELPER_FLAGS_3(pacia, TCG_CALL_NO_WG, i64, env, i64, i64)
|
|
|
|
DEF_HELPER_FLAGS_3(pacib, TCG_CALL_NO_WG, i64, env, i64, i64)
|
|
|
|
DEF_HELPER_FLAGS_3(pacda, TCG_CALL_NO_WG, i64, env, i64, i64)
|
|
|
|
DEF_HELPER_FLAGS_3(pacdb, TCG_CALL_NO_WG, i64, env, i64, i64)
|
|
|
|
DEF_HELPER_FLAGS_3(pacga, TCG_CALL_NO_WG, i64, env, i64, i64)
|
|
|
|
DEF_HELPER_FLAGS_3(autia, TCG_CALL_NO_WG, i64, env, i64, i64)
|
|
|
|
DEF_HELPER_FLAGS_3(autib, TCG_CALL_NO_WG, i64, env, i64, i64)
|
|
|
|
DEF_HELPER_FLAGS_3(autda, TCG_CALL_NO_WG, i64, env, i64, i64)
|
|
|
|
DEF_HELPER_FLAGS_3(autdb, TCG_CALL_NO_WG, i64, env, i64, i64)
|
|
|
|
DEF_HELPER_FLAGS_2(xpaci, TCG_CALL_NO_RWG_SE, i64, env, i64)
|
|
|
|
DEF_HELPER_FLAGS_2(xpacd, TCG_CALL_NO_RWG_SE, i64, env, i64)
|
2020-06-26 05:31:07 +02:00
|
|
|
|
2021-04-16 20:31:03 +02:00
|
|
|
DEF_HELPER_FLAGS_3(mte_check, TCG_CALL_NO_WG, i64, env, i32, i64)
|
2020-06-26 05:31:25 +02:00
|
|
|
DEF_HELPER_FLAGS_3(mte_check_zva, TCG_CALL_NO_WG, i64, env, i32, i64)
|
2020-06-26 05:31:07 +02:00
|
|
|
DEF_HELPER_FLAGS_3(irg, TCG_CALL_NO_RWG, i64, env, i64, i64)
|
2020-06-26 05:31:09 +02:00
|
|
|
DEF_HELPER_FLAGS_4(addsubg, TCG_CALL_NO_RWG_SE, i64, env, i64, s32, i32)
|
2020-06-26 05:31:13 +02:00
|
|
|
DEF_HELPER_FLAGS_3(ldg, TCG_CALL_NO_WG, i64, env, i64, i64)
|
|
|
|
DEF_HELPER_FLAGS_3(stg, TCG_CALL_NO_WG, void, env, i64, i64)
|
|
|
|
DEF_HELPER_FLAGS_3(stg_parallel, TCG_CALL_NO_WG, void, env, i64, i64)
|
|
|
|
DEF_HELPER_FLAGS_2(stg_stub, TCG_CALL_NO_WG, void, env, i64)
|
|
|
|
DEF_HELPER_FLAGS_3(st2g, TCG_CALL_NO_WG, void, env, i64, i64)
|
|
|
|
DEF_HELPER_FLAGS_3(st2g_parallel, TCG_CALL_NO_WG, void, env, i64, i64)
|
|
|
|
DEF_HELPER_FLAGS_2(st2g_stub, TCG_CALL_NO_WG, void, env, i64)
|
2020-06-26 05:31:17 +02:00
|
|
|
DEF_HELPER_FLAGS_2(ldgm, TCG_CALL_NO_WG, i64, env, i64)
|
|
|
|
DEF_HELPER_FLAGS_3(stgm, TCG_CALL_NO_WG, void, env, i64, i64)
|
|
|
|
DEF_HELPER_FLAGS_3(stzgm_tags, TCG_CALL_NO_WG, void, env, i64, i64)
|