target/arm: Use tcg_gen_atomic_cmpxchg_i128 for STXP
Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Message-Id: <20221112042555.2622152-2-richard.henderson@linaro.org>
This commit is contained in:
Richard Henderson
parent
d1beee4da1
commit
546789c7df
@ -505,110 +505,6 @@ uint64_t HELPER(crc32c_64)(uint64_t acc, uint64_t val, uint32_t bytes)
|
||||
return crc32c(acc, buf, bytes) ^ 0xffffffff;
|
||||
}
|
||||
|
||||
uint64_t HELPER(paired_cmpxchg64_le)(CPUARMState *env, uint64_t addr,
|
||||
uint64_t new_lo, uint64_t new_hi)
|
||||
{
|
||||
Int128 cmpv = int128_make128(env->exclusive_val, env->exclusive_high);
|
||||
Int128 newv = int128_make128(new_lo, new_hi);
|
||||
Int128 oldv;
|
||||
uintptr_t ra = GETPC();
|
||||
uint64_t o0, o1;
|
||||
bool success;
|
||||
int mem_idx = cpu_mmu_index(env, false);
|
||||
MemOpIdx oi0 = make_memop_idx(MO_LEUQ | MO_ALIGN_16, mem_idx);
|
||||
MemOpIdx oi1 = make_memop_idx(MO_LEUQ, mem_idx);
|
||||
|
||||
o0 = cpu_ldq_le_mmu(env, addr + 0, oi0, ra);
|
||||
o1 = cpu_ldq_le_mmu(env, addr + 8, oi1, ra);
|
||||
oldv = int128_make128(o0, o1);
|
||||
|
||||
success = int128_eq(oldv, cmpv);
|
||||
if (success) {
|
||||
cpu_stq_le_mmu(env, addr + 0, int128_getlo(newv), oi1, ra);
|
||||
cpu_stq_le_mmu(env, addr + 8, int128_gethi(newv), oi1, ra);
|
||||
}
|
||||
|
||||
return !success;
|
||||
}
|
||||
|
||||
uint64_t HELPER(paired_cmpxchg64_le_parallel)(CPUARMState *env, uint64_t addr,
|
||||
uint64_t new_lo, uint64_t new_hi)
|
||||
{
|
||||
Int128 oldv, cmpv, newv;
|
||||
uintptr_t ra = GETPC();
|
||||
bool success;
|
||||
int mem_idx;
|
||||
MemOpIdx oi;
|
||||
|
||||
assert(HAVE_CMPXCHG128);
|
||||
|
||||
mem_idx = cpu_mmu_index(env, false);
|
||||
oi = make_memop_idx(MO_LE | MO_128 | MO_ALIGN, mem_idx);
|
||||
|
||||
cmpv = int128_make128(env->exclusive_val, env->exclusive_high);
|
||||
newv = int128_make128(new_lo, new_hi);
|
||||
oldv = cpu_atomic_cmpxchgo_le_mmu(env, addr, cmpv, newv, oi, ra);
|
||||
|
||||
success = int128_eq(oldv, cmpv);
|
||||
return !success;
|
||||
}
|
||||
|
||||
uint64_t HELPER(paired_cmpxchg64_be)(CPUARMState *env, uint64_t addr,
|
||||
uint64_t new_lo, uint64_t new_hi)
|
||||
{
|
||||
/*
|
||||
* High and low need to be switched here because this is not actually a
|
||||
* 128bit store but two doublewords stored consecutively
|
||||
*/
|
||||
Int128 cmpv = int128_make128(env->exclusive_high, env->exclusive_val);
|
||||
Int128 newv = int128_make128(new_hi, new_lo);
|
||||
Int128 oldv;
|
||||
uintptr_t ra = GETPC();
|
||||
uint64_t o0, o1;
|
||||
bool success;
|
||||
int mem_idx = cpu_mmu_index(env, false);
|
||||
MemOpIdx oi0 = make_memop_idx(MO_BEUQ | MO_ALIGN_16, mem_idx);
|
||||
MemOpIdx oi1 = make_memop_idx(MO_BEUQ, mem_idx);
|
||||
|
||||
o1 = cpu_ldq_be_mmu(env, addr + 0, oi0, ra);
|
||||
o0 = cpu_ldq_be_mmu(env, addr + 8, oi1, ra);
|
||||
oldv = int128_make128(o0, o1);
|
||||
|
||||
success = int128_eq(oldv, cmpv);
|
||||
if (success) {
|
||||
cpu_stq_be_mmu(env, addr + 0, int128_gethi(newv), oi1, ra);
|
||||
cpu_stq_be_mmu(env, addr + 8, int128_getlo(newv), oi1, ra);
|
||||
}
|
||||
|
||||
return !success;
|
||||
}
|
||||
|
||||
uint64_t HELPER(paired_cmpxchg64_be_parallel)(CPUARMState *env, uint64_t addr,
|
||||
uint64_t new_lo, uint64_t new_hi)
|
||||
{
|
||||
Int128 oldv, cmpv, newv;
|
||||
uintptr_t ra = GETPC();
|
||||
bool success;
|
||||
int mem_idx;
|
||||
MemOpIdx oi;
|
||||
|
||||
assert(HAVE_CMPXCHG128);
|
||||
|
||||
mem_idx = cpu_mmu_index(env, false);
|
||||
oi = make_memop_idx(MO_BE | MO_128 | MO_ALIGN, mem_idx);
|
||||
|
||||
/*
|
||||
* High and low need to be switched here because this is not actually a
|
||||
* 128bit store but two doublewords stored consecutively
|
||||
*/
|
||||
cmpv = int128_make128(env->exclusive_high, env->exclusive_val);
|
||||
newv = int128_make128(new_hi, new_lo);
|
||||
oldv = cpu_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra);
|
||||
|
||||
success = int128_eq(oldv, cmpv);
|
||||
return !success;
|
||||
}
|
||||
|
||||
/* Writes back the old data into Rs. */
|
||||
void HELPER(casp_le_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr,
|
||||
uint64_t new_lo, uint64_t new_hi)
|
||||
|
||||
@ -50,12 +50,6 @@ DEF_HELPER_FLAGS_2(frecpx_f16, TCG_CALL_NO_RWG, f16, f16, ptr)
|
||||
DEF_HELPER_FLAGS_2(fcvtx_f64_to_f32, TCG_CALL_NO_RWG, f32, f64, env)
|
||||
DEF_HELPER_FLAGS_3(crc32_64, TCG_CALL_NO_RWG_SE, i64, i64, i64, i32)
|
||||
DEF_HELPER_FLAGS_3(crc32c_64, TCG_CALL_NO_RWG_SE, i64, i64, i64, i32)
|
||||
DEF_HELPER_FLAGS_4(paired_cmpxchg64_le, TCG_CALL_NO_WG, i64, env, i64, i64, i64)
|
||||
DEF_HELPER_FLAGS_4(paired_cmpxchg64_le_parallel, TCG_CALL_NO_WG,
|
||||
i64, env, i64, i64, i64)
|
||||
DEF_HELPER_FLAGS_4(paired_cmpxchg64_be, TCG_CALL_NO_WG, i64, env, i64, i64, i64)
|
||||
DEF_HELPER_FLAGS_4(paired_cmpxchg64_be_parallel, TCG_CALL_NO_WG,
|
||||
i64, env, i64, i64, i64)
|
||||
DEF_HELPER_5(casp_le_parallel, void, env, i32, i64, i64, i64)
|
||||
DEF_HELPER_5(casp_be_parallel, void, env, i32, i64, i64, i64)
|
||||
DEF_HELPER_FLAGS_3(advsimd_maxh, TCG_CALL_NO_RWG, f16, f16, f16, ptr)
|
||||
|
||||
@ -2601,32 +2601,42 @@ static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2,
|
||||
get_mem_index(s),
|
||||
MO_64 | MO_ALIGN | s->be_data);
|
||||
tcg_gen_setcond_i64(TCG_COND_NE, tmp, tmp, cpu_exclusive_val);
|
||||
} else if (tb_cflags(s->base.tb) & CF_PARALLEL) {
|
||||
if (!HAVE_CMPXCHG128) {
|
||||
gen_helper_exit_atomic(cpu_env);
|
||||
/*
|
||||
* Produce a result so we have a well-formed opcode
|
||||
* stream when the following (dead) code uses 'tmp'.
|
||||
* TCG will remove the dead ops for us.
|
||||
*/
|
||||
tcg_gen_movi_i64(tmp, 0);
|
||||
} else if (s->be_data == MO_LE) {
|
||||
gen_helper_paired_cmpxchg64_le_parallel(tmp, cpu_env,
|
||||
cpu_exclusive_addr,
|
||||
cpu_reg(s, rt),
|
||||
cpu_reg(s, rt2));
|
||||
} else {
|
||||
gen_helper_paired_cmpxchg64_be_parallel(tmp, cpu_env,
|
||||
cpu_exclusive_addr,
|
||||
cpu_reg(s, rt),
|
||||
cpu_reg(s, rt2));
|
||||
}
|
||||
} else if (s->be_data == MO_LE) {
|
||||
gen_helper_paired_cmpxchg64_le(tmp, cpu_env, cpu_exclusive_addr,
|
||||
cpu_reg(s, rt), cpu_reg(s, rt2));
|
||||
} else {
|
||||
gen_helper_paired_cmpxchg64_be(tmp, cpu_env, cpu_exclusive_addr,
|
||||
cpu_reg(s, rt), cpu_reg(s, rt2));
|
||||
TCGv_i128 t16 = tcg_temp_new_i128();
|
||||
TCGv_i128 c16 = tcg_temp_new_i128();
|
||||
TCGv_i64 a, b;
|
||||
|
||||
if (s->be_data == MO_LE) {
|
||||
tcg_gen_concat_i64_i128(t16, cpu_reg(s, rt), cpu_reg(s, rt2));
|
||||
tcg_gen_concat_i64_i128(c16, cpu_exclusive_val,
|
||||
cpu_exclusive_high);
|
||||
} else {
|
||||
tcg_gen_concat_i64_i128(t16, cpu_reg(s, rt2), cpu_reg(s, rt));
|
||||
tcg_gen_concat_i64_i128(c16, cpu_exclusive_high,
|
||||
cpu_exclusive_val);
|
||||
}
|
||||
|
||||
tcg_gen_atomic_cmpxchg_i128(t16, cpu_exclusive_addr, c16, t16,
|
||||
get_mem_index(s),
|
||||
MO_128 | MO_ALIGN | s->be_data);
|
||||
tcg_temp_free_i128(c16);
|
||||
|
||||
a = tcg_temp_new_i64();
|
||||
b = tcg_temp_new_i64();
|
||||
if (s->be_data == MO_LE) {
|
||||
tcg_gen_extr_i128_i64(a, b, t16);
|
||||
} else {
|
||||
tcg_gen_extr_i128_i64(b, a, t16);
|
||||
}
|
||||
|
||||
tcg_gen_xor_i64(a, a, cpu_exclusive_val);
|
||||
tcg_gen_xor_i64(b, b, cpu_exclusive_high);
|
||||
tcg_gen_or_i64(tmp, a, b);
|
||||
tcg_temp_free_i64(a);
|
||||
tcg_temp_free_i64(b);
|
||||
tcg_temp_free_i128(t16);
|
||||
|
||||
tcg_gen_setcondi_i64(TCG_COND_NE, tmp, tmp, 0);
|
||||
}
|
||||
} else {
|
||||
tcg_gen_atomic_cmpxchg_i64(tmp, cpu_exclusive_addr, cpu_exclusive_val,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user