OpenE2K
/
qemu-e2k
13
4
Fork 0
Code Issues 4 Pull Requests Projects 2 Releases Activity

tcg: Remove TCGv from tcg_gen_atomic_* 

Expand from TCGv to TCGTemp inline in the translators,
and validate that the size matches tcg_ctx->addr_type.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
This commit is contained in:
Richard Henderson 2023-03-28 17:25:10 -07:00
parent d5920b7280
commit 0700ceb393
2 changed files with 267 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

184
include/tcg/tcg-op.h
View File

@ -858,56 +858,148 @@ tcg_gen_qemu_st_i128(TCGv_i128 v, TCGv a, TCGArg i, MemOp m)
tcg_gen_qemu_st_i128_chk(v, tcgv_tl_temp(a), i, m, TCG_TYPE_TL); tcg_gen_qemu_st_i128_chk(v, tcgv_tl_temp(a), i, m, TCG_TYPE_TL);
} }
void tcg_gen_atomic_cmpxchg_i32(TCGv_i32, TCGv, TCGv_i32, TCGv_i32, void tcg_gen_atomic_cmpxchg_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, TCGv_i32,
TCGArg, MemOp); TCGArg, MemOp, TCGType);
void tcg_gen_atomic_cmpxchg_i64(TCGv_i64, TCGv, TCGv_i64, TCGv_i64, void tcg_gen_atomic_cmpxchg_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, TCGv_i64,
TCGArg, MemOp); TCGArg, MemOp, TCGType);
void tcg_gen_atomic_cmpxchg_i128(TCGv_i128, TCGv, TCGv_i128, TCGv_i128, void tcg_gen_atomic_cmpxchg_i128_chk(TCGv_i128, TCGTemp *, TCGv_i128,
TCGArg, MemOp); TCGv_i128, TCGArg, MemOp, TCGType);
void tcg_gen_nonatomic_cmpxchg_i32(TCGv_i32, TCGv, TCGv_i32, TCGv_i32, void tcg_gen_nonatomic_cmpxchg_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, TCGv_i32,
TCGArg, MemOp); TCGArg, MemOp, TCGType);
void tcg_gen_nonatomic_cmpxchg_i64(TCGv_i64, TCGv, TCGv_i64, TCGv_i64, void tcg_gen_nonatomic_cmpxchg_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, TCGv_i64,
TCGArg, MemOp); TCGArg, MemOp, TCGType);
void tcg_gen_nonatomic_cmpxchg_i128(TCGv_i128, TCGv, TCGv_i128, TCGv_i128, void tcg_gen_nonatomic_cmpxchg_i128_chk(TCGv_i128, TCGTemp *, TCGv_i128,
TCGArg, MemOp); TCGv_i128, TCGArg, MemOp, TCGType);
void tcg_gen_atomic_xchg_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); void tcg_gen_atomic_xchg_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32,
void tcg_gen_atomic_xchg_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); TCGArg, MemOp, TCGType);
void tcg_gen_atomic_xchg_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64,
TCGArg, MemOp, TCGType);
void tcg_gen_atomic_fetch_add_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); void tcg_gen_atomic_fetch_add_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32,
void tcg_gen_atomic_fetch_add_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); TCGArg, MemOp, TCGType);
void tcg_gen_atomic_fetch_and_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); void tcg_gen_atomic_fetch_add_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64,
void tcg_gen_atomic_fetch_and_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); TCGArg, MemOp, TCGType);
void tcg_gen_atomic_fetch_or_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); void tcg_gen_atomic_fetch_and_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32,
void tcg_gen_atomic_fetch_or_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); TCGArg, MemOp, TCGType);
void tcg_gen_atomic_fetch_xor_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); void tcg_gen_atomic_fetch_and_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64,
void tcg_gen_atomic_fetch_xor_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); TCGArg, MemOp, TCGType);
void tcg_gen_atomic_fetch_smin_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); void tcg_gen_atomic_fetch_or_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32,
void tcg_gen_atomic_fetch_smin_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); TCGArg, MemOp, TCGType);
void tcg_gen_atomic_fetch_umin_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); void tcg_gen_atomic_fetch_or_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64,
void tcg_gen_atomic_fetch_umin_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); TCGArg, MemOp, TCGType);
void tcg_gen_atomic_fetch_smax_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); void tcg_gen_atomic_fetch_xor_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32,
void tcg_gen_atomic_fetch_smax_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); TCGArg, MemOp, TCGType);
void tcg_gen_atomic_fetch_umax_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); void tcg_gen_atomic_fetch_xor_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64,
void tcg_gen_atomic_fetch_umax_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); TCGArg, MemOp, TCGType);
void tcg_gen_atomic_fetch_smin_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32,
TCGArg, MemOp, TCGType);
void tcg_gen_atomic_fetch_smin_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64,
TCGArg, MemOp, TCGType);
void tcg_gen_atomic_fetch_umin_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32,
TCGArg, MemOp, TCGType);
void tcg_gen_atomic_fetch_umin_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64,
TCGArg, MemOp, TCGType);
void tcg_gen_atomic_fetch_smax_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32,
TCGArg, MemOp, TCGType);
void tcg_gen_atomic_fetch_smax_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64,
TCGArg, MemOp, TCGType);
void tcg_gen_atomic_fetch_umax_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32,
TCGArg, MemOp, TCGType);
void tcg_gen_atomic_fetch_umax_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64,
TCGArg, MemOp, TCGType);
void tcg_gen_atomic_add_fetch_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); void tcg_gen_atomic_add_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32,
void tcg_gen_atomic_add_fetch_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); TCGArg, MemOp, TCGType);
void tcg_gen_atomic_and_fetch_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); void tcg_gen_atomic_add_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64,
void tcg_gen_atomic_and_fetch_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); TCGArg, MemOp, TCGType);
void tcg_gen_atomic_or_fetch_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); void tcg_gen_atomic_and_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32,
void tcg_gen_atomic_or_fetch_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); TCGArg, MemOp, TCGType);
void tcg_gen_atomic_xor_fetch_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); void tcg_gen_atomic_and_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64,
void tcg_gen_atomic_xor_fetch_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); TCGArg, MemOp, TCGType);
void tcg_gen_atomic_smin_fetch_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); void tcg_gen_atomic_or_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32,
void tcg_gen_atomic_smin_fetch_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); TCGArg, MemOp, TCGType);
void tcg_gen_atomic_umin_fetch_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); void tcg_gen_atomic_or_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64,
void tcg_gen_atomic_umin_fetch_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); TCGArg, MemOp, TCGType);
void tcg_gen_atomic_smax_fetch_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); void tcg_gen_atomic_xor_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32,
void tcg_gen_atomic_smax_fetch_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); TCGArg, MemOp, TCGType);
void tcg_gen_atomic_umax_fetch_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); void tcg_gen_atomic_xor_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64,
void tcg_gen_atomic_umax_fetch_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); TCGArg, MemOp, TCGType);
void tcg_gen_atomic_smin_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32,
TCGArg, MemOp, TCGType);
void tcg_gen_atomic_smin_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64,
TCGArg, MemOp, TCGType);
void tcg_gen_atomic_umin_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32,
TCGArg, MemOp, TCGType);
void tcg_gen_atomic_umin_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64,
TCGArg, MemOp, TCGType);
void tcg_gen_atomic_smax_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32,
TCGArg, MemOp, TCGType);
void tcg_gen_atomic_smax_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64,
TCGArg, MemOp, TCGType);
void tcg_gen_atomic_umax_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32,
TCGArg, MemOp, TCGType);
void tcg_gen_atomic_umax_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64,
TCGArg, MemOp, TCGType);
#define DEF_ATOMIC2(N, S) \
static inline void N##_##S(TCGv_##S r, TCGv a, TCGv_##S v, \
TCGArg i, MemOp m) \
{ N##_##S##_chk(r, tcgv_tl_temp(a), v, i, m, TCG_TYPE_TL); }
#define DEF_ATOMIC3(N, S) \
static inline void N##_##S(TCGv_##S r, TCGv a, TCGv_##S o, \
TCGv_##S n, TCGArg i, MemOp m) \
{ N##_##S##_chk(r, tcgv_tl_temp(a), o, n, i, m, TCG_TYPE_TL); }
DEF_ATOMIC3(tcg_gen_atomic_cmpxchg, i32)
DEF_ATOMIC3(tcg_gen_atomic_cmpxchg, i64)
DEF_ATOMIC3(tcg_gen_atomic_cmpxchg, i128)
DEF_ATOMIC3(tcg_gen_nonatomic_cmpxchg, i32)
DEF_ATOMIC3(tcg_gen_nonatomic_cmpxchg, i64)
DEF_ATOMIC3(tcg_gen_nonatomic_cmpxchg, i128)
DEF_ATOMIC2(tcg_gen_atomic_xchg, i32)
DEF_ATOMIC2(tcg_gen_atomic_xchg, i64)
DEF_ATOMIC2(tcg_gen_atomic_fetch_add, i32)
DEF_ATOMIC2(tcg_gen_atomic_fetch_add, i64)
DEF_ATOMIC2(tcg_gen_atomic_fetch_and, i32)
DEF_ATOMIC2(tcg_gen_atomic_fetch_and, i64)
DEF_ATOMIC2(tcg_gen_atomic_fetch_or, i32)
DEF_ATOMIC2(tcg_gen_atomic_fetch_or, i64)
DEF_ATOMIC2(tcg_gen_atomic_fetch_xor, i32)
DEF_ATOMIC2(tcg_gen_atomic_fetch_xor, i64)
DEF_ATOMIC2(tcg_gen_atomic_fetch_smin, i32)
DEF_ATOMIC2(tcg_gen_atomic_fetch_smin, i64)
DEF_ATOMIC2(tcg_gen_atomic_fetch_umin, i32)
DEF_ATOMIC2(tcg_gen_atomic_fetch_umin, i64)
DEF_ATOMIC2(tcg_gen_atomic_fetch_smax, i32)
DEF_ATOMIC2(tcg_gen_atomic_fetch_smax, i64)
DEF_ATOMIC2(tcg_gen_atomic_fetch_umax, i32)
DEF_ATOMIC2(tcg_gen_atomic_fetch_umax, i64)
DEF_ATOMIC2(tcg_gen_atomic_add_fetch, i32)
DEF_ATOMIC2(tcg_gen_atomic_add_fetch, i64)
DEF_ATOMIC2(tcg_gen_atomic_and_fetch, i32)
DEF_ATOMIC2(tcg_gen_atomic_and_fetch, i64)
DEF_ATOMIC2(tcg_gen_atomic_or_fetch, i32)
DEF_ATOMIC2(tcg_gen_atomic_or_fetch, i64)
DEF_ATOMIC2(tcg_gen_atomic_xor_fetch, i32)
DEF_ATOMIC2(tcg_gen_atomic_xor_fetch, i64)
DEF_ATOMIC2(tcg_gen_atomic_smin_fetch, i32)
DEF_ATOMIC2(tcg_gen_atomic_smin_fetch, i64)
DEF_ATOMIC2(tcg_gen_atomic_umin_fetch, i32)
DEF_ATOMIC2(tcg_gen_atomic_umin_fetch, i64)
DEF_ATOMIC2(tcg_gen_atomic_smax_fetch, i32)
DEF_ATOMIC2(tcg_gen_atomic_smax_fetch, i64)
DEF_ATOMIC2(tcg_gen_atomic_umax_fetch, i32)
DEF_ATOMIC2(tcg_gen_atomic_umax_fetch, i64)
#undef DEF_ATOMIC2
#undef DEF_ATOMIC3
void tcg_gen_mov_vec(TCGv_vec, TCGv_vec); void tcg_gen_mov_vec(TCGv_vec, TCGv_vec);
void tcg_gen_dup_i32_vec(unsigned vece, TCGv_vec, TCGv_i32); void tcg_gen_dup_i32_vec(unsigned vece, TCGv_vec, TCGv_i32);

198
tcg/tcg-op-ldst.c
View File

@ -443,22 +443,21 @@ static void canonicalize_memop_i128_as_i64(MemOp ret[2], MemOp orig)
ret[1] = mop_2; ret[1] = mop_2;
} }
static TCGv_i64 maybe_extend_addr64(TCGv addr) static TCGv_i64 maybe_extend_addr64(TCGTemp *addr)
{ {
#if TARGET_LONG_BITS == 32 if (tcg_ctx->addr_type == TCG_TYPE_I32) {
TCGv_i64 a64 = tcg_temp_ebb_new_i64(); TCGv_i64 a64 = tcg_temp_ebb_new_i64();
tcg_gen_extu_i32_i64(a64, addr); tcg_gen_extu_i32_i64(a64, temp_tcgv_i32(addr));
return a64; return a64;
#else }
return addr; return temp_tcgv_i64(addr);
#endif
} }
static void maybe_free_addr64(TCGv_i64 a64) static void maybe_free_addr64(TCGv_i64 a64)
{ {
#if TARGET_LONG_BITS == 32 if (tcg_ctx->addr_type == TCG_TYPE_I32) {
tcg_temp_free_i64(a64); tcg_temp_free_i64(a64);
#endif }
} }
static void tcg_gen_qemu_ld_i128_int(TCGv_i128 val, TCGTemp *addr, static void tcg_gen_qemu_ld_i128_int(TCGv_i128 val, TCGTemp *addr,
@ -742,17 +741,18 @@ static void * const table_cmpxchg[(MO_SIZE | MO_BSWAP) + 1] = {
WITH_ATOMIC128([MO_128 | MO_BE] = gen_helper_atomic_cmpxchgo_be) WITH_ATOMIC128([MO_128 | MO_BE] = gen_helper_atomic_cmpxchgo_be)
}; };
void tcg_gen_nonatomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv, static void tcg_gen_nonatomic_cmpxchg_i32_int(TCGv_i32 retv, TCGTemp *addr,
TCGv_i32 newv, TCGArg idx, MemOp memop) TCGv_i32 cmpv, TCGv_i32 newv,
TCGArg idx, MemOp memop)
{ {
TCGv_i32 t1 = tcg_temp_ebb_new_i32(); TCGv_i32 t1 = tcg_temp_ebb_new_i32();
TCGv_i32 t2 = tcg_temp_ebb_new_i32(); TCGv_i32 t2 = tcg_temp_ebb_new_i32();
tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE); tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE);
tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN); tcg_gen_qemu_ld_i32_int(t1, addr, idx, memop & ~MO_SIGN);
tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1); tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1);
tcg_gen_qemu_st_i32(t2, addr, idx, memop); tcg_gen_qemu_st_i32_int(t2, addr, idx, memop);
tcg_temp_free_i32(t2); tcg_temp_free_i32(t2);
if (memop & MO_SIGN) { if (memop & MO_SIGN) {
@ -763,15 +763,26 @@ void tcg_gen_nonatomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
tcg_temp_free_i32(t1); tcg_temp_free_i32(t1);
} }
void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv, void tcg_gen_nonatomic_cmpxchg_i32_chk(TCGv_i32 retv, TCGTemp *addr,
TCGv_i32 newv, TCGArg idx, MemOp memop) TCGv_i32 cmpv, TCGv_i32 newv,
TCGArg idx, MemOp memop,
TCGType addr_type)
{
tcg_debug_assert(addr_type == tcg_ctx->addr_type);
tcg_debug_assert((memop & MO_SIZE) <= MO_32);
tcg_gen_nonatomic_cmpxchg_i32_int(retv, addr, cmpv, newv, idx, memop);
}
static void tcg_gen_atomic_cmpxchg_i32_int(TCGv_i32 retv, TCGTemp *addr,
TCGv_i32 cmpv, TCGv_i32 newv,
TCGArg idx, MemOp memop)
{ {
gen_atomic_cx_i32 gen; gen_atomic_cx_i32 gen;
TCGv_i64 a64; TCGv_i64 a64;
MemOpIdx oi; MemOpIdx oi;
if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
tcg_gen_nonatomic_cmpxchg_i32(retv, addr, cmpv, newv, idx, memop); tcg_gen_nonatomic_cmpxchg_i32_int(retv, addr, cmpv, newv, idx, memop);
return; return;
} }
@ -789,14 +800,25 @@ void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
} }
} }
void tcg_gen_nonatomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv, void tcg_gen_atomic_cmpxchg_i32_chk(TCGv_i32 retv, TCGTemp *addr,
TCGv_i64 newv, TCGArg idx, MemOp memop) TCGv_i32 cmpv, TCGv_i32 newv,
TCGArg idx, MemOp memop,
TCGType addr_type)
{
tcg_debug_assert(addr_type == tcg_ctx->addr_type);
tcg_debug_assert((memop & MO_SIZE) <= MO_32);
tcg_gen_atomic_cmpxchg_i32_int(retv, addr, cmpv, newv, idx, memop);
}
static void tcg_gen_nonatomic_cmpxchg_i64_int(TCGv_i64 retv, TCGTemp *addr,
TCGv_i64 cmpv, TCGv_i64 newv,
TCGArg idx, MemOp memop)
{ {
TCGv_i64 t1, t2; TCGv_i64 t1, t2;
if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
tcg_gen_nonatomic_cmpxchg_i32(TCGV_LOW(retv), addr, TCGV_LOW(cmpv), tcg_gen_nonatomic_cmpxchg_i32_int(TCGV_LOW(retv), addr, TCGV_LOW(cmpv),
TCGV_LOW(newv), idx, memop); TCGV_LOW(newv), idx, memop);
if (memop & MO_SIGN) { if (memop & MO_SIGN) {
tcg_gen_sari_i32(TCGV_HIGH(retv), TCGV_LOW(retv), 31); tcg_gen_sari_i32(TCGV_HIGH(retv), TCGV_LOW(retv), 31);
} else { } else {
@ -810,9 +832,9 @@ void tcg_gen_nonatomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE); tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE);
tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN); tcg_gen_qemu_ld_i64_int(t1, addr, idx, memop & ~MO_SIGN);
tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1); tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1);
tcg_gen_qemu_st_i64(t2, addr, idx, memop); tcg_gen_qemu_st_i64_int(t2, addr, idx, memop);
tcg_temp_free_i64(t2); tcg_temp_free_i64(t2);
if (memop & MO_SIGN) { if (memop & MO_SIGN) {
@ -823,11 +845,22 @@ void tcg_gen_nonatomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
tcg_temp_free_i64(t1); tcg_temp_free_i64(t1);
} }
void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv, void tcg_gen_nonatomic_cmpxchg_i64_chk(TCGv_i64 retv, TCGTemp *addr,
TCGv_i64 newv, TCGArg idx, MemOp memop) TCGv_i64 cmpv, TCGv_i64 newv,
TCGArg idx, MemOp memop,
TCGType addr_type)
{
tcg_debug_assert(addr_type == tcg_ctx->addr_type);
tcg_debug_assert((memop & MO_SIZE) <= MO_64);
tcg_gen_nonatomic_cmpxchg_i64_int(retv, addr, cmpv, newv, idx, memop);
}
static void tcg_gen_atomic_cmpxchg_i64_int(TCGv_i64 retv, TCGTemp *addr,
TCGv_i64 cmpv, TCGv_i64 newv,
TCGArg idx, MemOp memop)
{ {
if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
tcg_gen_nonatomic_cmpxchg_i64(retv, addr, cmpv, newv, idx, memop); tcg_gen_nonatomic_cmpxchg_i64_int(retv, addr, cmpv, newv, idx, memop);
return; return;
} }
@ -856,8 +889,8 @@ void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
} }
if (TCG_TARGET_REG_BITS == 32) { if (TCG_TARGET_REG_BITS == 32) {
tcg_gen_atomic_cmpxchg_i32(TCGV_LOW(retv), addr, TCGV_LOW(cmpv), tcg_gen_atomic_cmpxchg_i32_int(TCGV_LOW(retv), addr, TCGV_LOW(cmpv),
TCGV_LOW(newv), idx, memop); TCGV_LOW(newv), idx, memop);
if (memop & MO_SIGN) { if (memop & MO_SIGN) {
tcg_gen_sari_i32(TCGV_HIGH(retv), TCGV_LOW(retv), 31); tcg_gen_sari_i32(TCGV_HIGH(retv), TCGV_LOW(retv), 31);
} else { } else {
@ -870,7 +903,8 @@ void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
tcg_gen_extrl_i64_i32(c32, cmpv); tcg_gen_extrl_i64_i32(c32, cmpv);
tcg_gen_extrl_i64_i32(n32, newv); tcg_gen_extrl_i64_i32(n32, newv);
tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN); tcg_gen_atomic_cmpxchg_i32_int(r32, addr, c32, n32,
idx, memop & ~MO_SIGN);
tcg_temp_free_i32(c32); tcg_temp_free_i32(c32);
tcg_temp_free_i32(n32); tcg_temp_free_i32(n32);
@ -883,8 +917,18 @@ void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
} }
} }
void tcg_gen_nonatomic_cmpxchg_i128(TCGv_i128 retv, TCGv addr, TCGv_i128 cmpv, void tcg_gen_atomic_cmpxchg_i64_chk(TCGv_i64 retv, TCGTemp *addr,
TCGv_i128 newv, TCGArg idx, MemOp memop) TCGv_i64 cmpv, TCGv_i64 newv,
TCGArg idx, MemOp memop, TCGType addr_type)
{
tcg_debug_assert(addr_type == tcg_ctx->addr_type);
tcg_debug_assert((memop & MO_SIZE) <= MO_64);
tcg_gen_atomic_cmpxchg_i64_int(retv, addr, cmpv, newv, idx, memop);
}
static void tcg_gen_nonatomic_cmpxchg_i128_int(TCGv_i128 retv, TCGTemp *addr,
TCGv_i128 cmpv, TCGv_i128 newv,
TCGArg idx, MemOp memop)
{ {
if (TCG_TARGET_REG_BITS == 32) { if (TCG_TARGET_REG_BITS == 32) {
/* Inline expansion below is simply too large for 32-bit hosts. */ /* Inline expansion below is simply too large for 32-bit hosts. */
@ -892,12 +936,8 @@ void tcg_gen_nonatomic_cmpxchg_i128(TCGv_i128 retv, TCGv addr, TCGv_i128 cmpv,
? gen_helper_nonatomic_cmpxchgo_le ? gen_helper_nonatomic_cmpxchgo_le
: gen_helper_nonatomic_cmpxchgo_be); : gen_helper_nonatomic_cmpxchgo_be);
MemOpIdx oi = make_memop_idx(memop, idx); MemOpIdx oi = make_memop_idx(memop, idx);
TCGv_i64 a64; TCGv_i64 a64 = maybe_extend_addr64(addr);
tcg_debug_assert((memop & MO_SIZE) == MO_128);
tcg_debug_assert((memop & MO_SIGN) == 0);
a64 = maybe_extend_addr64(addr);
gen(retv, cpu_env, a64, cmpv, newv, tcg_constant_i32(oi)); gen(retv, cpu_env, a64, cmpv, newv, tcg_constant_i32(oi));
maybe_free_addr64(a64); maybe_free_addr64(a64);
} else { } else {
@ -907,7 +947,7 @@ void tcg_gen_nonatomic_cmpxchg_i128(TCGv_i128 retv, TCGv addr, TCGv_i128 cmpv,
TCGv_i64 t1 = tcg_temp_ebb_new_i64(); TCGv_i64 t1 = tcg_temp_ebb_new_i64();
TCGv_i64 z = tcg_constant_i64(0); TCGv_i64 z = tcg_constant_i64(0);
tcg_gen_qemu_ld_i128(oldv, addr, idx, memop); tcg_gen_qemu_ld_i128_int(oldv, addr, idx, memop);
/* Compare i128 */ /* Compare i128 */
tcg_gen_xor_i64(t0, TCGV128_LOW(oldv), TCGV128_LOW(cmpv)); tcg_gen_xor_i64(t0, TCGV128_LOW(oldv), TCGV128_LOW(cmpv));
@ -921,7 +961,7 @@ void tcg_gen_nonatomic_cmpxchg_i128(TCGv_i128 retv, TCGv addr, TCGv_i128 cmpv,
TCGV128_HIGH(newv), TCGV128_HIGH(oldv)); TCGV128_HIGH(newv), TCGV128_HIGH(oldv));
/* Unconditional writeback. */ /* Unconditional writeback. */
tcg_gen_qemu_st_i128(tmpv, addr, idx, memop); tcg_gen_qemu_st_i128_int(tmpv, addr, idx, memop);
tcg_gen_mov_i128(retv, oldv); tcg_gen_mov_i128(retv, oldv);
tcg_temp_free_i64(t0); tcg_temp_free_i64(t0);
@ -931,20 +971,28 @@ void tcg_gen_nonatomic_cmpxchg_i128(TCGv_i128 retv, TCGv addr, TCGv_i128 cmpv,
} }
} }
void tcg_gen_atomic_cmpxchg_i128(TCGv_i128 retv, TCGv addr, TCGv_i128 cmpv, void tcg_gen_nonatomic_cmpxchg_i128_chk(TCGv_i128 retv, TCGTemp *addr,
TCGv_i128 newv, TCGArg idx, MemOp memop) TCGv_i128 cmpv, TCGv_i128 newv,
TCGArg idx, MemOp memop,
TCGType addr_type)
{
tcg_debug_assert(addr_type == tcg_ctx->addr_type);
tcg_debug_assert((memop & (MO_SIZE | MO_SIGN)) == MO_128);
tcg_gen_nonatomic_cmpxchg_i128_int(retv, addr, cmpv, newv, idx, memop);
}
static void tcg_gen_atomic_cmpxchg_i128_int(TCGv_i128 retv, TCGTemp *addr,
TCGv_i128 cmpv, TCGv_i128 newv,
TCGArg idx, MemOp memop)
{ {
gen_atomic_cx_i128 gen; gen_atomic_cx_i128 gen;
if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
tcg_gen_nonatomic_cmpxchg_i128(retv, addr, cmpv, newv, idx, memop); tcg_gen_nonatomic_cmpxchg_i128_int(retv, addr, cmpv, newv, idx, memop);
return; return;
} }
tcg_debug_assert((memop & MO_SIZE) == MO_128);
tcg_debug_assert((memop & MO_SIGN) == 0);
gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
if (gen) { if (gen) {
MemOpIdx oi = make_memop_idx(memop, idx); MemOpIdx oi = make_memop_idx(memop, idx);
TCGv_i64 a64 = maybe_extend_addr64(addr); TCGv_i64 a64 = maybe_extend_addr64(addr);
@ -964,7 +1012,17 @@ void tcg_gen_atomic_cmpxchg_i128(TCGv_i128 retv, TCGv addr, TCGv_i128 cmpv,
tcg_gen_movi_i64(TCGV128_HIGH(retv), 0); tcg_gen_movi_i64(TCGV128_HIGH(retv), 0);
} }
static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val, void tcg_gen_atomic_cmpxchg_i128_chk(TCGv_i128 retv, TCGTemp *addr,
TCGv_i128 cmpv, TCGv_i128 newv,
TCGArg idx, MemOp memop,
TCGType addr_type)
{
tcg_debug_assert(addr_type == tcg_ctx->addr_type);
tcg_debug_assert((memop & (MO_SIZE | MO_SIGN)) == MO_128);
tcg_gen_atomic_cmpxchg_i128_int(retv, addr, cmpv, newv, idx, memop);
}
static void do_nonatomic_op_i32(TCGv_i32 ret, TCGTemp *addr, TCGv_i32 val,
TCGArg idx, MemOp memop, bool new_val, TCGArg idx, MemOp memop, bool new_val,
void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32)) void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32))
{ {
@ -973,17 +1031,17 @@ static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
memop = tcg_canonicalize_memop(memop, 0, 0); memop = tcg_canonicalize_memop(memop, 0, 0);
tcg_gen_qemu_ld_i32(t1, addr, idx, memop); tcg_gen_qemu_ld_i32_int(t1, addr, idx, memop);
tcg_gen_ext_i32(t2, val, memop); tcg_gen_ext_i32(t2, val, memop);
gen(t2, t1, t2); gen(t2, t1, t2);
tcg_gen_qemu_st_i32(t2, addr, idx, memop); tcg_gen_qemu_st_i32_int(t2, addr, idx, memop);
tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop); tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop);
tcg_temp_free_i32(t1); tcg_temp_free_i32(t1);
tcg_temp_free_i32(t2); tcg_temp_free_i32(t2);
} }
static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val, static void do_atomic_op_i32(TCGv_i32 ret, TCGTemp *addr, TCGv_i32 val,
TCGArg idx, MemOp memop, void * const table[]) TCGArg idx, MemOp memop, void * const table[])
{ {
gen_atomic_op_i32 gen; gen_atomic_op_i32 gen;
@ -1005,7 +1063,7 @@ static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
} }
} }
static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val, static void do_nonatomic_op_i64(TCGv_i64 ret, TCGTemp *addr, TCGv_i64 val,
TCGArg idx, MemOp memop, bool new_val, TCGArg idx, MemOp memop, bool new_val,
void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64)) void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64))
{ {
@ -1014,40 +1072,36 @@ static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
memop = tcg_canonicalize_memop(memop, 1, 0); memop = tcg_canonicalize_memop(memop, 1, 0);
tcg_gen_qemu_ld_i64(t1, addr, idx, memop); tcg_gen_qemu_ld_i64_int(t1, addr, idx, memop);
tcg_gen_ext_i64(t2, val, memop); tcg_gen_ext_i64(t2, val, memop);
gen(t2, t1, t2); gen(t2, t1, t2);
tcg_gen_qemu_st_i64(t2, addr, idx, memop); tcg_gen_qemu_st_i64_int(t2, addr, idx, memop);
tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop); tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop);
tcg_temp_free_i64(t1); tcg_temp_free_i64(t1);
tcg_temp_free_i64(t2); tcg_temp_free_i64(t2);
} }
static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val, static void do_atomic_op_i64(TCGv_i64 ret, TCGTemp *addr, TCGv_i64 val,
TCGArg idx, MemOp memop, void * const table[]) TCGArg idx, MemOp memop, void * const table[])
{ {
memop = tcg_canonicalize_memop(memop, 1, 0); memop = tcg_canonicalize_memop(memop, 1, 0);
if ((memop & MO_SIZE) == MO_64) { if ((memop & MO_SIZE) == MO_64) {
#ifdef CONFIG_ATOMIC64 gen_atomic_op_i64 gen = table[memop & (MO_SIZE | MO_BSWAP)];
gen_atomic_op_i64 gen;
TCGv_i64 a64;
MemOpIdx oi;
gen = table[memop & (MO_SIZE | MO_BSWAP)]; if (gen) {
tcg_debug_assert(gen != NULL); MemOpIdx oi = make_memop_idx(memop & ~MO_SIGN, idx);
TCGv_i64 a64 = maybe_extend_addr64(addr);
gen(ret, cpu_env, a64, val, tcg_constant_i32(oi));
maybe_free_addr64(a64);
return;
}
oi = make_memop_idx(memop & ~MO_SIGN, idx);
a64 = maybe_extend_addr64(addr);
gen(ret, cpu_env, a64, val, tcg_constant_i32(oi));
maybe_free_addr64(a64);
#else
gen_helper_exit_atomic(cpu_env); gen_helper_exit_atomic(cpu_env);
/* Produce a result, so that we have a well-formed opcode stream /* Produce a result, so that we have a well-formed opcode stream
with respect to uses of the result in the (dead) code following. */ with respect to uses of the result in the (dead) code following. */
tcg_gen_movi_i64(ret, 0); tcg_gen_movi_i64(ret, 0);
#endif /* CONFIG_ATOMIC64 */
} else { } else {
TCGv_i32 v32 = tcg_temp_ebb_new_i32(); TCGv_i32 v32 = tcg_temp_ebb_new_i32();
TCGv_i32 r32 = tcg_temp_ebb_new_i32(); TCGv_i32 r32 = tcg_temp_ebb_new_i32();
@ -1075,9 +1129,12 @@ static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = { \
WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \ WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \
WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \ WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \
}; \ }; \
void tcg_gen_atomic_##NAME##_i32 \ void tcg_gen_atomic_##NAME##_i32_chk(TCGv_i32 ret, TCGTemp *addr, \
(TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, MemOp memop) \ TCGv_i32 val, TCGArg idx, \
MemOp memop, TCGType addr_type) \
{ \ { \
tcg_debug_assert(addr_type == tcg_ctx->addr_type); \
tcg_debug_assert((memop & MO_SIZE) <= MO_32); \
if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \ if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \
do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \ do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \
} else { \ } else { \
@ -1085,9 +1142,12 @@ void tcg_gen_atomic_##NAME##_i32 \
tcg_gen_##OP##_i32); \ tcg_gen_##OP##_i32); \
} \ } \
} \ } \
void tcg_gen_atomic_##NAME##_i64 \ void tcg_gen_atomic_##NAME##_i64_chk(TCGv_i64 ret, TCGTemp *addr, \
(TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, MemOp memop) \ TCGv_i64 val, TCGArg idx, \
MemOp memop, TCGType addr_type) \
{ \ { \
tcg_debug_assert(addr_type == tcg_ctx->addr_type); \
tcg_debug_assert((memop & MO_SIZE) <= MO_64); \
if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \ if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \
do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \ do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \
} else { \ } else { \