156 lines
4.9 KiB
C++
156 lines
4.9 KiB
C++
/*
|
|
* Simple interface for 128-bit atomic operations.
|
|
*
|
|
* Copyright (C) 2018 Linaro, Ltd.
|
|
*
|
|
* This work is licensed under the terms of the GNU GPL, version 2 or later.
|
|
* See the COPYING file in the top-level directory.
|
|
*
|
|
* See docs/devel/atomics.rst for discussion about the guarantees each
|
|
* atomic primitive is meant to provide.
|
|
*/
|
|
|
|
#ifndef QEMU_ATOMIC128_H
|
|
#define QEMU_ATOMIC128_H
|
|
|
|
#include "qemu/int128.h"
|
|
|
|
/*
|
|
* GCC is a house divided about supporting large atomic operations.
|
|
*
|
|
* For hosts that only have large compare-and-swap, a legalistic reading
|
|
* of the C++ standard means that one cannot implement __atomic_read on
|
|
* read-only memory, and thus all atomic operations must synchronize
|
|
* through libatomic.
|
|
*
|
|
* See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80878
|
|
*
|
|
* This interpretation is not especially helpful for QEMU.
|
|
* For softmmu, all RAM is always read/write from the hypervisor.
|
|
* For user-only, if the guest doesn't implement such an __atomic_read
|
|
* then the host need not worry about it either.
|
|
*
|
|
* Moreover, using libatomic is not an option, because its interface is
|
|
* built for std::atomic<T>, and requires that *all* accesses to such an
|
|
* object go through the library. In our case we do not have an object
|
|
* in the C/C++ sense, but a view of memory as seen by the guest.
|
|
* The guest may issue a large atomic operation and then access those
|
|
* pieces using word-sized accesses. From the hypervisor, we have no
|
|
* way to connect those two actions.
|
|
*
|
|
* Therefore, special case each platform.
|
|
*/
|
|
|
|
#if defined(CONFIG_ATOMIC128)
|
|
static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
|
|
{
|
|
return qatomic_cmpxchg__nocheck(ptr, cmp, new);
|
|
}
|
|
# define HAVE_CMPXCHG128 1
|
|
#elif defined(CONFIG_CMPXCHG128)
|
|
static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
|
|
{
|
|
return __sync_val_compare_and_swap_16(ptr, cmp, new);
|
|
}
|
|
# define HAVE_CMPXCHG128 1
|
|
#elif defined(__aarch64__)
|
|
/* Through gcc 8, aarch64 has no support for 128-bit at all. */
|
|
static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
|
|
{
|
|
uint64_t cmpl = int128_getlo(cmp), cmph = int128_gethi(cmp);
|
|
uint64_t newl = int128_getlo(new), newh = int128_gethi(new);
|
|
uint64_t oldl, oldh;
|
|
uint32_t tmp;
|
|
|
|
asm("0: ldaxp %[oldl], %[oldh], %[mem]\n\t"
|
|
"cmp %[oldl], %[cmpl]\n\t"
|
|
"ccmp %[oldh], %[cmph], #0, eq\n\t"
|
|
"b.ne 1f\n\t"
|
|
"stlxp %w[tmp], %[newl], %[newh], %[mem]\n\t"
|
|
"cbnz %w[tmp], 0b\n"
|
|
"1:"
|
|
: [mem] "+m"(*ptr), [tmp] "=&r"(tmp),
|
|
[oldl] "=&r"(oldl), [oldh] "=&r"(oldh)
|
|
: [cmpl] "r"(cmpl), [cmph] "r"(cmph),
|
|
[newl] "r"(newl), [newh] "r"(newh)
|
|
: "memory", "cc");
|
|
|
|
return int128_make128(oldl, oldh);
|
|
}
|
|
# define HAVE_CMPXCHG128 1
|
|
#else
|
|
/* Fallback definition that must be optimized away, or error. */
|
|
Int128 QEMU_ERROR("unsupported atomic")
|
|
atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new);
|
|
# define HAVE_CMPXCHG128 0
|
|
#endif /* Some definition for HAVE_CMPXCHG128 */
|
|
|
|
|
|
#if defined(CONFIG_ATOMIC128)
|
|
static inline Int128 atomic16_read(Int128 *ptr)
|
|
{
|
|
return qatomic_read__nocheck(ptr);
|
|
}
|
|
|
|
static inline void atomic16_set(Int128 *ptr, Int128 val)
|
|
{
|
|
qatomic_set__nocheck(ptr, val);
|
|
}
|
|
|
|
# define HAVE_ATOMIC128 1
|
|
#elif !defined(CONFIG_USER_ONLY) && defined(__aarch64__)
|
|
/* We can do better than cmpxchg for AArch64. */
|
|
static inline Int128 atomic16_read(Int128 *ptr)
|
|
{
|
|
uint64_t l, h;
|
|
uint32_t tmp;
|
|
|
|
/* The load must be paired with the store to guarantee not tearing. */
|
|
asm("0: ldxp %[l], %[h], %[mem]\n\t"
|
|
"stxp %w[tmp], %[l], %[h], %[mem]\n\t"
|
|
"cbnz %w[tmp], 0b"
|
|
: [mem] "+m"(*ptr), [tmp] "=r"(tmp), [l] "=r"(l), [h] "=r"(h));
|
|
|
|
return int128_make128(l, h);
|
|
}
|
|
|
|
static inline void atomic16_set(Int128 *ptr, Int128 val)
|
|
{
|
|
uint64_t l = int128_getlo(val), h = int128_gethi(val);
|
|
uint64_t t1, t2;
|
|
|
|
/* Load into temporaries to acquire the exclusive access lock. */
|
|
asm("0: ldxp %[t1], %[t2], %[mem]\n\t"
|
|
"stxp %w[t1], %[l], %[h], %[mem]\n\t"
|
|
"cbnz %w[t1], 0b"
|
|
: [mem] "+m"(*ptr), [t1] "=&r"(t1), [t2] "=&r"(t2)
|
|
: [l] "r"(l), [h] "r"(h));
|
|
}
|
|
|
|
# define HAVE_ATOMIC128 1
|
|
#elif !defined(CONFIG_USER_ONLY) && HAVE_CMPXCHG128
|
|
static inline Int128 atomic16_read(Int128 *ptr)
|
|
{
|
|
/* Maybe replace 0 with 0, returning the old value. */
|
|
return atomic16_cmpxchg(ptr, 0, 0);
|
|
}
|
|
|
|
static inline void atomic16_set(Int128 *ptr, Int128 val)
|
|
{
|
|
Int128 old = *ptr, cmp;
|
|
do {
|
|
cmp = old;
|
|
old = atomic16_cmpxchg(ptr, cmp, val);
|
|
} while (old != cmp);
|
|
}
|
|
|
|
# define HAVE_ATOMIC128 1
|
|
#else
|
|
/* Fallback definitions that must be optimized away, or error. */
|
|
Int128 QEMU_ERROR("unsupported atomic") atomic16_read(Int128 *ptr);
|
|
void QEMU_ERROR("unsupported atomic") atomic16_set(Int128 *ptr, Int128 val);
|
|
# define HAVE_ATOMIC128 0
|
|
#endif /* Some definition for HAVE_ATOMIC128 */
|
|
|
|
#endif /* QEMU_ATOMIC128_H */
|