522 lines
20 KiB
C++
522 lines
20 KiB
C++
//===-- tsan_interceptors_mac.cpp -----------------------------------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// This file is a part of ThreadSanitizer (TSan), a race detector.
|
|
//
|
|
// Mac-specific interceptors.
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "sanitizer_common/sanitizer_platform.h"
|
|
#if SANITIZER_MAC
|
|
|
|
#include "interception/interception.h"
|
|
#include "tsan_interceptors.h"
|
|
#include "tsan_interface.h"
|
|
#include "tsan_interface_ann.h"
|
|
#include "sanitizer_common/sanitizer_addrhashmap.h"
|
|
|
|
#include <errno.h>
|
|
#include <libkern/OSAtomic.h>
|
|
#include <objc/objc-sync.h>
|
|
#include <os/lock.h>
|
|
#include <sys/ucontext.h>
|
|
|
|
#if defined(__has_include) && __has_include(<xpc/xpc.h>)
|
|
#include <xpc/xpc.h>
|
|
#endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
|
|
|
|
typedef long long_t;
|
|
|
|
extern "C" {
|
|
int getcontext(ucontext_t *ucp) __attribute__((returns_twice));
|
|
int setcontext(const ucontext_t *ucp);
|
|
}
|
|
|
|
namespace __tsan {
|
|
|
|
// The non-barrier versions of OSAtomic* functions are semantically mo_relaxed,
|
|
// but the two variants (e.g. OSAtomicAdd32 and OSAtomicAdd32Barrier) are
|
|
// actually aliases of each other, and we cannot have different interceptors for
|
|
// them, because they're actually the same function. Thus, we have to stay
|
|
// conservative and treat the non-barrier versions as mo_acq_rel.
|
|
static constexpr morder kMacOrderBarrier = mo_acq_rel;
|
|
static constexpr morder kMacOrderNonBarrier = mo_acq_rel;
|
|
static constexpr morder kMacFailureOrder = mo_relaxed;
|
|
|
|
#define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
|
|
TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \
|
|
SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \
|
|
return tsan_atomic_f((volatile tsan_t *)ptr, x, mo); \
|
|
}
|
|
|
|
#define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
|
|
TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \
|
|
SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \
|
|
return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x; \
|
|
}
|
|
|
|
#define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
|
|
TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \
|
|
SCOPED_TSAN_INTERCEPTOR(f, ptr); \
|
|
return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1; \
|
|
}
|
|
|
|
#define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \
|
|
mo) \
|
|
TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \
|
|
SCOPED_TSAN_INTERCEPTOR(f, ptr); \
|
|
return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1; \
|
|
}
|
|
|
|
#define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m) \
|
|
m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \
|
|
kMacOrderNonBarrier) \
|
|
m(int32_t, int32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \
|
|
kMacOrderBarrier) \
|
|
m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f, \
|
|
kMacOrderNonBarrier) \
|
|
m(int64_t, int64_t, a64, f##64##Barrier, __tsan_atomic64_##tsan_atomic_f, \
|
|
kMacOrderBarrier)
|
|
|
|
#define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig) \
|
|
m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \
|
|
kMacOrderNonBarrier) \
|
|
m(int32_t, uint32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \
|
|
kMacOrderBarrier) \
|
|
m_orig(int32_t, uint32_t, a32, f##32##Orig, __tsan_atomic32_##tsan_atomic_f, \
|
|
kMacOrderNonBarrier) \
|
|
m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier, \
|
|
__tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier)
|
|
|
|
OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd, fetch_add,
|
|
OSATOMIC_INTERCEPTOR_PLUS_X)
|
|
OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicIncrement, fetch_add,
|
|
OSATOMIC_INTERCEPTOR_PLUS_1)
|
|
OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicDecrement, fetch_sub,
|
|
OSATOMIC_INTERCEPTOR_MINUS_1)
|
|
OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicOr, fetch_or, OSATOMIC_INTERCEPTOR_PLUS_X,
|
|
OSATOMIC_INTERCEPTOR)
|
|
OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicAnd, fetch_and,
|
|
OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
|
|
OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor,
|
|
OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
|
|
|
|
#define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t) \
|
|
TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \
|
|
SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \
|
|
return tsan_atomic_f##_compare_exchange_strong( \
|
|
(volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \
|
|
kMacOrderNonBarrier, kMacFailureOrder); \
|
|
} \
|
|
\
|
|
TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value, \
|
|
t volatile *ptr) { \
|
|
SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \
|
|
return tsan_atomic_f##_compare_exchange_strong( \
|
|
(volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \
|
|
kMacOrderBarrier, kMacFailureOrder); \
|
|
}
|
|
|
|
OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int)
|
|
OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapLong, __tsan_atomic64, a64,
|
|
long_t)
|
|
OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapPtr, __tsan_atomic64, a64,
|
|
void *)
|
|
OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32,
|
|
int32_t)
|
|
OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64,
|
|
int64_t)
|
|
|
|
#define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \
|
|
TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \
|
|
SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \
|
|
volatile char *byte_ptr = ((volatile char *)ptr) + (n >> 3); \
|
|
char bit = 0x80u >> (n & 7); \
|
|
char mask = clear ? ~bit : bit; \
|
|
char orig_byte = op((volatile a8 *)byte_ptr, mask, mo); \
|
|
return orig_byte & bit; \
|
|
}
|
|
|
|
#define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear) \
|
|
OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, kMacOrderNonBarrier) \
|
|
OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, clear, kMacOrderBarrier)
|
|
|
|
OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndSet, __tsan_atomic8_fetch_or, false)
|
|
OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndClear, __tsan_atomic8_fetch_and,
|
|
true)
|
|
|
|
TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead *list, void *item,
|
|
size_t offset) {
|
|
SCOPED_TSAN_INTERCEPTOR(OSAtomicEnqueue, list, item, offset);
|
|
__tsan_release(item);
|
|
REAL(OSAtomicEnqueue)(list, item, offset);
|
|
}
|
|
|
|
TSAN_INTERCEPTOR(void *, OSAtomicDequeue, OSQueueHead *list, size_t offset) {
|
|
SCOPED_TSAN_INTERCEPTOR(OSAtomicDequeue, list, offset);
|
|
void *item = REAL(OSAtomicDequeue)(list, offset);
|
|
if (item) __tsan_acquire(item);
|
|
return item;
|
|
}
|
|
|
|
// OSAtomicFifoEnqueue and OSAtomicFifoDequeue are only on OS X.
|
|
#if !SANITIZER_IOS
|
|
|
|
TSAN_INTERCEPTOR(void, OSAtomicFifoEnqueue, OSFifoQueueHead *list, void *item,
|
|
size_t offset) {
|
|
SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoEnqueue, list, item, offset);
|
|
__tsan_release(item);
|
|
REAL(OSAtomicFifoEnqueue)(list, item, offset);
|
|
}
|
|
|
|
TSAN_INTERCEPTOR(void *, OSAtomicFifoDequeue, OSFifoQueueHead *list,
|
|
size_t offset) {
|
|
SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoDequeue, list, offset);
|
|
void *item = REAL(OSAtomicFifoDequeue)(list, offset);
|
|
if (item) __tsan_acquire(item);
|
|
return item;
|
|
}
|
|
|
|
#endif
|
|
|
|
TSAN_INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) {
|
|
CHECK(!cur_thread()->is_dead);
|
|
if (!cur_thread()->is_inited) {
|
|
return REAL(OSSpinLockLock)(lock);
|
|
}
|
|
SCOPED_TSAN_INTERCEPTOR(OSSpinLockLock, lock);
|
|
REAL(OSSpinLockLock)(lock);
|
|
Acquire(thr, pc, (uptr)lock);
|
|
}
|
|
|
|
TSAN_INTERCEPTOR(bool, OSSpinLockTry, volatile OSSpinLock *lock) {
|
|
CHECK(!cur_thread()->is_dead);
|
|
if (!cur_thread()->is_inited) {
|
|
return REAL(OSSpinLockTry)(lock);
|
|
}
|
|
SCOPED_TSAN_INTERCEPTOR(OSSpinLockTry, lock);
|
|
bool result = REAL(OSSpinLockTry)(lock);
|
|
if (result)
|
|
Acquire(thr, pc, (uptr)lock);
|
|
return result;
|
|
}
|
|
|
|
TSAN_INTERCEPTOR(void, OSSpinLockUnlock, volatile OSSpinLock *lock) {
|
|
CHECK(!cur_thread()->is_dead);
|
|
if (!cur_thread()->is_inited) {
|
|
return REAL(OSSpinLockUnlock)(lock);
|
|
}
|
|
SCOPED_TSAN_INTERCEPTOR(OSSpinLockUnlock, lock);
|
|
Release(thr, pc, (uptr)lock);
|
|
REAL(OSSpinLockUnlock)(lock);
|
|
}
|
|
|
|
TSAN_INTERCEPTOR(void, os_lock_lock, void *lock) {
|
|
CHECK(!cur_thread()->is_dead);
|
|
if (!cur_thread()->is_inited) {
|
|
return REAL(os_lock_lock)(lock);
|
|
}
|
|
SCOPED_TSAN_INTERCEPTOR(os_lock_lock, lock);
|
|
REAL(os_lock_lock)(lock);
|
|
Acquire(thr, pc, (uptr)lock);
|
|
}
|
|
|
|
TSAN_INTERCEPTOR(bool, os_lock_trylock, void *lock) {
|
|
CHECK(!cur_thread()->is_dead);
|
|
if (!cur_thread()->is_inited) {
|
|
return REAL(os_lock_trylock)(lock);
|
|
}
|
|
SCOPED_TSAN_INTERCEPTOR(os_lock_trylock, lock);
|
|
bool result = REAL(os_lock_trylock)(lock);
|
|
if (result)
|
|
Acquire(thr, pc, (uptr)lock);
|
|
return result;
|
|
}
|
|
|
|
TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) {
|
|
CHECK(!cur_thread()->is_dead);
|
|
if (!cur_thread()->is_inited) {
|
|
return REAL(os_lock_unlock)(lock);
|
|
}
|
|
SCOPED_TSAN_INTERCEPTOR(os_lock_unlock, lock);
|
|
Release(thr, pc, (uptr)lock);
|
|
REAL(os_lock_unlock)(lock);
|
|
}
|
|
|
|
TSAN_INTERCEPTOR(void, os_unfair_lock_lock, os_unfair_lock_t lock) {
|
|
if (!cur_thread()->is_inited || cur_thread()->is_dead) {
|
|
return REAL(os_unfair_lock_lock)(lock);
|
|
}
|
|
SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock, lock);
|
|
REAL(os_unfair_lock_lock)(lock);
|
|
Acquire(thr, pc, (uptr)lock);
|
|
}
|
|
|
|
TSAN_INTERCEPTOR(void, os_unfair_lock_lock_with_options, os_unfair_lock_t lock,
|
|
u32 options) {
|
|
if (!cur_thread()->is_inited || cur_thread()->is_dead) {
|
|
return REAL(os_unfair_lock_lock_with_options)(lock, options);
|
|
}
|
|
SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock_with_options, lock, options);
|
|
REAL(os_unfair_lock_lock_with_options)(lock, options);
|
|
Acquire(thr, pc, (uptr)lock);
|
|
}
|
|
|
|
TSAN_INTERCEPTOR(bool, os_unfair_lock_trylock, os_unfair_lock_t lock) {
|
|
if (!cur_thread()->is_inited || cur_thread()->is_dead) {
|
|
return REAL(os_unfair_lock_trylock)(lock);
|
|
}
|
|
SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_trylock, lock);
|
|
bool result = REAL(os_unfair_lock_trylock)(lock);
|
|
if (result)
|
|
Acquire(thr, pc, (uptr)lock);
|
|
return result;
|
|
}
|
|
|
|
TSAN_INTERCEPTOR(void, os_unfair_lock_unlock, os_unfair_lock_t lock) {
|
|
if (!cur_thread()->is_inited || cur_thread()->is_dead) {
|
|
return REAL(os_unfair_lock_unlock)(lock);
|
|
}
|
|
SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_unlock, lock);
|
|
Release(thr, pc, (uptr)lock);
|
|
REAL(os_unfair_lock_unlock)(lock);
|
|
}
|
|
|
|
#if defined(__has_include) && __has_include(<xpc/xpc.h>)
|
|
|
|
TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler,
|
|
xpc_connection_t connection, xpc_handler_t handler) {
|
|
SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler, connection,
|
|
handler);
|
|
Release(thr, pc, (uptr)connection);
|
|
xpc_handler_t new_handler = ^(xpc_object_t object) {
|
|
{
|
|
SCOPED_INTERCEPTOR_RAW(xpc_connection_set_event_handler);
|
|
Acquire(thr, pc, (uptr)connection);
|
|
}
|
|
handler(object);
|
|
};
|
|
REAL(xpc_connection_set_event_handler)(connection, new_handler);
|
|
}
|
|
|
|
TSAN_INTERCEPTOR(void, xpc_connection_send_barrier, xpc_connection_t connection,
|
|
dispatch_block_t barrier) {
|
|
SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_barrier, connection, barrier);
|
|
Release(thr, pc, (uptr)connection);
|
|
dispatch_block_t new_barrier = ^() {
|
|
{
|
|
SCOPED_INTERCEPTOR_RAW(xpc_connection_send_barrier);
|
|
Acquire(thr, pc, (uptr)connection);
|
|
}
|
|
barrier();
|
|
};
|
|
REAL(xpc_connection_send_barrier)(connection, new_barrier);
|
|
}
|
|
|
|
TSAN_INTERCEPTOR(void, xpc_connection_send_message_with_reply,
|
|
xpc_connection_t connection, xpc_object_t message,
|
|
dispatch_queue_t replyq, xpc_handler_t handler) {
|
|
SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_message_with_reply, connection,
|
|
message, replyq, handler);
|
|
Release(thr, pc, (uptr)connection);
|
|
xpc_handler_t new_handler = ^(xpc_object_t object) {
|
|
{
|
|
SCOPED_INTERCEPTOR_RAW(xpc_connection_send_message_with_reply);
|
|
Acquire(thr, pc, (uptr)connection);
|
|
}
|
|
handler(object);
|
|
};
|
|
REAL(xpc_connection_send_message_with_reply)
|
|
(connection, message, replyq, new_handler);
|
|
}
|
|
|
|
TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) {
|
|
SCOPED_TSAN_INTERCEPTOR(xpc_connection_cancel, connection);
|
|
Release(thr, pc, (uptr)connection);
|
|
REAL(xpc_connection_cancel)(connection);
|
|
}
|
|
|
|
#endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
|
|
|
|
// Determines whether the Obj-C object pointer is a tagged pointer. Tagged
|
|
// pointers encode the object data directly in their pointer bits and do not
|
|
// have an associated memory allocation. The Obj-C runtime uses tagged pointers
|
|
// to transparently optimize small objects.
|
|
static bool IsTaggedObjCPointer(id obj) {
|
|
const uptr kPossibleTaggedBits = 0x8000000000000001ull;
|
|
return ((uptr)obj & kPossibleTaggedBits) != 0;
|
|
}
|
|
|
|
// Returns an address which can be used to inform TSan about synchronization
|
|
// points (MutexLock/Unlock). The TSan infrastructure expects this to be a valid
|
|
// address in the process space. We do a small allocation here to obtain a
|
|
// stable address (the array backing the hash map can change). The memory is
|
|
// never free'd (leaked) and allocation and locking are slow, but this code only
|
|
// runs for @synchronized with tagged pointers, which is very rare.
|
|
static uptr GetOrCreateSyncAddress(uptr addr, ThreadState *thr, uptr pc) {
|
|
typedef AddrHashMap<uptr, 5> Map;
|
|
static Map Addresses;
|
|
Map::Handle h(&Addresses, addr);
|
|
if (h.created()) {
|
|
ThreadIgnoreBegin(thr, pc);
|
|
*h = (uptr) user_alloc(thr, pc, /*size=*/1);
|
|
ThreadIgnoreEnd(thr);
|
|
}
|
|
return *h;
|
|
}
|
|
|
|
// Returns an address on which we can synchronize given an Obj-C object pointer.
|
|
// For normal object pointers, this is just the address of the object in memory.
|
|
// Tagged pointers are not backed by an actual memory allocation, so we need to
|
|
// synthesize a valid address.
|
|
static uptr SyncAddressForObjCObject(id obj, ThreadState *thr, uptr pc) {
|
|
if (IsTaggedObjCPointer(obj))
|
|
return GetOrCreateSyncAddress((uptr)obj, thr, pc);
|
|
return (uptr)obj;
|
|
}
|
|
|
|
TSAN_INTERCEPTOR(int, objc_sync_enter, id obj) {
|
|
SCOPED_TSAN_INTERCEPTOR(objc_sync_enter, obj);
|
|
if (!obj) return REAL(objc_sync_enter)(obj);
|
|
uptr addr = SyncAddressForObjCObject(obj, thr, pc);
|
|
MutexPreLock(thr, pc, addr, MutexFlagWriteReentrant);
|
|
int result = REAL(objc_sync_enter)(obj);
|
|
CHECK_EQ(result, OBJC_SYNC_SUCCESS);
|
|
MutexPostLock(thr, pc, addr, MutexFlagWriteReentrant);
|
|
return result;
|
|
}
|
|
|
|
TSAN_INTERCEPTOR(int, objc_sync_exit, id obj) {
|
|
SCOPED_TSAN_INTERCEPTOR(objc_sync_exit, obj);
|
|
if (!obj) return REAL(objc_sync_exit)(obj);
|
|
uptr addr = SyncAddressForObjCObject(obj, thr, pc);
|
|
MutexUnlock(thr, pc, addr);
|
|
int result = REAL(objc_sync_exit)(obj);
|
|
if (result != OBJC_SYNC_SUCCESS) MutexInvalidAccess(thr, pc, addr);
|
|
return result;
|
|
}
|
|
|
|
TSAN_INTERCEPTOR(int, swapcontext, ucontext_t *oucp, const ucontext_t *ucp) {
|
|
{
|
|
SCOPED_INTERCEPTOR_RAW(swapcontext, oucp, ucp);
|
|
}
|
|
// Because of swapcontext() semantics we have no option but to copy its
|
|
// implementation here
|
|
if (!oucp || !ucp) {
|
|
errno = EINVAL;
|
|
return -1;
|
|
}
|
|
ThreadState *thr = cur_thread();
|
|
const int UCF_SWAPPED = 0x80000000;
|
|
oucp->uc_onstack &= ~UCF_SWAPPED;
|
|
thr->ignore_interceptors++;
|
|
int ret = getcontext(oucp);
|
|
if (!(oucp->uc_onstack & UCF_SWAPPED)) {
|
|
thr->ignore_interceptors--;
|
|
if (!ret) {
|
|
oucp->uc_onstack |= UCF_SWAPPED;
|
|
ret = setcontext(ucp);
|
|
}
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
// On macOS, libc++ is always linked dynamically, so intercepting works the
|
|
// usual way.
|
|
#define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
|
|
|
|
namespace {
|
|
struct fake_shared_weak_count {
|
|
volatile a64 shared_owners;
|
|
volatile a64 shared_weak_owners;
|
|
virtual void _unused_0x0() = 0;
|
|
virtual void _unused_0x8() = 0;
|
|
virtual void on_zero_shared() = 0;
|
|
virtual void _unused_0x18() = 0;
|
|
virtual void on_zero_shared_weak() = 0;
|
|
virtual ~fake_shared_weak_count() = 0; // suppress -Wnon-virtual-dtor
|
|
};
|
|
} // namespace
|
|
|
|
// The following code adds libc++ interceptors for:
|
|
// void __shared_weak_count::__release_shared() _NOEXCEPT;
|
|
// bool __shared_count::__release_shared() _NOEXCEPT;
|
|
// Shared and weak pointers in C++ maintain reference counts via atomics in
|
|
// libc++.dylib, which are TSan-invisible, and this leads to false positives in
|
|
// destructor code. These interceptors re-implements the whole functions so that
|
|
// the mo_acq_rel semantics of the atomic decrement are visible.
|
|
//
|
|
// Unfortunately, the interceptors cannot simply Acquire/Release some sync
|
|
// object and call the original function, because it would have a race between
|
|
// the sync and the destruction of the object. Calling both under a lock will
|
|
// not work because the destructor can invoke this interceptor again (and even
|
|
// in a different thread, so recursive locks don't help).
|
|
|
|
STDCXX_INTERCEPTOR(void, _ZNSt3__119__shared_weak_count16__release_sharedEv,
|
|
fake_shared_weak_count *o) {
|
|
if (!flags()->shared_ptr_interceptor)
|
|
return REAL(_ZNSt3__119__shared_weak_count16__release_sharedEv)(o);
|
|
|
|
SCOPED_TSAN_INTERCEPTOR(_ZNSt3__119__shared_weak_count16__release_sharedEv,
|
|
o);
|
|
if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) {
|
|
Acquire(thr, pc, (uptr)&o->shared_owners);
|
|
o->on_zero_shared();
|
|
if (__tsan_atomic64_fetch_add(&o->shared_weak_owners, -1, mo_release) ==
|
|
0) {
|
|
Acquire(thr, pc, (uptr)&o->shared_weak_owners);
|
|
o->on_zero_shared_weak();
|
|
}
|
|
}
|
|
}
|
|
|
|
STDCXX_INTERCEPTOR(bool, _ZNSt3__114__shared_count16__release_sharedEv,
|
|
fake_shared_weak_count *o) {
|
|
if (!flags()->shared_ptr_interceptor)
|
|
return REAL(_ZNSt3__114__shared_count16__release_sharedEv)(o);
|
|
|
|
SCOPED_TSAN_INTERCEPTOR(_ZNSt3__114__shared_count16__release_sharedEv, o);
|
|
if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) {
|
|
Acquire(thr, pc, (uptr)&o->shared_owners);
|
|
o->on_zero_shared();
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
namespace {
|
|
struct call_once_callback_args {
|
|
void (*orig_func)(void *arg);
|
|
void *orig_arg;
|
|
void *flag;
|
|
};
|
|
|
|
void call_once_callback_wrapper(void *arg) {
|
|
call_once_callback_args *new_args = (call_once_callback_args *)arg;
|
|
new_args->orig_func(new_args->orig_arg);
|
|
__tsan_release(new_args->flag);
|
|
}
|
|
} // namespace
|
|
|
|
// This adds a libc++ interceptor for:
|
|
// void __call_once(volatile unsigned long&, void*, void(*)(void*));
|
|
// C++11 call_once is implemented via an internal function __call_once which is
|
|
// inside libc++.dylib, and the atomic release store inside it is thus
|
|
// TSan-invisible. To avoid false positives, this interceptor wraps the callback
|
|
// function and performs an explicit Release after the user code has run.
|
|
STDCXX_INTERCEPTOR(void, _ZNSt3__111__call_onceERVmPvPFvS2_E, void *flag,
|
|
void *arg, void (*func)(void *arg)) {
|
|
call_once_callback_args new_args = {func, arg, flag};
|
|
REAL(_ZNSt3__111__call_onceERVmPvPFvS2_E)(flag, &new_args,
|
|
call_once_callback_wrapper);
|
|
}
|
|
|
|
} // namespace __tsan
|
|
|
|
#endif // SANITIZER_MAC
|