461 lines
13 KiB
C++
461 lines
13 KiB
C++
// Copyright (C) 2002-2021 Free Software Foundation, Inc.
|
|
//
|
|
// This file is part of GCC.
|
|
//
|
|
// GCC is free software; you can redistribute it and/or modify
|
|
// it under the terms of the GNU General Public License as published by
|
|
// the Free Software Foundation; either version 3, or (at your option)
|
|
// any later version.
|
|
|
|
// GCC is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU General Public License for more details.
|
|
|
|
// Under Section 7 of GPL version 3, you are granted additional
|
|
// permissions described in the GCC Runtime Library Exception, version
|
|
// 3.1, as published by the Free Software Foundation.
|
|
|
|
// You should have received a copy of the GNU General Public License and
|
|
// a copy of the GCC Runtime Library Exception along with this program;
|
|
// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
|
|
// <http://www.gnu.org/licenses/>.
|
|
|
|
// Written by Mark Mitchell, CodeSourcery LLC, <mark@codesourcery.com>
|
|
// Thread support written by Jason Merrill, Red Hat Inc. <jason@redhat.com>
|
|
|
|
#include <bits/c++config.h>
|
|
#include <cxxabi.h>
|
|
#include <exception>
|
|
#include <new>
|
|
#include <ext/atomicity.h>
|
|
#include <ext/concurrence.h>
|
|
#include <bits/atomic_lockfree_defines.h>
|
|
#if defined(__GTHREADS) && defined(__GTHREAD_HAS_COND) \
|
|
&& (ATOMIC_INT_LOCK_FREE > 1) && defined(_GLIBCXX_HAVE_LINUX_FUTEX)
|
|
# include <climits>
|
|
# include <syscall.h>
|
|
# include <unistd.h>
|
|
# define _GLIBCXX_USE_FUTEX
|
|
# define _GLIBCXX_FUTEX_WAIT 0
|
|
# define _GLIBCXX_FUTEX_WAKE 1
|
|
#endif
|
|
|
|
// The IA64/generic ABI uses the first byte of the guard variable.
|
|
// The ARM EABI uses the least significant bit.
|
|
|
|
// Thread-safe static local initialization support.
|
|
#ifdef __GTHREADS
|
|
# ifndef _GLIBCXX_USE_FUTEX
|
|
namespace
|
|
{
|
|
// A single mutex controlling all static initializations.
|
|
static __gnu_cxx::__recursive_mutex* static_mutex;
|
|
|
|
typedef char fake_recursive_mutex[sizeof(__gnu_cxx::__recursive_mutex)]
|
|
__attribute__ ((aligned(__alignof__(__gnu_cxx::__recursive_mutex))));
|
|
fake_recursive_mutex fake_mutex;
|
|
|
|
static void init()
|
|
{ static_mutex = new (&fake_mutex) __gnu_cxx::__recursive_mutex(); }
|
|
|
|
__gnu_cxx::__recursive_mutex&
|
|
get_static_mutex()
|
|
{
|
|
static __gthread_once_t once = __GTHREAD_ONCE_INIT;
|
|
__gthread_once(&once, init);
|
|
return *static_mutex;
|
|
}
|
|
|
|
// Simple wrapper for exception safety.
|
|
struct mutex_wrapper
|
|
{
|
|
bool unlock;
|
|
mutex_wrapper() : unlock(true)
|
|
{ get_static_mutex().lock(); }
|
|
|
|
~mutex_wrapper()
|
|
{
|
|
if (unlock)
|
|
static_mutex->unlock();
|
|
}
|
|
};
|
|
}
|
|
# endif
|
|
|
|
# if defined(__GTHREAD_HAS_COND) && !defined(_GLIBCXX_USE_FUTEX)
|
|
namespace
|
|
{
|
|
// A single condition variable controlling all static initializations.
|
|
static __gnu_cxx::__cond* static_cond;
|
|
|
|
// using a fake type to avoid initializing a static class.
|
|
typedef char fake_cond_t[sizeof(__gnu_cxx::__cond)]
|
|
__attribute__ ((aligned(__alignof__(__gnu_cxx::__cond))));
|
|
fake_cond_t fake_cond;
|
|
|
|
static void init_static_cond()
|
|
{ static_cond = new (&fake_cond) __gnu_cxx::__cond(); }
|
|
|
|
__gnu_cxx::__cond&
|
|
get_static_cond()
|
|
{
|
|
static __gthread_once_t once = __GTHREAD_ONCE_INIT;
|
|
__gthread_once(&once, init_static_cond);
|
|
return *static_cond;
|
|
}
|
|
}
|
|
# endif
|
|
|
|
# ifndef _GLIBCXX_GUARD_TEST_AND_ACQUIRE
|
|
|
|
// Test the guard variable with a memory load with
|
|
// acquire semantics.
|
|
|
|
inline bool
|
|
__test_and_acquire (__cxxabiv1::__guard *g)
|
|
{
|
|
unsigned char __c;
|
|
unsigned char *__p = reinterpret_cast<unsigned char *>(g);
|
|
__atomic_load (__p, &__c, __ATOMIC_ACQUIRE);
|
|
(void) __p;
|
|
return _GLIBCXX_GUARD_TEST(&__c);
|
|
}
|
|
# define _GLIBCXX_GUARD_TEST_AND_ACQUIRE(G) __test_and_acquire (G)
|
|
# endif
|
|
|
|
# ifndef _GLIBCXX_GUARD_SET_AND_RELEASE
|
|
|
|
// Set the guard variable to 1 with memory order release semantics.
|
|
|
|
inline void
|
|
__set_and_release (__cxxabiv1::__guard *g)
|
|
{
|
|
unsigned char *__p = reinterpret_cast<unsigned char *>(g);
|
|
unsigned char val = 1;
|
|
__atomic_store (__p, &val, __ATOMIC_RELEASE);
|
|
(void) __p;
|
|
}
|
|
# define _GLIBCXX_GUARD_SET_AND_RELEASE(G) __set_and_release (G)
|
|
# endif
|
|
|
|
#else /* !__GTHREADS */
|
|
|
|
# undef _GLIBCXX_GUARD_TEST_AND_ACQUIRE
|
|
# undef _GLIBCXX_GUARD_SET_AND_RELEASE
|
|
# define _GLIBCXX_GUARD_SET_AND_RELEASE(G) _GLIBCXX_GUARD_SET (G)
|
|
|
|
#endif /* __GTHREADS */
|
|
|
|
//
|
|
// Here are C++ run-time routines for guarded initialization of static
|
|
// variables. There are 4 scenarios under which these routines are called:
|
|
//
|
|
// 1. Threads not supported (__GTHREADS not defined)
|
|
// 2. Threads are supported but not enabled at run-time.
|
|
// 3. Threads enabled at run-time but __gthreads_* are not fully POSIX.
|
|
// 4. Threads enabled at run-time and __gthreads_* support all POSIX threads
|
|
// primitives we need here.
|
|
//
|
|
// The old code supported scenarios 1-3 but was broken since it used a global
|
|
// mutex for all threads and had the mutex locked during the whole duration of
|
|
// initialization of a guarded static variable. The following created a
|
|
// dead-lock with the old code.
|
|
//
|
|
// Thread 1 acquires the global mutex.
|
|
// Thread 1 starts initializing static variable.
|
|
// Thread 1 creates thread 2 during initialization.
|
|
// Thread 2 attempts to acquire mutex to initialize another variable.
|
|
// Thread 2 blocks since thread 1 is locking the mutex.
|
|
// Thread 1 waits for result from thread 2 and also blocks. A deadlock.
|
|
//
|
|
// The new code here can handle this situation and thus is more robust. However,
|
|
// we need to use the POSIX thread condition variable, which is not supported
|
|
// in all platforms, notably older versions of Microsoft Windows. The gthr*.h
|
|
// headers define a symbol __GTHREAD_HAS_COND for platforms that support POSIX
|
|
// like condition variables. For platforms that do not support condition
|
|
// variables, we need to fall back to the old code.
|
|
|
|
// If _GLIBCXX_USE_FUTEX, no global mutex or condition variable is used,
|
|
// only atomic operations are used together with futex syscall.
|
|
// Valid values of the first integer in guard are:
|
|
// 0 No thread encountered the guarded init
|
|
// yet or it has been aborted.
|
|
// _GLIBCXX_GUARD_BIT The guarded static var has been successfully
|
|
// initialized.
|
|
// _GLIBCXX_GUARD_PENDING_BIT The guarded static var is being initialized
|
|
// and no other thread is waiting for its
|
|
// initialization.
|
|
// (_GLIBCXX_GUARD_PENDING_BIT The guarded static var is being initialized
|
|
// | _GLIBCXX_GUARD_WAITING_BIT) and some other threads are waiting until
|
|
// it is initialized.
|
|
|
|
namespace __cxxabiv1
|
|
{
|
|
#ifdef _GLIBCXX_USE_FUTEX
|
|
namespace
|
|
{
|
|
static inline int __guard_test_bit (const int __byte, const int __val)
|
|
{
|
|
union { int __i; char __c[sizeof (int)]; } __u = { 0 };
|
|
__u.__c[__byte] = __val;
|
|
return __u.__i;
|
|
}
|
|
}
|
|
#endif
|
|
|
|
static inline int
|
|
init_in_progress_flag(__guard* g)
|
|
{ return ((char *)g)[1]; }
|
|
|
|
static inline void
|
|
set_init_in_progress_flag(__guard* g, int v)
|
|
{ ((char *)g)[1] = v; }
|
|
|
|
static inline void
|
|
throw_recursive_init_exception()
|
|
{
|
|
#if __cpp_exceptions
|
|
throw __gnu_cxx::recursive_init_error();
|
|
#else
|
|
// Use __builtin_trap so we don't require abort().
|
|
__builtin_trap();
|
|
#endif
|
|
}
|
|
|
|
// acquire() is a helper function used to acquire guard if thread support is
|
|
// not compiled in or is compiled in but not enabled at run-time.
|
|
static int
|
|
acquire(__guard *g)
|
|
{
|
|
// Quit if the object is already initialized.
|
|
if (_GLIBCXX_GUARD_TEST(g))
|
|
return 0;
|
|
|
|
if (init_in_progress_flag(g))
|
|
throw_recursive_init_exception();
|
|
|
|
set_init_in_progress_flag(g, 1);
|
|
return 1;
|
|
}
|
|
|
|
extern "C"
|
|
int __cxa_guard_acquire (__guard *g)
|
|
{
|
|
#ifdef __GTHREADS
|
|
// If the target can reorder loads, we need to insert a read memory
|
|
// barrier so that accesses to the guarded variable happen after the
|
|
// guard test.
|
|
if (_GLIBCXX_GUARD_TEST_AND_ACQUIRE (g))
|
|
return 0;
|
|
|
|
# ifdef _GLIBCXX_USE_FUTEX
|
|
// If __atomic_* and futex syscall are supported, don't use any global
|
|
// mutex.
|
|
|
|
// Use the same bits in the guard variable whether single-threaded or not,
|
|
// so that __cxa_guard_release and __cxa_guard_abort match the logic here
|
|
// even if __libc_single_threaded becomes false between now and then.
|
|
|
|
if (__gnu_cxx::__is_single_threaded())
|
|
{
|
|
// No need to use atomics, and no need to wait for other threads.
|
|
int *gi = (int *) (void *) g;
|
|
if (*gi == 0)
|
|
{
|
|
*gi = _GLIBCXX_GUARD_PENDING_BIT;
|
|
return 1;
|
|
}
|
|
else
|
|
throw_recursive_init_exception();
|
|
}
|
|
else
|
|
{
|
|
int *gi = (int *) (void *) g;
|
|
const int guard_bit = _GLIBCXX_GUARD_BIT;
|
|
const int pending_bit = _GLIBCXX_GUARD_PENDING_BIT;
|
|
const int waiting_bit = _GLIBCXX_GUARD_WAITING_BIT;
|
|
|
|
while (1)
|
|
{
|
|
int expected(0);
|
|
if (__atomic_compare_exchange_n(gi, &expected, pending_bit, false,
|
|
__ATOMIC_ACQ_REL,
|
|
__ATOMIC_ACQUIRE))
|
|
{
|
|
// This thread should do the initialization.
|
|
return 1;
|
|
}
|
|
|
|
if (expected == guard_bit)
|
|
{
|
|
// Already initialized.
|
|
return 0;
|
|
}
|
|
|
|
if (expected == pending_bit)
|
|
{
|
|
// Use acquire here.
|
|
int newv = expected | waiting_bit;
|
|
if (!__atomic_compare_exchange_n(gi, &expected, newv, false,
|
|
__ATOMIC_ACQ_REL,
|
|
__ATOMIC_ACQUIRE))
|
|
{
|
|
if (expected == guard_bit)
|
|
{
|
|
// Make a thread that failed to set the
|
|
// waiting bit exit the function earlier,
|
|
// if it detects that another thread has
|
|
// successfully finished initialising.
|
|
return 0;
|
|
}
|
|
if (expected == 0)
|
|
continue;
|
|
}
|
|
|
|
expected = newv;
|
|
}
|
|
|
|
syscall (SYS_futex, gi, _GLIBCXX_FUTEX_WAIT, expected, 0);
|
|
}
|
|
}
|
|
# else // ! _GLIBCXX_USE_FUTEX
|
|
if (__gthread_active_p ())
|
|
{
|
|
mutex_wrapper mw;
|
|
|
|
while (1) // When this loop is executing, mutex is locked.
|
|
{
|
|
# ifdef __GTHREAD_HAS_COND
|
|
// The static is already initialized.
|
|
if (_GLIBCXX_GUARD_TEST(g))
|
|
return 0; // The mutex will be unlocked via wrapper
|
|
|
|
if (init_in_progress_flag(g))
|
|
{
|
|
// The guarded static is currently being initialized by
|
|
// another thread, so we release mutex and wait for the
|
|
// condition variable. We will lock the mutex again after
|
|
// this.
|
|
get_static_cond().wait_recursive(&get_static_mutex());
|
|
}
|
|
else
|
|
{
|
|
set_init_in_progress_flag(g, 1);
|
|
return 1; // The mutex will be unlocked via wrapper.
|
|
}
|
|
# else
|
|
// This provides compatibility with older systems not supporting
|
|
// POSIX like condition variables.
|
|
if (acquire(g))
|
|
{
|
|
mw.unlock = false;
|
|
return 1; // The mutex still locked.
|
|
}
|
|
return 0; // The mutex will be unlocked via wrapper.
|
|
# endif
|
|
}
|
|
}
|
|
# endif
|
|
#endif // ! __GTHREADS
|
|
|
|
return acquire (g);
|
|
}
|
|
|
|
extern "C"
|
|
void __cxa_guard_abort (__guard *g) noexcept
|
|
{
|
|
#ifdef _GLIBCXX_USE_FUTEX
|
|
// If __atomic_* and futex syscall are supported, don't use any global
|
|
// mutex.
|
|
|
|
if (__gnu_cxx::__is_single_threaded())
|
|
{
|
|
// No need to use atomics, and no other threads to wake.
|
|
int *gi = (int *) (void *) g;
|
|
*gi = 0;
|
|
return;
|
|
}
|
|
else
|
|
{
|
|
int *gi = (int *) (void *) g;
|
|
const int waiting_bit = _GLIBCXX_GUARD_WAITING_BIT;
|
|
int old = __atomic_exchange_n (gi, 0, __ATOMIC_ACQ_REL);
|
|
|
|
if ((old & waiting_bit) != 0)
|
|
syscall (SYS_futex, gi, _GLIBCXX_FUTEX_WAKE, INT_MAX);
|
|
return;
|
|
}
|
|
#elif defined(__GTHREAD_HAS_COND)
|
|
if (__gthread_active_p())
|
|
{
|
|
mutex_wrapper mw;
|
|
|
|
set_init_in_progress_flag(g, 0);
|
|
|
|
// If we abort, we still need to wake up all other threads waiting for
|
|
// the condition variable.
|
|
get_static_cond().broadcast();
|
|
return;
|
|
}
|
|
#endif
|
|
|
|
set_init_in_progress_flag(g, 0);
|
|
#if defined(__GTHREADS) && !defined(__GTHREAD_HAS_COND)
|
|
// This provides compatibility with older systems not supporting POSIX like
|
|
// condition variables.
|
|
if (__gthread_active_p ())
|
|
static_mutex->unlock();
|
|
#endif
|
|
}
|
|
|
|
extern "C"
|
|
void __cxa_guard_release (__guard *g) noexcept
|
|
{
|
|
#ifdef _GLIBCXX_USE_FUTEX
|
|
// If __atomic_* and futex syscall are supported, don't use any global
|
|
// mutex.
|
|
|
|
if (__gnu_cxx::__is_single_threaded())
|
|
{
|
|
int *gi = (int *) (void *) g;
|
|
*gi = _GLIBCXX_GUARD_BIT;
|
|
return;
|
|
}
|
|
else
|
|
{
|
|
int *gi = (int *) (void *) g;
|
|
const int guard_bit = _GLIBCXX_GUARD_BIT;
|
|
const int waiting_bit = _GLIBCXX_GUARD_WAITING_BIT;
|
|
int old = __atomic_exchange_n (gi, guard_bit, __ATOMIC_ACQ_REL);
|
|
|
|
if ((old & waiting_bit) != 0)
|
|
syscall (SYS_futex, gi, _GLIBCXX_FUTEX_WAKE, INT_MAX);
|
|
return;
|
|
}
|
|
|
|
#elif defined(__GTHREAD_HAS_COND)
|
|
if (__gthread_active_p())
|
|
{
|
|
mutex_wrapper mw;
|
|
|
|
set_init_in_progress_flag(g, 0);
|
|
_GLIBCXX_GUARD_SET_AND_RELEASE(g);
|
|
|
|
get_static_cond().broadcast();
|
|
return;
|
|
}
|
|
#endif
|
|
|
|
set_init_in_progress_flag(g, 0);
|
|
_GLIBCXX_GUARD_SET_AND_RELEASE (g);
|
|
|
|
#if defined(__GTHREADS) && !defined(__GTHREAD_HAS_COND)
|
|
// This provides compatibility with older systems not supporting POSIX like
|
|
// condition variables.
|
|
if (__gthread_active_p())
|
|
static_mutex->unlock();
|
|
#endif
|
|
}
|
|
}
|