Add HTM fastpath and use Intel RTM for it on x86.
* beginend.cc (htm_fastpath): New. (gtm_thread::begin_transaction, _ITM_commitTransaction, _ITM_commitTransactionEH): Add HTM fastpath handling. * config/linux/rwlock.h (gtm_rwlock.is_write_locked): New. * config/posix/rwlock.h (gtm_rwlock.is_write_locked): New. * config/x86/target.h (htm_available, htm_init, htm_begin_success, htm_begin, htm_commit, htm_abort, htm_abort_should_retry): New. * configure.tgt: Add -mrtm to XCFLAGS. * method-serial.cc (htm_mg, o_htm_mg, htm_dispatch, dispatch_htm): New. (gtm_thread::serialirr_mode): Add HTM fastpath handling. * libitm_i.h (htm_fastpath, dispatch_htm): Declare. * retry.cc (parse_default_method): Add HTM method parsing. (gtm_thread::number_of_threads_changed): Use HTM by default if available. From-SVN: r193369
This commit is contained in:
Torvald Riegel
Torvald Riegel
parent
2a28e76a78
commit
64fbcc74a3
|
|
@ -1,3 +1,20 @@
|
|||
2012-11-09 Torvald Riegel <triegel@redhat.com>
|
||||
|
||||
* beginend.cc (htm_fastpath): New.
|
||||
(gtm_thread::begin_transaction, _ITM_commitTransaction,
|
||||
_ITM_commitTransactionEH): Add HTM fastpath handling.
|
||||
* config/linux/rwlock.h (gtm_rwlock.is_write_locked): New.
|
||||
* config/posix/rwlock.h (gtm_rwlock.is_write_locked): New.
|
||||
* config/x86/target.h (htm_available, htm_init, htm_begin_success,
|
||||
htm_begin, htm_commit, htm_abort, htm_abort_should_retry): New.
|
||||
* configure.tgt: Add -mrtm to XCFLAGS.
|
||||
* method-serial.cc (htm_mg, o_htm_mg, htm_dispatch, dispatch_htm): New.
|
||||
(gtm_thread::serialirr_mode): Add HTM fastpath handling.
|
||||
* libitm_i.h (htm_fastpath, dispatch_htm): Declare.
|
||||
* retry.cc (parse_default_method): Add HTM method parsing.
|
||||
(gtm_thread::number_of_threads_changed): Use HTM by default if
|
||||
available.
|
||||
|
||||
2012-11-04 Thomas Schwinge <thomas@codesourcery.com>
|
||||
|
||||
* configure: Regenerate.
|
||||
|
|
|
|||
|
|
@ -54,6 +54,8 @@ static pthread_mutex_t global_tid_lock = PTHREAD_MUTEX_INITIALIZER;
|
|||
static pthread_key_t thr_release_key;
|
||||
static pthread_once_t thr_release_once = PTHREAD_ONCE_INIT;
|
||||
|
||||
// See gtm_thread::begin_transaction.
|
||||
uint32_t GTM::htm_fastpath = 0;
|
||||
|
||||
/* Allocate a transaction structure. */
|
||||
void *
|
||||
|
|
@ -163,6 +165,70 @@ GTM::gtm_thread::begin_transaction (uint32_t prop, const gtm_jmpbuf *jb)
|
|||
if (unlikely(prop & pr_undoLogCode))
|
||||
GTM_fatal("pr_undoLogCode not supported");
|
||||
|
||||
#if defined(USE_HTM_FASTPATH) && !defined(HTM_CUSTOM_FASTPATH)
|
||||
// HTM fastpath. Only chosen in the absence of transaction_cancel to allow
|
||||
// using an uninstrumented code path.
|
||||
// The fastpath is enabled only by dispatch_htm's method group, which uses
|
||||
// serial-mode methods as fallback. Serial-mode transactions cannot execute
|
||||
// concurrently with HW transactions because the latter monitor the serial
|
||||
// lock's writer flag and thus abort if another thread is or becomes a
|
||||
// serial transaction. Therefore, if the fastpath is enabled, then a
|
||||
// transaction is not executing as a HW transaction iff the serial lock is
|
||||
// write-locked. This allows us to use htm_fastpath and the serial lock's
|
||||
// writer flag to reliable determine whether the current thread runs a HW
|
||||
// transaction, and thus we do not need to maintain this information in
|
||||
// per-thread state.
|
||||
// If an uninstrumented code path is not available, we can still run
|
||||
// instrumented code from a HW transaction because the HTM fastpath kicks
|
||||
// in early in both begin and commit, and the transaction is not canceled.
|
||||
// HW transactions might get requests to switch to serial-irrevocable mode,
|
||||
// but these can be ignored because the HTM provides all necessary
|
||||
// correctness guarantees. Transactions cannot detect whether they are
|
||||
// indeed in serial mode, and HW transactions should never need serial mode
|
||||
// for any internal changes (e.g., they never abort visibly to the STM code
|
||||
// and thus do not trigger the standard retry handling).
|
||||
if (likely(htm_fastpath && (prop & pr_hasNoAbort)))
|
||||
{
|
||||
for (uint32_t t = htm_fastpath; t; t--)
|
||||
{
|
||||
uint32_t ret = htm_begin();
|
||||
if (htm_begin_success(ret))
|
||||
{
|
||||
// We are executing a transaction now.
|
||||
// Monitor the writer flag in the serial-mode lock, and abort
|
||||
// if there is an active or waiting serial-mode transaction.
|
||||
if (unlikely(serial_lock.is_write_locked()))
|
||||
htm_abort();
|
||||
else
|
||||
// We do not need to set a_saveLiveVariables because of HTM.
|
||||
return (prop & pr_uninstrumentedCode) ?
|
||||
a_runUninstrumentedCode : a_runInstrumentedCode;
|
||||
}
|
||||
// The transaction has aborted. Don't retry if it's unlikely that
|
||||
// retrying the transaction will be successful.
|
||||
if (!htm_abort_should_retry(ret))
|
||||
break;
|
||||
// Wait until any concurrent serial-mode transactions have finished.
|
||||
// This is an empty critical section, but won't be elided.
|
||||
if (serial_lock.is_write_locked())
|
||||
{
|
||||
tx = gtm_thr();
|
||||
if (unlikely(tx == NULL))
|
||||
{
|
||||
// See below.
|
||||
tx = new gtm_thread();
|
||||
set_gtm_thr(tx);
|
||||
}
|
||||
serial_lock.read_lock(tx);
|
||||
serial_lock.read_unlock(tx);
|
||||
// TODO We should probably reset the retry count t here, unless
|
||||
// we have retried so often that we should go serial to avoid
|
||||
// starvation.
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
tx = gtm_thr();
|
||||
if (unlikely(tx == NULL))
|
||||
{
|
||||
|
|
@ -537,6 +603,17 @@ GTM::gtm_thread::restart (gtm_restart_reason r, bool finish_serial_upgrade)
|
|||
void ITM_REGPARM
|
||||
_ITM_commitTransaction(void)
|
||||
{
|
||||
#if defined(USE_HTM_FASTPATH)
|
||||
// HTM fastpath. If we are not executing a HW transaction, then we will be
|
||||
// a serial-mode transaction. If we are, then there will be no other
|
||||
// concurrent serial-mode transaction.
|
||||
// See gtm_thread::begin_transaction.
|
||||
if (likely(htm_fastpath && !gtm_thread::serial_lock.is_write_locked()))
|
||||
{
|
||||
htm_commit();
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
gtm_thread *tx = gtm_thr();
|
||||
if (!tx->trycommit ())
|
||||
tx->restart (RESTART_VALIDATE_COMMIT);
|
||||
|
|
@ -545,6 +622,14 @@ _ITM_commitTransaction(void)
|
|||
void ITM_REGPARM
|
||||
_ITM_commitTransactionEH(void *exc_ptr)
|
||||
{
|
||||
#if defined(USE_HTM_FASTPATH)
|
||||
// See _ITM_commitTransaction.
|
||||
if (likely(htm_fastpath && !gtm_thread::serial_lock.is_write_locked()))
|
||||
{
|
||||
htm_commit();
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
gtm_thread *tx = gtm_thr();
|
||||
if (!tx->trycommit ())
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* Copyright (C) 2011 Free Software Foundation, Inc.
|
||||
/* Copyright (C) 2011, 2012 Free Software Foundation, Inc.
|
||||
Contributed by Torvald Riegel <triegel@redhat.com>.
|
||||
|
||||
This file is part of the GNU Transactional Memory Library (libitm).
|
||||
|
|
@ -59,6 +59,14 @@ class gtm_rwlock
|
|||
bool write_upgrade (gtm_thread *tx);
|
||||
void write_upgrade_finish (gtm_thread *tx);
|
||||
|
||||
// Returns true iff there is a concurrent active or waiting writer.
|
||||
// This is primarily useful for simple HyTM approaches, and the value being
|
||||
// checked is loaded with memory_order_relaxed.
|
||||
bool is_write_locked()
|
||||
{
|
||||
return writers.load (memory_order_relaxed) != 0;
|
||||
}
|
||||
|
||||
protected:
|
||||
bool write_lock_generic (gtm_thread *tx);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* Copyright (C) 2009, 2011 Free Software Foundation, Inc.
|
||||
/* Copyright (C) 2009, 2011, 2012 Free Software Foundation, Inc.
|
||||
Contributed by Richard Henderson <rth@redhat.com>.
|
||||
|
||||
This file is part of the GNU Transactional Memory Library (libitm).
|
||||
|
|
@ -74,6 +74,14 @@ class gtm_rwlock
|
|||
bool write_upgrade (gtm_thread *tx);
|
||||
void write_upgrade_finish (gtm_thread *tx);
|
||||
|
||||
// Returns true iff there is a concurrent active or waiting writer.
|
||||
// This is primarily useful for simple HyTM approaches, and the value being
|
||||
// checked is loaded with memory_order_relaxed.
|
||||
bool is_write_locked()
|
||||
{
|
||||
return summary.load (memory_order_relaxed) & (a_writer | w_writer);
|
||||
}
|
||||
|
||||
protected:
|
||||
bool write_lock_generic (gtm_thread *tx);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* Copyright (C) 2008, 2009, 2011 Free Software Foundation, Inc.
|
||||
/* Copyright (C) 2008, 2009, 2011, 2012 Free Software Foundation, Inc.
|
||||
Contributed by Richard Henderson <rth@redhat.com>.
|
||||
|
||||
This file is part of the GNU Transactional Memory Library (libitm).
|
||||
|
|
@ -22,6 +22,10 @@
|
|||
see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
|
||||
<http://www.gnu.org/licenses/>. */
|
||||
|
||||
// We'll be using some of the cpu builtins, and their associated types.
|
||||
#include <x86intrin.h>
|
||||
#include <cpuid.h>
|
||||
|
||||
namespace GTM HIDDEN {
|
||||
|
||||
/* ??? This doesn't work for Win64. */
|
||||
|
|
@ -62,19 +66,66 @@ cpu_relax (void)
|
|||
__builtin_ia32_pause ();
|
||||
}
|
||||
|
||||
} // namespace GTM
|
||||
// Use Intel RTM if supported by the assembler.
|
||||
// See gtm_thread::begin_transaction for how these functions are used.
|
||||
#ifdef HAVE_AS_RTM
|
||||
#define USE_HTM_FASTPATH
|
||||
|
||||
// We'll be using some of the cpu builtins, and their associated types.
|
||||
#ifndef __cplusplus
|
||||
/* ??? It's broken for C++. */
|
||||
#include <x86intrin.h>
|
||||
#else
|
||||
# ifdef __SSE2__
|
||||
# include <emmintrin.h>
|
||||
# elif defined(__SSE__)
|
||||
# include <xmmintrin.h>
|
||||
# endif
|
||||
# ifdef __AVX__
|
||||
# include <immintrin.h>
|
||||
# endif
|
||||
static inline bool
|
||||
htm_available ()
|
||||
{
|
||||
const unsigned cpuid_rtm = bit_RTM;
|
||||
if (__get_cpuid_max (0, NULL) >= 7)
|
||||
{
|
||||
unsigned a, b, c, d;
|
||||
__cpuid_count (7, 0, a, b, c, d);
|
||||
if (b & cpuid_rtm)
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline uint32_t
|
||||
htm_init ()
|
||||
{
|
||||
// Maximum number of times we try to execute a transaction as a HW
|
||||
// transaction.
|
||||
// ??? Why 2? Any offline or runtime tuning necessary?
|
||||
return htm_available () ? 2 : 0;
|
||||
}
|
||||
|
||||
static inline uint32_t
|
||||
htm_begin ()
|
||||
{
|
||||
return _xbegin();
|
||||
}
|
||||
|
||||
static inline bool
|
||||
htm_begin_success (uint32_t begin_ret)
|
||||
{
|
||||
return begin_ret == _XBEGIN_STARTED;
|
||||
}
|
||||
|
||||
static inline void
|
||||
htm_commit ()
|
||||
{
|
||||
_xend();
|
||||
}
|
||||
|
||||
static inline void
|
||||
htm_abort ()
|
||||
{
|
||||
// ??? According to a yet unpublished ABI rule, 0xff is reserved and
|
||||
// supposed to signal a busy lock. Source: andi.kleen@intel.com
|
||||
_xabort(0xff);
|
||||
}
|
||||
|
||||
static inline bool
|
||||
htm_abort_should_retry (uint32_t begin_ret)
|
||||
{
|
||||
return begin_ret & _XABORT_RETRY;
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
} // namespace GTM
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@ case "${target_cpu}" in
|
|||
XCFLAGS="${XCFLAGS} -fomit-frame-pointer"
|
||||
fi
|
||||
esac
|
||||
XCFLAGS="${XCFLAGS} -mrtm"
|
||||
ARCH=x86
|
||||
;;
|
||||
|
||||
|
|
@ -101,6 +102,7 @@ case "${target_cpu}" in
|
|||
XCFLAGS="${XCFLAGS} -fomit-frame-pointer"
|
||||
;;
|
||||
esac
|
||||
XCFLAGS="${XCFLAGS} -mrtm"
|
||||
ARCH=x86
|
||||
;;
|
||||
|
||||
|
|
|
|||
|
|
@ -332,9 +332,14 @@ extern abi_dispatch *dispatch_serialirr();
|
|||
extern abi_dispatch *dispatch_serialirr_onwrite();
|
||||
extern abi_dispatch *dispatch_gl_wt();
|
||||
extern abi_dispatch *dispatch_ml_wt();
|
||||
extern abi_dispatch *dispatch_htm();
|
||||
|
||||
extern gtm_cacheline_mask gtm_mask_stack(gtm_cacheline *, gtm_cacheline_mask);
|
||||
|
||||
// Control variable for the HTM fastpath that uses serial mode as fallback.
|
||||
// Non-zero if the HTM fastpath is enabled. See gtm_thread::begin_transaction.
|
||||
extern uint32_t htm_fastpath;
|
||||
|
||||
} // namespace GTM
|
||||
|
||||
#endif // LIBITM_I_H
|
||||
|
|
|
|||
|
|
@ -212,11 +212,46 @@ class serialirr_onwrite_dispatch : public serialirr_dispatch
|
|||
}
|
||||
};
|
||||
|
||||
// This group is pure HTM with serial mode as a fallback. There is no
|
||||
// difference to serial_mg except that we need to enable or disable the HTM
|
||||
// fastpath. See gtm_thread::begin_transaction.
|
||||
struct htm_mg : public method_group
|
||||
{
|
||||
virtual void init()
|
||||
{
|
||||
// Enable the HTM fastpath if the HW is available. The fastpath is
|
||||
// initially disabled.
|
||||
#ifdef USE_HTM_FASTPATH
|
||||
htm_fastpath = htm_init();
|
||||
#endif
|
||||
}
|
||||
virtual void fini()
|
||||
{
|
||||
// Disable the HTM fastpath.
|
||||
htm_fastpath = 0;
|
||||
}
|
||||
};
|
||||
|
||||
static htm_mg o_htm_mg;
|
||||
|
||||
// We just need the subclass to associate it with the HTM method group that
|
||||
// sets up the HTM fast path. This will use serial_dispatch as fallback for
|
||||
// transactions that might get canceled; it has a different method group, but
|
||||
// this is harmless for serial dispatchs because they never abort.
|
||||
class htm_dispatch : public serialirr_dispatch
|
||||
{
|
||||
public:
|
||||
htm_dispatch() : serialirr_dispatch(false, true, false, false,
|
||||
gtm_thread::STATE_SERIAL | gtm_thread::STATE_IRREVOCABLE, &o_htm_mg)
|
||||
{ }
|
||||
};
|
||||
|
||||
} // anon namespace
|
||||
|
||||
static const serialirr_dispatch o_serialirr_dispatch;
|
||||
static const serial_dispatch o_serial_dispatch;
|
||||
static const serialirr_onwrite_dispatch o_serialirr_onwrite_dispatch;
|
||||
static const htm_dispatch o_htm_dispatch;
|
||||
|
||||
abi_dispatch *
|
||||
GTM::dispatch_serialirr ()
|
||||
|
|
@ -237,6 +272,12 @@ GTM::dispatch_serialirr_onwrite ()
|
|||
const_cast<serialirr_onwrite_dispatch *>(&o_serialirr_onwrite_dispatch);
|
||||
}
|
||||
|
||||
abi_dispatch *
|
||||
GTM::dispatch_htm ()
|
||||
{
|
||||
return const_cast<htm_dispatch *>(&o_htm_dispatch);
|
||||
}
|
||||
|
||||
// Put the transaction into serial-irrevocable mode.
|
||||
|
||||
void
|
||||
|
|
@ -244,6 +285,13 @@ GTM::gtm_thread::serialirr_mode ()
|
|||
{
|
||||
struct abi_dispatch *disp = abi_disp ();
|
||||
|
||||
#if defined(USE_HTM_FASTPATH)
|
||||
// HTM fastpath. If we are executing a HW transaction, don't go serial but
|
||||
// continue. See gtm_thread::begin_transaction.
|
||||
if (likely(htm_fastpath && !gtm_thread::serial_lock.is_write_locked()))
|
||||
return;
|
||||
#endif
|
||||
|
||||
if (this->state & STATE_SERIAL)
|
||||
{
|
||||
if (this->state & STATE_IRREVOCABLE)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* Copyright (C) 2008, 2009, 2011 Free Software Foundation, Inc.
|
||||
/* Copyright (C) 2008, 2009, 2011, 2012 Free Software Foundation, Inc.
|
||||
Contributed by Richard Henderson <rth@redhat.com>.
|
||||
|
||||
This file is part of the GNU Transactional Memory Library (libitm).
|
||||
|
|
@ -254,6 +254,11 @@ parse_default_method()
|
|||
disp = GTM::dispatch_ml_wt();
|
||||
env += 5;
|
||||
}
|
||||
else if (strncmp(env, "htm", 3) == 0)
|
||||
{
|
||||
disp = GTM::dispatch_htm();
|
||||
env += 3;
|
||||
}
|
||||
else
|
||||
goto unknown;
|
||||
|
||||
|
|
@ -311,7 +316,15 @@ GTM::gtm_thread::number_of_threads_changed(unsigned previous, unsigned now)
|
|||
set_default_dispatch(default_dispatch_user);
|
||||
else
|
||||
{
|
||||
abi_dispatch* a = dispatch_ml_wt();
|
||||
// If HTM is available, use it by default with serial mode as
|
||||
// fallback. Otherwise, use ml_wt because it probably scales best.
|
||||
abi_dispatch* a;
|
||||
#ifdef USE_HTM_FASTPATH
|
||||
if (htm_available())
|
||||
a = dispatch_htm();
|
||||
else
|
||||
#endif
|
||||
a = dispatch_ml_wt();
|
||||
if (a->supports(now))
|
||||
set_default_dispatch(a);
|
||||
else
|
||||
|
|
|
|||
Loading…
Reference in New Issue