From bfee0724e0c7f3473bf17274cc750d65207db037 Mon Sep 17 00:00:00 2001 From: Jakub Jelinek Date: Fri, 15 Jul 2011 21:46:11 +0200 Subject: [PATCH] wait.h (do_spin): New inline... * config/linux/wait.h (do_spin): New inline, largely copied from do_wait, just don't do futex_wait here, instead return true if it should be done. (do_wait): Implement using do_spin. * config/linux/mutex.h (gomp_mutex_lock_slow): Add an int argument to prototype. (gomp_mutex_lock): Use __sync_val_compare_and_swap instead of __sync_bool_compare_and_swap, pass the oldval to gomp_mutex_lock_slow. * config/linux/mutex.c (gomp_mutex_lock_slow): Add oldval argument. If all mutex contenders are just spinning and not sleeping, don't change state to 2 unnecessarily. Optimize the loop when state has already become 2 to use just one atomic operation per loop instead of two. * config/linux/ia64/mutex.h (gomp_mutex_lock_slow): Add an int argument to prototype. (gomp_mutex_lock): Use __sync_val_compare_and_swap instead of __sync_bool_compare_and_swap, pass the oldval to gomp_mutex_lock_slow. From-SVN: r176326 --- libgomp/ChangeLog | 22 ++++++++++++++++++++++ libgomp/config/linux/ia64/mutex.h | 9 +++++---- libgomp/config/linux/mutex.c | 26 +++++++++++++++++++------- libgomp/config/linux/mutex.h | 9 +++++---- libgomp/config/linux/wait.h | 12 +++++++++--- 5 files changed, 60 insertions(+), 18 deletions(-) diff --git a/libgomp/ChangeLog b/libgomp/ChangeLog index 1d93273b919..36c716c518e 100644 --- a/libgomp/ChangeLog +++ b/libgomp/ChangeLog @@ -1,3 +1,25 @@ +2011-07-15 Jakub Jelinek + + * config/linux/wait.h (do_spin): New inline, largely copied + from do_wait, just don't do futex_wait here, instead return true if + it should be done. + (do_wait): Implement using do_spin. + * config/linux/mutex.h (gomp_mutex_lock_slow): Add an int argument + to prototype. + (gomp_mutex_lock): Use __sync_val_compare_and_swap instead of + __sync_bool_compare_and_swap, pass the oldval to + gomp_mutex_lock_slow. + * config/linux/mutex.c (gomp_mutex_lock_slow): Add oldval argument. + If all mutex contenders are just spinning and not sleeping, don't + change state to 2 unnecessarily. Optimize the loop when state has + already become 2 to use just one atomic operation per loop instead + of two. + * config/linux/ia64/mutex.h (gomp_mutex_lock_slow): Add an int argument + to prototype. + (gomp_mutex_lock): Use __sync_val_compare_and_swap instead of + __sync_bool_compare_and_swap, pass the oldval to + gomp_mutex_lock_slow. + 2011-06-22 Jakub Jelinek PR libgomp/49490 diff --git a/libgomp/config/linux/ia64/mutex.h b/libgomp/config/linux/ia64/mutex.h index 6e294059b23..8a67673df40 100644 --- a/libgomp/config/linux/ia64/mutex.h +++ b/libgomp/config/linux/ia64/mutex.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2005, 2008, 2009 Free Software Foundation, Inc. +/* Copyright (C) 2005, 2008, 2009, 2011 Free Software Foundation, Inc. Contributed by Richard Henderson . This file is part of the GNU OpenMP Library (libgomp). @@ -38,11 +38,12 @@ static inline void gomp_mutex_init (gomp_mutex_t *mutex) *mutex = 0; } -extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex); +extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex, int); static inline void gomp_mutex_lock (gomp_mutex_t *mutex) { - if (!__sync_bool_compare_and_swap (mutex, 0, 1)) - gomp_mutex_lock_slow (mutex); + int oldval = __sync_val_compare_and_swap (mutex, 0, 1); + if (__builtin_expect (oldval, 0)) + gomp_mutex_lock_slow (mutex, oldval); } extern void gomp_mutex_unlock_slow (gomp_mutex_t *mutex); diff --git a/libgomp/config/linux/mutex.c b/libgomp/config/linux/mutex.c index 3ca37c19f5f..2574f7be42a 100644 --- a/libgomp/config/linux/mutex.c +++ b/libgomp/config/linux/mutex.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2005, 2008, 2009 Free Software Foundation, Inc. +/* Copyright (C) 2005, 2008, 2009, 2011 Free Software Foundation, Inc. Contributed by Richard Henderson . This file is part of the GNU OpenMP Library (libgomp). @@ -32,15 +32,27 @@ long int gomp_futex_wake = FUTEX_WAKE | FUTEX_PRIVATE_FLAG; long int gomp_futex_wait = FUTEX_WAIT | FUTEX_PRIVATE_FLAG; void -gomp_mutex_lock_slow (gomp_mutex_t *mutex) +gomp_mutex_lock_slow (gomp_mutex_t *mutex, int oldval) { - do + while (oldval == 1) { - int oldval = __sync_val_compare_and_swap (mutex, 1, 2); - if (oldval != 0) - do_wait (mutex, 2); + if (do_spin (mutex, 1)) + { + oldval = __sync_lock_test_and_set (mutex, 2); + if (oldval == 0) + return; + futex_wait (mutex, 2); + break; + } + else + { + oldval = __sync_val_compare_and_swap (mutex, 0, 1); + if (oldval == 0) + return; + } } - while (!__sync_bool_compare_and_swap (mutex, 0, 2)); + while ((oldval = __sync_lock_test_and_set (mutex, 2))) + do_wait (mutex, 2); } void diff --git a/libgomp/config/linux/mutex.h b/libgomp/config/linux/mutex.h index 1905ce012fd..eafb0e76734 100644 --- a/libgomp/config/linux/mutex.h +++ b/libgomp/config/linux/mutex.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2005, 2009 Free Software Foundation, Inc. +/* Copyright (C) 2005, 2009, 2011 Free Software Foundation, Inc. Contributed by Richard Henderson . This file is part of the GNU OpenMP Library (libgomp). @@ -38,11 +38,12 @@ static inline void gomp_mutex_init (gomp_mutex_t *mutex) *mutex = 0; } -extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex); +extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex, int); static inline void gomp_mutex_lock (gomp_mutex_t *mutex) { - if (!__sync_bool_compare_and_swap (mutex, 0, 1)) - gomp_mutex_lock_slow (mutex); + int oldval = __sync_val_compare_and_swap (mutex, 0, 1); + if (__builtin_expect (oldval, 0)) + gomp_mutex_lock_slow (mutex, oldval); } extern void gomp_mutex_unlock_slow (gomp_mutex_t *mutex); diff --git a/libgomp/config/linux/wait.h b/libgomp/config/linux/wait.h index 0e8abf111b0..4f659855811 100644 --- a/libgomp/config/linux/wait.h +++ b/libgomp/config/linux/wait.h @@ -44,7 +44,7 @@ extern long int gomp_futex_wait, gomp_futex_wake; #include -static inline void do_wait (int *addr, int val) +static inline int do_spin (int *addr, int val) { unsigned long long i, count = gomp_spin_count_var; @@ -52,10 +52,16 @@ static inline void do_wait (int *addr, int val) count = gomp_throttled_spin_count_var; for (i = 0; i < count; i++) if (__builtin_expect (*addr != val, 0)) - return; + return 0; else cpu_relax (); - futex_wait (addr, val); + return 1; +} + +static inline void do_wait (int *addr, int val) +{ + if (do_spin (addr, val)) + futex_wait (addr, val); } #ifdef HAVE_ATTRIBUTE_VISIBILITY