wait.h (do_spin): New inline...

* config/linux/wait.h (do_spin): New inline, largely copied
	from do_wait, just don't do futex_wait here, instead return true if
	it should be done.
	(do_wait): Implement using do_spin.
	* config/linux/mutex.h (gomp_mutex_lock_slow): Add an int argument
	to prototype.
	(gomp_mutex_lock): Use __sync_val_compare_and_swap instead of
	__sync_bool_compare_and_swap, pass the oldval to
	gomp_mutex_lock_slow.
	* config/linux/mutex.c (gomp_mutex_lock_slow): Add oldval argument.
	If all mutex contenders are just spinning and not sleeping, don't
	change state to 2 unnecessarily.  Optimize the loop when state has
	already become 2 to use just one atomic operation per loop instead
	of two.
	* config/linux/ia64/mutex.h (gomp_mutex_lock_slow): Add an int argument
	to prototype.
	(gomp_mutex_lock): Use __sync_val_compare_and_swap instead of
	__sync_bool_compare_and_swap, pass the oldval to
	gomp_mutex_lock_slow.

From-SVN: r176326
This commit is contained in:
Jakub Jelinek 2011-07-15 21:46:11 +02:00 committed by Jakub Jelinek
parent 1be0310c76
commit bfee0724e0
5 changed files with 60 additions and 18 deletions

View File

@ -1,3 +1,25 @@
2011-07-15 Jakub Jelinek <jakub@redhat.com>
* config/linux/wait.h (do_spin): New inline, largely copied
from do_wait, just don't do futex_wait here, instead return true if
it should be done.
(do_wait): Implement using do_spin.
* config/linux/mutex.h (gomp_mutex_lock_slow): Add an int argument
to prototype.
(gomp_mutex_lock): Use __sync_val_compare_and_swap instead of
__sync_bool_compare_and_swap, pass the oldval to
gomp_mutex_lock_slow.
* config/linux/mutex.c (gomp_mutex_lock_slow): Add oldval argument.
If all mutex contenders are just spinning and not sleeping, don't
change state to 2 unnecessarily. Optimize the loop when state has
already become 2 to use just one atomic operation per loop instead
of two.
* config/linux/ia64/mutex.h (gomp_mutex_lock_slow): Add an int argument
to prototype.
(gomp_mutex_lock): Use __sync_val_compare_and_swap instead of
__sync_bool_compare_and_swap, pass the oldval to
gomp_mutex_lock_slow.
2011-06-22 Jakub Jelinek <jakub@redhat.com> 2011-06-22 Jakub Jelinek <jakub@redhat.com>
PR libgomp/49490 PR libgomp/49490

View File

@ -1,4 +1,4 @@
/* Copyright (C) 2005, 2008, 2009 Free Software Foundation, Inc. /* Copyright (C) 2005, 2008, 2009, 2011 Free Software Foundation, Inc.
Contributed by Richard Henderson <rth@redhat.com>. Contributed by Richard Henderson <rth@redhat.com>.
This file is part of the GNU OpenMP Library (libgomp). This file is part of the GNU OpenMP Library (libgomp).
@ -38,11 +38,12 @@ static inline void gomp_mutex_init (gomp_mutex_t *mutex)
*mutex = 0; *mutex = 0;
} }
extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex); extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex, int);
static inline void gomp_mutex_lock (gomp_mutex_t *mutex) static inline void gomp_mutex_lock (gomp_mutex_t *mutex)
{ {
if (!__sync_bool_compare_and_swap (mutex, 0, 1)) int oldval = __sync_val_compare_and_swap (mutex, 0, 1);
gomp_mutex_lock_slow (mutex); if (__builtin_expect (oldval, 0))
gomp_mutex_lock_slow (mutex, oldval);
} }
extern void gomp_mutex_unlock_slow (gomp_mutex_t *mutex); extern void gomp_mutex_unlock_slow (gomp_mutex_t *mutex);

View File

@ -1,4 +1,4 @@
/* Copyright (C) 2005, 2008, 2009 Free Software Foundation, Inc. /* Copyright (C) 2005, 2008, 2009, 2011 Free Software Foundation, Inc.
Contributed by Richard Henderson <rth@redhat.com>. Contributed by Richard Henderson <rth@redhat.com>.
This file is part of the GNU OpenMP Library (libgomp). This file is part of the GNU OpenMP Library (libgomp).
@ -32,15 +32,27 @@ long int gomp_futex_wake = FUTEX_WAKE | FUTEX_PRIVATE_FLAG;
long int gomp_futex_wait = FUTEX_WAIT | FUTEX_PRIVATE_FLAG; long int gomp_futex_wait = FUTEX_WAIT | FUTEX_PRIVATE_FLAG;
void void
gomp_mutex_lock_slow (gomp_mutex_t *mutex) gomp_mutex_lock_slow (gomp_mutex_t *mutex, int oldval)
{ {
do while (oldval == 1)
{ {
int oldval = __sync_val_compare_and_swap (mutex, 1, 2); if (do_spin (mutex, 1))
if (oldval != 0) {
do_wait (mutex, 2); oldval = __sync_lock_test_and_set (mutex, 2);
if (oldval == 0)
return;
futex_wait (mutex, 2);
break;
}
else
{
oldval = __sync_val_compare_and_swap (mutex, 0, 1);
if (oldval == 0)
return;
}
} }
while (!__sync_bool_compare_and_swap (mutex, 0, 2)); while ((oldval = __sync_lock_test_and_set (mutex, 2)))
do_wait (mutex, 2);
} }
void void

View File

@ -1,4 +1,4 @@
/* Copyright (C) 2005, 2009 Free Software Foundation, Inc. /* Copyright (C) 2005, 2009, 2011 Free Software Foundation, Inc.
Contributed by Richard Henderson <rth@redhat.com>. Contributed by Richard Henderson <rth@redhat.com>.
This file is part of the GNU OpenMP Library (libgomp). This file is part of the GNU OpenMP Library (libgomp).
@ -38,11 +38,12 @@ static inline void gomp_mutex_init (gomp_mutex_t *mutex)
*mutex = 0; *mutex = 0;
} }
extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex); extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex, int);
static inline void gomp_mutex_lock (gomp_mutex_t *mutex) static inline void gomp_mutex_lock (gomp_mutex_t *mutex)
{ {
if (!__sync_bool_compare_and_swap (mutex, 0, 1)) int oldval = __sync_val_compare_and_swap (mutex, 0, 1);
gomp_mutex_lock_slow (mutex); if (__builtin_expect (oldval, 0))
gomp_mutex_lock_slow (mutex, oldval);
} }
extern void gomp_mutex_unlock_slow (gomp_mutex_t *mutex); extern void gomp_mutex_unlock_slow (gomp_mutex_t *mutex);

View File

@ -44,7 +44,7 @@ extern long int gomp_futex_wait, gomp_futex_wake;
#include <futex.h> #include <futex.h>
static inline void do_wait (int *addr, int val) static inline int do_spin (int *addr, int val)
{ {
unsigned long long i, count = gomp_spin_count_var; unsigned long long i, count = gomp_spin_count_var;
@ -52,10 +52,16 @@ static inline void do_wait (int *addr, int val)
count = gomp_throttled_spin_count_var; count = gomp_throttled_spin_count_var;
for (i = 0; i < count; i++) for (i = 0; i < count; i++)
if (__builtin_expect (*addr != val, 0)) if (__builtin_expect (*addr != val, 0))
return; return 0;
else else
cpu_relax (); cpu_relax ();
futex_wait (addr, val); return 1;
}
static inline void do_wait (int *addr, int val)
{
if (do_spin (addr, val))
futex_wait (addr, val);
} }
#ifdef HAVE_ATTRIBUTE_VISIBILITY #ifdef HAVE_ATTRIBUTE_VISIBILITY