3405d230b3
This commit adds the architecture support required to enable the optimised implementation of lockrefs. That's as simple as defining arch_spin_value_unlocked() and selecting the Kconfig option. We also define cmpxchg64_relaxed(), because the lockref code does not need the cmpxchg to have barrier semantics. Using Linus' test case[1] on one system I see a 4x improvement for the basic enablement, and a further 1.3x for cmpxchg64_relaxed(), for a total of 5.3x vs the baseline. On another system I see more like 2x improvement. [1]: http://marc.info/?l=linux-fsdevel&m=137782380714721&w=4 Signed-off-by: Michael Ellerman <mpe@ellerman.id.au> Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
311 lines
6.4 KiB
C
311 lines
6.4 KiB
C
#ifndef _ASM_POWERPC_CMPXCHG_H_
|
|
#define _ASM_POWERPC_CMPXCHG_H_
|
|
|
|
#ifdef __KERNEL__
|
|
#include <linux/compiler.h>
|
|
#include <asm/synch.h>
|
|
#include <asm/asm-compat.h>
|
|
|
|
/*
|
|
* Atomic exchange
|
|
*
|
|
* Changes the memory location '*ptr' to be val and returns
|
|
* the previous value stored there.
|
|
*/
|
|
static __always_inline unsigned long
|
|
__xchg_u32(volatile void *p, unsigned long val)
|
|
{
|
|
unsigned long prev;
|
|
|
|
__asm__ __volatile__(
|
|
PPC_RELEASE_BARRIER
|
|
"1: lwarx %0,0,%2 \n"
|
|
PPC405_ERR77(0,%2)
|
|
" stwcx. %3,0,%2 \n\
|
|
bne- 1b"
|
|
PPC_ACQUIRE_BARRIER
|
|
: "=&r" (prev), "+m" (*(volatile unsigned int *)p)
|
|
: "r" (p), "r" (val)
|
|
: "cc", "memory");
|
|
|
|
return prev;
|
|
}
|
|
|
|
/*
|
|
* Atomic exchange
|
|
*
|
|
* Changes the memory location '*ptr' to be val and returns
|
|
* the previous value stored there.
|
|
*/
|
|
static __always_inline unsigned long
|
|
__xchg_u32_local(volatile void *p, unsigned long val)
|
|
{
|
|
unsigned long prev;
|
|
|
|
__asm__ __volatile__(
|
|
"1: lwarx %0,0,%2 \n"
|
|
PPC405_ERR77(0,%2)
|
|
" stwcx. %3,0,%2 \n\
|
|
bne- 1b"
|
|
: "=&r" (prev), "+m" (*(volatile unsigned int *)p)
|
|
: "r" (p), "r" (val)
|
|
: "cc", "memory");
|
|
|
|
return prev;
|
|
}
|
|
|
|
#ifdef CONFIG_PPC64
|
|
static __always_inline unsigned long
|
|
__xchg_u64(volatile void *p, unsigned long val)
|
|
{
|
|
unsigned long prev;
|
|
|
|
__asm__ __volatile__(
|
|
PPC_RELEASE_BARRIER
|
|
"1: ldarx %0,0,%2 \n"
|
|
PPC405_ERR77(0,%2)
|
|
" stdcx. %3,0,%2 \n\
|
|
bne- 1b"
|
|
PPC_ACQUIRE_BARRIER
|
|
: "=&r" (prev), "+m" (*(volatile unsigned long *)p)
|
|
: "r" (p), "r" (val)
|
|
: "cc", "memory");
|
|
|
|
return prev;
|
|
}
|
|
|
|
static __always_inline unsigned long
|
|
__xchg_u64_local(volatile void *p, unsigned long val)
|
|
{
|
|
unsigned long prev;
|
|
|
|
__asm__ __volatile__(
|
|
"1: ldarx %0,0,%2 \n"
|
|
PPC405_ERR77(0,%2)
|
|
" stdcx. %3,0,%2 \n\
|
|
bne- 1b"
|
|
: "=&r" (prev), "+m" (*(volatile unsigned long *)p)
|
|
: "r" (p), "r" (val)
|
|
: "cc", "memory");
|
|
|
|
return prev;
|
|
}
|
|
#endif
|
|
|
|
/*
|
|
* This function doesn't exist, so you'll get a linker error
|
|
* if something tries to do an invalid xchg().
|
|
*/
|
|
extern void __xchg_called_with_bad_pointer(void);
|
|
|
|
static __always_inline unsigned long
|
|
__xchg(volatile void *ptr, unsigned long x, unsigned int size)
|
|
{
|
|
switch (size) {
|
|
case 4:
|
|
return __xchg_u32(ptr, x);
|
|
#ifdef CONFIG_PPC64
|
|
case 8:
|
|
return __xchg_u64(ptr, x);
|
|
#endif
|
|
}
|
|
__xchg_called_with_bad_pointer();
|
|
return x;
|
|
}
|
|
|
|
static __always_inline unsigned long
|
|
__xchg_local(volatile void *ptr, unsigned long x, unsigned int size)
|
|
{
|
|
switch (size) {
|
|
case 4:
|
|
return __xchg_u32_local(ptr, x);
|
|
#ifdef CONFIG_PPC64
|
|
case 8:
|
|
return __xchg_u64_local(ptr, x);
|
|
#endif
|
|
}
|
|
__xchg_called_with_bad_pointer();
|
|
return x;
|
|
}
|
|
#define xchg(ptr,x) \
|
|
({ \
|
|
__typeof__(*(ptr)) _x_ = (x); \
|
|
(__typeof__(*(ptr))) __xchg((ptr), (unsigned long)_x_, sizeof(*(ptr))); \
|
|
})
|
|
|
|
#define xchg_local(ptr,x) \
|
|
({ \
|
|
__typeof__(*(ptr)) _x_ = (x); \
|
|
(__typeof__(*(ptr))) __xchg_local((ptr), \
|
|
(unsigned long)_x_, sizeof(*(ptr))); \
|
|
})
|
|
|
|
/*
|
|
* Compare and exchange - if *p == old, set it to new,
|
|
* and return the old value of *p.
|
|
*/
|
|
#define __HAVE_ARCH_CMPXCHG 1
|
|
|
|
static __always_inline unsigned long
|
|
__cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new)
|
|
{
|
|
unsigned int prev;
|
|
|
|
__asm__ __volatile__ (
|
|
PPC_RELEASE_BARRIER
|
|
"1: lwarx %0,0,%2 # __cmpxchg_u32\n\
|
|
cmpw 0,%0,%3\n\
|
|
bne- 2f\n"
|
|
PPC405_ERR77(0,%2)
|
|
" stwcx. %4,0,%2\n\
|
|
bne- 1b"
|
|
PPC_ACQUIRE_BARRIER
|
|
"\n\
|
|
2:"
|
|
: "=&r" (prev), "+m" (*p)
|
|
: "r" (p), "r" (old), "r" (new)
|
|
: "cc", "memory");
|
|
|
|
return prev;
|
|
}
|
|
|
|
static __always_inline unsigned long
|
|
__cmpxchg_u32_local(volatile unsigned int *p, unsigned long old,
|
|
unsigned long new)
|
|
{
|
|
unsigned int prev;
|
|
|
|
__asm__ __volatile__ (
|
|
"1: lwarx %0,0,%2 # __cmpxchg_u32\n\
|
|
cmpw 0,%0,%3\n\
|
|
bne- 2f\n"
|
|
PPC405_ERR77(0,%2)
|
|
" stwcx. %4,0,%2\n\
|
|
bne- 1b"
|
|
"\n\
|
|
2:"
|
|
: "=&r" (prev), "+m" (*p)
|
|
: "r" (p), "r" (old), "r" (new)
|
|
: "cc", "memory");
|
|
|
|
return prev;
|
|
}
|
|
|
|
#ifdef CONFIG_PPC64
|
|
static __always_inline unsigned long
|
|
__cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new)
|
|
{
|
|
unsigned long prev;
|
|
|
|
__asm__ __volatile__ (
|
|
PPC_RELEASE_BARRIER
|
|
"1: ldarx %0,0,%2 # __cmpxchg_u64\n\
|
|
cmpd 0,%0,%3\n\
|
|
bne- 2f\n\
|
|
stdcx. %4,0,%2\n\
|
|
bne- 1b"
|
|
PPC_ACQUIRE_BARRIER
|
|
"\n\
|
|
2:"
|
|
: "=&r" (prev), "+m" (*p)
|
|
: "r" (p), "r" (old), "r" (new)
|
|
: "cc", "memory");
|
|
|
|
return prev;
|
|
}
|
|
|
|
static __always_inline unsigned long
|
|
__cmpxchg_u64_local(volatile unsigned long *p, unsigned long old,
|
|
unsigned long new)
|
|
{
|
|
unsigned long prev;
|
|
|
|
__asm__ __volatile__ (
|
|
"1: ldarx %0,0,%2 # __cmpxchg_u64\n\
|
|
cmpd 0,%0,%3\n\
|
|
bne- 2f\n\
|
|
stdcx. %4,0,%2\n\
|
|
bne- 1b"
|
|
"\n\
|
|
2:"
|
|
: "=&r" (prev), "+m" (*p)
|
|
: "r" (p), "r" (old), "r" (new)
|
|
: "cc", "memory");
|
|
|
|
return prev;
|
|
}
|
|
#endif
|
|
|
|
/* This function doesn't exist, so you'll get a linker error
|
|
if something tries to do an invalid cmpxchg(). */
|
|
extern void __cmpxchg_called_with_bad_pointer(void);
|
|
|
|
static __always_inline unsigned long
|
|
__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
|
|
unsigned int size)
|
|
{
|
|
switch (size) {
|
|
case 4:
|
|
return __cmpxchg_u32(ptr, old, new);
|
|
#ifdef CONFIG_PPC64
|
|
case 8:
|
|
return __cmpxchg_u64(ptr, old, new);
|
|
#endif
|
|
}
|
|
__cmpxchg_called_with_bad_pointer();
|
|
return old;
|
|
}
|
|
|
|
static __always_inline unsigned long
|
|
__cmpxchg_local(volatile void *ptr, unsigned long old, unsigned long new,
|
|
unsigned int size)
|
|
{
|
|
switch (size) {
|
|
case 4:
|
|
return __cmpxchg_u32_local(ptr, old, new);
|
|
#ifdef CONFIG_PPC64
|
|
case 8:
|
|
return __cmpxchg_u64_local(ptr, old, new);
|
|
#endif
|
|
}
|
|
__cmpxchg_called_with_bad_pointer();
|
|
return old;
|
|
}
|
|
|
|
#define cmpxchg(ptr, o, n) \
|
|
({ \
|
|
__typeof__(*(ptr)) _o_ = (o); \
|
|
__typeof__(*(ptr)) _n_ = (n); \
|
|
(__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \
|
|
(unsigned long)_n_, sizeof(*(ptr))); \
|
|
})
|
|
|
|
|
|
#define cmpxchg_local(ptr, o, n) \
|
|
({ \
|
|
__typeof__(*(ptr)) _o_ = (o); \
|
|
__typeof__(*(ptr)) _n_ = (n); \
|
|
(__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_, \
|
|
(unsigned long)_n_, sizeof(*(ptr))); \
|
|
})
|
|
|
|
#ifdef CONFIG_PPC64
|
|
#define cmpxchg64(ptr, o, n) \
|
|
({ \
|
|
BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
|
|
cmpxchg((ptr), (o), (n)); \
|
|
})
|
|
#define cmpxchg64_local(ptr, o, n) \
|
|
({ \
|
|
BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
|
|
cmpxchg_local((ptr), (o), (n)); \
|
|
})
|
|
#define cmpxchg64_relaxed cmpxchg64_local
|
|
#else
|
|
#include <asm-generic/cmpxchg-local.h>
|
|
#define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n))
|
|
#endif
|
|
|
|
#endif /* __KERNEL__ */
|
|
#endif /* _ASM_POWERPC_CMPXCHG_H_ */
|