MIPS: Allow kernel use of LL/SC to be separate from the presence of LL/SC.
On some CPUs, it is more efficient to disable and enable interrupts in the kernel rather than use ll/sc for atomic operations. But if we were to set cpu_has_llsc to false, we would break the userspace futex interface (in asm/futex.h). We separate the two concepts, with a new predicate kernel_uses_llsc, that lets us disable the kernel's use of ll/sc while still allowing the futex code to use it. Also there were a couple of cases in bitops.h where we were using ll/sc unconditionally even if cpu_has_llsc were false. Signed-off-by: David Daney <ddaney@caviumnetworks.com> Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
This commit is contained in:
David Daney
Ralf Baechle
parent
f7ade3c168
commit
b791d1193a
|
|
@ -49,7 +49,7 @@
|
|||
*/
|
||||
static __inline__ void atomic_add(int i, atomic_t * v)
|
||||
{
|
||||
if (cpu_has_llsc && R10000_LLSC_WAR) {
|
||||
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
||||
int temp;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
@ -61,7 +61,7 @@ static __inline__ void atomic_add(int i, atomic_t * v)
|
|||
" .set mips0 \n"
|
||||
: "=&r" (temp), "=m" (v->counter)
|
||||
: "Ir" (i), "m" (v->counter));
|
||||
} else if (cpu_has_llsc) {
|
||||
} else if (kernel_uses_llsc) {
|
||||
int temp;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
@ -94,7 +94,7 @@ static __inline__ void atomic_add(int i, atomic_t * v)
|
|||
*/
|
||||
static __inline__ void atomic_sub(int i, atomic_t * v)
|
||||
{
|
||||
if (cpu_has_llsc && R10000_LLSC_WAR) {
|
||||
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
||||
int temp;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
@ -106,7 +106,7 @@ static __inline__ void atomic_sub(int i, atomic_t * v)
|
|||
" .set mips0 \n"
|
||||
: "=&r" (temp), "=m" (v->counter)
|
||||
: "Ir" (i), "m" (v->counter));
|
||||
} else if (cpu_has_llsc) {
|
||||
} else if (kernel_uses_llsc) {
|
||||
int temp;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
@ -139,7 +139,7 @@ static __inline__ int atomic_add_return(int i, atomic_t * v)
|
|||
|
||||
smp_llsc_mb();
|
||||
|
||||
if (cpu_has_llsc && R10000_LLSC_WAR) {
|
||||
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
||||
int temp;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
@ -153,7 +153,7 @@ static __inline__ int atomic_add_return(int i, atomic_t * v)
|
|||
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
|
||||
: "Ir" (i), "m" (v->counter)
|
||||
: "memory");
|
||||
} else if (cpu_has_llsc) {
|
||||
} else if (kernel_uses_llsc) {
|
||||
int temp;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
@ -191,7 +191,7 @@ static __inline__ int atomic_sub_return(int i, atomic_t * v)
|
|||
|
||||
smp_llsc_mb();
|
||||
|
||||
if (cpu_has_llsc && R10000_LLSC_WAR) {
|
||||
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
||||
int temp;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
@ -205,7 +205,7 @@ static __inline__ int atomic_sub_return(int i, atomic_t * v)
|
|||
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
|
||||
: "Ir" (i), "m" (v->counter)
|
||||
: "memory");
|
||||
} else if (cpu_has_llsc) {
|
||||
} else if (kernel_uses_llsc) {
|
||||
int temp;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
@ -251,7 +251,7 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
|
|||
|
||||
smp_llsc_mb();
|
||||
|
||||
if (cpu_has_llsc && R10000_LLSC_WAR) {
|
||||
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
||||
int temp;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
@ -269,7 +269,7 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
|
|||
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
|
||||
: "Ir" (i), "m" (v->counter)
|
||||
: "memory");
|
||||
} else if (cpu_has_llsc) {
|
||||
} else if (kernel_uses_llsc) {
|
||||
int temp;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
@ -428,7 +428,7 @@ static __inline__ int atomic_add_unless(atomic_t *v, int a, int u)
|
|||
*/
|
||||
static __inline__ void atomic64_add(long i, atomic64_t * v)
|
||||
{
|
||||
if (cpu_has_llsc && R10000_LLSC_WAR) {
|
||||
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
||||
long temp;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
@ -440,7 +440,7 @@ static __inline__ void atomic64_add(long i, atomic64_t * v)
|
|||
" .set mips0 \n"
|
||||
: "=&r" (temp), "=m" (v->counter)
|
||||
: "Ir" (i), "m" (v->counter));
|
||||
} else if (cpu_has_llsc) {
|
||||
} else if (kernel_uses_llsc) {
|
||||
long temp;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
@ -473,7 +473,7 @@ static __inline__ void atomic64_add(long i, atomic64_t * v)
|
|||
*/
|
||||
static __inline__ void atomic64_sub(long i, atomic64_t * v)
|
||||
{
|
||||
if (cpu_has_llsc && R10000_LLSC_WAR) {
|
||||
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
||||
long temp;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
@ -485,7 +485,7 @@ static __inline__ void atomic64_sub(long i, atomic64_t * v)
|
|||
" .set mips0 \n"
|
||||
: "=&r" (temp), "=m" (v->counter)
|
||||
: "Ir" (i), "m" (v->counter));
|
||||
} else if (cpu_has_llsc) {
|
||||
} else if (kernel_uses_llsc) {
|
||||
long temp;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
@ -518,7 +518,7 @@ static __inline__ long atomic64_add_return(long i, atomic64_t * v)
|
|||
|
||||
smp_llsc_mb();
|
||||
|
||||
if (cpu_has_llsc && R10000_LLSC_WAR) {
|
||||
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
||||
long temp;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
@ -532,7 +532,7 @@ static __inline__ long atomic64_add_return(long i, atomic64_t * v)
|
|||
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
|
||||
: "Ir" (i), "m" (v->counter)
|
||||
: "memory");
|
||||
} else if (cpu_has_llsc) {
|
||||
} else if (kernel_uses_llsc) {
|
||||
long temp;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
@ -570,7 +570,7 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t * v)
|
|||
|
||||
smp_llsc_mb();
|
||||
|
||||
if (cpu_has_llsc && R10000_LLSC_WAR) {
|
||||
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
||||
long temp;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
@ -584,7 +584,7 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t * v)
|
|||
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
|
||||
: "Ir" (i), "m" (v->counter)
|
||||
: "memory");
|
||||
} else if (cpu_has_llsc) {
|
||||
} else if (kernel_uses_llsc) {
|
||||
long temp;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
@ -630,7 +630,7 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
|
|||
|
||||
smp_llsc_mb();
|
||||
|
||||
if (cpu_has_llsc && R10000_LLSC_WAR) {
|
||||
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
||||
long temp;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
@ -648,7 +648,7 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
|
|||
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
|
||||
: "Ir" (i), "m" (v->counter)
|
||||
: "memory");
|
||||
} else if (cpu_has_llsc) {
|
||||
} else if (kernel_uses_llsc) {
|
||||
long temp;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
|
|||
unsigned short bit = nr & SZLONG_MASK;
|
||||
unsigned long temp;
|
||||
|
||||
if (cpu_has_llsc && R10000_LLSC_WAR) {
|
||||
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
||||
__asm__ __volatile__(
|
||||
" .set mips3 \n"
|
||||
"1: " __LL "%0, %1 # set_bit \n"
|
||||
|
|
@ -72,7 +72,7 @@ static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
|
|||
: "=&r" (temp), "=m" (*m)
|
||||
: "ir" (1UL << bit), "m" (*m));
|
||||
#ifdef CONFIG_CPU_MIPSR2
|
||||
} else if (__builtin_constant_p(bit)) {
|
||||
} else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
|
||||
__asm__ __volatile__(
|
||||
"1: " __LL "%0, %1 # set_bit \n"
|
||||
" " __INS "%0, %4, %2, 1 \n"
|
||||
|
|
@ -84,7 +84,7 @@ static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
|
|||
: "=&r" (temp), "=m" (*m)
|
||||
: "ir" (bit), "m" (*m), "r" (~0));
|
||||
#endif /* CONFIG_CPU_MIPSR2 */
|
||||
} else if (cpu_has_llsc) {
|
||||
} else if (kernel_uses_llsc) {
|
||||
__asm__ __volatile__(
|
||||
" .set mips3 \n"
|
||||
"1: " __LL "%0, %1 # set_bit \n"
|
||||
|
|
@ -126,7 +126,7 @@ static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
|
|||
unsigned short bit = nr & SZLONG_MASK;
|
||||
unsigned long temp;
|
||||
|
||||
if (cpu_has_llsc && R10000_LLSC_WAR) {
|
||||
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
||||
__asm__ __volatile__(
|
||||
" .set mips3 \n"
|
||||
"1: " __LL "%0, %1 # clear_bit \n"
|
||||
|
|
@ -137,7 +137,7 @@ static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
|
|||
: "=&r" (temp), "=m" (*m)
|
||||
: "ir" (~(1UL << bit)), "m" (*m));
|
||||
#ifdef CONFIG_CPU_MIPSR2
|
||||
} else if (__builtin_constant_p(bit)) {
|
||||
} else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
|
||||
__asm__ __volatile__(
|
||||
"1: " __LL "%0, %1 # clear_bit \n"
|
||||
" " __INS "%0, $0, %2, 1 \n"
|
||||
|
|
@ -149,7 +149,7 @@ static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
|
|||
: "=&r" (temp), "=m" (*m)
|
||||
: "ir" (bit), "m" (*m));
|
||||
#endif /* CONFIG_CPU_MIPSR2 */
|
||||
} else if (cpu_has_llsc) {
|
||||
} else if (kernel_uses_llsc) {
|
||||
__asm__ __volatile__(
|
||||
" .set mips3 \n"
|
||||
"1: " __LL "%0, %1 # clear_bit \n"
|
||||
|
|
@ -202,7 +202,7 @@ static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
|
|||
{
|
||||
unsigned short bit = nr & SZLONG_MASK;
|
||||
|
||||
if (cpu_has_llsc && R10000_LLSC_WAR) {
|
||||
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
||||
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
|
||||
unsigned long temp;
|
||||
|
||||
|
|
@ -215,7 +215,7 @@ static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
|
|||
" .set mips0 \n"
|
||||
: "=&r" (temp), "=m" (*m)
|
||||
: "ir" (1UL << bit), "m" (*m));
|
||||
} else if (cpu_has_llsc) {
|
||||
} else if (kernel_uses_llsc) {
|
||||
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
|
||||
unsigned long temp;
|
||||
|
||||
|
|
@ -260,7 +260,7 @@ static inline int test_and_set_bit(unsigned long nr,
|
|||
|
||||
smp_llsc_mb();
|
||||
|
||||
if (cpu_has_llsc && R10000_LLSC_WAR) {
|
||||
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
||||
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
|
||||
unsigned long temp;
|
||||
|
||||
|
|
@ -275,7 +275,7 @@ static inline int test_and_set_bit(unsigned long nr,
|
|||
: "=&r" (temp), "=m" (*m), "=&r" (res)
|
||||
: "r" (1UL << bit), "m" (*m)
|
||||
: "memory");
|
||||
} else if (cpu_has_llsc) {
|
||||
} else if (kernel_uses_llsc) {
|
||||
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
|
||||
unsigned long temp;
|
||||
|
||||
|
|
@ -328,7 +328,7 @@ static inline int test_and_set_bit_lock(unsigned long nr,
|
|||
unsigned short bit = nr & SZLONG_MASK;
|
||||
unsigned long res;
|
||||
|
||||
if (cpu_has_llsc && R10000_LLSC_WAR) {
|
||||
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
||||
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
|
||||
unsigned long temp;
|
||||
|
||||
|
|
@ -343,7 +343,7 @@ static inline int test_and_set_bit_lock(unsigned long nr,
|
|||
: "=&r" (temp), "=m" (*m), "=&r" (res)
|
||||
: "r" (1UL << bit), "m" (*m)
|
||||
: "memory");
|
||||
} else if (cpu_has_llsc) {
|
||||
} else if (kernel_uses_llsc) {
|
||||
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
|
||||
unsigned long temp;
|
||||
|
||||
|
|
@ -397,7 +397,7 @@ static inline int test_and_clear_bit(unsigned long nr,
|
|||
|
||||
smp_llsc_mb();
|
||||
|
||||
if (cpu_has_llsc && R10000_LLSC_WAR) {
|
||||
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
||||
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
|
||||
unsigned long temp;
|
||||
|
||||
|
|
@ -414,7 +414,7 @@ static inline int test_and_clear_bit(unsigned long nr,
|
|||
: "r" (1UL << bit), "m" (*m)
|
||||
: "memory");
|
||||
#ifdef CONFIG_CPU_MIPSR2
|
||||
} else if (__builtin_constant_p(nr)) {
|
||||
} else if (kernel_uses_llsc && __builtin_constant_p(nr)) {
|
||||
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
|
||||
unsigned long temp;
|
||||
|
||||
|
|
@ -431,7 +431,7 @@ static inline int test_and_clear_bit(unsigned long nr,
|
|||
: "ir" (bit), "m" (*m)
|
||||
: "memory");
|
||||
#endif
|
||||
} else if (cpu_has_llsc) {
|
||||
} else if (kernel_uses_llsc) {
|
||||
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
|
||||
unsigned long temp;
|
||||
|
||||
|
|
@ -487,7 +487,7 @@ static inline int test_and_change_bit(unsigned long nr,
|
|||
|
||||
smp_llsc_mb();
|
||||
|
||||
if (cpu_has_llsc && R10000_LLSC_WAR) {
|
||||
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
||||
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
|
||||
unsigned long temp;
|
||||
|
||||
|
|
@ -502,7 +502,7 @@ static inline int test_and_change_bit(unsigned long nr,
|
|||
: "=&r" (temp), "=m" (*m), "=&r" (res)
|
||||
: "r" (1UL << bit), "m" (*m)
|
||||
: "memory");
|
||||
} else if (cpu_has_llsc) {
|
||||
} else if (kernel_uses_llsc) {
|
||||
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
|
||||
unsigned long temp;
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@
|
|||
({ \
|
||||
__typeof(*(m)) __ret; \
|
||||
\
|
||||
if (cpu_has_llsc && R10000_LLSC_WAR) { \
|
||||
if (kernel_uses_llsc && R10000_LLSC_WAR) { \
|
||||
__asm__ __volatile__( \
|
||||
" .set push \n" \
|
||||
" .set noat \n" \
|
||||
|
|
@ -33,7 +33,7 @@
|
|||
: "=&r" (__ret), "=R" (*m) \
|
||||
: "R" (*m), "Jr" (old), "Jr" (new) \
|
||||
: "memory"); \
|
||||
} else if (cpu_has_llsc) { \
|
||||
} else if (kernel_uses_llsc) { \
|
||||
__asm__ __volatile__( \
|
||||
" .set push \n" \
|
||||
" .set noat \n" \
|
||||
|
|
|
|||
|
|
@ -80,6 +80,9 @@
|
|||
#ifndef cpu_has_llsc
|
||||
#define cpu_has_llsc (cpu_data[0].options & MIPS_CPU_LLSC)
|
||||
#endif
|
||||
#ifndef kernel_uses_llsc
|
||||
#define kernel_uses_llsc cpu_has_llsc
|
||||
#endif
|
||||
#ifndef cpu_has_mips16
|
||||
#define cpu_has_mips16 (cpu_data[0].ases & MIPS_ASE_MIPS16)
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ static __inline__ long local_add_return(long i, local_t * l)
|
|||
{
|
||||
unsigned long result;
|
||||
|
||||
if (cpu_has_llsc && R10000_LLSC_WAR) {
|
||||
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
||||
unsigned long temp;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
@ -43,7 +43,7 @@ static __inline__ long local_add_return(long i, local_t * l)
|
|||
: "=&r" (result), "=&r" (temp), "=m" (l->a.counter)
|
||||
: "Ir" (i), "m" (l->a.counter)
|
||||
: "memory");
|
||||
} else if (cpu_has_llsc) {
|
||||
} else if (kernel_uses_llsc) {
|
||||
unsigned long temp;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
@ -74,7 +74,7 @@ static __inline__ long local_sub_return(long i, local_t * l)
|
|||
{
|
||||
unsigned long result;
|
||||
|
||||
if (cpu_has_llsc && R10000_LLSC_WAR) {
|
||||
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
||||
unsigned long temp;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
@ -88,7 +88,7 @@ static __inline__ long local_sub_return(long i, local_t * l)
|
|||
: "=&r" (result), "=&r" (temp), "=m" (l->a.counter)
|
||||
: "Ir" (i), "m" (l->a.counter)
|
||||
: "memory");
|
||||
} else if (cpu_has_llsc) {
|
||||
} else if (kernel_uses_llsc) {
|
||||
unsigned long temp;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
|
|||
|
|
@ -94,7 +94,7 @@ static inline unsigned long __xchg_u32(volatile int * m, unsigned int val)
|
|||
{
|
||||
__u32 retval;
|
||||
|
||||
if (cpu_has_llsc && R10000_LLSC_WAR) {
|
||||
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
||||
unsigned long dummy;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
@ -109,7 +109,7 @@ static inline unsigned long __xchg_u32(volatile int * m, unsigned int val)
|
|||
: "=&r" (retval), "=m" (*m), "=&r" (dummy)
|
||||
: "R" (*m), "Jr" (val)
|
||||
: "memory");
|
||||
} else if (cpu_has_llsc) {
|
||||
} else if (kernel_uses_llsc) {
|
||||
unsigned long dummy;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
@ -146,7 +146,7 @@ static inline __u64 __xchg_u64(volatile __u64 * m, __u64 val)
|
|||
{
|
||||
__u64 retval;
|
||||
|
||||
if (cpu_has_llsc && R10000_LLSC_WAR) {
|
||||
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
||||
unsigned long dummy;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
@ -159,7 +159,7 @@ static inline __u64 __xchg_u64(volatile __u64 * m, __u64 val)
|
|||
: "=&r" (retval), "=m" (*m), "=&r" (dummy)
|
||||
: "R" (*m), "Jr" (val)
|
||||
: "memory");
|
||||
} else if (cpu_has_llsc) {
|
||||
} else if (kernel_uses_llsc) {
|
||||
unsigned long dummy;
|
||||
|
||||
__asm__ __volatile__(
|
||||
|
|
|
|||
Loading…
Reference in New Issue