rcu: New rcu_user_enter_after_irq() and rcu_user_exit_after_irq() APIs
In some cases, it is necessary to enter or exit userspace-RCU-idle mode from an interrupt handler, for example, if some other CPU sends this CPU a resched IPI. In this case, the current CPU would enter the IPI handler in userspace-RCU-idle mode, but would need to exit the IPI handler after having exited that mode. To allow this to work, this commit adds two new APIs to TREE_RCU: - rcu_user_enter_after_irq(). This must be called from an interrupt between rcu_irq_enter() and rcu_irq_exit(). After the irq calls rcu_irq_exit(), the irq handler will return into an RCU extended quiescent state. In theory, this interrupt is never a nested interrupt, but in practice it might interrupt softirq, which looks to RCU like a nested interrupt. - rcu_user_exit_after_irq(). This must be called from a non-nesting interrupt, interrupting an RCU extended quiescent state, also between rcu_irq_enter() and rcu_irq_exit(). After the irq calls rcu_irq_exit(), the irq handler will return in an RCU non-quiescent state. [ Combined with "Allow calls to rcu_exit_user_irq from nesting irqs." ] Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Reviewed-by: Josh Triplett <josh@joshtriplett.org>
This commit is contained in:
parent
adf5091e6c
commit
19dd1591fc
|
@ -193,6 +193,8 @@ extern void rcu_irq_enter(void);
|
||||||
extern void rcu_irq_exit(void);
|
extern void rcu_irq_exit(void);
|
||||||
extern void rcu_user_enter(void);
|
extern void rcu_user_enter(void);
|
||||||
extern void rcu_user_exit(void);
|
extern void rcu_user_exit(void);
|
||||||
|
extern void rcu_user_enter_after_irq(void);
|
||||||
|
extern void rcu_user_exit_after_irq(void);
|
||||||
extern void exit_rcu(void);
|
extern void exit_rcu(void);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -425,6 +425,27 @@ void rcu_user_enter(void)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* rcu_user_enter_after_irq - inform RCU that we are going to resume userspace
|
||||||
|
* after the current irq returns.
|
||||||
|
*
|
||||||
|
* This is similar to rcu_user_enter() but in the context of a non-nesting
|
||||||
|
* irq. After this call, RCU enters into idle mode when the interrupt
|
||||||
|
* returns.
|
||||||
|
*/
|
||||||
|
void rcu_user_enter_after_irq(void)
|
||||||
|
{
|
||||||
|
unsigned long flags;
|
||||||
|
struct rcu_dynticks *rdtp;
|
||||||
|
|
||||||
|
local_irq_save(flags);
|
||||||
|
rdtp = &__get_cpu_var(rcu_dynticks);
|
||||||
|
/* Ensure this irq is interrupting a non-idle RCU state. */
|
||||||
|
WARN_ON_ONCE(!(rdtp->dynticks_nesting & DYNTICK_TASK_MASK));
|
||||||
|
rdtp->dynticks_nesting = 1;
|
||||||
|
local_irq_restore(flags);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* rcu_irq_exit - inform RCU that current CPU is exiting irq towards idle
|
* rcu_irq_exit - inform RCU that current CPU is exiting irq towards idle
|
||||||
*
|
*
|
||||||
|
@ -549,6 +570,28 @@ void rcu_user_exit(void)
|
||||||
rcu_eqs_exit(1);
|
rcu_eqs_exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* rcu_user_exit_after_irq - inform RCU that we won't resume to userspace
|
||||||
|
* idle mode after the current non-nesting irq returns.
|
||||||
|
*
|
||||||
|
* This is similar to rcu_user_exit() but in the context of an irq.
|
||||||
|
* This is called when the irq has interrupted a userspace RCU idle mode
|
||||||
|
* context. When the current non-nesting interrupt returns after this call,
|
||||||
|
* the CPU won't restore the RCU idle mode.
|
||||||
|
*/
|
||||||
|
void rcu_user_exit_after_irq(void)
|
||||||
|
{
|
||||||
|
unsigned long flags;
|
||||||
|
struct rcu_dynticks *rdtp;
|
||||||
|
|
||||||
|
local_irq_save(flags);
|
||||||
|
rdtp = &__get_cpu_var(rcu_dynticks);
|
||||||
|
/* Ensure we are interrupting an RCU idle mode. */
|
||||||
|
WARN_ON_ONCE(rdtp->dynticks_nesting & DYNTICK_TASK_NEST_MASK);
|
||||||
|
rdtp->dynticks_nesting += DYNTICK_TASK_EXIT_IDLE;
|
||||||
|
local_irq_restore(flags);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* rcu_irq_enter - inform RCU that current CPU is entering irq away from idle
|
* rcu_irq_enter - inform RCU that current CPU is entering irq away from idle
|
||||||
*
|
*
|
||||||
|
|
Loading…
Reference in New Issue