rcu: Further cleanups of use of lastcomp
Now that a copy of the rsp->completed flag is available in all rcu_node structures, make full use of it. It is still legitimate to access rsp->completed while holding the root rcu_node structure's lock, however. Also, tighten up force_quiescent_state()'s checks for end of current grace period. Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Cc: laijs@cn.fujitsu.com Cc: dipankar@in.ibm.com Cc: mathieu.desnoyers@polymtl.ca Cc: josh@joshtriplett.org Cc: dvhltc@us.ibm.com Cc: niv@us.ibm.com Cc: peterz@infradead.org Cc: rostedt@goodmis.org Cc: Valdis.Kletnieks@vt.edu Cc: dhowells@redhat.com LKML-Reference: <1258170699933-git-send-email-> Signed-off-by: Ingo Molnar <mingo@elte.hu>
This commit is contained in:
parent
8e9aa8f067
commit
560d4bc0df
|
@ -817,7 +817,7 @@ cpu_quiet(int cpu, struct rcu_state *rsp, struct rcu_data *rdp, long lastcomp)
|
||||||
|
|
||||||
rnp = rdp->mynode;
|
rnp = rdp->mynode;
|
||||||
spin_lock_irqsave(&rnp->lock, flags);
|
spin_lock_irqsave(&rnp->lock, flags);
|
||||||
if (lastcomp != ACCESS_ONCE(rsp->completed)) {
|
if (lastcomp != rnp->completed) {
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Someone beat us to it for this grace period, so leave.
|
* Someone beat us to it for this grace period, so leave.
|
||||||
|
@ -935,7 +935,6 @@ static void rcu_adopt_orphan_cbs(struct rcu_state *rsp)
|
||||||
static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
|
static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
|
||||||
{
|
{
|
||||||
unsigned long flags;
|
unsigned long flags;
|
||||||
long lastcomp;
|
|
||||||
unsigned long mask;
|
unsigned long mask;
|
||||||
struct rcu_data *rdp = rsp->rda[cpu];
|
struct rcu_data *rdp = rsp->rda[cpu];
|
||||||
struct rcu_node *rnp;
|
struct rcu_node *rnp;
|
||||||
|
@ -971,7 +970,6 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
|
||||||
spin_unlock(&rnp->lock); /* irqs remain disabled. */
|
spin_unlock(&rnp->lock); /* irqs remain disabled. */
|
||||||
rnp = rnp->parent;
|
rnp = rnp->parent;
|
||||||
} while (rnp != NULL);
|
} while (rnp != NULL);
|
||||||
lastcomp = rsp->completed;
|
|
||||||
|
|
||||||
spin_unlock_irqrestore(&rsp->onofflock, flags);
|
spin_unlock_irqrestore(&rsp->onofflock, flags);
|
||||||
|
|
||||||
|
@ -1145,7 +1143,7 @@ static int rcu_process_dyntick(struct rcu_state *rsp, long lastcomp,
|
||||||
rcu_for_each_leaf_node(rsp, rnp) {
|
rcu_for_each_leaf_node(rsp, rnp) {
|
||||||
mask = 0;
|
mask = 0;
|
||||||
spin_lock_irqsave(&rnp->lock, flags);
|
spin_lock_irqsave(&rnp->lock, flags);
|
||||||
if (rsp->completed != lastcomp) {
|
if (rnp->completed != lastcomp) {
|
||||||
spin_unlock_irqrestore(&rnp->lock, flags);
|
spin_unlock_irqrestore(&rnp->lock, flags);
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
|
@ -1159,7 +1157,7 @@ static int rcu_process_dyntick(struct rcu_state *rsp, long lastcomp,
|
||||||
if ((rnp->qsmask & bit) != 0 && f(rsp->rda[cpu]))
|
if ((rnp->qsmask & bit) != 0 && f(rsp->rda[cpu]))
|
||||||
mask |= bit;
|
mask |= bit;
|
||||||
}
|
}
|
||||||
if (mask != 0 && rsp->completed == lastcomp) {
|
if (mask != 0 && rnp->completed == lastcomp) {
|
||||||
|
|
||||||
/* cpu_quiet_msk() releases rnp->lock. */
|
/* cpu_quiet_msk() releases rnp->lock. */
|
||||||
cpu_quiet_msk(mask, rsp, rnp, flags);
|
cpu_quiet_msk(mask, rsp, rnp, flags);
|
||||||
|
@ -1196,7 +1194,7 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed)
|
||||||
lastcomp = rsp->gpnum - 1;
|
lastcomp = rsp->gpnum - 1;
|
||||||
signaled = rsp->signaled;
|
signaled = rsp->signaled;
|
||||||
rsp->jiffies_force_qs = jiffies + RCU_JIFFIES_TILL_FORCE_QS;
|
rsp->jiffies_force_qs = jiffies + RCU_JIFFIES_TILL_FORCE_QS;
|
||||||
if (lastcomp == rsp->gpnum) {
|
if(!rcu_gp_in_progress(rsp)) {
|
||||||
rsp->n_force_qs_ngp++;
|
rsp->n_force_qs_ngp++;
|
||||||
spin_unlock(&rnp->lock);
|
spin_unlock(&rnp->lock);
|
||||||
goto unlock_ret; /* no GP in progress, time updated. */
|
goto unlock_ret; /* no GP in progress, time updated. */
|
||||||
|
@ -1224,7 +1222,8 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed)
|
||||||
/* Update state, record completion counter. */
|
/* Update state, record completion counter. */
|
||||||
forcenow = 0;
|
forcenow = 0;
|
||||||
spin_lock(&rnp->lock);
|
spin_lock(&rnp->lock);
|
||||||
if (lastcomp == rsp->completed &&
|
if (lastcomp + 1 == rsp->gpnum &&
|
||||||
|
lastcomp == rsp->completed &&
|
||||||
rsp->signaled == signaled) {
|
rsp->signaled == signaled) {
|
||||||
rsp->signaled = RCU_FORCE_QS;
|
rsp->signaled = RCU_FORCE_QS;
|
||||||
rsp->completed_fqs = lastcomp;
|
rsp->completed_fqs = lastcomp;
|
||||||
|
|
Loading…
Reference in New Issue