rcu: Make rcutorture's batches-completed API use ->gp_seq
The rcutorture test invokes rcu_batches_started(), rcu_batches_completed(), rcu_batches_started_bh(), rcu_batches_completed_bh(), rcu_batches_started_sched(), and rcu_batches_completed_sched() to do grace-period consistency checks, and rcuperf uses the _completed variants for statistics. These functions use ->gpnum and ->completed. This commit therefore replaces them with rcu_get_gp_seq(), rcu_bh_get_gp_seq(), and rcu_sched_get_gp_seq(), adjusting rcutorture and rcuperf to make use of them. Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
This commit is contained in:
Paul E. McKenney
parent
dee4f42298
commit
17ef2fe97c
|
|
@ -463,12 +463,9 @@ void srcutorture_get_gp_data(enum rcutorture_type test_type,
|
|||
#endif
|
||||
|
||||
#ifdef CONFIG_TINY_RCU
|
||||
static inline unsigned long rcu_batches_started(void) { return 0; }
|
||||
static inline unsigned long rcu_batches_started_bh(void) { return 0; }
|
||||
static inline unsigned long rcu_batches_started_sched(void) { return 0; }
|
||||
static inline unsigned long rcu_batches_completed(void) { return 0; }
|
||||
static inline unsigned long rcu_batches_completed_bh(void) { return 0; }
|
||||
static inline unsigned long rcu_batches_completed_sched(void) { return 0; }
|
||||
static inline unsigned long rcu_get_gp_seq(void) { return 0; }
|
||||
static inline unsigned long rcu_bh_get_gp_seq(void) { return 0; }
|
||||
static inline unsigned long rcu_sched_get_gp_seq(void) { return 0; }
|
||||
static inline unsigned long rcu_exp_batches_completed(void) { return 0; }
|
||||
static inline unsigned long rcu_exp_batches_completed_sched(void) { return 0; }
|
||||
static inline unsigned long
|
||||
|
|
@ -480,12 +477,9 @@ static inline void show_rcu_gp_kthreads(void) { }
|
|||
#else /* #ifdef CONFIG_TINY_RCU */
|
||||
extern unsigned long rcutorture_testseq;
|
||||
extern unsigned long rcutorture_vernum;
|
||||
unsigned long rcu_batches_started(void);
|
||||
unsigned long rcu_batches_started_bh(void);
|
||||
unsigned long rcu_batches_started_sched(void);
|
||||
unsigned long rcu_batches_completed(void);
|
||||
unsigned long rcu_batches_completed_bh(void);
|
||||
unsigned long rcu_batches_completed_sched(void);
|
||||
unsigned long rcu_get_gp_seq(void);
|
||||
unsigned long rcu_bh_get_gp_seq(void);
|
||||
unsigned long rcu_sched_get_gp_seq(void);
|
||||
unsigned long rcu_exp_batches_completed(void);
|
||||
unsigned long rcu_exp_batches_completed_sched(void);
|
||||
unsigned long srcu_batches_completed(struct srcu_struct *sp);
|
||||
|
|
|
|||
|
|
@ -138,8 +138,7 @@ struct rcu_perf_ops {
|
|||
void (*cleanup)(void);
|
||||
int (*readlock)(void);
|
||||
void (*readunlock)(int idx);
|
||||
unsigned long (*started)(void);
|
||||
unsigned long (*completed)(void);
|
||||
unsigned long (*get_gp_seq)(void);
|
||||
unsigned long (*exp_completed)(void);
|
||||
void (*async)(struct rcu_head *head, rcu_callback_t func);
|
||||
void (*gp_barrier)(void);
|
||||
|
|
@ -179,8 +178,7 @@ static struct rcu_perf_ops rcu_ops = {
|
|||
.init = rcu_sync_perf_init,
|
||||
.readlock = rcu_perf_read_lock,
|
||||
.readunlock = rcu_perf_read_unlock,
|
||||
.started = rcu_batches_started,
|
||||
.completed = rcu_batches_completed,
|
||||
.get_gp_seq = rcu_get_gp_seq,
|
||||
.exp_completed = rcu_exp_batches_completed,
|
||||
.async = call_rcu,
|
||||
.gp_barrier = rcu_barrier,
|
||||
|
|
@ -209,8 +207,7 @@ static struct rcu_perf_ops rcu_bh_ops = {
|
|||
.init = rcu_sync_perf_init,
|
||||
.readlock = rcu_bh_perf_read_lock,
|
||||
.readunlock = rcu_bh_perf_read_unlock,
|
||||
.started = rcu_batches_started_bh,
|
||||
.completed = rcu_batches_completed_bh,
|
||||
.get_gp_seq = rcu_bh_get_gp_seq,
|
||||
.exp_completed = rcu_exp_batches_completed_sched,
|
||||
.async = call_rcu_bh,
|
||||
.gp_barrier = rcu_barrier_bh,
|
||||
|
|
@ -266,8 +263,7 @@ static struct rcu_perf_ops srcu_ops = {
|
|||
.init = rcu_sync_perf_init,
|
||||
.readlock = srcu_perf_read_lock,
|
||||
.readunlock = srcu_perf_read_unlock,
|
||||
.started = NULL,
|
||||
.completed = srcu_perf_completed,
|
||||
.get_gp_seq = srcu_perf_completed,
|
||||
.exp_completed = srcu_perf_completed,
|
||||
.async = srcu_call_rcu,
|
||||
.gp_barrier = srcu_rcu_barrier,
|
||||
|
|
@ -295,8 +291,7 @@ static struct rcu_perf_ops srcud_ops = {
|
|||
.cleanup = srcu_sync_perf_cleanup,
|
||||
.readlock = srcu_perf_read_lock,
|
||||
.readunlock = srcu_perf_read_unlock,
|
||||
.started = NULL,
|
||||
.completed = srcu_perf_completed,
|
||||
.get_gp_seq = srcu_perf_completed,
|
||||
.exp_completed = srcu_perf_completed,
|
||||
.async = srcu_call_rcu,
|
||||
.gp_barrier = srcu_rcu_barrier,
|
||||
|
|
@ -325,8 +320,7 @@ static struct rcu_perf_ops sched_ops = {
|
|||
.init = rcu_sync_perf_init,
|
||||
.readlock = sched_perf_read_lock,
|
||||
.readunlock = sched_perf_read_unlock,
|
||||
.started = rcu_batches_started_sched,
|
||||
.completed = rcu_batches_completed_sched,
|
||||
.get_gp_seq = rcu_sched_get_gp_seq,
|
||||
.exp_completed = rcu_exp_batches_completed_sched,
|
||||
.async = call_rcu_sched,
|
||||
.gp_barrier = rcu_barrier_sched,
|
||||
|
|
@ -353,8 +347,7 @@ static struct rcu_perf_ops tasks_ops = {
|
|||
.init = rcu_sync_perf_init,
|
||||
.readlock = tasks_perf_read_lock,
|
||||
.readunlock = tasks_perf_read_unlock,
|
||||
.started = rcu_no_completed,
|
||||
.completed = rcu_no_completed,
|
||||
.get_gp_seq = rcu_no_completed,
|
||||
.async = call_rcu_tasks,
|
||||
.gp_barrier = rcu_barrier_tasks,
|
||||
.sync = synchronize_rcu_tasks,
|
||||
|
|
@ -447,8 +440,7 @@ rcu_perf_writer(void *arg)
|
|||
b_rcu_perf_writer_started =
|
||||
cur_ops->exp_completed() / 2;
|
||||
} else {
|
||||
b_rcu_perf_writer_started =
|
||||
cur_ops->completed();
|
||||
b_rcu_perf_writer_started = cur_ops->get_gp_seq();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -505,7 +497,7 @@ retry:
|
|||
cur_ops->exp_completed() / 2;
|
||||
} else {
|
||||
b_rcu_perf_writer_finished =
|
||||
cur_ops->completed();
|
||||
cur_ops->get_gp_seq();
|
||||
}
|
||||
if (shutdown) {
|
||||
smp_mb(); /* Assign before wake. */
|
||||
|
|
|
|||
|
|
@ -264,8 +264,7 @@ struct rcu_torture_ops {
|
|||
int (*readlock)(void);
|
||||
void (*read_delay)(struct torture_random_state *rrsp);
|
||||
void (*readunlock)(int idx);
|
||||
unsigned long (*started)(void);
|
||||
unsigned long (*completed)(void);
|
||||
unsigned long (*get_gp_seq)(void);
|
||||
void (*deferred_free)(struct rcu_torture *p);
|
||||
void (*sync)(void);
|
||||
void (*exp_sync)(void);
|
||||
|
|
@ -305,10 +304,10 @@ static void rcu_read_delay(struct torture_random_state *rrsp)
|
|||
* force_quiescent_state. */
|
||||
|
||||
if (!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) {
|
||||
started = cur_ops->completed();
|
||||
started = cur_ops->get_gp_seq();
|
||||
ts = rcu_trace_clock_local();
|
||||
mdelay(longdelay_ms);
|
||||
completed = cur_ops->completed();
|
||||
completed = cur_ops->get_gp_seq();
|
||||
do_trace_rcu_torture_read(cur_ops->name, NULL, ts,
|
||||
started, completed);
|
||||
}
|
||||
|
|
@ -400,8 +399,7 @@ static struct rcu_torture_ops rcu_ops = {
|
|||
.readlock = rcu_torture_read_lock,
|
||||
.read_delay = rcu_read_delay,
|
||||
.readunlock = rcu_torture_read_unlock,
|
||||
.started = rcu_batches_started,
|
||||
.completed = rcu_batches_completed,
|
||||
.get_gp_seq = rcu_get_gp_seq,
|
||||
.deferred_free = rcu_torture_deferred_free,
|
||||
.sync = synchronize_rcu,
|
||||
.exp_sync = synchronize_rcu_expedited,
|
||||
|
|
@ -442,8 +440,7 @@ static struct rcu_torture_ops rcu_bh_ops = {
|
|||
.readlock = rcu_bh_torture_read_lock,
|
||||
.read_delay = rcu_read_delay, /* just reuse rcu's version. */
|
||||
.readunlock = rcu_bh_torture_read_unlock,
|
||||
.started = rcu_batches_started_bh,
|
||||
.completed = rcu_batches_completed_bh,
|
||||
.get_gp_seq = rcu_bh_get_gp_seq,
|
||||
.deferred_free = rcu_bh_torture_deferred_free,
|
||||
.sync = synchronize_rcu_bh,
|
||||
.exp_sync = synchronize_rcu_bh_expedited,
|
||||
|
|
@ -486,8 +483,7 @@ static struct rcu_torture_ops rcu_busted_ops = {
|
|||
.readlock = rcu_torture_read_lock,
|
||||
.read_delay = rcu_read_delay, /* just reuse rcu's version. */
|
||||
.readunlock = rcu_torture_read_unlock,
|
||||
.started = rcu_no_completed,
|
||||
.completed = rcu_no_completed,
|
||||
.get_gp_seq = rcu_no_completed,
|
||||
.deferred_free = rcu_busted_torture_deferred_free,
|
||||
.sync = synchronize_rcu_busted,
|
||||
.exp_sync = synchronize_rcu_busted,
|
||||
|
|
@ -575,8 +571,7 @@ static struct rcu_torture_ops srcu_ops = {
|
|||
.readlock = srcu_torture_read_lock,
|
||||
.read_delay = srcu_read_delay,
|
||||
.readunlock = srcu_torture_read_unlock,
|
||||
.started = NULL,
|
||||
.completed = srcu_torture_completed,
|
||||
.get_gp_seq = srcu_torture_completed,
|
||||
.deferred_free = srcu_torture_deferred_free,
|
||||
.sync = srcu_torture_synchronize,
|
||||
.exp_sync = srcu_torture_synchronize_expedited,
|
||||
|
|
@ -613,8 +608,7 @@ static struct rcu_torture_ops srcud_ops = {
|
|||
.readlock = srcu_torture_read_lock,
|
||||
.read_delay = srcu_read_delay,
|
||||
.readunlock = srcu_torture_read_unlock,
|
||||
.started = NULL,
|
||||
.completed = srcu_torture_completed,
|
||||
.get_gp_seq = srcu_torture_completed,
|
||||
.deferred_free = srcu_torture_deferred_free,
|
||||
.sync = srcu_torture_synchronize,
|
||||
.exp_sync = srcu_torture_synchronize_expedited,
|
||||
|
|
@ -651,8 +645,7 @@ static struct rcu_torture_ops sched_ops = {
|
|||
.readlock = sched_torture_read_lock,
|
||||
.read_delay = rcu_read_delay, /* just reuse rcu's version. */
|
||||
.readunlock = sched_torture_read_unlock,
|
||||
.started = rcu_batches_started_sched,
|
||||
.completed = rcu_batches_completed_sched,
|
||||
.get_gp_seq = rcu_sched_get_gp_seq,
|
||||
.deferred_free = rcu_sched_torture_deferred_free,
|
||||
.sync = synchronize_sched,
|
||||
.exp_sync = synchronize_sched_expedited,
|
||||
|
|
@ -690,8 +683,7 @@ static struct rcu_torture_ops tasks_ops = {
|
|||
.readlock = tasks_torture_read_lock,
|
||||
.read_delay = rcu_read_delay, /* just reuse rcu's version. */
|
||||
.readunlock = tasks_torture_read_unlock,
|
||||
.started = rcu_no_completed,
|
||||
.completed = rcu_no_completed,
|
||||
.get_gp_seq = rcu_no_completed,
|
||||
.deferred_free = rcu_tasks_torture_deferred_free,
|
||||
.sync = synchronize_rcu_tasks,
|
||||
.exp_sync = synchronize_rcu_tasks,
|
||||
|
|
@ -1104,10 +1096,7 @@ static void rcu_torture_timer(struct timer_list *unused)
|
|||
unsigned long long ts;
|
||||
|
||||
idx = cur_ops->readlock();
|
||||
if (cur_ops->started)
|
||||
started = cur_ops->started();
|
||||
else
|
||||
started = cur_ops->completed();
|
||||
started = cur_ops->get_gp_seq();
|
||||
ts = rcu_trace_clock_local();
|
||||
p = rcu_dereference_check(rcu_torture_current,
|
||||
rcu_read_lock_bh_held() ||
|
||||
|
|
@ -1131,7 +1120,7 @@ static void rcu_torture_timer(struct timer_list *unused)
|
|||
/* Should not happen, but... */
|
||||
pipe_count = RCU_TORTURE_PIPE_LEN;
|
||||
}
|
||||
completed = cur_ops->completed();
|
||||
completed = cur_ops->get_gp_seq();
|
||||
if (pipe_count > 1) {
|
||||
do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, ts,
|
||||
started, completed);
|
||||
|
|
@ -1139,8 +1128,8 @@ static void rcu_torture_timer(struct timer_list *unused)
|
|||
}
|
||||
__this_cpu_inc(rcu_torture_count[pipe_count]);
|
||||
completed = completed - started;
|
||||
if (cur_ops->started)
|
||||
completed++;
|
||||
if (completed > ULONG_MAX >> 1)
|
||||
completed = 0; /* Not all gp_seq have full range. */
|
||||
if (completed > RCU_TORTURE_PIPE_LEN) {
|
||||
/* Should not happen, but... */
|
||||
completed = RCU_TORTURE_PIPE_LEN;
|
||||
|
|
@ -1187,10 +1176,7 @@ rcu_torture_reader(void *arg)
|
|||
mod_timer(&t, jiffies + 1);
|
||||
}
|
||||
idx = cur_ops->readlock();
|
||||
if (cur_ops->started)
|
||||
started = cur_ops->started();
|
||||
else
|
||||
started = cur_ops->completed();
|
||||
started = cur_ops->get_gp_seq();
|
||||
ts = rcu_trace_clock_local();
|
||||
p = rcu_dereference_check(rcu_torture_current,
|
||||
rcu_read_lock_bh_held() ||
|
||||
|
|
@ -1212,7 +1198,7 @@ rcu_torture_reader(void *arg)
|
|||
/* Should not happen, but... */
|
||||
pipe_count = RCU_TORTURE_PIPE_LEN;
|
||||
}
|
||||
completed = cur_ops->completed();
|
||||
completed = cur_ops->get_gp_seq();
|
||||
if (pipe_count > 1) {
|
||||
do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu,
|
||||
ts, started, completed);
|
||||
|
|
@ -1220,8 +1206,8 @@ rcu_torture_reader(void *arg)
|
|||
}
|
||||
__this_cpu_inc(rcu_torture_count[pipe_count]);
|
||||
completed = completed - started;
|
||||
if (cur_ops->started)
|
||||
completed++;
|
||||
if (completed > ULONG_MAX >> 1)
|
||||
completed = 0; /* Not all gp_seq have full range. */
|
||||
if (completed > RCU_TORTURE_PIPE_LEN) {
|
||||
/* Should not happen, but... */
|
||||
completed = RCU_TORTURE_PIPE_LEN;
|
||||
|
|
|
|||
|
|
@ -530,58 +530,31 @@ static void force_quiescent_state(struct rcu_state *rsp);
|
|||
static int rcu_pending(void);
|
||||
|
||||
/*
|
||||
* Return the number of RCU batches started thus far for debug & stats.
|
||||
* Return the number of RCU GPs completed thus far for debug & stats.
|
||||
*/
|
||||
unsigned long rcu_batches_started(void)
|
||||
unsigned long rcu_get_gp_seq(void)
|
||||
{
|
||||
return rcu_state_p->gpnum;
|
||||
return rcu_seq_ctr(READ_ONCE(rcu_state_p->gp_seq));
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(rcu_batches_started);
|
||||
EXPORT_SYMBOL_GPL(rcu_get_gp_seq);
|
||||
|
||||
/*
|
||||
* Return the number of RCU-sched batches started thus far for debug & stats.
|
||||
* Return the number of RCU-sched GPs completed thus far for debug & stats.
|
||||
*/
|
||||
unsigned long rcu_batches_started_sched(void)
|
||||
unsigned long rcu_sched_get_gp_seq(void)
|
||||
{
|
||||
return rcu_sched_state.gpnum;
|
||||
return rcu_seq_ctr(READ_ONCE(rcu_sched_state.gp_seq));
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(rcu_batches_started_sched);
|
||||
EXPORT_SYMBOL_GPL(rcu_sched_get_gp_seq);
|
||||
|
||||
/*
|
||||
* Return the number of RCU BH batches started thus far for debug & stats.
|
||||
* Return the number of RCU-bh GPs completed thus far for debug & stats.
|
||||
*/
|
||||
unsigned long rcu_batches_started_bh(void)
|
||||
unsigned long rcu_bh_get_gp_seq(void)
|
||||
{
|
||||
return rcu_bh_state.gpnum;
|
||||
return rcu_seq_ctr(READ_ONCE(rcu_bh_state.gp_seq));
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(rcu_batches_started_bh);
|
||||
|
||||
/*
|
||||
* Return the number of RCU batches completed thus far for debug & stats.
|
||||
*/
|
||||
unsigned long rcu_batches_completed(void)
|
||||
{
|
||||
return rcu_state_p->completed;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(rcu_batches_completed);
|
||||
|
||||
/*
|
||||
* Return the number of RCU-sched batches completed thus far for debug & stats.
|
||||
*/
|
||||
unsigned long rcu_batches_completed_sched(void)
|
||||
{
|
||||
return rcu_sched_state.completed;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(rcu_batches_completed_sched);
|
||||
|
||||
/*
|
||||
* Return the number of RCU BH batches completed thus far for debug & stats.
|
||||
*/
|
||||
unsigned long rcu_batches_completed_bh(void)
|
||||
{
|
||||
return rcu_bh_state.completed;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(rcu_batches_completed_bh);
|
||||
EXPORT_SYMBOL_GPL(rcu_bh_get_gp_seq);
|
||||
|
||||
/*
|
||||
* Return the number of RCU expedited batches completed thus far for
|
||||
|
|
|
|||
Loading…
Reference in New Issue