sched: Push put_prev_task() into pick_next_task()
In order to avoid having to do put/set on a whole cgroup hierarchy when we context switch, push the put into pick_next_task() so that both operations are in the same function. Further changes then allow us to possibly optimize away redundant work. Signed-off-by: Peter Zijlstra <peterz@infradead.org> Link: http://lkml.kernel.org/r/1328936700.2476.17.camel@laptop Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
parent
fed14d45f9
commit
606dba2e28
|
@ -2579,18 +2579,11 @@ static inline void schedule_debug(struct task_struct *prev)
|
||||||
schedstat_inc(this_rq(), sched_count);
|
schedstat_inc(this_rq(), sched_count);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void put_prev_task(struct rq *rq, struct task_struct *prev)
|
|
||||||
{
|
|
||||||
if (prev->on_rq || rq->skip_clock_update < 0)
|
|
||||||
update_rq_clock(rq);
|
|
||||||
prev->sched_class->put_prev_task(rq, prev);
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Pick up the highest-prio task:
|
* Pick up the highest-prio task:
|
||||||
*/
|
*/
|
||||||
static inline struct task_struct *
|
static inline struct task_struct *
|
||||||
pick_next_task(struct rq *rq)
|
pick_next_task(struct rq *rq, struct task_struct *prev)
|
||||||
{
|
{
|
||||||
const struct sched_class *class;
|
const struct sched_class *class;
|
||||||
struct task_struct *p;
|
struct task_struct *p;
|
||||||
|
@ -2600,13 +2593,13 @@ pick_next_task(struct rq *rq)
|
||||||
* the fair class we can call that function directly:
|
* the fair class we can call that function directly:
|
||||||
*/
|
*/
|
||||||
if (likely(rq->nr_running == rq->cfs.h_nr_running)) {
|
if (likely(rq->nr_running == rq->cfs.h_nr_running)) {
|
||||||
p = fair_sched_class.pick_next_task(rq);
|
p = fair_sched_class.pick_next_task(rq, prev);
|
||||||
if (likely(p))
|
if (likely(p))
|
||||||
return p;
|
return p;
|
||||||
}
|
}
|
||||||
|
|
||||||
for_each_class(class) {
|
for_each_class(class) {
|
||||||
p = class->pick_next_task(rq);
|
p = class->pick_next_task(rq, prev);
|
||||||
if (p)
|
if (p)
|
||||||
return p;
|
return p;
|
||||||
}
|
}
|
||||||
|
@ -2714,8 +2707,10 @@ need_resched:
|
||||||
rq->idle_stamp = 0;
|
rq->idle_stamp = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
put_prev_task(rq, prev);
|
if (prev->on_rq || rq->skip_clock_update < 0)
|
||||||
next = pick_next_task(rq);
|
update_rq_clock(rq);
|
||||||
|
|
||||||
|
next = pick_next_task(rq, prev);
|
||||||
clear_tsk_need_resched(prev);
|
clear_tsk_need_resched(prev);
|
||||||
clear_preempt_need_resched();
|
clear_preempt_need_resched();
|
||||||
rq->skip_clock_update = 0;
|
rq->skip_clock_update = 0;
|
||||||
|
@ -4748,7 +4743,7 @@ static void migrate_tasks(unsigned int dead_cpu)
|
||||||
if (rq->nr_running == 1)
|
if (rq->nr_running == 1)
|
||||||
break;
|
break;
|
||||||
|
|
||||||
next = pick_next_task(rq);
|
next = pick_next_task(rq, NULL);
|
||||||
BUG_ON(!next);
|
BUG_ON(!next);
|
||||||
next->sched_class->put_prev_task(rq, next);
|
next->sched_class->put_prev_task(rq, next);
|
||||||
|
|
||||||
|
|
|
@ -990,7 +990,7 @@ static struct sched_dl_entity *pick_next_dl_entity(struct rq *rq,
|
||||||
return rb_entry(left, struct sched_dl_entity, rb_node);
|
return rb_entry(left, struct sched_dl_entity, rb_node);
|
||||||
}
|
}
|
||||||
|
|
||||||
struct task_struct *pick_next_task_dl(struct rq *rq)
|
struct task_struct *pick_next_task_dl(struct rq *rq, struct task_struct *prev)
|
||||||
{
|
{
|
||||||
struct sched_dl_entity *dl_se;
|
struct sched_dl_entity *dl_se;
|
||||||
struct task_struct *p;
|
struct task_struct *p;
|
||||||
|
@ -1001,6 +1001,9 @@ struct task_struct *pick_next_task_dl(struct rq *rq)
|
||||||
if (unlikely(!dl_rq->dl_nr_running))
|
if (unlikely(!dl_rq->dl_nr_running))
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
|
if (prev)
|
||||||
|
prev->sched_class->put_prev_task(rq, prev);
|
||||||
|
|
||||||
dl_se = pick_next_dl_entity(rq, dl_rq);
|
dl_se = pick_next_dl_entity(rq, dl_rq);
|
||||||
BUG_ON(!dl_se);
|
BUG_ON(!dl_se);
|
||||||
|
|
||||||
|
|
|
@ -4655,7 +4655,8 @@ preempt:
|
||||||
set_last_buddy(se);
|
set_last_buddy(se);
|
||||||
}
|
}
|
||||||
|
|
||||||
static struct task_struct *pick_next_task_fair(struct rq *rq)
|
static struct task_struct *
|
||||||
|
pick_next_task_fair(struct rq *rq, struct task_struct *prev)
|
||||||
{
|
{
|
||||||
struct task_struct *p;
|
struct task_struct *p;
|
||||||
struct cfs_rq *cfs_rq = &rq->cfs;
|
struct cfs_rq *cfs_rq = &rq->cfs;
|
||||||
|
@ -4664,6 +4665,9 @@ static struct task_struct *pick_next_task_fair(struct rq *rq)
|
||||||
if (!cfs_rq->nr_running)
|
if (!cfs_rq->nr_running)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
|
if (prev)
|
||||||
|
prev->sched_class->put_prev_task(rq, prev);
|
||||||
|
|
||||||
do {
|
do {
|
||||||
se = pick_next_entity(cfs_rq);
|
se = pick_next_entity(cfs_rq);
|
||||||
set_next_entity(cfs_rq, se);
|
set_next_entity(cfs_rq, se);
|
||||||
|
|
|
@ -33,8 +33,12 @@ static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int fl
|
||||||
resched_task(rq->idle);
|
resched_task(rq->idle);
|
||||||
}
|
}
|
||||||
|
|
||||||
static struct task_struct *pick_next_task_idle(struct rq *rq)
|
static struct task_struct *
|
||||||
|
pick_next_task_idle(struct rq *rq, struct task_struct *prev)
|
||||||
{
|
{
|
||||||
|
if (prev)
|
||||||
|
prev->sched_class->put_prev_task(rq, prev);
|
||||||
|
|
||||||
schedstat_inc(rq, sched_goidle);
|
schedstat_inc(rq, sched_goidle);
|
||||||
#ifdef CONFIG_SMP
|
#ifdef CONFIG_SMP
|
||||||
/* Trigger the post schedule to do an idle_enter for CFS */
|
/* Trigger the post schedule to do an idle_enter for CFS */
|
||||||
|
|
|
@ -1310,15 +1310,7 @@ static struct task_struct *_pick_next_task_rt(struct rq *rq)
|
||||||
{
|
{
|
||||||
struct sched_rt_entity *rt_se;
|
struct sched_rt_entity *rt_se;
|
||||||
struct task_struct *p;
|
struct task_struct *p;
|
||||||
struct rt_rq *rt_rq;
|
struct rt_rq *rt_rq = &rq->rt;
|
||||||
|
|
||||||
rt_rq = &rq->rt;
|
|
||||||
|
|
||||||
if (!rt_rq->rt_nr_running)
|
|
||||||
return NULL;
|
|
||||||
|
|
||||||
if (rt_rq_throttled(rt_rq))
|
|
||||||
return NULL;
|
|
||||||
|
|
||||||
do {
|
do {
|
||||||
rt_se = pick_next_rt_entity(rq, rt_rq);
|
rt_se = pick_next_rt_entity(rq, rt_rq);
|
||||||
|
@ -1332,9 +1324,22 @@ static struct task_struct *_pick_next_task_rt(struct rq *rq)
|
||||||
return p;
|
return p;
|
||||||
}
|
}
|
||||||
|
|
||||||
static struct task_struct *pick_next_task_rt(struct rq *rq)
|
static struct task_struct *
|
||||||
|
pick_next_task_rt(struct rq *rq, struct task_struct *prev)
|
||||||
{
|
{
|
||||||
struct task_struct *p = _pick_next_task_rt(rq);
|
struct task_struct *p;
|
||||||
|
struct rt_rq *rt_rq = &rq->rt;
|
||||||
|
|
||||||
|
if (!rt_rq->rt_nr_running)
|
||||||
|
return NULL;
|
||||||
|
|
||||||
|
if (rt_rq_throttled(rt_rq))
|
||||||
|
return NULL;
|
||||||
|
|
||||||
|
if (prev)
|
||||||
|
prev->sched_class->put_prev_task(rq, prev);
|
||||||
|
|
||||||
|
p = _pick_next_task_rt(rq);
|
||||||
|
|
||||||
/* The running task is never eligible for pushing */
|
/* The running task is never eligible for pushing */
|
||||||
if (p)
|
if (p)
|
||||||
|
|
|
@ -1105,7 +1105,13 @@ struct sched_class {
|
||||||
|
|
||||||
void (*check_preempt_curr) (struct rq *rq, struct task_struct *p, int flags);
|
void (*check_preempt_curr) (struct rq *rq, struct task_struct *p, int flags);
|
||||||
|
|
||||||
struct task_struct * (*pick_next_task) (struct rq *rq);
|
/*
|
||||||
|
* It is the responsibility of the pick_next_task() method that will
|
||||||
|
* return the next task to call put_prev_task() on the @prev task or
|
||||||
|
* something equivalent.
|
||||||
|
*/
|
||||||
|
struct task_struct * (*pick_next_task) (struct rq *rq,
|
||||||
|
struct task_struct *prev);
|
||||||
void (*put_prev_task) (struct rq *rq, struct task_struct *p);
|
void (*put_prev_task) (struct rq *rq, struct task_struct *p);
|
||||||
|
|
||||||
#ifdef CONFIG_SMP
|
#ifdef CONFIG_SMP
|
||||||
|
|
|
@ -23,16 +23,20 @@ check_preempt_curr_stop(struct rq *rq, struct task_struct *p, int flags)
|
||||||
/* we're never preempted */
|
/* we're never preempted */
|
||||||
}
|
}
|
||||||
|
|
||||||
static struct task_struct *pick_next_task_stop(struct rq *rq)
|
static struct task_struct *
|
||||||
|
pick_next_task_stop(struct rq *rq, struct task_struct *prev)
|
||||||
{
|
{
|
||||||
struct task_struct *stop = rq->stop;
|
struct task_struct *stop = rq->stop;
|
||||||
|
|
||||||
if (stop && stop->on_rq) {
|
if (!stop || !stop->on_rq)
|
||||||
stop->se.exec_start = rq_clock_task(rq);
|
return NULL;
|
||||||
return stop;
|
|
||||||
}
|
|
||||||
|
|
||||||
return NULL;
|
if (prev)
|
||||||
|
prev->sched_class->put_prev_task(rq, prev);
|
||||||
|
|
||||||
|
stop->se.exec_start = rq_clock_task(rq);
|
||||||
|
|
||||||
|
return stop;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void
|
static void
|
||||||
|
|
Loading…
Reference in New Issue