sched/fair: Make it possible to account fair load avg consistently
The current code accounts for the time a task was absent from the fair class (per ATTACH_AGE_LOAD). However it does not work correctly when a task got migrated or moved to another cgroup while outside of the fair class. This patch tries to address that by aging on migration. We locklessly read the 'last_update_time' stamp from both the old and new cfs_rq, ages the load upto the old time, and sets it to the new time. These timestamps should in general not be more than 1 tick apart from one another, so there is a definite bound on things. Signed-off-by: Byungchul Park <byungchul.park@lge.com> [ Changelog, a few edits and !SMP build fix ] Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Mike Galbraith <efault@gmx.de> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Link: http://lkml.kernel.org/r/1445616981-29904-2-git-send-email-byungchul.park@lge.com Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
parent
467386fbbf
commit
ad936d8658
|
@ -2120,6 +2120,10 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
|
||||||
p->se.vruntime = 0;
|
p->se.vruntime = 0;
|
||||||
INIT_LIST_HEAD(&p->se.group_node);
|
INIT_LIST_HEAD(&p->se.group_node);
|
||||||
|
|
||||||
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
||||||
|
p->se.cfs_rq = NULL;
|
||||||
|
#endif
|
||||||
|
|
||||||
#ifdef CONFIG_SCHEDSTATS
|
#ifdef CONFIG_SCHEDSTATS
|
||||||
memset(&p->se.statistics, 0, sizeof(p->se.statistics));
|
memset(&p->se.statistics, 0, sizeof(p->se.statistics));
|
||||||
#endif
|
#endif
|
||||||
|
|
|
@ -2715,6 +2715,52 @@ static inline void update_tg_load_avg(struct cfs_rq *cfs_rq, int force)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Called within set_task_rq() right before setting a task's cpu. The
|
||||||
|
* caller only guarantees p->pi_lock is held; no other assumptions,
|
||||||
|
* including the state of rq->lock, should be made.
|
||||||
|
*/
|
||||||
|
void set_task_rq_fair(struct sched_entity *se,
|
||||||
|
struct cfs_rq *prev, struct cfs_rq *next)
|
||||||
|
{
|
||||||
|
if (!sched_feat(ATTACH_AGE_LOAD))
|
||||||
|
return;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* We are supposed to update the task to "current" time, then its up to
|
||||||
|
* date and ready to go to new CPU/cfs_rq. But we have difficulty in
|
||||||
|
* getting what current time is, so simply throw away the out-of-date
|
||||||
|
* time. This will result in the wakee task is less decayed, but giving
|
||||||
|
* the wakee more load sounds not bad.
|
||||||
|
*/
|
||||||
|
if (se->avg.last_update_time && prev) {
|
||||||
|
u64 p_last_update_time;
|
||||||
|
u64 n_last_update_time;
|
||||||
|
|
||||||
|
#ifndef CONFIG_64BIT
|
||||||
|
u64 p_last_update_time_copy;
|
||||||
|
u64 n_last_update_time_copy;
|
||||||
|
|
||||||
|
do {
|
||||||
|
p_last_update_time_copy = prev->load_last_update_time_copy;
|
||||||
|
n_last_update_time_copy = next->load_last_update_time_copy;
|
||||||
|
|
||||||
|
smp_rmb();
|
||||||
|
|
||||||
|
p_last_update_time = prev->avg.last_update_time;
|
||||||
|
n_last_update_time = next->avg.last_update_time;
|
||||||
|
|
||||||
|
} while (p_last_update_time != p_last_update_time_copy ||
|
||||||
|
n_last_update_time != n_last_update_time_copy);
|
||||||
|
#else
|
||||||
|
p_last_update_time = prev->avg.last_update_time;
|
||||||
|
n_last_update_time = next->avg.last_update_time;
|
||||||
|
#endif
|
||||||
|
__update_load_avg(p_last_update_time, cpu_of(rq_of(prev)),
|
||||||
|
&se->avg, 0, 0, NULL);
|
||||||
|
se->avg.last_update_time = n_last_update_time;
|
||||||
|
}
|
||||||
|
}
|
||||||
#else /* CONFIG_FAIR_GROUP_SCHED */
|
#else /* CONFIG_FAIR_GROUP_SCHED */
|
||||||
static inline void update_tg_load_avg(struct cfs_rq *cfs_rq, int force) {}
|
static inline void update_tg_load_avg(struct cfs_rq *cfs_rq, int force) {}
|
||||||
#endif /* CONFIG_FAIR_GROUP_SCHED */
|
#endif /* CONFIG_FAIR_GROUP_SCHED */
|
||||||
|
|
|
@ -335,7 +335,15 @@ extern void sched_move_task(struct task_struct *tsk);
|
||||||
|
|
||||||
#ifdef CONFIG_FAIR_GROUP_SCHED
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
||||||
extern int sched_group_set_shares(struct task_group *tg, unsigned long shares);
|
extern int sched_group_set_shares(struct task_group *tg, unsigned long shares);
|
||||||
#endif
|
|
||||||
|
#ifdef CONFIG_SMP
|
||||||
|
extern void set_task_rq_fair(struct sched_entity *se,
|
||||||
|
struct cfs_rq *prev, struct cfs_rq *next);
|
||||||
|
#else /* !CONFIG_SMP */
|
||||||
|
static inline void set_task_rq_fair(struct sched_entity *se,
|
||||||
|
struct cfs_rq *prev, struct cfs_rq *next) { }
|
||||||
|
#endif /* CONFIG_SMP */
|
||||||
|
#endif /* CONFIG_FAIR_GROUP_SCHED */
|
||||||
|
|
||||||
#else /* CONFIG_CGROUP_SCHED */
|
#else /* CONFIG_CGROUP_SCHED */
|
||||||
|
|
||||||
|
@ -933,6 +941,7 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#ifdef CONFIG_FAIR_GROUP_SCHED
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
||||||
|
set_task_rq_fair(&p->se, p->se.cfs_rq, tg->cfs_rq[cpu]);
|
||||||
p->se.cfs_rq = tg->cfs_rq[cpu];
|
p->se.cfs_rq = tg->cfs_rq[cpu];
|
||||||
p->se.parent = tg->se[cpu];
|
p->se.parent = tg->se[cpu];
|
||||||
#endif
|
#endif
|
||||||
|
|
Loading…
Reference in New Issue