sched/fair: Optimize update_blocked_averages()
Removing a cfs_rq from rq->leaf_cfs_rq_list can break the parent/child
ordering of the list when it will be added back. In order to remove an
empty and fully decayed cfs_rq, we must remove its children too, so they
will be added back in the right order next time.
With a normal decay of PELT, a parent will be empty and fully decayed
if all children are empty and fully decayed too. In such a case, we just
have to ensure that the whole branch will be added when a new task is
enqueued. This is default behavior since :
commit f678331973
("sched/fair: Fix insertion in rq->leaf_cfs_rq_list")
In case of throttling, the PELT of throttled cfs_rq will not be updated
whereas the parent will. This breaks the assumption made above unless we
remove the children of a cfs_rq that is throttled. Then, they will be
added back when unthrottled and a sched_entity will be enqueued.
As throttled cfs_rq are now removed from the list, we can remove the
associated test in update_blocked_averages().
Signed-off-by: Vincent Guittot <vincent.guittot@linaro.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: sargun@sargun.me
Cc: tj@kernel.org
Cc: xiexiuqi@huawei.com
Cc: xiezhipeng1@huawei.com
Link: https://lkml.kernel.org/r/1549469662-13614-2-git-send-email-vincent.guittot@linaro.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
parent
c9ba7560c5
commit
31bc6aeaab
|
@ -346,6 +346,18 @@ static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
|
||||||
static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq)
|
static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq)
|
||||||
{
|
{
|
||||||
if (cfs_rq->on_list) {
|
if (cfs_rq->on_list) {
|
||||||
|
struct rq *rq = rq_of(cfs_rq);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* With cfs_rq being unthrottled/throttled during an enqueue,
|
||||||
|
* it can happen the tmp_alone_branch points the a leaf that
|
||||||
|
* we finally want to del. In this case, tmp_alone_branch moves
|
||||||
|
* to the prev element but it will point to rq->leaf_cfs_rq_list
|
||||||
|
* at the end of the enqueue.
|
||||||
|
*/
|
||||||
|
if (rq->tmp_alone_branch == &cfs_rq->leaf_cfs_rq_list)
|
||||||
|
rq->tmp_alone_branch = cfs_rq->leaf_cfs_rq_list.prev;
|
||||||
|
|
||||||
list_del_rcu(&cfs_rq->leaf_cfs_rq_list);
|
list_del_rcu(&cfs_rq->leaf_cfs_rq_list);
|
||||||
cfs_rq->on_list = 0;
|
cfs_rq->on_list = 0;
|
||||||
}
|
}
|
||||||
|
@ -4438,6 +4450,10 @@ static int tg_unthrottle_up(struct task_group *tg, void *data)
|
||||||
/* adjust cfs_rq_clock_task() */
|
/* adjust cfs_rq_clock_task() */
|
||||||
cfs_rq->throttled_clock_task_time += rq_clock_task(rq) -
|
cfs_rq->throttled_clock_task_time += rq_clock_task(rq) -
|
||||||
cfs_rq->throttled_clock_task;
|
cfs_rq->throttled_clock_task;
|
||||||
|
|
||||||
|
/* Add cfs_rq with already running entity in the list */
|
||||||
|
if (cfs_rq->nr_running >= 1)
|
||||||
|
list_add_leaf_cfs_rq(cfs_rq);
|
||||||
}
|
}
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
|
@ -4449,8 +4465,10 @@ static int tg_throttle_down(struct task_group *tg, void *data)
|
||||||
struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
|
struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
|
||||||
|
|
||||||
/* group is entering throttled state, stop time */
|
/* group is entering throttled state, stop time */
|
||||||
if (!cfs_rq->throttle_count)
|
if (!cfs_rq->throttle_count) {
|
||||||
cfs_rq->throttled_clock_task = rq_clock_task(rq);
|
cfs_rq->throttled_clock_task = rq_clock_task(rq);
|
||||||
|
list_del_leaf_cfs_rq(cfs_rq);
|
||||||
|
}
|
||||||
cfs_rq->throttle_count++;
|
cfs_rq->throttle_count++;
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
|
@ -4553,6 +4571,8 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
assert_list_leaf_cfs_rq(rq);
|
||||||
|
|
||||||
if (!se)
|
if (!se)
|
||||||
add_nr_running(rq, task_delta);
|
add_nr_running(rq, task_delta);
|
||||||
|
|
||||||
|
@ -7700,10 +7720,6 @@ static void update_blocked_averages(int cpu)
|
||||||
for_each_leaf_cfs_rq(rq, cfs_rq) {
|
for_each_leaf_cfs_rq(rq, cfs_rq) {
|
||||||
struct sched_entity *se;
|
struct sched_entity *se;
|
||||||
|
|
||||||
/* throttled entities do not contribute to load */
|
|
||||||
if (throttled_hierarchy(cfs_rq))
|
|
||||||
continue;
|
|
||||||
|
|
||||||
if (update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq))
|
if (update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq))
|
||||||
update_tg_load_avg(cfs_rq, 0);
|
update_tg_load_avg(cfs_rq, 0);
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue