sched/fair: Beef up wake_wide()
Josef Bacik reported that Facebook sees better performance with their 1:N load (1 dispatch/node, N workers/node) when carrying an old patch to try very hard to wake to an idle CPU. While looking at wake_wide(), I noticed that it doesn't pay attention to the wakeup of a many partner waker, returning 1 only when waking one of its many partners. Correct that, letting explicit domain flags override the heuristic. While at it, adjust task_struct bits, we don't need a 64-bit counter. Tested-by: Josef Bacik <jbacik@fb.com> Signed-off-by: Mike Galbraith <umgwanakikbuti@gmail.com> [ Tidy things up. ] Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Mike Galbraith <efault@gmx.de> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: kernel-team<Kernel-team@fb.com> Cc: morten.rasmussen@arm.com Cc: riel@redhat.com Link: http://lkml.kernel.org/r/1436888390.7983.49.camel@gmail.com Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
parent
fbd705a0c6
commit
63b0e9edce
|
@ -1359,9 +1359,9 @@ struct task_struct {
|
||||||
#ifdef CONFIG_SMP
|
#ifdef CONFIG_SMP
|
||||||
struct llist_node wake_entry;
|
struct llist_node wake_entry;
|
||||||
int on_cpu;
|
int on_cpu;
|
||||||
struct task_struct *last_wakee;
|
unsigned int wakee_flips;
|
||||||
unsigned long wakee_flips;
|
|
||||||
unsigned long wakee_flip_decay_ts;
|
unsigned long wakee_flip_decay_ts;
|
||||||
|
struct task_struct *last_wakee;
|
||||||
|
|
||||||
int wake_cpu;
|
int wake_cpu;
|
||||||
#endif
|
#endif
|
||||||
|
|
|
@ -4726,26 +4726,29 @@ static long effective_load(struct task_group *tg, int cpu, long wl, long wg)
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Detect M:N waker/wakee relationships via a switching-frequency heuristic.
|
||||||
|
* A waker of many should wake a different task than the one last awakened
|
||||||
|
* at a frequency roughly N times higher than one of its wakees. In order
|
||||||
|
* to determine whether we should let the load spread vs consolodating to
|
||||||
|
* shared cache, we look for a minimum 'flip' frequency of llc_size in one
|
||||||
|
* partner, and a factor of lls_size higher frequency in the other. With
|
||||||
|
* both conditions met, we can be relatively sure that the relationship is
|
||||||
|
* non-monogamous, with partner count exceeding socket size. Waker/wakee
|
||||||
|
* being client/server, worker/dispatcher, interrupt source or whatever is
|
||||||
|
* irrelevant, spread criteria is apparent partner count exceeds socket size.
|
||||||
|
*/
|
||||||
static int wake_wide(struct task_struct *p)
|
static int wake_wide(struct task_struct *p)
|
||||||
{
|
{
|
||||||
|
unsigned int master = current->wakee_flips;
|
||||||
|
unsigned int slave = p->wakee_flips;
|
||||||
int factor = this_cpu_read(sd_llc_size);
|
int factor = this_cpu_read(sd_llc_size);
|
||||||
|
|
||||||
/*
|
if (master < slave)
|
||||||
* Yeah, it's the switching-frequency, could means many wakee or
|
swap(master, slave);
|
||||||
* rapidly switch, use factor here will just help to automatically
|
if (slave < factor || master < slave * factor)
|
||||||
* adjust the loose-degree, so bigger node will lead to more pull.
|
return 0;
|
||||||
*/
|
return 1;
|
||||||
if (p->wakee_flips > factor) {
|
|
||||||
/*
|
|
||||||
* wakee is somewhat hot, it needs certain amount of cpu
|
|
||||||
* resource, so if waker is far more hot, prefer to leave
|
|
||||||
* it alone.
|
|
||||||
*/
|
|
||||||
if (current->wakee_flips > (factor * p->wakee_flips))
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync)
|
static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync)
|
||||||
|
@ -4757,13 +4760,6 @@ static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync)
|
||||||
unsigned long weight;
|
unsigned long weight;
|
||||||
int balanced;
|
int balanced;
|
||||||
|
|
||||||
/*
|
|
||||||
* If we wake multiple tasks be careful to not bounce
|
|
||||||
* ourselves around too much.
|
|
||||||
*/
|
|
||||||
if (wake_wide(p))
|
|
||||||
return 0;
|
|
||||||
|
|
||||||
idx = sd->wake_idx;
|
idx = sd->wake_idx;
|
||||||
this_cpu = smp_processor_id();
|
this_cpu = smp_processor_id();
|
||||||
prev_cpu = task_cpu(p);
|
prev_cpu = task_cpu(p);
|
||||||
|
@ -5017,17 +5013,17 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f
|
||||||
{
|
{
|
||||||
struct sched_domain *tmp, *affine_sd = NULL, *sd = NULL;
|
struct sched_domain *tmp, *affine_sd = NULL, *sd = NULL;
|
||||||
int cpu = smp_processor_id();
|
int cpu = smp_processor_id();
|
||||||
int new_cpu = cpu;
|
int new_cpu = prev_cpu;
|
||||||
int want_affine = 0;
|
int want_affine = 0;
|
||||||
int sync = wake_flags & WF_SYNC;
|
int sync = wake_flags & WF_SYNC;
|
||||||
|
|
||||||
if (sd_flag & SD_BALANCE_WAKE)
|
if (sd_flag & SD_BALANCE_WAKE)
|
||||||
want_affine = cpumask_test_cpu(cpu, tsk_cpus_allowed(p));
|
want_affine = !wake_wide(p) && cpumask_test_cpu(cpu, tsk_cpus_allowed(p));
|
||||||
|
|
||||||
rcu_read_lock();
|
rcu_read_lock();
|
||||||
for_each_domain(cpu, tmp) {
|
for_each_domain(cpu, tmp) {
|
||||||
if (!(tmp->flags & SD_LOAD_BALANCE))
|
if (!(tmp->flags & SD_LOAD_BALANCE))
|
||||||
continue;
|
break;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* If both cpu and prev_cpu are part of this domain,
|
* If both cpu and prev_cpu are part of this domain,
|
||||||
|
@ -5041,17 +5037,21 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f
|
||||||
|
|
||||||
if (tmp->flags & sd_flag)
|
if (tmp->flags & sd_flag)
|
||||||
sd = tmp;
|
sd = tmp;
|
||||||
|
else if (!want_affine)
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (affine_sd && cpu != prev_cpu && wake_affine(affine_sd, p, sync))
|
if (affine_sd) {
|
||||||
prev_cpu = cpu;
|
sd = NULL; /* Prefer wake_affine over balance flags */
|
||||||
|
if (cpu != prev_cpu && wake_affine(affine_sd, p, sync))
|
||||||
if (sd_flag & SD_BALANCE_WAKE) {
|
new_cpu = cpu;
|
||||||
new_cpu = select_idle_sibling(p, prev_cpu);
|
|
||||||
goto unlock;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
while (sd) {
|
if (!sd) {
|
||||||
|
if (sd_flag & SD_BALANCE_WAKE) /* XXX always ? */
|
||||||
|
new_cpu = select_idle_sibling(p, new_cpu);
|
||||||
|
|
||||||
|
} else while (sd) {
|
||||||
struct sched_group *group;
|
struct sched_group *group;
|
||||||
int weight;
|
int weight;
|
||||||
|
|
||||||
|
@ -5085,7 +5085,6 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f
|
||||||
}
|
}
|
||||||
/* while loop will break here if sd == NULL */
|
/* while loop will break here if sd == NULL */
|
||||||
}
|
}
|
||||||
unlock:
|
|
||||||
rcu_read_unlock();
|
rcu_read_unlock();
|
||||||
|
|
||||||
return new_cpu;
|
return new_cpu;
|
||||||
|
|
Loading…
Reference in New Issue