sched/fair: Add NOHZ_STATS_KICK
Split the NOHZ idle balancer into doing two separate actions: - update blocked load statistic - actually load-balance Since the latter requires the former, ensure this happens. For now always tag both bits at the same time. Prepares for a future where we can toggle only the STATS bit. Suggested-by: Vincent Guittot <vincent.guittot@linaro.org> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Mike Galbraith <efault@gmx.de> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: linux-kernel@vger.kernel.org Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
parent
a22e47a4e3
commit
b7031a02ec
|
@ -583,7 +583,7 @@ static inline bool got_nohz_idle_kick(void)
|
||||||
{
|
{
|
||||||
int cpu = smp_processor_id();
|
int cpu = smp_processor_id();
|
||||||
|
|
||||||
if (!(atomic_read(nohz_flags(cpu)) & NOHZ_BALANCE_KICK))
|
if (!(atomic_read(nohz_flags(cpu)) & NOHZ_KICK_MASK))
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
if (idle_cpu(cpu) && !need_resched())
|
if (idle_cpu(cpu) && !need_resched())
|
||||||
|
@ -593,7 +593,7 @@ static inline bool got_nohz_idle_kick(void)
|
||||||
* We can't run Idle Load Balance on this CPU for this time so we
|
* We can't run Idle Load Balance on this CPU for this time so we
|
||||||
* cancel it and clear NOHZ_BALANCE_KICK
|
* cancel it and clear NOHZ_BALANCE_KICK
|
||||||
*/
|
*/
|
||||||
atomic_andnot(NOHZ_BALANCE_KICK, nohz_flags(cpu));
|
atomic_andnot(NOHZ_KICK_MASK, nohz_flags(cpu));
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -9082,8 +9082,8 @@ static void nohz_balancer_kick(void)
|
||||||
if (ilb_cpu >= nr_cpu_ids)
|
if (ilb_cpu >= nr_cpu_ids)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
flags = atomic_fetch_or(NOHZ_BALANCE_KICK, nohz_flags(ilb_cpu));
|
flags = atomic_fetch_or(NOHZ_KICK_MASK, nohz_flags(ilb_cpu));
|
||||||
if (flags & NOHZ_BALANCE_KICK)
|
if (flags & NOHZ_KICK_MASK)
|
||||||
return;
|
return;
|
||||||
/*
|
/*
|
||||||
* Use smp_send_reschedule() instead of resched_cpu().
|
* Use smp_send_reschedule() instead of resched_cpu().
|
||||||
|
@ -9202,8 +9202,6 @@ static void rebalance_domains(struct rq *rq, enum cpu_idle_type idle)
|
||||||
int need_serialize, need_decay = 0;
|
int need_serialize, need_decay = 0;
|
||||||
u64 max_cost = 0;
|
u64 max_cost = 0;
|
||||||
|
|
||||||
update_blocked_averages(cpu);
|
|
||||||
|
|
||||||
rcu_read_lock();
|
rcu_read_lock();
|
||||||
for_each_domain(cpu, sd) {
|
for_each_domain(cpu, sd) {
|
||||||
/*
|
/*
|
||||||
|
@ -9298,20 +9296,27 @@ out:
|
||||||
* In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the
|
* In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the
|
||||||
* rebalancing for all the CPUs for whom scheduler ticks are stopped.
|
* rebalancing for all the CPUs for whom scheduler ticks are stopped.
|
||||||
*/
|
*/
|
||||||
static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
|
static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
|
||||||
{
|
{
|
||||||
int this_cpu = this_rq->cpu;
|
|
||||||
struct rq *rq;
|
|
||||||
int balance_cpu;
|
|
||||||
/* Earliest time when we have to do rebalance again */
|
/* Earliest time when we have to do rebalance again */
|
||||||
unsigned long next_balance = jiffies + 60*HZ;
|
unsigned long next_balance = jiffies + 60*HZ;
|
||||||
int update_next_balance = 0;
|
int update_next_balance = 0;
|
||||||
|
int this_cpu = this_rq->cpu;
|
||||||
|
unsigned int flags;
|
||||||
|
int balance_cpu;
|
||||||
|
struct rq *rq;
|
||||||
|
|
||||||
if (!(atomic_read(nohz_flags(this_cpu)) & NOHZ_BALANCE_KICK))
|
if (!(atomic_read(nohz_flags(this_cpu)) & NOHZ_KICK_MASK))
|
||||||
return;
|
return false;
|
||||||
|
|
||||||
if (idle != CPU_IDLE)
|
if (idle != CPU_IDLE) {
|
||||||
goto end;
|
atomic_andnot(NOHZ_KICK_MASK, nohz_flags(this_cpu));
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
flags = atomic_fetch_andnot(NOHZ_KICK_MASK, nohz_flags(this_cpu));
|
||||||
|
|
||||||
|
SCHED_WARN_ON((flags & NOHZ_KICK_MASK) == NOHZ_BALANCE_KICK);
|
||||||
|
|
||||||
for_each_cpu(balance_cpu, nohz.idle_cpus_mask) {
|
for_each_cpu(balance_cpu, nohz.idle_cpus_mask) {
|
||||||
if (balance_cpu == this_cpu || !idle_cpu(balance_cpu))
|
if (balance_cpu == this_cpu || !idle_cpu(balance_cpu))
|
||||||
|
@ -9339,7 +9344,9 @@ static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
|
||||||
cpu_load_update_idle(rq);
|
cpu_load_update_idle(rq);
|
||||||
rq_unlock_irq(rq, &rf);
|
rq_unlock_irq(rq, &rf);
|
||||||
|
|
||||||
rebalance_domains(rq, CPU_IDLE);
|
update_blocked_averages(rq->cpu);
|
||||||
|
if (flags & NOHZ_BALANCE_KICK)
|
||||||
|
rebalance_domains(rq, CPU_IDLE);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (time_after(next_balance, rq->next_balance)) {
|
if (time_after(next_balance, rq->next_balance)) {
|
||||||
|
@ -9348,6 +9355,10 @@ static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
update_blocked_averages(this_cpu);
|
||||||
|
if (flags & NOHZ_BALANCE_KICK)
|
||||||
|
rebalance_domains(this_rq, CPU_IDLE);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* next_balance will be updated only when there is a need.
|
* next_balance will be updated only when there is a need.
|
||||||
* When the CPU is attached to null domain for ex, it will not be
|
* When the CPU is attached to null domain for ex, it will not be
|
||||||
|
@ -9355,8 +9366,8 @@ static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
|
||||||
*/
|
*/
|
||||||
if (likely(update_next_balance))
|
if (likely(update_next_balance))
|
||||||
nohz.next_balance = next_balance;
|
nohz.next_balance = next_balance;
|
||||||
end:
|
|
||||||
atomic_andnot(NOHZ_BALANCE_KICK, nohz_flags(this_cpu));
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -9443,7 +9454,10 @@ unlock:
|
||||||
return kick;
|
return kick;
|
||||||
}
|
}
|
||||||
#else
|
#else
|
||||||
static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) { }
|
static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -9464,7 +9478,11 @@ static __latent_entropy void run_rebalance_domains(struct softirq_action *h)
|
||||||
* load balance only within the local sched_domain hierarchy
|
* load balance only within the local sched_domain hierarchy
|
||||||
* and abort nohz_idle_balance altogether if we pull some load.
|
* and abort nohz_idle_balance altogether if we pull some load.
|
||||||
*/
|
*/
|
||||||
nohz_idle_balance(this_rq, idle);
|
if (nohz_idle_balance(this_rq, idle))
|
||||||
|
return;
|
||||||
|
|
||||||
|
/* normal load balance */
|
||||||
|
update_blocked_averages(this_rq->cpu);
|
||||||
rebalance_domains(this_rq, idle);
|
rebalance_domains(this_rq, idle);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -2036,9 +2036,13 @@ extern void cfs_bandwidth_usage_dec(void);
|
||||||
#ifdef CONFIG_NO_HZ_COMMON
|
#ifdef CONFIG_NO_HZ_COMMON
|
||||||
#define NOHZ_TICK_STOPPED_BIT 0
|
#define NOHZ_TICK_STOPPED_BIT 0
|
||||||
#define NOHZ_BALANCE_KICK_BIT 1
|
#define NOHZ_BALANCE_KICK_BIT 1
|
||||||
|
#define NOHZ_STATS_KICK_BIT 2
|
||||||
|
|
||||||
#define NOHZ_TICK_STOPPED BIT(NOHZ_TICK_STOPPED_BIT)
|
#define NOHZ_TICK_STOPPED BIT(NOHZ_TICK_STOPPED_BIT)
|
||||||
#define NOHZ_BALANCE_KICK BIT(NOHZ_BALANCE_KICK_BIT)
|
#define NOHZ_BALANCE_KICK BIT(NOHZ_BALANCE_KICK_BIT)
|
||||||
|
#define NOHZ_STATS_KICK BIT(NOHZ_STATS_KICK_BIT)
|
||||||
|
|
||||||
|
#define NOHZ_KICK_MASK (NOHZ_BALANCE_KICK | NOHZ_STATS_KICK)
|
||||||
|
|
||||||
#define nohz_flags(cpu) (&cpu_rq(cpu)->nohz_flags)
|
#define nohz_flags(cpu) (&cpu_rq(cpu)->nohz_flags)
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue