2007-07-09 18:51:58 +02:00
|
|
|
/*
|
|
|
|
* Completely Fair Scheduling (CFS) Class (SCHED_NORMAL/SCHED_BATCH)
|
|
|
|
*
|
|
|
|
* Copyright (C) 2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
|
|
|
|
*
|
|
|
|
* Interactivity improvements by Mike Galbraith
|
|
|
|
* (C) 2007 Mike Galbraith <efault@gmx.de>
|
|
|
|
*
|
|
|
|
* Various enhancements by Dmitry Adamushko.
|
|
|
|
* (C) 2007 Dmitry Adamushko <dmitry.adamushko@gmail.com>
|
|
|
|
*
|
|
|
|
* Group scheduling enhancements by Srivatsa Vaddagiri
|
|
|
|
* Copyright IBM Corporation, 2007
|
|
|
|
* Author: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com>
|
|
|
|
*
|
|
|
|
* Scaled math optimizations by Thomas Gleixner
|
|
|
|
* Copyright (C) 2007, Thomas Gleixner <tglx@linutronix.de>
|
2007-08-25 18:41:53 +02:00
|
|
|
*
|
|
|
|
* Adaptive scheduling granularity, math enhancements by Peter Zijlstra
|
|
|
|
* Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
|
2007-07-09 18:51:58 +02:00
|
|
|
*/
|
|
|
|
|
2008-01-25 21:08:34 +01:00
|
|
|
#include <linux/latencytop.h>
|
2009-11-30 12:16:47 +01:00
|
|
|
#include <linux/sched.h>
|
2011-03-26 13:52:55 +01:00
|
|
|
#include <linux/cpumask.h>
|
2011-10-25 10:00:11 +02:00
|
|
|
#include <linux/slab.h>
|
|
|
|
#include <linux/profile.h>
|
|
|
|
#include <linux/interrupt.h>
|
|
|
|
|
|
|
|
#include <trace/events/sched.h>
|
|
|
|
|
|
|
|
#include "sched.h"
|
2008-01-25 21:08:34 +01:00
|
|
|
|
2007-07-09 18:51:58 +02:00
|
|
|
/*
|
2007-08-25 18:41:53 +02:00
|
|
|
* Targeted preemption latency for CPU-bound tasks:
|
2010-10-14 09:09:13 +02:00
|
|
|
* (default: 6ms * (1 + ilog(ncpus)), units: nanoseconds)
|
2007-07-09 18:51:58 +02:00
|
|
|
*
|
2007-08-25 18:41:53 +02:00
|
|
|
* NOTE: this latency value is not the same as the concept of
|
2007-10-15 17:00:14 +02:00
|
|
|
* 'timeslice length' - timeslices in CFS are of variable length
|
|
|
|
* and have no persistent notion like in traditional, time-slice
|
|
|
|
* based scheduling concepts.
|
2007-07-09 18:51:58 +02:00
|
|
|
*
|
2007-10-15 17:00:14 +02:00
|
|
|
* (to see the precise effective timeslice length of your workload,
|
|
|
|
* run vmstat and monitor the context-switches (cs) field)
|
2007-07-09 18:51:58 +02:00
|
|
|
*/
|
2010-03-11 17:17:15 +01:00
|
|
|
unsigned int sysctl_sched_latency = 6000000ULL;
|
|
|
|
unsigned int normalized_sysctl_sched_latency = 6000000ULL;
|
2007-10-15 17:00:02 +02:00
|
|
|
|
2009-11-30 12:16:47 +01:00
|
|
|
/*
|
|
|
|
* The initial- and re-scaling of tunables is configurable
|
|
|
|
* (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus))
|
|
|
|
*
|
|
|
|
* Options are:
|
|
|
|
* SCHED_TUNABLESCALING_NONE - unscaled, always *1
|
|
|
|
* SCHED_TUNABLESCALING_LOG - scaled logarithmical, *1+ilog(ncpus)
|
|
|
|
* SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus
|
|
|
|
*/
|
|
|
|
enum sched_tunable_scaling sysctl_sched_tunable_scaling
|
|
|
|
= SCHED_TUNABLESCALING_LOG;
|
|
|
|
|
2007-10-15 17:00:02 +02:00
|
|
|
/*
|
2007-11-09 22:39:37 +01:00
|
|
|
* Minimal preemption granularity for CPU-bound tasks:
|
2010-10-14 09:09:13 +02:00
|
|
|
* (default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds)
|
2007-10-15 17:00:02 +02:00
|
|
|
*/
|
2010-09-12 08:14:52 +02:00
|
|
|
unsigned int sysctl_sched_min_granularity = 750000ULL;
|
|
|
|
unsigned int normalized_sysctl_sched_min_granularity = 750000ULL;
|
2007-08-25 18:41:53 +02:00
|
|
|
|
|
|
|
/*
|
2007-11-09 22:39:37 +01:00
|
|
|
* is kept at sysctl_sched_latency / sysctl_sched_min_granularity
|
|
|
|
*/
|
2010-09-12 08:14:52 +02:00
|
|
|
static unsigned int sched_nr_latency = 8;
|
2007-11-09 22:39:37 +01:00
|
|
|
|
|
|
|
/*
|
2009-09-09 15:41:37 +02:00
|
|
|
* After fork, child runs first. If set to 0 (default) then
|
2007-11-09 22:39:37 +01:00
|
|
|
* parent will (try to) run first.
|
2007-08-25 18:41:53 +02:00
|
|
|
*/
|
2009-09-09 15:41:37 +02:00
|
|
|
unsigned int sysctl_sched_child_runs_first __read_mostly;
|
2007-07-09 18:51:58 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* SCHED_OTHER wake-up granularity.
|
2009-09-09 15:41:37 +02:00
|
|
|
* (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds)
|
2007-07-09 18:51:58 +02:00
|
|
|
*
|
|
|
|
* This option delays the preemption effects of decoupled workloads
|
|
|
|
* and reduces their over-scheduling. Synchronous workloads will still
|
|
|
|
* have immediate wakeup/sleep latencies.
|
|
|
|
*/
|
2009-09-09 15:41:37 +02:00
|
|
|
unsigned int sysctl_sched_wakeup_granularity = 1000000UL;
|
2009-11-30 12:16:46 +01:00
|
|
|
unsigned int normalized_sysctl_sched_wakeup_granularity = 1000000UL;
|
2007-07-09 18:51:58 +02:00
|
|
|
|
2007-10-15 17:00:18 +02:00
|
|
|
const_debug unsigned int sysctl_sched_migration_cost = 500000UL;
|
|
|
|
|
2010-11-16 00:47:06 +01:00
|
|
|
/*
|
|
|
|
* The exponential sliding window over which load is averaged for shares
|
|
|
|
* distribution.
|
|
|
|
* (default: 10msec)
|
|
|
|
*/
|
|
|
|
unsigned int __read_mostly sysctl_sched_shares_window = 10000000UL;
|
|
|
|
|
2011-07-21 18:43:30 +02:00
|
|
|
#ifdef CONFIG_CFS_BANDWIDTH
|
|
|
|
/*
|
|
|
|
* Amount of runtime to allocate from global (tg) to local (per-cfs_rq) pool
|
|
|
|
* each time a cfs_rq requests quota.
|
|
|
|
*
|
|
|
|
* Note: in the case that the slice exceeds the runtime remaining (either due
|
|
|
|
* to consumption or the quota being specified to be smaller than the slice)
|
|
|
|
* we will always only issue the remaining available time.
|
|
|
|
*
|
|
|
|
* default: 5 msec, units: microseconds
|
|
|
|
*/
|
|
|
|
unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL;
|
|
|
|
#endif
|
|
|
|
|
2011-10-25 10:00:11 +02:00
|
|
|
/*
|
|
|
|
* Increase the granularity value when there are more CPUs,
|
|
|
|
* because with more CPUs the 'effective latency' as visible
|
|
|
|
* to users decreases. But the relationship is not linear,
|
|
|
|
* so pick a second-best guess by going with the log2 of the
|
|
|
|
* number of CPUs.
|
|
|
|
*
|
|
|
|
* This idea comes from the SD scheduler of Con Kolivas:
|
|
|
|
*/
|
|
|
|
static int get_update_sysctl_factor(void)
|
|
|
|
{
|
|
|
|
unsigned int cpus = min_t(int, num_online_cpus(), 8);
|
|
|
|
unsigned int factor;
|
|
|
|
|
|
|
|
switch (sysctl_sched_tunable_scaling) {
|
|
|
|
case SCHED_TUNABLESCALING_NONE:
|
|
|
|
factor = 1;
|
|
|
|
break;
|
|
|
|
case SCHED_TUNABLESCALING_LINEAR:
|
|
|
|
factor = cpus;
|
|
|
|
break;
|
|
|
|
case SCHED_TUNABLESCALING_LOG:
|
|
|
|
default:
|
|
|
|
factor = 1 + ilog2(cpus);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return factor;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void update_sysctl(void)
|
|
|
|
{
|
|
|
|
unsigned int factor = get_update_sysctl_factor();
|
|
|
|
|
|
|
|
#define SET_SYSCTL(name) \
|
|
|
|
(sysctl_##name = (factor) * normalized_sysctl_##name)
|
|
|
|
SET_SYSCTL(sched_min_granularity);
|
|
|
|
SET_SYSCTL(sched_latency);
|
|
|
|
SET_SYSCTL(sched_wakeup_granularity);
|
|
|
|
#undef SET_SYSCTL
|
|
|
|
}
|
|
|
|
|
|
|
|
void sched_init_granularity(void)
|
|
|
|
{
|
|
|
|
update_sysctl();
|
|
|
|
}
|
|
|
|
|
|
|
|
#if BITS_PER_LONG == 32
|
|
|
|
# define WMULT_CONST (~0UL)
|
|
|
|
#else
|
|
|
|
# define WMULT_CONST (1UL << 32)
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#define WMULT_SHIFT 32
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Shift right and round:
|
|
|
|
*/
|
|
|
|
#define SRR(x, y) (((x) + (1UL << ((y) - 1))) >> (y))
|
|
|
|
|
|
|
|
/*
|
|
|
|
* delta *= weight / lw
|
|
|
|
*/
|
|
|
|
static unsigned long
|
|
|
|
calc_delta_mine(unsigned long delta_exec, unsigned long weight,
|
|
|
|
struct load_weight *lw)
|
|
|
|
{
|
|
|
|
u64 tmp;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* weight can be less than 2^SCHED_LOAD_RESOLUTION for task group sched
|
|
|
|
* entities since MIN_SHARES = 2. Treat weight as 1 if less than
|
|
|
|
* 2^SCHED_LOAD_RESOLUTION.
|
|
|
|
*/
|
|
|
|
if (likely(weight > (1UL << SCHED_LOAD_RESOLUTION)))
|
|
|
|
tmp = (u64)delta_exec * scale_load_down(weight);
|
|
|
|
else
|
|
|
|
tmp = (u64)delta_exec;
|
|
|
|
|
|
|
|
if (!lw->inv_weight) {
|
|
|
|
unsigned long w = scale_load_down(lw->weight);
|
|
|
|
|
|
|
|
if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST))
|
|
|
|
lw->inv_weight = 1;
|
|
|
|
else if (unlikely(!w))
|
|
|
|
lw->inv_weight = WMULT_CONST;
|
|
|
|
else
|
|
|
|
lw->inv_weight = WMULT_CONST / w;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Check whether we'd overflow the 64-bit multiplication:
|
|
|
|
*/
|
|
|
|
if (unlikely(tmp > WMULT_CONST))
|
|
|
|
tmp = SRR(SRR(tmp, WMULT_SHIFT/2) * lw->inv_weight,
|
|
|
|
WMULT_SHIFT/2);
|
|
|
|
else
|
|
|
|
tmp = SRR(tmp * lw->inv_weight, WMULT_SHIFT);
|
|
|
|
|
|
|
|
return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
const struct sched_class fair_sched_class;
|
2008-10-17 19:27:03 +02:00
|
|
|
|
2007-07-09 18:51:58 +02:00
|
|
|
/**************************************************************
|
|
|
|
* CFS operations on generic schedulable entities:
|
|
|
|
*/
|
|
|
|
|
2007-10-15 17:00:03 +02:00
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
2007-07-09 18:51:58 +02:00
|
|
|
|
2007-10-15 17:00:03 +02:00
|
|
|
/* cpu runqueue to which this cfs_rq is attached */
|
2007-07-09 18:51:58 +02:00
|
|
|
static inline struct rq *rq_of(struct cfs_rq *cfs_rq)
|
|
|
|
{
|
2007-10-15 17:00:03 +02:00
|
|
|
return cfs_rq->rq;
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
2007-10-15 17:00:03 +02:00
|
|
|
/* An entity is a task if it doesn't "own" a runqueue */
|
|
|
|
#define entity_is_task(se) (!se->my_q)
|
2007-07-09 18:51:58 +02:00
|
|
|
|
2009-07-24 12:25:30 +02:00
|
|
|
static inline struct task_struct *task_of(struct sched_entity *se)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_SCHED_DEBUG
|
|
|
|
WARN_ON_ONCE(!entity_is_task(se));
|
|
|
|
#endif
|
|
|
|
return container_of(se, struct task_struct, se);
|
|
|
|
}
|
|
|
|
|
2008-04-19 19:45:00 +02:00
|
|
|
/* Walk up scheduling entities hierarchy */
|
|
|
|
#define for_each_sched_entity(se) \
|
|
|
|
for (; se; se = se->parent)
|
|
|
|
|
|
|
|
static inline struct cfs_rq *task_cfs_rq(struct task_struct *p)
|
|
|
|
{
|
|
|
|
return p->se.cfs_rq;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* runqueue on which this entity is (to be) queued */
|
|
|
|
static inline struct cfs_rq *cfs_rq_of(struct sched_entity *se)
|
|
|
|
{
|
|
|
|
return se->cfs_rq;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* runqueue "owned" by this group */
|
|
|
|
static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp)
|
|
|
|
{
|
|
|
|
return grp->my_q;
|
|
|
|
}
|
|
|
|
|
2010-11-16 00:47:01 +01:00
|
|
|
static inline void list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
|
|
|
|
{
|
|
|
|
if (!cfs_rq->on_list) {
|
2010-11-16 00:47:05 +01:00
|
|
|
/*
|
|
|
|
* Ensure we either appear before our parent (if already
|
|
|
|
* enqueued) or force our parent to appear after us when it is
|
|
|
|
* enqueued. The fact that we always enqueue bottom-up
|
|
|
|
* reduces this to two cases.
|
|
|
|
*/
|
|
|
|
if (cfs_rq->tg->parent &&
|
|
|
|
cfs_rq->tg->parent->cfs_rq[cpu_of(rq_of(cfs_rq))]->on_list) {
|
|
|
|
list_add_rcu(&cfs_rq->leaf_cfs_rq_list,
|
|
|
|
&rq_of(cfs_rq)->leaf_cfs_rq_list);
|
|
|
|
} else {
|
|
|
|
list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list,
|
2010-11-16 00:47:01 +01:00
|
|
|
&rq_of(cfs_rq)->leaf_cfs_rq_list);
|
2010-11-16 00:47:05 +01:00
|
|
|
}
|
2010-11-16 00:47:01 +01:00
|
|
|
|
|
|
|
cfs_rq->on_list = 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq)
|
|
|
|
{
|
|
|
|
if (cfs_rq->on_list) {
|
|
|
|
list_del_rcu(&cfs_rq->leaf_cfs_rq_list);
|
|
|
|
cfs_rq->on_list = 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2008-04-19 19:45:00 +02:00
|
|
|
/* Iterate thr' all leaf cfs_rq's on a runqueue */
|
|
|
|
#define for_each_leaf_cfs_rq(rq, cfs_rq) \
|
|
|
|
list_for_each_entry_rcu(cfs_rq, &rq->leaf_cfs_rq_list, leaf_cfs_rq_list)
|
|
|
|
|
|
|
|
/* Do the two (enqueued) entities belong to the same group ? */
|
|
|
|
static inline int
|
|
|
|
is_same_group(struct sched_entity *se, struct sched_entity *pse)
|
|
|
|
{
|
|
|
|
if (se->cfs_rq == pse->cfs_rq)
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline struct sched_entity *parent_entity(struct sched_entity *se)
|
|
|
|
{
|
|
|
|
return se->parent;
|
|
|
|
}
|
|
|
|
|
2008-10-24 11:06:15 +02:00
|
|
|
/* return depth at which a sched entity is present in the hierarchy */
|
|
|
|
static inline int depth_se(struct sched_entity *se)
|
|
|
|
{
|
|
|
|
int depth = 0;
|
|
|
|
|
|
|
|
for_each_sched_entity(se)
|
|
|
|
depth++;
|
|
|
|
|
|
|
|
return depth;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
find_matching_se(struct sched_entity **se, struct sched_entity **pse)
|
|
|
|
{
|
|
|
|
int se_depth, pse_depth;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* preemption test can be made between sibling entities who are in the
|
|
|
|
* same cfs_rq i.e who have a common parent. Walk up the hierarchy of
|
|
|
|
* both tasks until we find their ancestors who are siblings of common
|
|
|
|
* parent.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/* First walk up until both entities are at same depth */
|
|
|
|
se_depth = depth_se(*se);
|
|
|
|
pse_depth = depth_se(*pse);
|
|
|
|
|
|
|
|
while (se_depth > pse_depth) {
|
|
|
|
se_depth--;
|
|
|
|
*se = parent_entity(*se);
|
|
|
|
}
|
|
|
|
|
|
|
|
while (pse_depth > se_depth) {
|
|
|
|
pse_depth--;
|
|
|
|
*pse = parent_entity(*pse);
|
|
|
|
}
|
|
|
|
|
|
|
|
while (!is_same_group(*se, *pse)) {
|
|
|
|
*se = parent_entity(*se);
|
|
|
|
*pse = parent_entity(*pse);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2009-07-24 12:25:30 +02:00
|
|
|
#else /* !CONFIG_FAIR_GROUP_SCHED */
|
|
|
|
|
|
|
|
static inline struct task_struct *task_of(struct sched_entity *se)
|
|
|
|
{
|
|
|
|
return container_of(se, struct task_struct, se);
|
|
|
|
}
|
2007-07-09 18:51:58 +02:00
|
|
|
|
2007-10-15 17:00:03 +02:00
|
|
|
static inline struct rq *rq_of(struct cfs_rq *cfs_rq)
|
|
|
|
{
|
|
|
|
return container_of(cfs_rq, struct rq, cfs);
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
#define entity_is_task(se) 1
|
|
|
|
|
2008-04-19 19:45:00 +02:00
|
|
|
#define for_each_sched_entity(se) \
|
|
|
|
for (; se; se = NULL)
|
2007-07-09 18:51:58 +02:00
|
|
|
|
2008-04-19 19:45:00 +02:00
|
|
|
static inline struct cfs_rq *task_cfs_rq(struct task_struct *p)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
2008-04-19 19:45:00 +02:00
|
|
|
return &task_rq(p)->cfs;
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
2008-04-19 19:45:00 +02:00
|
|
|
static inline struct cfs_rq *cfs_rq_of(struct sched_entity *se)
|
|
|
|
{
|
|
|
|
struct task_struct *p = task_of(se);
|
|
|
|
struct rq *rq = task_rq(p);
|
|
|
|
|
|
|
|
return &rq->cfs;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* runqueue "owned" by this group */
|
|
|
|
static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp)
|
|
|
|
{
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2010-11-16 00:47:01 +01:00
|
|
|
static inline void list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2008-04-19 19:45:00 +02:00
|
|
|
#define for_each_leaf_cfs_rq(rq, cfs_rq) \
|
|
|
|
for (cfs_rq = &rq->cfs; cfs_rq; cfs_rq = NULL)
|
|
|
|
|
|
|
|
static inline int
|
|
|
|
is_same_group(struct sched_entity *se, struct sched_entity *pse)
|
|
|
|
{
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline struct sched_entity *parent_entity(struct sched_entity *se)
|
|
|
|
{
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2008-10-24 11:06:15 +02:00
|
|
|
static inline void
|
|
|
|
find_matching_se(struct sched_entity **se, struct sched_entity **pse)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2008-04-19 19:45:00 +02:00
|
|
|
#endif /* CONFIG_FAIR_GROUP_SCHED */
|
|
|
|
|
2011-07-21 18:43:30 +02:00
|
|
|
static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
|
|
|
|
unsigned long delta_exec);
|
2007-07-09 18:51:58 +02:00
|
|
|
|
|
|
|
/**************************************************************
|
|
|
|
* Scheduling class tree data structure manipulation methods:
|
|
|
|
*/
|
|
|
|
|
2007-10-15 17:00:14 +02:00
|
|
|
static inline u64 max_vruntime(u64 min_vruntime, u64 vruntime)
|
2007-10-15 17:00:07 +02:00
|
|
|
{
|
2007-10-15 17:00:11 +02:00
|
|
|
s64 delta = (s64)(vruntime - min_vruntime);
|
|
|
|
if (delta > 0)
|
2007-10-15 17:00:07 +02:00
|
|
|
min_vruntime = vruntime;
|
|
|
|
|
|
|
|
return min_vruntime;
|
|
|
|
}
|
|
|
|
|
2007-10-15 17:00:14 +02:00
|
|
|
static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime)
|
2007-10-15 17:00:12 +02:00
|
|
|
{
|
|
|
|
s64 delta = (s64)(vruntime - min_vruntime);
|
|
|
|
if (delta < 0)
|
|
|
|
min_vruntime = vruntime;
|
|
|
|
|
|
|
|
return min_vruntime;
|
|
|
|
}
|
|
|
|
|
2009-07-16 12:32:27 +02:00
|
|
|
static inline int entity_before(struct sched_entity *a,
|
|
|
|
struct sched_entity *b)
|
|
|
|
{
|
|
|
|
return (s64)(a->vruntime - b->vruntime) < 0;
|
|
|
|
}
|
|
|
|
|
2008-10-24 11:06:13 +02:00
|
|
|
static void update_min_vruntime(struct cfs_rq *cfs_rq)
|
|
|
|
{
|
|
|
|
u64 vruntime = cfs_rq->min_vruntime;
|
|
|
|
|
|
|
|
if (cfs_rq->curr)
|
|
|
|
vruntime = cfs_rq->curr->vruntime;
|
|
|
|
|
|
|
|
if (cfs_rq->rb_leftmost) {
|
|
|
|
struct sched_entity *se = rb_entry(cfs_rq->rb_leftmost,
|
|
|
|
struct sched_entity,
|
|
|
|
run_node);
|
|
|
|
|
sched: fix update_min_vruntime
Impact: fix SCHED_IDLE latency problems
OK, so we have 1 running task A (which is obviously curr and the tree is
equally obviously empty).
'A' nicely chugs along, doing its thing, carrying min_vruntime along as it
goes.
Then some whacko speed freak SCHED_IDLE task gets inserted due to SMP
balancing, which is very likely far right, in that case
update_curr
update_min_vruntime
cfs_rq->rb_leftmost := true (the crazy task sitting in a tree)
vruntime = se->vruntime
and voila, min_vruntime is waaay right of where it ought to be.
OK, so why did I write it like that to begin with...
Aah, yes.
Say we've just dequeued current
schedule
deactivate_task(prev)
dequeue_entity
update_min_vruntime
Then we'll set
vruntime = cfs_rq->min_vruntime;
we find !cfs_rq->curr, but do find someone in the tree. Then we _must_
do vruntime = se->vruntime, because
vruntime = min_vruntime(vruntime := cfs_rq->min_vruntime, se->vruntime)
will not advance vruntime, and cause lags the other way around (which we
fixed with that initial patch: 1af5f730fc1bf7c62ec9fb2d307206e18bf40a69
(sched: more accurate min_vruntime accounting).
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Tested-by: Mike Galbraith <efault@gmx.de>
Acked-by: Mike Galbraith <efault@gmx.de>
Cc: <stable@kernel.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-01-15 14:53:39 +01:00
|
|
|
if (!cfs_rq->curr)
|
2008-10-24 11:06:13 +02:00
|
|
|
vruntime = se->vruntime;
|
|
|
|
else
|
|
|
|
vruntime = min_vruntime(vruntime, se->vruntime);
|
|
|
|
}
|
|
|
|
|
|
|
|
cfs_rq->min_vruntime = max_vruntime(cfs_rq->min_vruntime, vruntime);
|
2011-04-05 17:23:48 +02:00
|
|
|
#ifndef CONFIG_64BIT
|
|
|
|
smp_wmb();
|
|
|
|
cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime;
|
|
|
|
#endif
|
2008-10-24 11:06:13 +02:00
|
|
|
}
|
|
|
|
|
2007-07-09 18:51:58 +02:00
|
|
|
/*
|
|
|
|
* Enqueue an entity into the rb-tree:
|
|
|
|
*/
|
2007-10-15 17:00:14 +02:00
|
|
|
static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
|
|
|
struct rb_node **link = &cfs_rq->tasks_timeline.rb_node;
|
|
|
|
struct rb_node *parent = NULL;
|
|
|
|
struct sched_entity *entry;
|
|
|
|
int leftmost = 1;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Find the right place in the rbtree:
|
|
|
|
*/
|
|
|
|
while (*link) {
|
|
|
|
parent = *link;
|
|
|
|
entry = rb_entry(parent, struct sched_entity, run_node);
|
|
|
|
/*
|
|
|
|
* We dont care about collisions. Nodes with
|
|
|
|
* the same key stay together.
|
|
|
|
*/
|
2011-07-20 14:46:59 +02:00
|
|
|
if (entity_before(se, entry)) {
|
2007-07-09 18:51:58 +02:00
|
|
|
link = &parent->rb_left;
|
|
|
|
} else {
|
|
|
|
link = &parent->rb_right;
|
|
|
|
leftmost = 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Maintain a cache of leftmost tree entries (it is frequently
|
|
|
|
* used):
|
|
|
|
*/
|
2008-10-24 11:06:13 +02:00
|
|
|
if (leftmost)
|
2007-10-15 17:00:11 +02:00
|
|
|
cfs_rq->rb_leftmost = &se->run_node;
|
2007-07-09 18:51:58 +02:00
|
|
|
|
|
|
|
rb_link_node(&se->run_node, parent, link);
|
|
|
|
rb_insert_color(&se->run_node, &cfs_rq->tasks_timeline);
|
|
|
|
}
|
|
|
|
|
2007-10-15 17:00:14 +02:00
|
|
|
static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
2008-03-14 20:55:51 +01:00
|
|
|
if (cfs_rq->rb_leftmost == &se->run_node) {
|
|
|
|
struct rb_node *next_node;
|
|
|
|
|
|
|
|
next_node = rb_next(&se->run_node);
|
|
|
|
cfs_rq->rb_leftmost = next_node;
|
|
|
|
}
|
2007-10-15 17:00:04 +02:00
|
|
|
|
2007-07-09 18:51:58 +02:00
|
|
|
rb_erase(&se->run_node, &cfs_rq->tasks_timeline);
|
|
|
|
}
|
|
|
|
|
2011-10-25 10:00:11 +02:00
|
|
|
struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
2008-11-04 21:25:07 +01:00
|
|
|
struct rb_node *left = cfs_rq->rb_leftmost;
|
|
|
|
|
|
|
|
if (!left)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
return rb_entry(left, struct sched_entity, run_node);
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
2011-02-01 15:51:03 +01:00
|
|
|
static struct sched_entity *__pick_next_entity(struct sched_entity *se)
|
|
|
|
{
|
|
|
|
struct rb_node *next = rb_next(&se->run_node);
|
|
|
|
|
|
|
|
if (!next)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
return rb_entry(next, struct sched_entity, run_node);
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_SCHED_DEBUG
|
2011-10-25 10:00:11 +02:00
|
|
|
struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq)
|
2007-10-15 17:00:05 +02:00
|
|
|
{
|
2008-02-22 10:32:21 +01:00
|
|
|
struct rb_node *last = rb_last(&cfs_rq->tasks_timeline);
|
2007-10-15 17:00:05 +02:00
|
|
|
|
2008-02-22 08:55:53 +01:00
|
|
|
if (!last)
|
|
|
|
return NULL;
|
2008-02-22 10:32:21 +01:00
|
|
|
|
|
|
|
return rb_entry(last, struct sched_entity, run_node);
|
2007-10-15 17:00:05 +02:00
|
|
|
}
|
|
|
|
|
2007-07-09 18:51:58 +02:00
|
|
|
/**************************************************************
|
|
|
|
* Scheduling class statistics methods:
|
|
|
|
*/
|
|
|
|
|
2009-11-30 12:16:48 +01:00
|
|
|
int sched_proc_update_handler(struct ctl_table *table, int write,
|
2009-09-24 00:57:19 +02:00
|
|
|
void __user *buffer, size_t *lenp,
|
2007-11-09 22:39:37 +01:00
|
|
|
loff_t *ppos)
|
|
|
|
{
|
2009-09-24 00:57:19 +02:00
|
|
|
int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
|
2009-11-30 12:16:48 +01:00
|
|
|
int factor = get_update_sysctl_factor();
|
2007-11-09 22:39:37 +01:00
|
|
|
|
|
|
|
if (ret || !write)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
sched_nr_latency = DIV_ROUND_UP(sysctl_sched_latency,
|
|
|
|
sysctl_sched_min_granularity);
|
|
|
|
|
2009-11-30 12:16:48 +01:00
|
|
|
#define WRT_SYSCTL(name) \
|
|
|
|
(normalized_sysctl_##name = sysctl_##name / (factor))
|
|
|
|
WRT_SYSCTL(sched_min_granularity);
|
|
|
|
WRT_SYSCTL(sched_latency);
|
|
|
|
WRT_SYSCTL(sched_wakeup_granularity);
|
|
|
|
#undef WRT_SYSCTL
|
|
|
|
|
2007-11-09 22:39:37 +01:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
#endif
|
2007-10-15 17:00:13 +02:00
|
|
|
|
2008-06-27 13:41:11 +02:00
|
|
|
/*
|
2008-10-17 19:27:04 +02:00
|
|
|
* delta /= w
|
2008-06-27 13:41:11 +02:00
|
|
|
*/
|
|
|
|
static inline unsigned long
|
|
|
|
calc_delta_fair(unsigned long delta, struct sched_entity *se)
|
|
|
|
{
|
2008-10-17 19:27:04 +02:00
|
|
|
if (unlikely(se->load.weight != NICE_0_LOAD))
|
|
|
|
delta = calc_delta_mine(delta, NICE_0_LOAD, &se->load);
|
2008-06-27 13:41:11 +02:00
|
|
|
|
|
|
|
return delta;
|
|
|
|
}
|
|
|
|
|
2007-10-15 17:00:13 +02:00
|
|
|
/*
|
|
|
|
* The idea is to set a period in which each task runs once.
|
|
|
|
*
|
|
|
|
* When there are too many tasks (sysctl_sched_nr_latency) we have to stretch
|
|
|
|
* this period because otherwise the slices get too small.
|
|
|
|
*
|
|
|
|
* p = (nr <= nl) ? l : l*nr/nl
|
|
|
|
*/
|
2007-10-15 17:00:04 +02:00
|
|
|
static u64 __sched_period(unsigned long nr_running)
|
|
|
|
{
|
|
|
|
u64 period = sysctl_sched_latency;
|
2007-11-09 22:39:37 +01:00
|
|
|
unsigned long nr_latency = sched_nr_latency;
|
2007-10-15 17:00:04 +02:00
|
|
|
|
|
|
|
if (unlikely(nr_running > nr_latency)) {
|
2008-01-25 21:08:21 +01:00
|
|
|
period = sysctl_sched_min_granularity;
|
2007-10-15 17:00:04 +02:00
|
|
|
period *= nr_running;
|
|
|
|
}
|
|
|
|
|
|
|
|
return period;
|
|
|
|
}
|
|
|
|
|
2007-10-15 17:00:13 +02:00
|
|
|
/*
|
|
|
|
* We calculate the wall-time slice from the period by taking a part
|
|
|
|
* proportional to the weight.
|
|
|
|
*
|
2008-10-17 19:27:04 +02:00
|
|
|
* s = p*P[w/rw]
|
2007-10-15 17:00:13 +02:00
|
|
|
*/
|
2007-10-15 17:00:05 +02:00
|
|
|
static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
2007-08-25 18:41:53 +02:00
|
|
|
{
|
2009-01-02 12:16:42 +01:00
|
|
|
u64 slice = __sched_period(cfs_rq->nr_running + !se->on_rq);
|
2008-10-17 19:27:04 +02:00
|
|
|
|
2009-01-02 12:16:42 +01:00
|
|
|
for_each_sched_entity(se) {
|
2009-01-15 17:17:15 +01:00
|
|
|
struct load_weight *load;
|
2009-06-16 10:35:12 +02:00
|
|
|
struct load_weight lw;
|
2009-01-15 17:17:15 +01:00
|
|
|
|
|
|
|
cfs_rq = cfs_rq_of(se);
|
|
|
|
load = &cfs_rq->load;
|
2008-10-17 19:27:04 +02:00
|
|
|
|
2009-01-02 12:16:42 +01:00
|
|
|
if (unlikely(!se->on_rq)) {
|
2009-06-16 10:35:12 +02:00
|
|
|
lw = cfs_rq->load;
|
2009-01-02 12:16:42 +01:00
|
|
|
|
|
|
|
update_load_add(&lw, se->load.weight);
|
|
|
|
load = &lw;
|
|
|
|
}
|
|
|
|
slice = calc_delta_mine(slice, se->load.weight, load);
|
|
|
|
}
|
|
|
|
return slice;
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
2007-10-15 17:00:13 +02:00
|
|
|
/*
|
2008-04-19 19:45:00 +02:00
|
|
|
* We calculate the vruntime slice of a to be inserted task
|
2007-10-15 17:00:13 +02:00
|
|
|
*
|
2008-10-17 19:27:04 +02:00
|
|
|
* vs = s/w
|
2007-10-15 17:00:13 +02:00
|
|
|
*/
|
2008-10-17 19:27:04 +02:00
|
|
|
static u64 sched_vslice(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
2007-10-15 17:00:10 +02:00
|
|
|
{
|
2008-10-17 19:27:04 +02:00
|
|
|
return calc_delta_fair(sched_slice(cfs_rq, se), se);
|
2008-06-27 13:41:11 +02:00
|
|
|
}
|
|
|
|
|
2010-11-16 00:47:09 +01:00
|
|
|
static void update_cfs_load(struct cfs_rq *cfs_rq, int global_update);
|
2011-01-22 05:45:01 +01:00
|
|
|
static void update_cfs_shares(struct cfs_rq *cfs_rq);
|
2010-11-16 00:47:08 +01:00
|
|
|
|
2007-07-09 18:51:58 +02:00
|
|
|
/*
|
|
|
|
* Update the current task's runtime statistics. Skip current tasks that
|
|
|
|
* are not in our scheduling class.
|
|
|
|
*/
|
|
|
|
static inline void
|
2007-10-15 17:00:03 +02:00
|
|
|
__update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr,
|
|
|
|
unsigned long delta_exec)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
2007-10-15 17:00:06 +02:00
|
|
|
unsigned long delta_exec_weighted;
|
2007-07-09 18:51:58 +02:00
|
|
|
|
2010-03-11 03:37:45 +01:00
|
|
|
schedstat_set(curr->statistics.exec_max,
|
|
|
|
max((u64)delta_exec, curr->statistics.exec_max));
|
2007-07-09 18:51:58 +02:00
|
|
|
|
|
|
|
curr->sum_exec_runtime += delta_exec;
|
2007-10-15 17:00:06 +02:00
|
|
|
schedstat_add(cfs_rq, exec_clock, delta_exec);
|
2008-06-27 13:41:11 +02:00
|
|
|
delta_exec_weighted = calc_delta_fair(delta_exec, curr);
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 18:04:41 +01:00
|
|
|
|
2007-10-15 17:00:04 +02:00
|
|
|
curr->vruntime += delta_exec_weighted;
|
2008-10-24 11:06:13 +02:00
|
|
|
update_min_vruntime(cfs_rq);
|
2010-11-16 00:47:08 +01:00
|
|
|
|
2010-11-20 00:53:51 +01:00
|
|
|
#if defined CONFIG_SMP && defined CONFIG_FAIR_GROUP_SCHED
|
2010-11-16 00:47:08 +01:00
|
|
|
cfs_rq->load_unacc_exec_time += delta_exec;
|
|
|
|
#endif
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
2007-08-09 11:16:47 +02:00
|
|
|
static void update_curr(struct cfs_rq *cfs_rq)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
2007-10-15 17:00:03 +02:00
|
|
|
struct sched_entity *curr = cfs_rq->curr;
|
2010-10-05 02:03:21 +02:00
|
|
|
u64 now = rq_of(cfs_rq)->clock_task;
|
2007-07-09 18:51:58 +02:00
|
|
|
unsigned long delta_exec;
|
|
|
|
|
|
|
|
if (unlikely(!curr))
|
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Get the amount of time the current task was running
|
|
|
|
* since the last time we changed load (this cannot
|
|
|
|
* overflow on 32 bits):
|
|
|
|
*/
|
2007-10-15 17:00:03 +02:00
|
|
|
delta_exec = (unsigned long)(now - curr->exec_start);
|
2008-12-16 08:45:31 +01:00
|
|
|
if (!delta_exec)
|
|
|
|
return;
|
2007-07-09 18:51:58 +02:00
|
|
|
|
2007-10-15 17:00:03 +02:00
|
|
|
__update_curr(cfs_rq, curr, delta_exec);
|
|
|
|
curr->exec_start = now;
|
2007-12-02 20:04:49 +01:00
|
|
|
|
|
|
|
if (entity_is_task(curr)) {
|
|
|
|
struct task_struct *curtask = task_of(curr);
|
|
|
|
|
2009-09-13 18:15:54 +02:00
|
|
|
trace_sched_stat_runtime(curtask, delta_exec, curr->vruntime);
|
2007-12-02 20:04:49 +01:00
|
|
|
cpuacct_charge(curtask, delta_exec);
|
timers: fix itimer/many thread hang
Overview
This patch reworks the handling of POSIX CPU timers, including the
ITIMER_PROF, ITIMER_VIRT timers and rlimit handling. It was put together
with the help of Roland McGrath, the owner and original writer of this code.
The problem we ran into, and the reason for this rework, has to do with using
a profiling timer in a process with a large number of threads. It appears
that the performance of the old implementation of run_posix_cpu_timers() was
at least O(n*3) (where "n" is the number of threads in a process) or worse.
Everything is fine with an increasing number of threads until the time taken
for that routine to run becomes the same as or greater than the tick time, at
which point things degrade rather quickly.
This patch fixes bug 9906, "Weird hang with NPTL and SIGPROF."
Code Changes
This rework corrects the implementation of run_posix_cpu_timers() to make it
run in constant time for a particular machine. (Performance may vary between
one machine and another depending upon whether the kernel is built as single-
or multiprocessor and, in the latter case, depending upon the number of
running processors.) To do this, at each tick we now update fields in
signal_struct as well as task_struct. The run_posix_cpu_timers() function
uses those fields to make its decisions.
We define a new structure, "task_cputime," to contain user, system and
scheduler times and use these in appropriate places:
struct task_cputime {
cputime_t utime;
cputime_t stime;
unsigned long long sum_exec_runtime;
};
This is included in the structure "thread_group_cputime," which is a new
substructure of signal_struct and which varies for uniprocessor versus
multiprocessor kernels. For uniprocessor kernels, it uses "task_cputime" as
a simple substructure, while for multiprocessor kernels it is a pointer:
struct thread_group_cputime {
struct task_cputime totals;
};
struct thread_group_cputime {
struct task_cputime *totals;
};
We also add a new task_cputime substructure directly to signal_struct, to
cache the earliest expiration of process-wide timers, and task_cputime also
replaces the it_*_expires fields of task_struct (used for earliest expiration
of thread timers). The "thread_group_cputime" structure contains process-wide
timers that are updated via account_user_time() and friends. In the non-SMP
case the structure is a simple aggregator; unfortunately in the SMP case that
simplicity was not achievable due to cache-line contention between CPUs (in
one measured case performance was actually _worse_ on a 16-cpu system than
the same test on a 4-cpu system, due to this contention). For SMP, the
thread_group_cputime counters are maintained as a per-cpu structure allocated
using alloc_percpu(). The timer functions update only the timer field in
the structure corresponding to the running CPU, obtained using per_cpu_ptr().
We define a set of inline functions in sched.h that we use to maintain the
thread_group_cputime structure and hide the differences between UP and SMP
implementations from the rest of the kernel. The thread_group_cputime_init()
function initializes the thread_group_cputime structure for the given task.
The thread_group_cputime_alloc() is a no-op for UP; for SMP it calls the
out-of-line function thread_group_cputime_alloc_smp() to allocate and fill
in the per-cpu structures and fields. The thread_group_cputime_free()
function, also a no-op for UP, in SMP frees the per-cpu structures. The
thread_group_cputime_clone_thread() function (also a UP no-op) for SMP calls
thread_group_cputime_alloc() if the per-cpu structures haven't yet been
allocated. The thread_group_cputime() function fills the task_cputime
structure it is passed with the contents of the thread_group_cputime fields;
in UP it's that simple but in SMP it must also safely check that tsk->signal
is non-NULL (if it is it just uses the appropriate fields of task_struct) and,
if so, sums the per-cpu values for each online CPU. Finally, the three
functions account_group_user_time(), account_group_system_time() and
account_group_exec_runtime() are used by timer functions to update the
respective fields of the thread_group_cputime structure.
Non-SMP operation is trivial and will not be mentioned further.
The per-cpu structure is always allocated when a task creates its first new
thread, via a call to thread_group_cputime_clone_thread() from copy_signal().
It is freed at process exit via a call to thread_group_cputime_free() from
cleanup_signal().
All functions that formerly summed utime/stime/sum_sched_runtime values from
from all threads in the thread group now use thread_group_cputime() to
snapshot the values in the thread_group_cputime structure or the values in
the task structure itself if the per-cpu structure hasn't been allocated.
Finally, the code in kernel/posix-cpu-timers.c has changed quite a bit.
The run_posix_cpu_timers() function has been split into a fast path and a
slow path; the former safely checks whether there are any expired thread
timers and, if not, just returns, while the slow path does the heavy lifting.
With the dedicated thread group fields, timers are no longer "rebalanced" and
the process_timer_rebalance() function and related code has gone away. All
summing loops are gone and all code that used them now uses the
thread_group_cputime() inline. When process-wide timers are set, the new
task_cputime structure in signal_struct is used to cache the earliest
expiration; this is checked in the fast path.
Performance
The fix appears not to add significant overhead to existing operations. It
generally performs the same as the current code except in two cases, one in
which it performs slightly worse (Case 5 below) and one in which it performs
very significantly better (Case 2 below). Overall it's a wash except in those
two cases.
I've since done somewhat more involved testing on a dual-core Opteron system.
Case 1: With no itimer running, for a test with 100,000 threads, the fixed
kernel took 1428.5 seconds, 513 seconds more than the unfixed system,
all of which was spent in the system. There were twice as many
voluntary context switches with the fix as without it.
Case 2: With an itimer running at .01 second ticks and 4000 threads (the most
an unmodified kernel can handle), the fixed kernel ran the test in
eight percent of the time (5.8 seconds as opposed to 70 seconds) and
had better tick accuracy (.012 seconds per tick as opposed to .023
seconds per tick).
Case 3: A 4000-thread test with an initial timer tick of .01 second and an
interval of 10,000 seconds (i.e. a timer that ticks only once) had
very nearly the same performance in both cases: 6.3 seconds elapsed
for the fixed kernel versus 5.5 seconds for the unfixed kernel.
With fewer threads (eight in these tests), the Case 1 test ran in essentially
the same time on both the modified and unmodified kernels (5.2 seconds versus
5.8 seconds). The Case 2 test ran in about the same time as well, 5.9 seconds
versus 5.4 seconds but again with much better tick accuracy, .013 seconds per
tick versus .025 seconds per tick for the unmodified kernel.
Since the fix affected the rlimit code, I also tested soft and hard CPU limits.
Case 4: With a hard CPU limit of 20 seconds and eight threads (and an itimer
running), the modified kernel was very slightly favored in that while
it killed the process in 19.997 seconds of CPU time (5.002 seconds of
wall time), only .003 seconds of that was system time, the rest was
user time. The unmodified kernel killed the process in 20.001 seconds
of CPU (5.014 seconds of wall time) of which .016 seconds was system
time. Really, though, the results were too close to call. The results
were essentially the same with no itimer running.
Case 5: With a soft limit of 20 seconds and a hard limit of 2000 seconds
(where the hard limit would never be reached) and an itimer running,
the modified kernel exhibited worse tick accuracy than the unmodified
kernel: .050 seconds/tick versus .028 seconds/tick. Otherwise,
performance was almost indistinguishable. With no itimer running this
test exhibited virtually identical behavior and times in both cases.
In times past I did some limited performance testing. those results are below.
On a four-cpu Opteron system without this fix, a sixteen-thread test executed
in 3569.991 seconds, of which user was 3568.435s and system was 1.556s. On
the same system with the fix, user and elapsed time were about the same, but
system time dropped to 0.007 seconds. Performance with eight, four and one
thread were comparable. Interestingly, the timer ticks with the fix seemed
more accurate: The sixteen-thread test with the fix received 149543 ticks
for 0.024 seconds per tick, while the same test without the fix received 58720
for 0.061 seconds per tick. Both cases were configured for an interval of
0.01 seconds. Again, the other tests were comparable. Each thread in this
test computed the primes up to 25,000,000.
I also did a test with a large number of threads, 100,000 threads, which is
impossible without the fix. In this case each thread computed the primes only
up to 10,000 (to make the runtime manageable). System time dominated, at
1546.968 seconds out of a total 2176.906 seconds (giving a user time of
629.938s). It received 147651 ticks for 0.015 seconds per tick, still quite
accurate. There is obviously no comparable test without the fix.
Signed-off-by: Frank Mayhar <fmayhar@google.com>
Cc: Roland McGrath <roland@redhat.com>
Cc: Alexey Dobriyan <adobriyan@gmail.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-09-12 18:54:39 +02:00
|
|
|
account_group_exec_runtime(curtask, delta_exec);
|
2007-12-02 20:04:49 +01:00
|
|
|
}
|
2011-07-21 18:43:30 +02:00
|
|
|
|
|
|
|
account_cfs_rq_runtime(cfs_rq, delta_exec);
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
2007-08-09 11:16:47 +02:00
|
|
|
update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
2010-03-11 03:37:45 +01:00
|
|
|
schedstat_set(se->statistics.wait_start, rq_of(cfs_rq)->clock);
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Task is being enqueued - update stats:
|
|
|
|
*/
|
2007-08-09 11:16:47 +02:00
|
|
|
static void update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Are we enqueueing a waiting task? (for current tasks
|
|
|
|
* a dequeue/enqueue event is a NOP)
|
|
|
|
*/
|
2007-10-15 17:00:03 +02:00
|
|
|
if (se != cfs_rq->curr)
|
2007-08-09 11:16:47 +02:00
|
|
|
update_stats_wait_start(cfs_rq, se);
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
2007-08-09 11:16:47 +02:00
|
|
|
update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
2010-03-11 03:37:45 +01:00
|
|
|
schedstat_set(se->statistics.wait_max, max(se->statistics.wait_max,
|
|
|
|
rq_of(cfs_rq)->clock - se->statistics.wait_start));
|
|
|
|
schedstat_set(se->statistics.wait_count, se->statistics.wait_count + 1);
|
|
|
|
schedstat_set(se->statistics.wait_sum, se->statistics.wait_sum +
|
|
|
|
rq_of(cfs_rq)->clock - se->statistics.wait_start);
|
2009-07-23 20:13:26 +02:00
|
|
|
#ifdef CONFIG_SCHEDSTATS
|
|
|
|
if (entity_is_task(se)) {
|
|
|
|
trace_sched_stat_wait(task_of(se),
|
2010-03-11 03:37:45 +01:00
|
|
|
rq_of(cfs_rq)->clock - se->statistics.wait_start);
|
2009-07-23 20:13:26 +02:00
|
|
|
}
|
|
|
|
#endif
|
2010-03-11 03:37:45 +01:00
|
|
|
schedstat_set(se->statistics.wait_start, 0);
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
2007-08-09 11:16:48 +02:00
|
|
|
update_stats_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Mark the end of the wait period if dequeueing a
|
|
|
|
* waiting task:
|
|
|
|
*/
|
2007-10-15 17:00:03 +02:00
|
|
|
if (se != cfs_rq->curr)
|
2007-08-09 11:16:47 +02:00
|
|
|
update_stats_wait_end(cfs_rq, se);
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We are picking a new current task - update its stats:
|
|
|
|
*/
|
|
|
|
static inline void
|
2007-08-09 11:16:47 +02:00
|
|
|
update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
|
|
|
/*
|
|
|
|
* We are starting a new run period:
|
|
|
|
*/
|
2010-10-05 02:03:21 +02:00
|
|
|
se->exec_start = rq_of(cfs_rq)->clock_task;
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/**************************************************
|
|
|
|
* Scheduling class queueing methods:
|
|
|
|
*/
|
|
|
|
|
2008-06-27 13:41:14 +02:00
|
|
|
#if defined CONFIG_SMP && defined CONFIG_FAIR_GROUP_SCHED
|
|
|
|
static void
|
|
|
|
add_cfs_task_weight(struct cfs_rq *cfs_rq, unsigned long weight)
|
|
|
|
{
|
|
|
|
cfs_rq->task_weight += weight;
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
static inline void
|
|
|
|
add_cfs_task_weight(struct cfs_rq *cfs_rq, unsigned long weight)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2007-10-15 17:00:07 +02:00
|
|
|
static void
|
|
|
|
account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
|
|
|
{
|
|
|
|
update_load_add(&cfs_rq->load, se->load.weight);
|
2008-06-27 13:41:14 +02:00
|
|
|
if (!parent_entity(se))
|
2011-10-25 10:00:11 +02:00
|
|
|
update_load_add(&rq_of(cfs_rq)->load, se->load.weight);
|
2008-09-25 06:23:54 +02:00
|
|
|
if (entity_is_task(se)) {
|
2008-06-27 13:41:14 +02:00
|
|
|
add_cfs_task_weight(cfs_rq, se->load.weight);
|
2008-09-25 06:23:54 +02:00
|
|
|
list_add(&se->group_node, &cfs_rq->tasks);
|
|
|
|
}
|
2007-10-15 17:00:07 +02:00
|
|
|
cfs_rq->nr_running++;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
|
|
|
{
|
|
|
|
update_load_sub(&cfs_rq->load, se->load.weight);
|
2008-06-27 13:41:14 +02:00
|
|
|
if (!parent_entity(se))
|
2011-10-25 10:00:11 +02:00
|
|
|
update_load_sub(&rq_of(cfs_rq)->load, se->load.weight);
|
2008-09-25 06:23:54 +02:00
|
|
|
if (entity_is_task(se)) {
|
2008-06-27 13:41:14 +02:00
|
|
|
add_cfs_task_weight(cfs_rq, -se->load.weight);
|
2008-09-25 06:23:54 +02:00
|
|
|
list_del_init(&se->group_node);
|
|
|
|
}
|
2007-10-15 17:00:07 +02:00
|
|
|
cfs_rq->nr_running--;
|
|
|
|
}
|
|
|
|
|
2011-01-24 08:33:52 +01:00
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
2011-07-21 18:43:36 +02:00
|
|
|
/* we need this in update_cfs_load and load-balance functions below */
|
|
|
|
static inline int throttled_hierarchy(struct cfs_rq *cfs_rq);
|
2011-01-24 08:33:52 +01:00
|
|
|
# ifdef CONFIG_SMP
|
2010-11-16 00:47:09 +01:00
|
|
|
static void update_cfs_rq_load_contribution(struct cfs_rq *cfs_rq,
|
|
|
|
int global_update)
|
|
|
|
{
|
|
|
|
struct task_group *tg = cfs_rq->tg;
|
|
|
|
long load_avg;
|
|
|
|
|
|
|
|
load_avg = div64_u64(cfs_rq->load_avg, cfs_rq->load_period+1);
|
|
|
|
load_avg -= cfs_rq->load_contribution;
|
|
|
|
|
|
|
|
if (global_update || abs(load_avg) > cfs_rq->load_contribution / 8) {
|
|
|
|
atomic_add(load_avg, &tg->load_weight);
|
|
|
|
cfs_rq->load_contribution += load_avg;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void update_cfs_load(struct cfs_rq *cfs_rq, int global_update)
|
2010-11-16 00:47:00 +01:00
|
|
|
{
|
2010-11-16 00:47:06 +01:00
|
|
|
u64 period = sysctl_sched_shares_window;
|
2010-11-16 00:47:00 +01:00
|
|
|
u64 now, delta;
|
2010-11-16 00:47:04 +01:00
|
|
|
unsigned long load = cfs_rq->load.weight;
|
2010-11-16 00:47:00 +01:00
|
|
|
|
2011-07-21 18:43:36 +02:00
|
|
|
if (cfs_rq->tg == &root_task_group || throttled_hierarchy(cfs_rq))
|
2010-11-16 00:47:00 +01:00
|
|
|
return;
|
|
|
|
|
2011-01-22 05:45:02 +01:00
|
|
|
now = rq_of(cfs_rq)->clock_task;
|
2010-11-16 00:47:00 +01:00
|
|
|
delta = now - cfs_rq->load_stamp;
|
|
|
|
|
2010-11-16 00:47:04 +01:00
|
|
|
/* truncate load history at 4 idle periods */
|
|
|
|
if (cfs_rq->load_stamp > cfs_rq->load_last &&
|
|
|
|
now - cfs_rq->load_last > 4 * period) {
|
|
|
|
cfs_rq->load_period = 0;
|
|
|
|
cfs_rq->load_avg = 0;
|
2011-01-22 05:45:03 +01:00
|
|
|
delta = period - 1;
|
2010-11-16 00:47:04 +01:00
|
|
|
}
|
|
|
|
|
2010-11-16 00:47:00 +01:00
|
|
|
cfs_rq->load_stamp = now;
|
2010-11-16 00:47:08 +01:00
|
|
|
cfs_rq->load_unacc_exec_time = 0;
|
2010-11-16 00:47:00 +01:00
|
|
|
cfs_rq->load_period += delta;
|
2010-11-16 00:47:04 +01:00
|
|
|
if (load) {
|
|
|
|
cfs_rq->load_last = now;
|
|
|
|
cfs_rq->load_avg += delta * load;
|
|
|
|
}
|
2010-11-16 00:47:00 +01:00
|
|
|
|
2010-11-16 00:47:09 +01:00
|
|
|
/* consider updating load contribution on each fold or truncate */
|
|
|
|
if (global_update || cfs_rq->load_period > period
|
|
|
|
|| !cfs_rq->load_period)
|
|
|
|
update_cfs_rq_load_contribution(cfs_rq, global_update);
|
|
|
|
|
2010-11-16 00:47:00 +01:00
|
|
|
while (cfs_rq->load_period > period) {
|
|
|
|
/*
|
|
|
|
* Inline assembly required to prevent the compiler
|
|
|
|
* optimising this loop into a divmod call.
|
|
|
|
* See __iter_div_u64_rem() for another example of this.
|
|
|
|
*/
|
|
|
|
asm("" : "+rm" (cfs_rq->load_period));
|
|
|
|
cfs_rq->load_period /= 2;
|
|
|
|
cfs_rq->load_avg /= 2;
|
|
|
|
}
|
2010-11-16 00:47:01 +01:00
|
|
|
|
2010-11-16 00:47:04 +01:00
|
|
|
if (!cfs_rq->curr && !cfs_rq->nr_running && !cfs_rq->load_avg)
|
|
|
|
list_del_leaf_cfs_rq(cfs_rq);
|
2010-11-16 00:47:00 +01:00
|
|
|
}
|
|
|
|
|
2011-10-13 16:52:28 +02:00
|
|
|
static inline long calc_tg_weight(struct task_group *tg, struct cfs_rq *cfs_rq)
|
|
|
|
{
|
|
|
|
long tg_weight;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Use this CPU's actual weight instead of the last load_contribution
|
|
|
|
* to gain a more accurate current total weight. See
|
|
|
|
* update_cfs_rq_load_contribution().
|
|
|
|
*/
|
|
|
|
tg_weight = atomic_read(&tg->load_weight);
|
|
|
|
tg_weight -= cfs_rq->load_contribution;
|
|
|
|
tg_weight += cfs_rq->load.weight;
|
|
|
|
|
|
|
|
return tg_weight;
|
|
|
|
}
|
|
|
|
|
2011-01-22 05:45:01 +01:00
|
|
|
static long calc_cfs_shares(struct cfs_rq *cfs_rq, struct task_group *tg)
|
2011-01-24 08:33:52 +01:00
|
|
|
{
|
2011-10-13 16:52:28 +02:00
|
|
|
long tg_weight, load, shares;
|
2011-01-24 08:33:52 +01:00
|
|
|
|
2011-10-13 16:52:28 +02:00
|
|
|
tg_weight = calc_tg_weight(tg, cfs_rq);
|
2011-01-22 05:45:01 +01:00
|
|
|
load = cfs_rq->load.weight;
|
2011-01-24 08:33:52 +01:00
|
|
|
|
|
|
|
shares = (tg->shares * load);
|
2011-10-13 16:52:28 +02:00
|
|
|
if (tg_weight)
|
|
|
|
shares /= tg_weight;
|
2011-01-24 08:33:52 +01:00
|
|
|
|
|
|
|
if (shares < MIN_SHARES)
|
|
|
|
shares = MIN_SHARES;
|
|
|
|
if (shares > tg->shares)
|
|
|
|
shares = tg->shares;
|
|
|
|
|
|
|
|
return shares;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void update_entity_shares_tick(struct cfs_rq *cfs_rq)
|
|
|
|
{
|
|
|
|
if (cfs_rq->load_unacc_exec_time > sysctl_sched_shares_window) {
|
|
|
|
update_cfs_load(cfs_rq, 0);
|
2011-01-22 05:45:01 +01:00
|
|
|
update_cfs_shares(cfs_rq);
|
2011-01-24 08:33:52 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
# else /* CONFIG_SMP */
|
|
|
|
static void update_cfs_load(struct cfs_rq *cfs_rq, int global_update)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2011-01-22 05:45:01 +01:00
|
|
|
static inline long calc_cfs_shares(struct cfs_rq *cfs_rq, struct task_group *tg)
|
2011-01-24 08:33:52 +01:00
|
|
|
{
|
|
|
|
return tg->shares;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void update_entity_shares_tick(struct cfs_rq *cfs_rq)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
# endif /* CONFIG_SMP */
|
2010-11-16 00:47:00 +01:00
|
|
|
static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se,
|
|
|
|
unsigned long weight)
|
|
|
|
{
|
2010-12-16 04:10:18 +01:00
|
|
|
if (se->on_rq) {
|
|
|
|
/* commit outstanding execution time */
|
|
|
|
if (cfs_rq->curr == se)
|
|
|
|
update_curr(cfs_rq);
|
2010-11-16 00:47:00 +01:00
|
|
|
account_entity_dequeue(cfs_rq, se);
|
2010-12-16 04:10:18 +01:00
|
|
|
}
|
2010-11-16 00:47:00 +01:00
|
|
|
|
|
|
|
update_load_set(&se->load, weight);
|
|
|
|
|
|
|
|
if (se->on_rq)
|
|
|
|
account_entity_enqueue(cfs_rq, se);
|
|
|
|
}
|
|
|
|
|
2011-01-22 05:45:01 +01:00
|
|
|
static void update_cfs_shares(struct cfs_rq *cfs_rq)
|
2010-11-16 00:47:00 +01:00
|
|
|
{
|
|
|
|
struct task_group *tg;
|
|
|
|
struct sched_entity *se;
|
2011-01-24 08:33:52 +01:00
|
|
|
long shares;
|
2010-11-16 00:47:00 +01:00
|
|
|
|
|
|
|
tg = cfs_rq->tg;
|
|
|
|
se = tg->se[cpu_of(rq_of(cfs_rq))];
|
2011-07-21 18:43:36 +02:00
|
|
|
if (!se || throttled_hierarchy(cfs_rq))
|
2010-11-16 00:47:00 +01:00
|
|
|
return;
|
2011-01-24 08:33:52 +01:00
|
|
|
#ifndef CONFIG_SMP
|
|
|
|
if (likely(se->load.weight == tg->shares))
|
|
|
|
return;
|
|
|
|
#endif
|
2011-01-22 05:45:01 +01:00
|
|
|
shares = calc_cfs_shares(cfs_rq, tg);
|
2010-11-16 00:47:00 +01:00
|
|
|
|
|
|
|
reweight_entity(cfs_rq_of(se), se, shares);
|
|
|
|
}
|
|
|
|
#else /* CONFIG_FAIR_GROUP_SCHED */
|
2010-11-16 00:47:09 +01:00
|
|
|
static void update_cfs_load(struct cfs_rq *cfs_rq, int global_update)
|
2010-11-16 00:47:00 +01:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2011-01-22 05:45:01 +01:00
|
|
|
static inline void update_cfs_shares(struct cfs_rq *cfs_rq)
|
2010-11-16 00:47:00 +01:00
|
|
|
{
|
|
|
|
}
|
2010-12-16 04:10:17 +01:00
|
|
|
|
|
|
|
static inline void update_entity_shares_tick(struct cfs_rq *cfs_rq)
|
|
|
|
{
|
|
|
|
}
|
2010-11-16 00:47:00 +01:00
|
|
|
#endif /* CONFIG_FAIR_GROUP_SCHED */
|
|
|
|
|
2007-08-09 11:16:48 +02:00
|
|
|
static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
|
|
|
#ifdef CONFIG_SCHEDSTATS
|
2009-07-23 20:13:26 +02:00
|
|
|
struct task_struct *tsk = NULL;
|
|
|
|
|
|
|
|
if (entity_is_task(se))
|
|
|
|
tsk = task_of(se);
|
|
|
|
|
2010-03-11 03:37:45 +01:00
|
|
|
if (se->statistics.sleep_start) {
|
|
|
|
u64 delta = rq_of(cfs_rq)->clock - se->statistics.sleep_start;
|
2007-07-09 18:51:58 +02:00
|
|
|
|
|
|
|
if ((s64)delta < 0)
|
|
|
|
delta = 0;
|
|
|
|
|
2010-03-11 03:37:45 +01:00
|
|
|
if (unlikely(delta > se->statistics.sleep_max))
|
|
|
|
se->statistics.sleep_max = delta;
|
2007-07-09 18:51:58 +02:00
|
|
|
|
2010-03-11 03:37:45 +01:00
|
|
|
se->statistics.sleep_start = 0;
|
|
|
|
se->statistics.sum_sleep_runtime += delta;
|
2008-01-25 21:08:34 +01:00
|
|
|
|
2009-07-23 20:13:26 +02:00
|
|
|
if (tsk) {
|
2009-07-23 20:13:26 +02:00
|
|
|
account_scheduler_latency(tsk, delta >> 10, 1);
|
2009-07-23 20:13:26 +02:00
|
|
|
trace_sched_stat_sleep(tsk, delta);
|
|
|
|
}
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
2010-03-11 03:37:45 +01:00
|
|
|
if (se->statistics.block_start) {
|
|
|
|
u64 delta = rq_of(cfs_rq)->clock - se->statistics.block_start;
|
2007-07-09 18:51:58 +02:00
|
|
|
|
|
|
|
if ((s64)delta < 0)
|
|
|
|
delta = 0;
|
|
|
|
|
2010-03-11 03:37:45 +01:00
|
|
|
if (unlikely(delta > se->statistics.block_max))
|
|
|
|
se->statistics.block_max = delta;
|
2007-07-09 18:51:58 +02:00
|
|
|
|
2010-03-11 03:37:45 +01:00
|
|
|
se->statistics.block_start = 0;
|
|
|
|
se->statistics.sum_sleep_runtime += delta;
|
2007-10-02 14:13:08 +02:00
|
|
|
|
2009-07-23 20:13:26 +02:00
|
|
|
if (tsk) {
|
2009-07-20 20:26:58 +02:00
|
|
|
if (tsk->in_iowait) {
|
2010-03-11 03:37:45 +01:00
|
|
|
se->statistics.iowait_sum += delta;
|
|
|
|
se->statistics.iowait_count++;
|
2009-07-23 20:13:26 +02:00
|
|
|
trace_sched_stat_iowait(tsk, delta);
|
2009-07-20 20:26:58 +02:00
|
|
|
}
|
|
|
|
|
2011-11-28 10:03:35 +01:00
|
|
|
trace_sched_stat_blocked(tsk, delta);
|
|
|
|
|
2009-07-23 20:13:26 +02:00
|
|
|
/*
|
|
|
|
* Blocking time is in units of nanosecs, so shift by
|
|
|
|
* 20 to get a milliseconds-range estimation of the
|
|
|
|
* amount of time that the task spent sleeping:
|
|
|
|
*/
|
|
|
|
if (unlikely(prof_on == SLEEP_PROFILING)) {
|
|
|
|
profile_hits(SLEEP_PROFILING,
|
|
|
|
(void *)get_wchan(tsk),
|
|
|
|
delta >> 20);
|
|
|
|
}
|
|
|
|
account_scheduler_latency(tsk, delta >> 10, 0);
|
2007-10-02 14:13:08 +02:00
|
|
|
}
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2007-10-15 17:00:10 +02:00
|
|
|
static void check_spread(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_SCHED_DEBUG
|
|
|
|
s64 d = se->vruntime - cfs_rq->min_vruntime;
|
|
|
|
|
|
|
|
if (d < 0)
|
|
|
|
d = -d;
|
|
|
|
|
|
|
|
if (d > 3*sysctl_sched_latency)
|
|
|
|
schedstat_inc(cfs_rq, nr_spread_over);
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2007-10-15 17:00:05 +02:00
|
|
|
static void
|
|
|
|
place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
|
|
|
|
{
|
2008-10-24 11:06:13 +02:00
|
|
|
u64 vruntime = cfs_rq->min_vruntime;
|
2007-10-15 17:00:05 +02:00
|
|
|
|
2007-11-09 22:39:37 +01:00
|
|
|
/*
|
|
|
|
* The 'current' period is already promised to the current tasks,
|
|
|
|
* however the extra weight of the new task will slow them down a
|
|
|
|
* little, place the new task so that it fits in the slot that
|
|
|
|
* stays open at the end.
|
|
|
|
*/
|
2007-10-15 17:00:05 +02:00
|
|
|
if (initial && sched_feat(START_DEBIT))
|
2008-10-17 19:27:04 +02:00
|
|
|
vruntime += sched_vslice(cfs_rq, se);
|
2007-10-15 17:00:05 +02:00
|
|
|
|
2009-09-18 09:19:25 +02:00
|
|
|
/* sleeps up to a single latency don't count. */
|
2010-03-11 17:17:17 +01:00
|
|
|
if (!initial) {
|
2009-09-18 09:19:25 +02:00
|
|
|
unsigned long thresh = sysctl_sched_latency;
|
2008-06-27 13:41:11 +02:00
|
|
|
|
2009-09-18 09:19:25 +02:00
|
|
|
/*
|
|
|
|
* Halve their sleep time's effect, to allow
|
|
|
|
* for a gentler effect of sleepers:
|
|
|
|
*/
|
|
|
|
if (sched_feat(GENTLE_FAIR_SLEEPERS))
|
|
|
|
thresh >>= 1;
|
2009-09-16 08:54:45 +02:00
|
|
|
|
2009-09-18 09:19:25 +02:00
|
|
|
vruntime -= thresh;
|
2007-10-15 17:00:05 +02:00
|
|
|
}
|
|
|
|
|
sched: Ensure that a child can't gain time over it's parent after fork()
A fork/exec load is usually "pass the baton", so the child
should never be placed behind the parent. With START_DEBIT we
make room for the new task, but with child_runs_first, that
room comes out of the _parent's_ hide. There's nothing to say
that the parent wasn't ahead of min_vruntime at fork() time,
which means that the "baton carrier", who is essentially the
parent in drag, can gain time and increase scheduling latencies
for waiters.
With NEW_FAIR_SLEEPERS + START_DEBIT + child_runs_first
enabled, we essentially pass the sleeper fairness off to the
child, which is fine, but if we don't base placement on the
parent's updated vruntime, we can end up compounding latency
woes if the child itself then does fork/exec. The debit
incurred at fork doesn't hurt the parent who is then going to
sleep and maybe exit, but the child who acquires the error
harms all comers.
This improves latencies of make -j<n> kernel build workloads.
Reported-by: Jens Axboe <jens.axboe@oracle.com>
Signed-off-by: Mike Galbraith <efault@gmx.de>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-08 11:12:28 +02:00
|
|
|
/* ensure we never gain time by being placed backwards. */
|
|
|
|
vruntime = max_vruntime(se->vruntime, vruntime);
|
|
|
|
|
2007-10-15 17:00:10 +02:00
|
|
|
se->vruntime = vruntime;
|
2007-10-15 17:00:05 +02:00
|
|
|
}
|
|
|
|
|
2011-07-21 18:43:39 +02:00
|
|
|
static void check_enqueue_throttle(struct cfs_rq *cfs_rq);
|
|
|
|
|
2007-07-09 18:51:58 +02:00
|
|
|
static void
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 18:04:41 +01:00
|
|
|
enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 18:04:41 +01:00
|
|
|
/*
|
|
|
|
* Update the normalized vruntime before updating min_vruntime
|
|
|
|
* through callig update_curr().
|
|
|
|
*/
|
2010-03-24 16:38:48 +01:00
|
|
|
if (!(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_WAKING))
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 18:04:41 +01:00
|
|
|
se->vruntime += cfs_rq->min_vruntime;
|
|
|
|
|
2007-07-09 18:51:58 +02:00
|
|
|
/*
|
2007-10-15 17:00:13 +02:00
|
|
|
* Update run-time statistics of the 'current'.
|
2007-07-09 18:51:58 +02:00
|
|
|
*/
|
2007-08-09 11:16:47 +02:00
|
|
|
update_curr(cfs_rq);
|
2010-11-16 00:47:09 +01:00
|
|
|
update_cfs_load(cfs_rq, 0);
|
2008-05-05 23:56:17 +02:00
|
|
|
account_entity_enqueue(cfs_rq, se);
|
2011-01-22 05:45:01 +01:00
|
|
|
update_cfs_shares(cfs_rq);
|
2007-07-09 18:51:58 +02:00
|
|
|
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 18:04:41 +01:00
|
|
|
if (flags & ENQUEUE_WAKEUP) {
|
2007-10-15 17:00:05 +02:00
|
|
|
place_entity(cfs_rq, se, 0);
|
2007-08-09 11:16:48 +02:00
|
|
|
enqueue_sleeper(cfs_rq, se);
|
2007-10-15 17:00:04 +02:00
|
|
|
}
|
2007-07-09 18:51:58 +02:00
|
|
|
|
2007-08-09 11:16:47 +02:00
|
|
|
update_stats_enqueue(cfs_rq, se);
|
2007-10-15 17:00:10 +02:00
|
|
|
check_spread(cfs_rq, se);
|
2007-10-15 17:00:08 +02:00
|
|
|
if (se != cfs_rq->curr)
|
|
|
|
__enqueue_entity(cfs_rq, se);
|
2010-11-16 00:47:00 +01:00
|
|
|
se->on_rq = 1;
|
2010-11-16 00:47:01 +01:00
|
|
|
|
2011-07-21 18:43:39 +02:00
|
|
|
if (cfs_rq->nr_running == 1) {
|
2010-11-16 00:47:01 +01:00
|
|
|
list_add_leaf_cfs_rq(cfs_rq);
|
2011-07-21 18:43:39 +02:00
|
|
|
check_enqueue_throttle(cfs_rq);
|
|
|
|
}
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
2011-02-01 15:48:37 +01:00
|
|
|
static void __clear_buddies_last(struct sched_entity *se)
|
2008-11-11 11:52:33 +01:00
|
|
|
{
|
2011-02-01 15:48:37 +01:00
|
|
|
for_each_sched_entity(se) {
|
|
|
|
struct cfs_rq *cfs_rq = cfs_rq_of(se);
|
|
|
|
if (cfs_rq->last == se)
|
|
|
|
cfs_rq->last = NULL;
|
|
|
|
else
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2008-11-11 11:52:33 +01:00
|
|
|
|
2011-02-01 15:48:37 +01:00
|
|
|
static void __clear_buddies_next(struct sched_entity *se)
|
|
|
|
{
|
|
|
|
for_each_sched_entity(se) {
|
|
|
|
struct cfs_rq *cfs_rq = cfs_rq_of(se);
|
|
|
|
if (cfs_rq->next == se)
|
|
|
|
cfs_rq->next = NULL;
|
|
|
|
else
|
|
|
|
break;
|
|
|
|
}
|
2008-11-11 11:52:33 +01:00
|
|
|
}
|
|
|
|
|
2011-02-01 15:51:03 +01:00
|
|
|
static void __clear_buddies_skip(struct sched_entity *se)
|
|
|
|
{
|
|
|
|
for_each_sched_entity(se) {
|
|
|
|
struct cfs_rq *cfs_rq = cfs_rq_of(se);
|
|
|
|
if (cfs_rq->skip == se)
|
|
|
|
cfs_rq->skip = NULL;
|
|
|
|
else
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2009-01-28 14:51:40 +01:00
|
|
|
static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
|
|
|
{
|
2011-02-01 15:48:37 +01:00
|
|
|
if (cfs_rq->last == se)
|
|
|
|
__clear_buddies_last(se);
|
|
|
|
|
|
|
|
if (cfs_rq->next == se)
|
|
|
|
__clear_buddies_next(se);
|
2011-02-01 15:51:03 +01:00
|
|
|
|
|
|
|
if (cfs_rq->skip == se)
|
|
|
|
__clear_buddies_skip(se);
|
2009-01-28 14:51:40 +01:00
|
|
|
}
|
|
|
|
|
2011-07-21 18:43:41 +02:00
|
|
|
static void return_cfs_rq_runtime(struct cfs_rq *cfs_rq);
|
|
|
|
|
2007-07-09 18:51:58 +02:00
|
|
|
static void
|
2010-03-24 16:38:48 +01:00
|
|
|
dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
2007-10-15 17:00:13 +02:00
|
|
|
/*
|
|
|
|
* Update run-time statistics of the 'current'.
|
|
|
|
*/
|
|
|
|
update_curr(cfs_rq);
|
|
|
|
|
2007-08-09 11:16:48 +02:00
|
|
|
update_stats_dequeue(cfs_rq, se);
|
2010-03-24 16:38:48 +01:00
|
|
|
if (flags & DEQUEUE_SLEEP) {
|
2007-10-15 17:00:10 +02:00
|
|
|
#ifdef CONFIG_SCHEDSTATS
|
2007-07-09 18:51:58 +02:00
|
|
|
if (entity_is_task(se)) {
|
|
|
|
struct task_struct *tsk = task_of(se);
|
|
|
|
|
|
|
|
if (tsk->state & TASK_INTERRUPTIBLE)
|
2010-03-11 03:37:45 +01:00
|
|
|
se->statistics.sleep_start = rq_of(cfs_rq)->clock;
|
2007-07-09 18:51:58 +02:00
|
|
|
if (tsk->state & TASK_UNINTERRUPTIBLE)
|
2010-03-11 03:37:45 +01:00
|
|
|
se->statistics.block_start = rq_of(cfs_rq)->clock;
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
2007-10-15 17:00:06 +02:00
|
|
|
#endif
|
2007-10-15 17:00:10 +02:00
|
|
|
}
|
|
|
|
|
2008-11-11 11:52:33 +01:00
|
|
|
clear_buddies(cfs_rq, se);
|
sched: backward looking buddy
Impact: improve/change/fix wakeup-buddy scheduling
Currently we only have a forward looking buddy, that is, we prefer to
schedule to the task we last woke up, under the presumption that its
going to consume the data we just produced, and therefore will have
cache hot benefits.
This allows co-waking producer/consumer task pairs to run ahead of the
pack for a little while, keeping their cache warm. Without this, we
would interleave all pairs, utterly trashing the cache.
This patch introduces a backward looking buddy, that is, suppose that
in the above scenario, the consumer preempts the producer before it
can go to sleep, we will therefore miss the wakeup from consumer to
producer (its already running, after all), breaking the cycle and
reverting to the cache-trashing interleaved schedule pattern.
The backward buddy will try to schedule back to the task that woke us
up in case the forward buddy is not available, under the assumption
that the last task will be the one with the most cache hot task around
barring current.
This will basically allow a task to continue after it got preempted.
In order to avoid starvation, we allow either buddy to get wakeup_gran
ahead of the pack.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Acked-by: Mike Galbraith <efault@gmx.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-11-04 21:25:09 +01:00
|
|
|
|
2007-10-15 17:00:08 +02:00
|
|
|
if (se != cfs_rq->curr)
|
2007-10-15 17:00:07 +02:00
|
|
|
__dequeue_entity(cfs_rq, se);
|
2010-11-16 00:47:00 +01:00
|
|
|
se->on_rq = 0;
|
2010-11-16 00:47:09 +01:00
|
|
|
update_cfs_load(cfs_rq, 0);
|
2007-10-15 17:00:07 +02:00
|
|
|
account_entity_dequeue(cfs_rq, se);
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 18:04:41 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Normalize the entity after updating the min_vruntime because the
|
|
|
|
* update can refer to the ->curr item and we need to reflect this
|
|
|
|
* movement in our normalized position.
|
|
|
|
*/
|
2010-03-24 16:38:48 +01:00
|
|
|
if (!(flags & DEQUEUE_SLEEP))
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 18:04:41 +01:00
|
|
|
se->vruntime -= cfs_rq->min_vruntime;
|
2011-05-18 01:21:10 +02:00
|
|
|
|
2011-07-21 18:43:41 +02:00
|
|
|
/* return excess runtime on last dequeue */
|
|
|
|
return_cfs_rq_runtime(cfs_rq);
|
|
|
|
|
2011-05-18 01:21:10 +02:00
|
|
|
update_min_vruntime(cfs_rq);
|
|
|
|
update_cfs_shares(cfs_rq);
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Preempt the current task with a newly woken task if needed:
|
|
|
|
*/
|
2007-09-05 14:32:49 +02:00
|
|
|
static void
|
2007-10-15 17:00:05 +02:00
|
|
|
check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
2007-09-05 14:32:49 +02:00
|
|
|
unsigned long ideal_runtime, delta_exec;
|
2011-09-16 19:35:52 +02:00
|
|
|
struct sched_entity *se;
|
|
|
|
s64 delta;
|
2007-09-05 14:32:49 +02:00
|
|
|
|
2007-10-15 17:00:05 +02:00
|
|
|
ideal_runtime = sched_slice(cfs_rq, curr);
|
2007-09-05 14:32:49 +02:00
|
|
|
delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime;
|
2009-01-28 14:51:39 +01:00
|
|
|
if (delta_exec > ideal_runtime) {
|
2007-07-09 18:51:58 +02:00
|
|
|
resched_task(rq_of(cfs_rq)->curr);
|
2009-01-28 14:51:39 +01:00
|
|
|
/*
|
|
|
|
* The current task ran long enough, ensure it doesn't get
|
|
|
|
* re-elected due to buddy favours.
|
|
|
|
*/
|
|
|
|
clear_buddies(cfs_rq, curr);
|
sched: Strengthen buddies and mitigate buddy induced latencies
This patch restores the effectiveness of LAST_BUDDY in preventing
pgsql+oltp from collapsing due to wakeup preemption. It also
switches LAST_BUDDY to exclusively do what it does best, namely
mitigate the effects of aggressive wakeup preemption, which
improves vmark throughput markedly, and restores mysql+oltp
scalability.
Since buddies are about scalability, enable them beginning at the
point where we begin expanding sched_latency, namely
sched_nr_latency. Previously, buddies were cleared aggressively,
which seriously reduced their effectiveness. Not clearing
aggressively however, produces a small drop in mysql+oltp
throughput immediately after peak, indicating that LAST_BUDDY is
actually doing some harm. This is right at the point where X on the
desktop in competition with another load wants low latency service.
Ergo, do not enable until we need to scale.
To mitigate latency induced by buddies, or by a task just missing
wakeup preemption, check latency at tick time.
Last hunk prevents buddies from stymieing BALANCE_NEWIDLE via
CACHE_HOT_BUDDY.
Supporting performance tests:
tip = v2.6.32-rc5-1497-ga525b32
tipx = NO_GENTLE_FAIR_SLEEPERS NEXT_BUDDY granularity knobs = 31 knobs + 31 buddies
tip+x = NO_GENTLE_FAIR_SLEEPERS granularity knobs = 31 knobs
(Three run averages except where noted.)
vmark:
------
tip 108466 messages per second
tip+ 125307 messages per second
tip+x 125335 messages per second
tipx 117781 messages per second
2.6.31.3 122729 messages per second
mysql+oltp:
-----------
clients 1 2 4 8 16 32 64 128 256
..........................................................................................
tip 9949.89 18690.20 34801.24 34460.04 32682.88 30765.97 28305.27 25059.64 19548.08
tip+ 10013.90 18526.84 34900.38 34420.14 33069.83 32083.40 30578.30 28010.71 25605.47
tipx 9698.71 18002.70 34477.56 33420.01 32634.30 31657.27 29932.67 26827.52 21487.18
2.6.31.3 8243.11 18784.20 34404.83 33148.38 31900.32 31161.90 29663.81 25995.94 18058.86
pgsql+oltp:
-----------
clients 1 2 4 8 16 32 64 128 256
..........................................................................................
tip 13686.37 26609.25 51934.28 51347.81 49479.51 45312.65 36691.91 26851.57 24145.35
tip+ (1x) 13907.85 27135.87 52951.98 52514.04 51742.52 50705.43 49947.97 48374.19 46227.94
tip+x 13906.78 27065.81 52951.19 52542.59 52176.11 51815.94 50838.90 49439.46 46891.00
tipx 13742.46 26769.81 52351.99 51891.73 51320.79 50938.98 50248.65 48908.70 46553.84
2.6.31.3 13815.35 26906.46 52683.34 52061.31 51937.10 51376.80 50474.28 49394.47 47003.25
Signed-off-by: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-10-23 23:09:22 +02:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Ensure that a task that missed wakeup preemption by a
|
|
|
|
* narrow margin doesn't have to wait for a full slice.
|
|
|
|
* This also mitigates buddy induced latencies under load.
|
|
|
|
*/
|
|
|
|
if (delta_exec < sysctl_sched_min_granularity)
|
|
|
|
return;
|
|
|
|
|
2011-09-16 19:35:52 +02:00
|
|
|
se = __pick_first_entity(cfs_rq);
|
|
|
|
delta = curr->vruntime - se->vruntime;
|
sched: Strengthen buddies and mitigate buddy induced latencies
This patch restores the effectiveness of LAST_BUDDY in preventing
pgsql+oltp from collapsing due to wakeup preemption. It also
switches LAST_BUDDY to exclusively do what it does best, namely
mitigate the effects of aggressive wakeup preemption, which
improves vmark throughput markedly, and restores mysql+oltp
scalability.
Since buddies are about scalability, enable them beginning at the
point where we begin expanding sched_latency, namely
sched_nr_latency. Previously, buddies were cleared aggressively,
which seriously reduced their effectiveness. Not clearing
aggressively however, produces a small drop in mysql+oltp
throughput immediately after peak, indicating that LAST_BUDDY is
actually doing some harm. This is right at the point where X on the
desktop in competition with another load wants low latency service.
Ergo, do not enable until we need to scale.
To mitigate latency induced by buddies, or by a task just missing
wakeup preemption, check latency at tick time.
Last hunk prevents buddies from stymieing BALANCE_NEWIDLE via
CACHE_HOT_BUDDY.
Supporting performance tests:
tip = v2.6.32-rc5-1497-ga525b32
tipx = NO_GENTLE_FAIR_SLEEPERS NEXT_BUDDY granularity knobs = 31 knobs + 31 buddies
tip+x = NO_GENTLE_FAIR_SLEEPERS granularity knobs = 31 knobs
(Three run averages except where noted.)
vmark:
------
tip 108466 messages per second
tip+ 125307 messages per second
tip+x 125335 messages per second
tipx 117781 messages per second
2.6.31.3 122729 messages per second
mysql+oltp:
-----------
clients 1 2 4 8 16 32 64 128 256
..........................................................................................
tip 9949.89 18690.20 34801.24 34460.04 32682.88 30765.97 28305.27 25059.64 19548.08
tip+ 10013.90 18526.84 34900.38 34420.14 33069.83 32083.40 30578.30 28010.71 25605.47
tipx 9698.71 18002.70 34477.56 33420.01 32634.30 31657.27 29932.67 26827.52 21487.18
2.6.31.3 8243.11 18784.20 34404.83 33148.38 31900.32 31161.90 29663.81 25995.94 18058.86
pgsql+oltp:
-----------
clients 1 2 4 8 16 32 64 128 256
..........................................................................................
tip 13686.37 26609.25 51934.28 51347.81 49479.51 45312.65 36691.91 26851.57 24145.35
tip+ (1x) 13907.85 27135.87 52951.98 52514.04 51742.52 50705.43 49947.97 48374.19 46227.94
tip+x 13906.78 27065.81 52951.19 52542.59 52176.11 51815.94 50838.90 49439.46 46891.00
tipx 13742.46 26769.81 52351.99 51891.73 51320.79 50938.98 50248.65 48908.70 46553.84
2.6.31.3 13815.35 26906.46 52683.34 52061.31 51937.10 51376.80 50474.28 49394.47 47003.25
Signed-off-by: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-10-23 23:09:22 +02:00
|
|
|
|
2011-09-16 19:35:52 +02:00
|
|
|
if (delta < 0)
|
|
|
|
return;
|
2011-01-05 05:41:17 +01:00
|
|
|
|
2011-09-16 19:35:52 +02:00
|
|
|
if (delta > ideal_runtime)
|
|
|
|
resched_task(rq_of(cfs_rq)->curr);
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
2007-10-15 17:00:08 +02:00
|
|
|
static void
|
2007-08-09 11:16:48 +02:00
|
|
|
set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
2007-10-15 17:00:08 +02:00
|
|
|
/* 'current' is not kept within the tree. */
|
|
|
|
if (se->on_rq) {
|
|
|
|
/*
|
|
|
|
* Any task has to be enqueued before it get to execute on
|
|
|
|
* a CPU. So account for the time it spent waiting on the
|
|
|
|
* runqueue.
|
|
|
|
*/
|
|
|
|
update_stats_wait_end(cfs_rq, se);
|
|
|
|
__dequeue_entity(cfs_rq, se);
|
|
|
|
}
|
|
|
|
|
2007-08-09 11:16:47 +02:00
|
|
|
update_stats_curr_start(cfs_rq, se);
|
2007-10-15 17:00:03 +02:00
|
|
|
cfs_rq->curr = se;
|
2007-10-15 17:00:02 +02:00
|
|
|
#ifdef CONFIG_SCHEDSTATS
|
|
|
|
/*
|
|
|
|
* Track our maximum slice length, if the CPU's load is at
|
|
|
|
* least twice that of our own weight (i.e. dont track it
|
|
|
|
* when there are only lesser-weight tasks around):
|
|
|
|
*/
|
2007-10-15 17:00:06 +02:00
|
|
|
if (rq_of(cfs_rq)->load.weight >= 2*se->load.weight) {
|
2010-03-11 03:37:45 +01:00
|
|
|
se->statistics.slice_max = max(se->statistics.slice_max,
|
2007-10-15 17:00:02 +02:00
|
|
|
se->sum_exec_runtime - se->prev_sum_exec_runtime);
|
|
|
|
}
|
|
|
|
#endif
|
2007-09-05 14:32:49 +02:00
|
|
|
se->prev_sum_exec_runtime = se->sum_exec_runtime;
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
2008-10-24 11:06:16 +02:00
|
|
|
static int
|
|
|
|
wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se);
|
|
|
|
|
2011-02-01 15:51:03 +01:00
|
|
|
/*
|
|
|
|
* Pick the next process, keeping these things in mind, in this order:
|
|
|
|
* 1) keep things fair between processes/task groups
|
|
|
|
* 2) pick the "next" process, since someone really wants that to run
|
|
|
|
* 3) pick the "last" process, for cache locality
|
|
|
|
* 4) do not run the "skip" process, if something else is available
|
|
|
|
*/
|
2008-11-04 21:25:07 +01:00
|
|
|
static struct sched_entity *pick_next_entity(struct cfs_rq *cfs_rq)
|
2008-03-14 21:12:12 +01:00
|
|
|
{
|
2011-02-01 15:51:03 +01:00
|
|
|
struct sched_entity *se = __pick_first_entity(cfs_rq);
|
sched: Strengthen buddies and mitigate buddy induced latencies
This patch restores the effectiveness of LAST_BUDDY in preventing
pgsql+oltp from collapsing due to wakeup preemption. It also
switches LAST_BUDDY to exclusively do what it does best, namely
mitigate the effects of aggressive wakeup preemption, which
improves vmark throughput markedly, and restores mysql+oltp
scalability.
Since buddies are about scalability, enable them beginning at the
point where we begin expanding sched_latency, namely
sched_nr_latency. Previously, buddies were cleared aggressively,
which seriously reduced their effectiveness. Not clearing
aggressively however, produces a small drop in mysql+oltp
throughput immediately after peak, indicating that LAST_BUDDY is
actually doing some harm. This is right at the point where X on the
desktop in competition with another load wants low latency service.
Ergo, do not enable until we need to scale.
To mitigate latency induced by buddies, or by a task just missing
wakeup preemption, check latency at tick time.
Last hunk prevents buddies from stymieing BALANCE_NEWIDLE via
CACHE_HOT_BUDDY.
Supporting performance tests:
tip = v2.6.32-rc5-1497-ga525b32
tipx = NO_GENTLE_FAIR_SLEEPERS NEXT_BUDDY granularity knobs = 31 knobs + 31 buddies
tip+x = NO_GENTLE_FAIR_SLEEPERS granularity knobs = 31 knobs
(Three run averages except where noted.)
vmark:
------
tip 108466 messages per second
tip+ 125307 messages per second
tip+x 125335 messages per second
tipx 117781 messages per second
2.6.31.3 122729 messages per second
mysql+oltp:
-----------
clients 1 2 4 8 16 32 64 128 256
..........................................................................................
tip 9949.89 18690.20 34801.24 34460.04 32682.88 30765.97 28305.27 25059.64 19548.08
tip+ 10013.90 18526.84 34900.38 34420.14 33069.83 32083.40 30578.30 28010.71 25605.47
tipx 9698.71 18002.70 34477.56 33420.01 32634.30 31657.27 29932.67 26827.52 21487.18
2.6.31.3 8243.11 18784.20 34404.83 33148.38 31900.32 31161.90 29663.81 25995.94 18058.86
pgsql+oltp:
-----------
clients 1 2 4 8 16 32 64 128 256
..........................................................................................
tip 13686.37 26609.25 51934.28 51347.81 49479.51 45312.65 36691.91 26851.57 24145.35
tip+ (1x) 13907.85 27135.87 52951.98 52514.04 51742.52 50705.43 49947.97 48374.19 46227.94
tip+x 13906.78 27065.81 52951.19 52542.59 52176.11 51815.94 50838.90 49439.46 46891.00
tipx 13742.46 26769.81 52351.99 51891.73 51320.79 50938.98 50248.65 48908.70 46553.84
2.6.31.3 13815.35 26906.46 52683.34 52061.31 51937.10 51376.80 50474.28 49394.47 47003.25
Signed-off-by: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-10-23 23:09:22 +02:00
|
|
|
struct sched_entity *left = se;
|
2008-11-04 21:25:07 +01:00
|
|
|
|
2011-02-01 15:51:03 +01:00
|
|
|
/*
|
|
|
|
* Avoid running the skip buddy, if running something else can
|
|
|
|
* be done without getting too unfair.
|
|
|
|
*/
|
|
|
|
if (cfs_rq->skip == se) {
|
|
|
|
struct sched_entity *second = __pick_next_entity(se);
|
|
|
|
if (second && wakeup_preempt_entity(second, left) < 1)
|
|
|
|
se = second;
|
|
|
|
}
|
2008-03-14 21:12:12 +01:00
|
|
|
|
sched: Strengthen buddies and mitigate buddy induced latencies
This patch restores the effectiveness of LAST_BUDDY in preventing
pgsql+oltp from collapsing due to wakeup preemption. It also
switches LAST_BUDDY to exclusively do what it does best, namely
mitigate the effects of aggressive wakeup preemption, which
improves vmark throughput markedly, and restores mysql+oltp
scalability.
Since buddies are about scalability, enable them beginning at the
point where we begin expanding sched_latency, namely
sched_nr_latency. Previously, buddies were cleared aggressively,
which seriously reduced their effectiveness. Not clearing
aggressively however, produces a small drop in mysql+oltp
throughput immediately after peak, indicating that LAST_BUDDY is
actually doing some harm. This is right at the point where X on the
desktop in competition with another load wants low latency service.
Ergo, do not enable until we need to scale.
To mitigate latency induced by buddies, or by a task just missing
wakeup preemption, check latency at tick time.
Last hunk prevents buddies from stymieing BALANCE_NEWIDLE via
CACHE_HOT_BUDDY.
Supporting performance tests:
tip = v2.6.32-rc5-1497-ga525b32
tipx = NO_GENTLE_FAIR_SLEEPERS NEXT_BUDDY granularity knobs = 31 knobs + 31 buddies
tip+x = NO_GENTLE_FAIR_SLEEPERS granularity knobs = 31 knobs
(Three run averages except where noted.)
vmark:
------
tip 108466 messages per second
tip+ 125307 messages per second
tip+x 125335 messages per second
tipx 117781 messages per second
2.6.31.3 122729 messages per second
mysql+oltp:
-----------
clients 1 2 4 8 16 32 64 128 256
..........................................................................................
tip 9949.89 18690.20 34801.24 34460.04 32682.88 30765.97 28305.27 25059.64 19548.08
tip+ 10013.90 18526.84 34900.38 34420.14 33069.83 32083.40 30578.30 28010.71 25605.47
tipx 9698.71 18002.70 34477.56 33420.01 32634.30 31657.27 29932.67 26827.52 21487.18
2.6.31.3 8243.11 18784.20 34404.83 33148.38 31900.32 31161.90 29663.81 25995.94 18058.86
pgsql+oltp:
-----------
clients 1 2 4 8 16 32 64 128 256
..........................................................................................
tip 13686.37 26609.25 51934.28 51347.81 49479.51 45312.65 36691.91 26851.57 24145.35
tip+ (1x) 13907.85 27135.87 52951.98 52514.04 51742.52 50705.43 49947.97 48374.19 46227.94
tip+x 13906.78 27065.81 52951.19 52542.59 52176.11 51815.94 50838.90 49439.46 46891.00
tipx 13742.46 26769.81 52351.99 51891.73 51320.79 50938.98 50248.65 48908.70 46553.84
2.6.31.3 13815.35 26906.46 52683.34 52061.31 51937.10 51376.80 50474.28 49394.47 47003.25
Signed-off-by: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-10-23 23:09:22 +02:00
|
|
|
/*
|
|
|
|
* Prefer last buddy, try to return the CPU to a preempted task.
|
|
|
|
*/
|
|
|
|
if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, left) < 1)
|
|
|
|
se = cfs_rq->last;
|
|
|
|
|
2011-02-01 15:51:03 +01:00
|
|
|
/*
|
|
|
|
* Someone really wants this to run. If it's not unfair, run it.
|
|
|
|
*/
|
|
|
|
if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, left) < 1)
|
|
|
|
se = cfs_rq->next;
|
|
|
|
|
sched: Strengthen buddies and mitigate buddy induced latencies
This patch restores the effectiveness of LAST_BUDDY in preventing
pgsql+oltp from collapsing due to wakeup preemption. It also
switches LAST_BUDDY to exclusively do what it does best, namely
mitigate the effects of aggressive wakeup preemption, which
improves vmark throughput markedly, and restores mysql+oltp
scalability.
Since buddies are about scalability, enable them beginning at the
point where we begin expanding sched_latency, namely
sched_nr_latency. Previously, buddies were cleared aggressively,
which seriously reduced their effectiveness. Not clearing
aggressively however, produces a small drop in mysql+oltp
throughput immediately after peak, indicating that LAST_BUDDY is
actually doing some harm. This is right at the point where X on the
desktop in competition with another load wants low latency service.
Ergo, do not enable until we need to scale.
To mitigate latency induced by buddies, or by a task just missing
wakeup preemption, check latency at tick time.
Last hunk prevents buddies from stymieing BALANCE_NEWIDLE via
CACHE_HOT_BUDDY.
Supporting performance tests:
tip = v2.6.32-rc5-1497-ga525b32
tipx = NO_GENTLE_FAIR_SLEEPERS NEXT_BUDDY granularity knobs = 31 knobs + 31 buddies
tip+x = NO_GENTLE_FAIR_SLEEPERS granularity knobs = 31 knobs
(Three run averages except where noted.)
vmark:
------
tip 108466 messages per second
tip+ 125307 messages per second
tip+x 125335 messages per second
tipx 117781 messages per second
2.6.31.3 122729 messages per second
mysql+oltp:
-----------
clients 1 2 4 8 16 32 64 128 256
..........................................................................................
tip 9949.89 18690.20 34801.24 34460.04 32682.88 30765.97 28305.27 25059.64 19548.08
tip+ 10013.90 18526.84 34900.38 34420.14 33069.83 32083.40 30578.30 28010.71 25605.47
tipx 9698.71 18002.70 34477.56 33420.01 32634.30 31657.27 29932.67 26827.52 21487.18
2.6.31.3 8243.11 18784.20 34404.83 33148.38 31900.32 31161.90 29663.81 25995.94 18058.86
pgsql+oltp:
-----------
clients 1 2 4 8 16 32 64 128 256
..........................................................................................
tip 13686.37 26609.25 51934.28 51347.81 49479.51 45312.65 36691.91 26851.57 24145.35
tip+ (1x) 13907.85 27135.87 52951.98 52514.04 51742.52 50705.43 49947.97 48374.19 46227.94
tip+x 13906.78 27065.81 52951.19 52542.59 52176.11 51815.94 50838.90 49439.46 46891.00
tipx 13742.46 26769.81 52351.99 51891.73 51320.79 50938.98 50248.65 48908.70 46553.84
2.6.31.3 13815.35 26906.46 52683.34 52061.31 51937.10 51376.80 50474.28 49394.47 47003.25
Signed-off-by: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-10-23 23:09:22 +02:00
|
|
|
clear_buddies(cfs_rq, se);
|
sched: backward looking buddy
Impact: improve/change/fix wakeup-buddy scheduling
Currently we only have a forward looking buddy, that is, we prefer to
schedule to the task we last woke up, under the presumption that its
going to consume the data we just produced, and therefore will have
cache hot benefits.
This allows co-waking producer/consumer task pairs to run ahead of the
pack for a little while, keeping their cache warm. Without this, we
would interleave all pairs, utterly trashing the cache.
This patch introduces a backward looking buddy, that is, suppose that
in the above scenario, the consumer preempts the producer before it
can go to sleep, we will therefore miss the wakeup from consumer to
producer (its already running, after all), breaking the cycle and
reverting to the cache-trashing interleaved schedule pattern.
The backward buddy will try to schedule back to the task that woke us
up in case the forward buddy is not available, under the assumption
that the last task will be the one with the most cache hot task around
barring current.
This will basically allow a task to continue after it got preempted.
In order to avoid starvation, we allow either buddy to get wakeup_gran
ahead of the pack.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Acked-by: Mike Galbraith <efault@gmx.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-11-04 21:25:09 +01:00
|
|
|
|
|
|
|
return se;
|
2008-03-14 21:12:12 +01:00
|
|
|
}
|
|
|
|
|
2011-07-21 18:43:39 +02:00
|
|
|
static void check_cfs_rq_runtime(struct cfs_rq *cfs_rq);
|
|
|
|
|
2007-08-09 11:16:48 +02:00
|
|
|
static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
|
|
|
/*
|
|
|
|
* If still on the runqueue then deactivate_task()
|
|
|
|
* was not called and update_curr() has to be done:
|
|
|
|
*/
|
|
|
|
if (prev->on_rq)
|
2007-08-09 11:16:47 +02:00
|
|
|
update_curr(cfs_rq);
|
2007-07-09 18:51:58 +02:00
|
|
|
|
2011-07-21 18:43:39 +02:00
|
|
|
/* throttle cfs_rqs exceeding runtime */
|
|
|
|
check_cfs_rq_runtime(cfs_rq);
|
|
|
|
|
2007-10-15 17:00:10 +02:00
|
|
|
check_spread(cfs_rq, prev);
|
2007-10-15 17:00:07 +02:00
|
|
|
if (prev->on_rq) {
|
2007-08-09 11:16:47 +02:00
|
|
|
update_stats_wait_start(cfs_rq, prev);
|
2007-10-15 17:00:07 +02:00
|
|
|
/* Put 'current' back into the tree. */
|
|
|
|
__enqueue_entity(cfs_rq, prev);
|
|
|
|
}
|
2007-10-15 17:00:03 +02:00
|
|
|
cfs_rq->curr = NULL;
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
2008-01-25 21:08:29 +01:00
|
|
|
static void
|
|
|
|
entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
|
|
|
/*
|
2007-10-15 17:00:07 +02:00
|
|
|
* Update run-time statistics of the 'current'.
|
2007-07-09 18:51:58 +02:00
|
|
|
*/
|
2007-10-15 17:00:07 +02:00
|
|
|
update_curr(cfs_rq);
|
2007-07-09 18:51:58 +02:00
|
|
|
|
2010-12-16 04:10:17 +01:00
|
|
|
/*
|
|
|
|
* Update share accounting for long-running entities.
|
|
|
|
*/
|
|
|
|
update_entity_shares_tick(cfs_rq);
|
|
|
|
|
2008-01-25 21:08:29 +01:00
|
|
|
#ifdef CONFIG_SCHED_HRTICK
|
|
|
|
/*
|
|
|
|
* queued ticks are scheduled to match the slice, so don't bother
|
|
|
|
* validating it and just reschedule.
|
|
|
|
*/
|
2008-04-25 03:17:55 +02:00
|
|
|
if (queued) {
|
|
|
|
resched_task(rq_of(cfs_rq)->curr);
|
|
|
|
return;
|
|
|
|
}
|
2008-01-25 21:08:29 +01:00
|
|
|
/*
|
|
|
|
* don't let the period tick interfere with the hrtick preemption
|
|
|
|
*/
|
|
|
|
if (!sched_feat(DOUBLE_TICK) &&
|
|
|
|
hrtimer_active(&rq_of(cfs_rq)->hrtick_timer))
|
|
|
|
return;
|
|
|
|
#endif
|
|
|
|
|
2011-07-29 10:20:33 +02:00
|
|
|
if (cfs_rq->nr_running > 1)
|
2007-10-15 17:00:05 +02:00
|
|
|
check_preempt_tick(cfs_rq, curr);
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
2011-07-21 18:43:28 +02:00
|
|
|
|
|
|
|
/**************************************************
|
|
|
|
* CFS bandwidth control machinery
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifdef CONFIG_CFS_BANDWIDTH
|
2011-10-25 10:00:11 +02:00
|
|
|
|
|
|
|
#ifdef HAVE_JUMP_LABEL
|
|
|
|
static struct jump_label_key __cfs_bandwidth_used;
|
|
|
|
|
|
|
|
static inline bool cfs_bandwidth_used(void)
|
|
|
|
{
|
|
|
|
return static_branch(&__cfs_bandwidth_used);
|
|
|
|
}
|
|
|
|
|
|
|
|
void account_cfs_bandwidth_used(int enabled, int was_enabled)
|
|
|
|
{
|
|
|
|
/* only need to count groups transitioning between enabled/!enabled */
|
|
|
|
if (enabled && !was_enabled)
|
|
|
|
jump_label_inc(&__cfs_bandwidth_used);
|
|
|
|
else if (!enabled && was_enabled)
|
|
|
|
jump_label_dec(&__cfs_bandwidth_used);
|
|
|
|
}
|
|
|
|
#else /* HAVE_JUMP_LABEL */
|
|
|
|
static bool cfs_bandwidth_used(void)
|
|
|
|
{
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
void account_cfs_bandwidth_used(int enabled, int was_enabled) {}
|
|
|
|
#endif /* HAVE_JUMP_LABEL */
|
|
|
|
|
2011-07-21 18:43:28 +02:00
|
|
|
/*
|
|
|
|
* default period for cfs group bandwidth.
|
|
|
|
* default: 0.1s, units: nanoseconds
|
|
|
|
*/
|
|
|
|
static inline u64 default_cfs_period(void)
|
|
|
|
{
|
|
|
|
return 100000000ULL;
|
|
|
|
}
|
2011-07-21 18:43:30 +02:00
|
|
|
|
|
|
|
static inline u64 sched_cfs_bandwidth_slice(void)
|
|
|
|
{
|
|
|
|
return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC;
|
|
|
|
}
|
|
|
|
|
2011-07-21 18:43:32 +02:00
|
|
|
/*
|
|
|
|
* Replenish runtime according to assigned quota and update expiration time.
|
|
|
|
* We use sched_clock_cpu directly instead of rq->clock to avoid adding
|
|
|
|
* additional synchronization around rq->lock.
|
|
|
|
*
|
|
|
|
* requires cfs_b->lock
|
|
|
|
*/
|
2011-10-25 10:00:11 +02:00
|
|
|
void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b)
|
2011-07-21 18:43:32 +02:00
|
|
|
{
|
|
|
|
u64 now;
|
|
|
|
|
|
|
|
if (cfs_b->quota == RUNTIME_INF)
|
|
|
|
return;
|
|
|
|
|
|
|
|
now = sched_clock_cpu(smp_processor_id());
|
|
|
|
cfs_b->runtime = cfs_b->quota;
|
|
|
|
cfs_b->runtime_expires = now + ktime_to_ns(cfs_b->period);
|
|
|
|
}
|
|
|
|
|
2011-10-25 10:00:11 +02:00
|
|
|
static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
|
|
|
|
{
|
|
|
|
return &tg->cfs_bandwidth;
|
|
|
|
}
|
|
|
|
|
2011-07-21 18:43:33 +02:00
|
|
|
/* returns 0 on failure to allocate runtime */
|
|
|
|
static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)
|
2011-07-21 18:43:30 +02:00
|
|
|
{
|
|
|
|
struct task_group *tg = cfs_rq->tg;
|
|
|
|
struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg);
|
2011-07-21 18:43:32 +02:00
|
|
|
u64 amount = 0, min_amount, expires;
|
2011-07-21 18:43:30 +02:00
|
|
|
|
|
|
|
/* note: this is a positive sum as runtime_remaining <= 0 */
|
|
|
|
min_amount = sched_cfs_bandwidth_slice() - cfs_rq->runtime_remaining;
|
|
|
|
|
|
|
|
raw_spin_lock(&cfs_b->lock);
|
|
|
|
if (cfs_b->quota == RUNTIME_INF)
|
|
|
|
amount = min_amount;
|
2011-07-21 18:43:31 +02:00
|
|
|
else {
|
2011-07-21 18:43:32 +02:00
|
|
|
/*
|
|
|
|
* If the bandwidth pool has become inactive, then at least one
|
|
|
|
* period must have elapsed since the last consumption.
|
|
|
|
* Refresh the global state and ensure bandwidth timer becomes
|
|
|
|
* active.
|
|
|
|
*/
|
|
|
|
if (!cfs_b->timer_active) {
|
|
|
|
__refill_cfs_bandwidth_runtime(cfs_b);
|
2011-07-21 18:43:31 +02:00
|
|
|
__start_cfs_bandwidth(cfs_b);
|
2011-07-21 18:43:32 +02:00
|
|
|
}
|
2011-07-21 18:43:31 +02:00
|
|
|
|
|
|
|
if (cfs_b->runtime > 0) {
|
|
|
|
amount = min(cfs_b->runtime, min_amount);
|
|
|
|
cfs_b->runtime -= amount;
|
|
|
|
cfs_b->idle = 0;
|
|
|
|
}
|
2011-07-21 18:43:30 +02:00
|
|
|
}
|
2011-07-21 18:43:32 +02:00
|
|
|
expires = cfs_b->runtime_expires;
|
2011-07-21 18:43:30 +02:00
|
|
|
raw_spin_unlock(&cfs_b->lock);
|
|
|
|
|
|
|
|
cfs_rq->runtime_remaining += amount;
|
2011-07-21 18:43:32 +02:00
|
|
|
/*
|
|
|
|
* we may have advanced our local expiration to account for allowed
|
|
|
|
* spread between our sched_clock and the one on which runtime was
|
|
|
|
* issued.
|
|
|
|
*/
|
|
|
|
if ((s64)(expires - cfs_rq->runtime_expires) > 0)
|
|
|
|
cfs_rq->runtime_expires = expires;
|
2011-07-21 18:43:33 +02:00
|
|
|
|
|
|
|
return cfs_rq->runtime_remaining > 0;
|
2011-07-21 18:43:30 +02:00
|
|
|
}
|
|
|
|
|
2011-07-21 18:43:32 +02:00
|
|
|
/*
|
|
|
|
* Note: This depends on the synchronization provided by sched_clock and the
|
|
|
|
* fact that rq->clock snapshots this value.
|
|
|
|
*/
|
|
|
|
static void expire_cfs_rq_runtime(struct cfs_rq *cfs_rq)
|
2011-07-21 18:43:30 +02:00
|
|
|
{
|
2011-07-21 18:43:32 +02:00
|
|
|
struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
|
|
|
|
struct rq *rq = rq_of(cfs_rq);
|
|
|
|
|
|
|
|
/* if the deadline is ahead of our clock, nothing to do */
|
|
|
|
if (likely((s64)(rq->clock - cfs_rq->runtime_expires) < 0))
|
2011-07-21 18:43:30 +02:00
|
|
|
return;
|
|
|
|
|
2011-07-21 18:43:32 +02:00
|
|
|
if (cfs_rq->runtime_remaining < 0)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If the local deadline has passed we have to consider the
|
|
|
|
* possibility that our sched_clock is 'fast' and the global deadline
|
|
|
|
* has not truly expired.
|
|
|
|
*
|
|
|
|
* Fortunately we can check determine whether this the case by checking
|
|
|
|
* whether the global deadline has advanced.
|
|
|
|
*/
|
|
|
|
|
|
|
|
if ((s64)(cfs_rq->runtime_expires - cfs_b->runtime_expires) >= 0) {
|
|
|
|
/* extend local deadline, drift is bounded above by 2 ticks */
|
|
|
|
cfs_rq->runtime_expires += TICK_NSEC;
|
|
|
|
} else {
|
|
|
|
/* global deadline is ahead, expiration has passed */
|
|
|
|
cfs_rq->runtime_remaining = 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
|
|
|
|
unsigned long delta_exec)
|
|
|
|
{
|
|
|
|
/* dock delta_exec before expiring quota (as it could span periods) */
|
2011-07-21 18:43:30 +02:00
|
|
|
cfs_rq->runtime_remaining -= delta_exec;
|
2011-07-21 18:43:32 +02:00
|
|
|
expire_cfs_rq_runtime(cfs_rq);
|
|
|
|
|
|
|
|
if (likely(cfs_rq->runtime_remaining > 0))
|
2011-07-21 18:43:30 +02:00
|
|
|
return;
|
|
|
|
|
2011-07-21 18:43:33 +02:00
|
|
|
/*
|
|
|
|
* if we're unable to extend our runtime we resched so that the active
|
|
|
|
* hierarchy can be throttled
|
|
|
|
*/
|
|
|
|
if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr))
|
|
|
|
resched_task(rq_of(cfs_rq)->curr);
|
2011-07-21 18:43:30 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static __always_inline void account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
|
|
|
|
unsigned long delta_exec)
|
|
|
|
{
|
2011-11-08 05:26:33 +01:00
|
|
|
if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled)
|
2011-07-21 18:43:30 +02:00
|
|
|
return;
|
|
|
|
|
|
|
|
__account_cfs_rq_runtime(cfs_rq, delta_exec);
|
|
|
|
}
|
|
|
|
|
2011-07-21 18:43:33 +02:00
|
|
|
static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
|
|
|
|
{
|
2011-11-08 05:26:33 +01:00
|
|
|
return cfs_bandwidth_used() && cfs_rq->throttled;
|
2011-07-21 18:43:33 +02:00
|
|
|
}
|
|
|
|
|
2011-07-21 18:43:36 +02:00
|
|
|
/* check whether cfs_rq, or any parent, is throttled */
|
|
|
|
static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
|
|
|
|
{
|
2011-11-08 05:26:33 +01:00
|
|
|
return cfs_bandwidth_used() && cfs_rq->throttle_count;
|
2011-07-21 18:43:36 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Ensure that neither of the group entities corresponding to src_cpu or
|
|
|
|
* dest_cpu are members of a throttled hierarchy when performing group
|
|
|
|
* load-balance operations.
|
|
|
|
*/
|
|
|
|
static inline int throttled_lb_pair(struct task_group *tg,
|
|
|
|
int src_cpu, int dest_cpu)
|
|
|
|
{
|
|
|
|
struct cfs_rq *src_cfs_rq, *dest_cfs_rq;
|
|
|
|
|
|
|
|
src_cfs_rq = tg->cfs_rq[src_cpu];
|
|
|
|
dest_cfs_rq = tg->cfs_rq[dest_cpu];
|
|
|
|
|
|
|
|
return throttled_hierarchy(src_cfs_rq) ||
|
|
|
|
throttled_hierarchy(dest_cfs_rq);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* updated child weight may affect parent so we have to do this bottom up */
|
|
|
|
static int tg_unthrottle_up(struct task_group *tg, void *data)
|
|
|
|
{
|
|
|
|
struct rq *rq = data;
|
|
|
|
struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
|
|
|
|
|
|
|
|
cfs_rq->throttle_count--;
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
if (!cfs_rq->throttle_count) {
|
|
|
|
u64 delta = rq->clock_task - cfs_rq->load_stamp;
|
|
|
|
|
|
|
|
/* leaving throttled state, advance shares averaging windows */
|
|
|
|
cfs_rq->load_stamp += delta;
|
|
|
|
cfs_rq->load_last += delta;
|
|
|
|
|
|
|
|
/* update entity weight now that we are on_rq again */
|
|
|
|
update_cfs_shares(cfs_rq);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int tg_throttle_down(struct task_group *tg, void *data)
|
|
|
|
{
|
|
|
|
struct rq *rq = data;
|
|
|
|
struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
|
|
|
|
|
|
|
|
/* group is entering throttled state, record last load */
|
|
|
|
if (!cfs_rq->throttle_count)
|
|
|
|
update_cfs_load(cfs_rq, 0);
|
|
|
|
cfs_rq->throttle_count++;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2011-07-21 18:43:39 +02:00
|
|
|
static void throttle_cfs_rq(struct cfs_rq *cfs_rq)
|
2011-07-21 18:43:33 +02:00
|
|
|
{
|
|
|
|
struct rq *rq = rq_of(cfs_rq);
|
|
|
|
struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
|
|
|
|
struct sched_entity *se;
|
|
|
|
long task_delta, dequeue = 1;
|
|
|
|
|
|
|
|
se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))];
|
|
|
|
|
|
|
|
/* account load preceding throttle */
|
2011-07-21 18:43:36 +02:00
|
|
|
rcu_read_lock();
|
|
|
|
walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq);
|
|
|
|
rcu_read_unlock();
|
2011-07-21 18:43:33 +02:00
|
|
|
|
|
|
|
task_delta = cfs_rq->h_nr_running;
|
|
|
|
for_each_sched_entity(se) {
|
|
|
|
struct cfs_rq *qcfs_rq = cfs_rq_of(se);
|
|
|
|
/* throttled entity or throttle-on-deactivate */
|
|
|
|
if (!se->on_rq)
|
|
|
|
break;
|
|
|
|
|
|
|
|
if (dequeue)
|
|
|
|
dequeue_entity(qcfs_rq, se, DEQUEUE_SLEEP);
|
|
|
|
qcfs_rq->h_nr_running -= task_delta;
|
|
|
|
|
|
|
|
if (qcfs_rq->load.weight)
|
|
|
|
dequeue = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!se)
|
|
|
|
rq->nr_running -= task_delta;
|
|
|
|
|
|
|
|
cfs_rq->throttled = 1;
|
2011-07-21 18:43:40 +02:00
|
|
|
cfs_rq->throttled_timestamp = rq->clock;
|
2011-07-21 18:43:33 +02:00
|
|
|
raw_spin_lock(&cfs_b->lock);
|
|
|
|
list_add_tail_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq);
|
|
|
|
raw_spin_unlock(&cfs_b->lock);
|
|
|
|
}
|
|
|
|
|
2011-10-25 10:00:11 +02:00
|
|
|
void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
|
2011-07-21 18:43:34 +02:00
|
|
|
{
|
|
|
|
struct rq *rq = rq_of(cfs_rq);
|
|
|
|
struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
|
|
|
|
struct sched_entity *se;
|
|
|
|
int enqueue = 1;
|
|
|
|
long task_delta;
|
|
|
|
|
|
|
|
se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))];
|
|
|
|
|
|
|
|
cfs_rq->throttled = 0;
|
|
|
|
raw_spin_lock(&cfs_b->lock);
|
2011-07-21 18:43:40 +02:00
|
|
|
cfs_b->throttled_time += rq->clock - cfs_rq->throttled_timestamp;
|
2011-07-21 18:43:34 +02:00
|
|
|
list_del_rcu(&cfs_rq->throttled_list);
|
|
|
|
raw_spin_unlock(&cfs_b->lock);
|
2011-07-21 18:43:40 +02:00
|
|
|
cfs_rq->throttled_timestamp = 0;
|
2011-07-21 18:43:34 +02:00
|
|
|
|
2011-07-21 18:43:36 +02:00
|
|
|
update_rq_clock(rq);
|
|
|
|
/* update hierarchical throttle state */
|
|
|
|
walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq);
|
|
|
|
|
2011-07-21 18:43:34 +02:00
|
|
|
if (!cfs_rq->load.weight)
|
|
|
|
return;
|
|
|
|
|
|
|
|
task_delta = cfs_rq->h_nr_running;
|
|
|
|
for_each_sched_entity(se) {
|
|
|
|
if (se->on_rq)
|
|
|
|
enqueue = 0;
|
|
|
|
|
|
|
|
cfs_rq = cfs_rq_of(se);
|
|
|
|
if (enqueue)
|
|
|
|
enqueue_entity(cfs_rq, se, ENQUEUE_WAKEUP);
|
|
|
|
cfs_rq->h_nr_running += task_delta;
|
|
|
|
|
|
|
|
if (cfs_rq_throttled(cfs_rq))
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!se)
|
|
|
|
rq->nr_running += task_delta;
|
|
|
|
|
|
|
|
/* determine whether we need to wake up potentially idle cpu */
|
|
|
|
if (rq->curr == rq->idle && rq->cfs.nr_running)
|
|
|
|
resched_task(rq->curr);
|
|
|
|
}
|
|
|
|
|
|
|
|
static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b,
|
|
|
|
u64 remaining, u64 expires)
|
|
|
|
{
|
|
|
|
struct cfs_rq *cfs_rq;
|
|
|
|
u64 runtime = remaining;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq,
|
|
|
|
throttled_list) {
|
|
|
|
struct rq *rq = rq_of(cfs_rq);
|
|
|
|
|
|
|
|
raw_spin_lock(&rq->lock);
|
|
|
|
if (!cfs_rq_throttled(cfs_rq))
|
|
|
|
goto next;
|
|
|
|
|
|
|
|
runtime = -cfs_rq->runtime_remaining + 1;
|
|
|
|
if (runtime > remaining)
|
|
|
|
runtime = remaining;
|
|
|
|
remaining -= runtime;
|
|
|
|
|
|
|
|
cfs_rq->runtime_remaining += runtime;
|
|
|
|
cfs_rq->runtime_expires = expires;
|
|
|
|
|
|
|
|
/* we check whether we're throttled above */
|
|
|
|
if (cfs_rq->runtime_remaining > 0)
|
|
|
|
unthrottle_cfs_rq(cfs_rq);
|
|
|
|
|
|
|
|
next:
|
|
|
|
raw_spin_unlock(&rq->lock);
|
|
|
|
|
|
|
|
if (!remaining)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
|
|
|
return remaining;
|
|
|
|
}
|
|
|
|
|
2011-07-21 18:43:31 +02:00
|
|
|
/*
|
|
|
|
* Responsible for refilling a task_group's bandwidth and unthrottling its
|
|
|
|
* cfs_rqs as appropriate. If there has been no activity within the last
|
|
|
|
* period the timer is deactivated until scheduling resumes; cfs_b->idle is
|
|
|
|
* used to track this state.
|
|
|
|
*/
|
|
|
|
static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun)
|
|
|
|
{
|
2011-07-21 18:43:34 +02:00
|
|
|
u64 runtime, runtime_expires;
|
|
|
|
int idle = 1, throttled;
|
2011-07-21 18:43:31 +02:00
|
|
|
|
|
|
|
raw_spin_lock(&cfs_b->lock);
|
|
|
|
/* no need to continue the timer with no bandwidth constraint */
|
|
|
|
if (cfs_b->quota == RUNTIME_INF)
|
|
|
|
goto out_unlock;
|
|
|
|
|
2011-07-21 18:43:34 +02:00
|
|
|
throttled = !list_empty(&cfs_b->throttled_cfs_rq);
|
|
|
|
/* idle depends on !throttled (for the case of a large deficit) */
|
|
|
|
idle = cfs_b->idle && !throttled;
|
2011-07-21 18:43:40 +02:00
|
|
|
cfs_b->nr_periods += overrun;
|
2011-07-21 18:43:34 +02:00
|
|
|
|
2011-07-21 18:43:32 +02:00
|
|
|
/* if we're going inactive then everything else can be deferred */
|
|
|
|
if (idle)
|
|
|
|
goto out_unlock;
|
|
|
|
|
|
|
|
__refill_cfs_bandwidth_runtime(cfs_b);
|
|
|
|
|
2011-07-21 18:43:34 +02:00
|
|
|
if (!throttled) {
|
|
|
|
/* mark as potentially idle for the upcoming period */
|
|
|
|
cfs_b->idle = 1;
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
|
|
|
|
2011-07-21 18:43:40 +02:00
|
|
|
/* account preceding periods in which throttling occurred */
|
|
|
|
cfs_b->nr_throttled += overrun;
|
|
|
|
|
2011-07-21 18:43:34 +02:00
|
|
|
/*
|
|
|
|
* There are throttled entities so we must first use the new bandwidth
|
|
|
|
* to unthrottle them before making it generally available. This
|
|
|
|
* ensures that all existing debts will be paid before a new cfs_rq is
|
|
|
|
* allowed to run.
|
|
|
|
*/
|
|
|
|
runtime = cfs_b->runtime;
|
|
|
|
runtime_expires = cfs_b->runtime_expires;
|
|
|
|
cfs_b->runtime = 0;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This check is repeated as we are holding onto the new bandwidth
|
|
|
|
* while we unthrottle. This can potentially race with an unthrottled
|
|
|
|
* group trying to acquire new bandwidth from the global pool.
|
|
|
|
*/
|
|
|
|
while (throttled && runtime > 0) {
|
|
|
|
raw_spin_unlock(&cfs_b->lock);
|
|
|
|
/* we can't nest cfs_b->lock while distributing bandwidth */
|
|
|
|
runtime = distribute_cfs_runtime(cfs_b, runtime,
|
|
|
|
runtime_expires);
|
|
|
|
raw_spin_lock(&cfs_b->lock);
|
|
|
|
|
|
|
|
throttled = !list_empty(&cfs_b->throttled_cfs_rq);
|
|
|
|
}
|
2011-07-21 18:43:31 +02:00
|
|
|
|
2011-07-21 18:43:34 +02:00
|
|
|
/* return (any) remaining runtime */
|
|
|
|
cfs_b->runtime = runtime;
|
|
|
|
/*
|
|
|
|
* While we are ensured activity in the period following an
|
|
|
|
* unthrottle, this also covers the case in which the new bandwidth is
|
|
|
|
* insufficient to cover the existing bandwidth deficit. (Forcing the
|
|
|
|
* timer to remain active while there are any throttled entities.)
|
|
|
|
*/
|
|
|
|
cfs_b->idle = 0;
|
2011-07-21 18:43:31 +02:00
|
|
|
out_unlock:
|
|
|
|
if (idle)
|
|
|
|
cfs_b->timer_active = 0;
|
|
|
|
raw_spin_unlock(&cfs_b->lock);
|
|
|
|
|
|
|
|
return idle;
|
|
|
|
}
|
2011-07-21 18:43:39 +02:00
|
|
|
|
2011-07-21 18:43:41 +02:00
|
|
|
/* a cfs_rq won't donate quota below this amount */
|
|
|
|
static const u64 min_cfs_rq_runtime = 1 * NSEC_PER_MSEC;
|
|
|
|
/* minimum remaining period time to redistribute slack quota */
|
|
|
|
static const u64 min_bandwidth_expiration = 2 * NSEC_PER_MSEC;
|
|
|
|
/* how long we wait to gather additional slack before distributing */
|
|
|
|
static const u64 cfs_bandwidth_slack_period = 5 * NSEC_PER_MSEC;
|
|
|
|
|
|
|
|
/* are we near the end of the current quota period? */
|
|
|
|
static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire)
|
|
|
|
{
|
|
|
|
struct hrtimer *refresh_timer = &cfs_b->period_timer;
|
|
|
|
u64 remaining;
|
|
|
|
|
|
|
|
/* if the call-back is running a quota refresh is already occurring */
|
|
|
|
if (hrtimer_callback_running(refresh_timer))
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
/* is a quota refresh about to occur? */
|
|
|
|
remaining = ktime_to_ns(hrtimer_expires_remaining(refresh_timer));
|
|
|
|
if (remaining < min_expire)
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b)
|
|
|
|
{
|
|
|
|
u64 min_left = cfs_bandwidth_slack_period + min_bandwidth_expiration;
|
|
|
|
|
|
|
|
/* if there's a quota refresh soon don't bother with slack */
|
|
|
|
if (runtime_refresh_within(cfs_b, min_left))
|
|
|
|
return;
|
|
|
|
|
|
|
|
start_bandwidth_timer(&cfs_b->slack_timer,
|
|
|
|
ns_to_ktime(cfs_bandwidth_slack_period));
|
|
|
|
}
|
|
|
|
|
|
|
|
/* we know any runtime found here is valid as update_curr() precedes return */
|
|
|
|
static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
|
|
|
|
{
|
|
|
|
struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
|
|
|
|
s64 slack_runtime = cfs_rq->runtime_remaining - min_cfs_rq_runtime;
|
|
|
|
|
|
|
|
if (slack_runtime <= 0)
|
|
|
|
return;
|
|
|
|
|
|
|
|
raw_spin_lock(&cfs_b->lock);
|
|
|
|
if (cfs_b->quota != RUNTIME_INF &&
|
|
|
|
cfs_rq->runtime_expires == cfs_b->runtime_expires) {
|
|
|
|
cfs_b->runtime += slack_runtime;
|
|
|
|
|
|
|
|
/* we are under rq->lock, defer unthrottling using a timer */
|
|
|
|
if (cfs_b->runtime > sched_cfs_bandwidth_slice() &&
|
|
|
|
!list_empty(&cfs_b->throttled_cfs_rq))
|
|
|
|
start_cfs_slack_bandwidth(cfs_b);
|
|
|
|
}
|
|
|
|
raw_spin_unlock(&cfs_b->lock);
|
|
|
|
|
|
|
|
/* even if it's not valid for return we don't want to try again */
|
|
|
|
cfs_rq->runtime_remaining -= slack_runtime;
|
|
|
|
}
|
|
|
|
|
|
|
|
static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
|
|
|
|
{
|
2011-11-08 05:26:33 +01:00
|
|
|
if (!cfs_bandwidth_used())
|
|
|
|
return;
|
|
|
|
|
2011-11-08 05:26:34 +01:00
|
|
|
if (!cfs_rq->runtime_enabled || cfs_rq->nr_running)
|
2011-07-21 18:43:41 +02:00
|
|
|
return;
|
|
|
|
|
|
|
|
__return_cfs_rq_runtime(cfs_rq);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This is done with a timer (instead of inline with bandwidth return) since
|
|
|
|
* it's necessary to juggle rq->locks to unthrottle their respective cfs_rqs.
|
|
|
|
*/
|
|
|
|
static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)
|
|
|
|
{
|
|
|
|
u64 runtime = 0, slice = sched_cfs_bandwidth_slice();
|
|
|
|
u64 expires;
|
|
|
|
|
|
|
|
/* confirm we're still not at a refresh boundary */
|
|
|
|
if (runtime_refresh_within(cfs_b, min_bandwidth_expiration))
|
|
|
|
return;
|
|
|
|
|
|
|
|
raw_spin_lock(&cfs_b->lock);
|
|
|
|
if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice) {
|
|
|
|
runtime = cfs_b->runtime;
|
|
|
|
cfs_b->runtime = 0;
|
|
|
|
}
|
|
|
|
expires = cfs_b->runtime_expires;
|
|
|
|
raw_spin_unlock(&cfs_b->lock);
|
|
|
|
|
|
|
|
if (!runtime)
|
|
|
|
return;
|
|
|
|
|
|
|
|
runtime = distribute_cfs_runtime(cfs_b, runtime, expires);
|
|
|
|
|
|
|
|
raw_spin_lock(&cfs_b->lock);
|
|
|
|
if (expires == cfs_b->runtime_expires)
|
|
|
|
cfs_b->runtime = runtime;
|
|
|
|
raw_spin_unlock(&cfs_b->lock);
|
|
|
|
}
|
|
|
|
|
2011-07-21 18:43:39 +02:00
|
|
|
/*
|
|
|
|
* When a group wakes up we want to make sure that its quota is not already
|
|
|
|
* expired/exceeded, otherwise it may be allowed to steal additional ticks of
|
|
|
|
* runtime as update_curr() throttling can not not trigger until it's on-rq.
|
|
|
|
*/
|
|
|
|
static void check_enqueue_throttle(struct cfs_rq *cfs_rq)
|
|
|
|
{
|
2011-11-08 05:26:33 +01:00
|
|
|
if (!cfs_bandwidth_used())
|
|
|
|
return;
|
|
|
|
|
2011-07-21 18:43:39 +02:00
|
|
|
/* an active group must be handled by the update_curr()->put() path */
|
|
|
|
if (!cfs_rq->runtime_enabled || cfs_rq->curr)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/* ensure the group is not already throttled */
|
|
|
|
if (cfs_rq_throttled(cfs_rq))
|
|
|
|
return;
|
|
|
|
|
|
|
|
/* update runtime allocation */
|
|
|
|
account_cfs_rq_runtime(cfs_rq, 0);
|
|
|
|
if (cfs_rq->runtime_remaining <= 0)
|
|
|
|
throttle_cfs_rq(cfs_rq);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* conditionally throttle active cfs_rq's from put_prev_entity() */
|
|
|
|
static void check_cfs_rq_runtime(struct cfs_rq *cfs_rq)
|
|
|
|
{
|
2011-11-08 05:26:33 +01:00
|
|
|
if (!cfs_bandwidth_used())
|
|
|
|
return;
|
|
|
|
|
2011-07-21 18:43:39 +02:00
|
|
|
if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0))
|
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* it's possible for a throttled entity to be forced into a running
|
|
|
|
* state (e.g. set_curr_task), in this case we're finished.
|
|
|
|
*/
|
|
|
|
if (cfs_rq_throttled(cfs_rq))
|
|
|
|
return;
|
|
|
|
|
|
|
|
throttle_cfs_rq(cfs_rq);
|
|
|
|
}
|
2011-10-25 10:00:11 +02:00
|
|
|
|
|
|
|
static inline u64 default_cfs_period(void);
|
|
|
|
static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun);
|
|
|
|
static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b);
|
|
|
|
|
|
|
|
static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer)
|
|
|
|
{
|
|
|
|
struct cfs_bandwidth *cfs_b =
|
|
|
|
container_of(timer, struct cfs_bandwidth, slack_timer);
|
|
|
|
do_sched_cfs_slack_timer(cfs_b);
|
|
|
|
|
|
|
|
return HRTIMER_NORESTART;
|
|
|
|
}
|
|
|
|
|
|
|
|
static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer)
|
|
|
|
{
|
|
|
|
struct cfs_bandwidth *cfs_b =
|
|
|
|
container_of(timer, struct cfs_bandwidth, period_timer);
|
|
|
|
ktime_t now;
|
|
|
|
int overrun;
|
|
|
|
int idle = 0;
|
|
|
|
|
|
|
|
for (;;) {
|
|
|
|
now = hrtimer_cb_get_time(timer);
|
|
|
|
overrun = hrtimer_forward(timer, now, cfs_b->period);
|
|
|
|
|
|
|
|
if (!overrun)
|
|
|
|
break;
|
|
|
|
|
|
|
|
idle = do_sched_cfs_period_timer(cfs_b, overrun);
|
|
|
|
}
|
|
|
|
|
|
|
|
return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
|
|
|
|
}
|
|
|
|
|
|
|
|
void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
|
|
|
|
{
|
|
|
|
raw_spin_lock_init(&cfs_b->lock);
|
|
|
|
cfs_b->runtime = 0;
|
|
|
|
cfs_b->quota = RUNTIME_INF;
|
|
|
|
cfs_b->period = ns_to_ktime(default_cfs_period());
|
|
|
|
|
|
|
|
INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq);
|
|
|
|
hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
|
|
|
|
cfs_b->period_timer.function = sched_cfs_period_timer;
|
|
|
|
hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
|
|
|
|
cfs_b->slack_timer.function = sched_cfs_slack_timer;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq)
|
|
|
|
{
|
|
|
|
cfs_rq->runtime_enabled = 0;
|
|
|
|
INIT_LIST_HEAD(&cfs_rq->throttled_list);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* requires cfs_b->lock, may release to reprogram timer */
|
|
|
|
void __start_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* The timer may be active because we're trying to set a new bandwidth
|
|
|
|
* period or because we're racing with the tear-down path
|
|
|
|
* (timer_active==0 becomes visible before the hrtimer call-back
|
|
|
|
* terminates). In either case we ensure that it's re-programmed
|
|
|
|
*/
|
|
|
|
while (unlikely(hrtimer_active(&cfs_b->period_timer))) {
|
|
|
|
raw_spin_unlock(&cfs_b->lock);
|
|
|
|
/* ensure cfs_b->lock is available while we wait */
|
|
|
|
hrtimer_cancel(&cfs_b->period_timer);
|
|
|
|
|
|
|
|
raw_spin_lock(&cfs_b->lock);
|
|
|
|
/* if someone else restarted the timer then we're done */
|
|
|
|
if (cfs_b->timer_active)
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
cfs_b->timer_active = 1;
|
|
|
|
start_bandwidth_timer(&cfs_b->period_timer, cfs_b->period);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
|
|
|
|
{
|
|
|
|
hrtimer_cancel(&cfs_b->period_timer);
|
|
|
|
hrtimer_cancel(&cfs_b->slack_timer);
|
|
|
|
}
|
|
|
|
|
|
|
|
void unthrottle_offline_cfs_rqs(struct rq *rq)
|
|
|
|
{
|
|
|
|
struct cfs_rq *cfs_rq;
|
|
|
|
|
|
|
|
for_each_leaf_cfs_rq(rq, cfs_rq) {
|
|
|
|
struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
|
|
|
|
|
|
|
|
if (!cfs_rq->runtime_enabled)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* clock_task is not advancing so we just need to make sure
|
|
|
|
* there's some valid quota amount
|
|
|
|
*/
|
|
|
|
cfs_rq->runtime_remaining = cfs_b->quota;
|
|
|
|
if (cfs_rq_throttled(cfs_rq))
|
|
|
|
unthrottle_cfs_rq(cfs_rq);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#else /* CONFIG_CFS_BANDWIDTH */
|
2011-07-21 18:43:30 +02:00
|
|
|
static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
|
|
|
|
unsigned long delta_exec) {}
|
2011-07-21 18:43:39 +02:00
|
|
|
static void check_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
|
|
|
|
static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {}
|
2011-07-21 18:43:41 +02:00
|
|
|
static void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
|
2011-07-21 18:43:33 +02:00
|
|
|
|
|
|
|
static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
2011-07-21 18:43:36 +02:00
|
|
|
|
|
|
|
static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int throttled_lb_pair(struct task_group *tg,
|
|
|
|
int src_cpu, int dest_cpu)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
2011-10-25 10:00:11 +02:00
|
|
|
|
|
|
|
void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
|
|
|
|
|
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
|
|
|
static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
|
2011-07-21 18:43:28 +02:00
|
|
|
#endif
|
|
|
|
|
2011-10-25 10:00:11 +02:00
|
|
|
static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
|
|
|
|
{
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
|
|
|
|
void unthrottle_offline_cfs_rqs(struct rq *rq) {}
|
|
|
|
|
|
|
|
#endif /* CONFIG_CFS_BANDWIDTH */
|
|
|
|
|
2007-07-09 18:51:58 +02:00
|
|
|
/**************************************************
|
|
|
|
* CFS operations on tasks:
|
|
|
|
*/
|
|
|
|
|
2008-01-25 21:08:29 +01:00
|
|
|
#ifdef CONFIG_SCHED_HRTICK
|
|
|
|
static void hrtick_start_fair(struct rq *rq, struct task_struct *p)
|
|
|
|
{
|
|
|
|
struct sched_entity *se = &p->se;
|
|
|
|
struct cfs_rq *cfs_rq = cfs_rq_of(se);
|
|
|
|
|
|
|
|
WARN_ON(task_rq(p) != rq);
|
|
|
|
|
2011-11-22 15:20:07 +01:00
|
|
|
if (cfs_rq->nr_running > 1) {
|
2008-01-25 21:08:29 +01:00
|
|
|
u64 slice = sched_slice(cfs_rq, se);
|
|
|
|
u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime;
|
|
|
|
s64 delta = slice - ran;
|
|
|
|
|
|
|
|
if (delta < 0) {
|
|
|
|
if (rq->curr == p)
|
|
|
|
resched_task(p);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Don't schedule slices shorter than 10000ns, that just
|
|
|
|
* doesn't make sense. Rely on vruntime for fairness.
|
|
|
|
*/
|
2008-07-18 18:01:23 +02:00
|
|
|
if (rq->curr != p)
|
2008-07-28 11:53:11 +02:00
|
|
|
delta = max_t(s64, 10000LL, delta);
|
2008-01-25 21:08:29 +01:00
|
|
|
|
2008-07-18 18:01:23 +02:00
|
|
|
hrtick_start(rq, delta);
|
2008-01-25 21:08:29 +01:00
|
|
|
}
|
|
|
|
}
|
2008-10-17 19:27:03 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* called from enqueue/dequeue and updates the hrtick when the
|
|
|
|
* current task is from our class and nr_running is low enough
|
|
|
|
* to matter.
|
|
|
|
*/
|
|
|
|
static void hrtick_update(struct rq *rq)
|
|
|
|
{
|
|
|
|
struct task_struct *curr = rq->curr;
|
|
|
|
|
2011-11-22 15:20:07 +01:00
|
|
|
if (!hrtick_enabled(rq) || curr->sched_class != &fair_sched_class)
|
2008-10-17 19:27:03 +02:00
|
|
|
return;
|
|
|
|
|
|
|
|
if (cfs_rq_of(&curr->se)->nr_running < sched_nr_latency)
|
|
|
|
hrtick_start_fair(rq, curr);
|
|
|
|
}
|
2008-06-24 20:09:43 +02:00
|
|
|
#else /* !CONFIG_SCHED_HRTICK */
|
2008-01-25 21:08:29 +01:00
|
|
|
static inline void
|
|
|
|
hrtick_start_fair(struct rq *rq, struct task_struct *p)
|
|
|
|
{
|
|
|
|
}
|
2008-10-17 19:27:03 +02:00
|
|
|
|
|
|
|
static inline void hrtick_update(struct rq *rq)
|
|
|
|
{
|
|
|
|
}
|
2008-01-25 21:08:29 +01:00
|
|
|
#endif
|
|
|
|
|
2007-07-09 18:51:58 +02:00
|
|
|
/*
|
|
|
|
* The enqueue_task method is called before nr_running is
|
|
|
|
* increased. Here we update the fair scheduling stats and
|
|
|
|
* then put the task into the rbtree:
|
|
|
|
*/
|
2010-01-20 21:58:57 +01:00
|
|
|
static void
|
2010-03-24 16:38:48 +01:00
|
|
|
enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
|
|
|
struct cfs_rq *cfs_rq;
|
2008-02-25 17:34:02 +01:00
|
|
|
struct sched_entity *se = &p->se;
|
2007-07-09 18:51:58 +02:00
|
|
|
|
|
|
|
for_each_sched_entity(se) {
|
2008-02-25 17:34:02 +01:00
|
|
|
if (se->on_rq)
|
2007-07-09 18:51:58 +02:00
|
|
|
break;
|
|
|
|
cfs_rq = cfs_rq_of(se);
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 18:04:41 +01:00
|
|
|
enqueue_entity(cfs_rq, se, flags);
|
2011-07-21 18:43:33 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* end evaluation on encountering a throttled cfs_rq
|
|
|
|
*
|
|
|
|
* note: in the case of encountering a throttled cfs_rq we will
|
|
|
|
* post the final h_nr_running increment below.
|
|
|
|
*/
|
|
|
|
if (cfs_rq_throttled(cfs_rq))
|
|
|
|
break;
|
2011-07-21 18:43:27 +02:00
|
|
|
cfs_rq->h_nr_running++;
|
2011-07-21 18:43:33 +02:00
|
|
|
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 18:04:41 +01:00
|
|
|
flags = ENQUEUE_WAKEUP;
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
2008-01-25 21:08:29 +01:00
|
|
|
|
2010-11-16 00:47:00 +01:00
|
|
|
for_each_sched_entity(se) {
|
2011-07-22 03:14:31 +02:00
|
|
|
cfs_rq = cfs_rq_of(se);
|
2011-07-21 18:43:27 +02:00
|
|
|
cfs_rq->h_nr_running++;
|
2010-11-16 00:47:00 +01:00
|
|
|
|
2011-07-21 18:43:33 +02:00
|
|
|
if (cfs_rq_throttled(cfs_rq))
|
|
|
|
break;
|
|
|
|
|
2010-11-16 00:47:09 +01:00
|
|
|
update_cfs_load(cfs_rq, 0);
|
2011-01-22 05:45:01 +01:00
|
|
|
update_cfs_shares(cfs_rq);
|
2010-11-16 00:47:00 +01:00
|
|
|
}
|
|
|
|
|
2011-07-21 18:43:33 +02:00
|
|
|
if (!se)
|
|
|
|
inc_nr_running(rq);
|
2008-10-17 19:27:03 +02:00
|
|
|
hrtick_update(rq);
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
2011-04-14 19:30:53 +02:00
|
|
|
static void set_next_buddy(struct sched_entity *se);
|
|
|
|
|
2007-07-09 18:51:58 +02:00
|
|
|
/*
|
|
|
|
* The dequeue_task method is called before nr_running is
|
|
|
|
* decreased. We remove the task from the rbtree and
|
|
|
|
* update the fair scheduling stats:
|
|
|
|
*/
|
2010-03-24 16:38:48 +01:00
|
|
|
static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
|
|
|
struct cfs_rq *cfs_rq;
|
2008-02-25 17:34:02 +01:00
|
|
|
struct sched_entity *se = &p->se;
|
2011-04-14 19:30:53 +02:00
|
|
|
int task_sleep = flags & DEQUEUE_SLEEP;
|
2007-07-09 18:51:58 +02:00
|
|
|
|
|
|
|
for_each_sched_entity(se) {
|
|
|
|
cfs_rq = cfs_rq_of(se);
|
2010-03-24 16:38:48 +01:00
|
|
|
dequeue_entity(cfs_rq, se, flags);
|
2011-07-21 18:43:33 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* end evaluation on encountering a throttled cfs_rq
|
|
|
|
*
|
|
|
|
* note: in the case of encountering a throttled cfs_rq we will
|
|
|
|
* post the final h_nr_running decrement below.
|
|
|
|
*/
|
|
|
|
if (cfs_rq_throttled(cfs_rq))
|
|
|
|
break;
|
2011-07-21 18:43:27 +02:00
|
|
|
cfs_rq->h_nr_running--;
|
2010-11-16 00:47:00 +01:00
|
|
|
|
2007-07-09 18:51:58 +02:00
|
|
|
/* Don't dequeue parent if it has other entities besides us */
|
2011-04-14 19:30:53 +02:00
|
|
|
if (cfs_rq->load.weight) {
|
|
|
|
/*
|
|
|
|
* Bias pick_next to pick a task from this cfs_rq, as
|
|
|
|
* p is sleeping when it is within its sched_slice.
|
|
|
|
*/
|
|
|
|
if (task_sleep && parent_entity(se))
|
|
|
|
set_next_buddy(parent_entity(se));
|
2011-07-07 07:30:37 +02:00
|
|
|
|
|
|
|
/* avoid re-evaluating load for this entity */
|
|
|
|
se = parent_entity(se);
|
2007-07-09 18:51:58 +02:00
|
|
|
break;
|
2011-04-14 19:30:53 +02:00
|
|
|
}
|
2010-03-24 16:38:48 +01:00
|
|
|
flags |= DEQUEUE_SLEEP;
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
2008-01-25 21:08:29 +01:00
|
|
|
|
2010-11-16 00:47:00 +01:00
|
|
|
for_each_sched_entity(se) {
|
2011-07-22 03:14:31 +02:00
|
|
|
cfs_rq = cfs_rq_of(se);
|
2011-07-21 18:43:27 +02:00
|
|
|
cfs_rq->h_nr_running--;
|
2010-11-16 00:47:00 +01:00
|
|
|
|
2011-07-21 18:43:33 +02:00
|
|
|
if (cfs_rq_throttled(cfs_rq))
|
|
|
|
break;
|
|
|
|
|
2010-11-16 00:47:09 +01:00
|
|
|
update_cfs_load(cfs_rq, 0);
|
2011-01-22 05:45:01 +01:00
|
|
|
update_cfs_shares(cfs_rq);
|
2010-11-16 00:47:00 +01:00
|
|
|
}
|
|
|
|
|
2011-07-21 18:43:33 +02:00
|
|
|
if (!se)
|
|
|
|
dec_nr_running(rq);
|
2008-10-17 19:27:03 +02:00
|
|
|
hrtick_update(rq);
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
2008-01-25 21:08:09 +01:00
|
|
|
#ifdef CONFIG_SMP
|
2011-10-25 10:00:11 +02:00
|
|
|
/* Used instead of source_load when we know the type == 0 */
|
|
|
|
static unsigned long weighted_cpuload(const int cpu)
|
|
|
|
{
|
|
|
|
return cpu_rq(cpu)->load.weight;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Return a low guess at the load of a migration-source cpu weighted
|
|
|
|
* according to the scheduling class and "nice" value.
|
|
|
|
*
|
|
|
|
* We want to under-estimate the load of migration sources, to
|
|
|
|
* balance conservatively.
|
|
|
|
*/
|
|
|
|
static unsigned long source_load(int cpu, int type)
|
|
|
|
{
|
|
|
|
struct rq *rq = cpu_rq(cpu);
|
|
|
|
unsigned long total = weighted_cpuload(cpu);
|
|
|
|
|
|
|
|
if (type == 0 || !sched_feat(LB_BIAS))
|
|
|
|
return total;
|
|
|
|
|
|
|
|
return min(rq->cpu_load[type-1], total);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Return a high guess at the load of a migration-target cpu weighted
|
|
|
|
* according to the scheduling class and "nice" value.
|
|
|
|
*/
|
|
|
|
static unsigned long target_load(int cpu, int type)
|
|
|
|
{
|
|
|
|
struct rq *rq = cpu_rq(cpu);
|
|
|
|
unsigned long total = weighted_cpuload(cpu);
|
|
|
|
|
|
|
|
if (type == 0 || !sched_feat(LB_BIAS))
|
|
|
|
return total;
|
|
|
|
|
|
|
|
return max(rq->cpu_load[type-1], total);
|
|
|
|
}
|
|
|
|
|
|
|
|
static unsigned long power_of(int cpu)
|
|
|
|
{
|
|
|
|
return cpu_rq(cpu)->cpu_power;
|
|
|
|
}
|
|
|
|
|
|
|
|
static unsigned long cpu_avg_load_per_task(int cpu)
|
|
|
|
{
|
|
|
|
struct rq *rq = cpu_rq(cpu);
|
|
|
|
unsigned long nr_running = ACCESS_ONCE(rq->nr_running);
|
|
|
|
|
|
|
|
if (nr_running)
|
|
|
|
return rq->load.weight / nr_running;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2008-03-16 20:36:10 +01:00
|
|
|
|
2011-04-05 17:23:47 +02:00
|
|
|
static void task_waking_fair(struct task_struct *p)
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 18:04:41 +01:00
|
|
|
{
|
|
|
|
struct sched_entity *se = &p->se;
|
|
|
|
struct cfs_rq *cfs_rq = cfs_rq_of(se);
|
2011-04-05 17:23:48 +02:00
|
|
|
u64 min_vruntime;
|
|
|
|
|
|
|
|
#ifndef CONFIG_64BIT
|
|
|
|
u64 min_vruntime_copy;
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 18:04:41 +01:00
|
|
|
|
2011-04-05 17:23:48 +02:00
|
|
|
do {
|
|
|
|
min_vruntime_copy = cfs_rq->min_vruntime_copy;
|
|
|
|
smp_rmb();
|
|
|
|
min_vruntime = cfs_rq->min_vruntime;
|
|
|
|
} while (min_vruntime != min_vruntime_copy);
|
|
|
|
#else
|
|
|
|
min_vruntime = cfs_rq->min_vruntime;
|
|
|
|
#endif
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 18:04:41 +01:00
|
|
|
|
2011-04-05 17:23:48 +02:00
|
|
|
se->vruntime -= min_vruntime;
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 18:04:41 +01:00
|
|
|
}
|
|
|
|
|
2008-06-27 13:41:27 +02:00
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
2008-06-27 13:41:39 +02:00
|
|
|
/*
|
|
|
|
* effective_load() calculates the load change as seen from the root_task_group
|
|
|
|
*
|
|
|
|
* Adding load to a group doesn't make a group heavier, but can cause movement
|
|
|
|
* of group shares between cpus. Assuming the shares were perfectly aligned one
|
|
|
|
* can calculate the shift in shares.
|
2011-10-13 16:52:28 +02:00
|
|
|
*
|
|
|
|
* Calculate the effective load difference if @wl is added (subtracted) to @tg
|
|
|
|
* on this @cpu and results in a total addition (subtraction) of @wg to the
|
|
|
|
* total group weight.
|
|
|
|
*
|
|
|
|
* Given a runqueue weight distribution (rw_i) we can compute a shares
|
|
|
|
* distribution (s_i) using:
|
|
|
|
*
|
|
|
|
* s_i = rw_i / \Sum rw_j (1)
|
|
|
|
*
|
|
|
|
* Suppose we have 4 CPUs and our @tg is a direct child of the root group and
|
|
|
|
* has 7 equal weight tasks, distributed as below (rw_i), with the resulting
|
|
|
|
* shares distribution (s_i):
|
|
|
|
*
|
|
|
|
* rw_i = { 2, 4, 1, 0 }
|
|
|
|
* s_i = { 2/7, 4/7, 1/7, 0 }
|
|
|
|
*
|
|
|
|
* As per wake_affine() we're interested in the load of two CPUs (the CPU the
|
|
|
|
* task used to run on and the CPU the waker is running on), we need to
|
|
|
|
* compute the effect of waking a task on either CPU and, in case of a sync
|
|
|
|
* wakeup, compute the effect of the current task going to sleep.
|
|
|
|
*
|
|
|
|
* So for a change of @wl to the local @cpu with an overall group weight change
|
|
|
|
* of @wl we can compute the new shares distribution (s'_i) using:
|
|
|
|
*
|
|
|
|
* s'_i = (rw_i + @wl) / (@wg + \Sum rw_j) (2)
|
|
|
|
*
|
|
|
|
* Suppose we're interested in CPUs 0 and 1, and want to compute the load
|
|
|
|
* differences in waking a task to CPU 0. The additional task changes the
|
|
|
|
* weight and shares distributions like:
|
|
|
|
*
|
|
|
|
* rw'_i = { 3, 4, 1, 0 }
|
|
|
|
* s'_i = { 3/8, 4/8, 1/8, 0 }
|
|
|
|
*
|
|
|
|
* We can then compute the difference in effective weight by using:
|
|
|
|
*
|
|
|
|
* dw_i = S * (s'_i - s_i) (3)
|
|
|
|
*
|
|
|
|
* Where 'S' is the group weight as seen by its parent.
|
|
|
|
*
|
|
|
|
* Therefore the effective change in loads on CPU 0 would be 5/56 (3/8 - 2/7)
|
|
|
|
* times the weight of the group. The effect on CPU 1 would be -4/56 (4/8 -
|
|
|
|
* 4/7) times the weight of the group.
|
2008-06-27 13:41:39 +02:00
|
|
|
*/
|
2010-11-16 00:47:00 +01:00
|
|
|
static long effective_load(struct task_group *tg, int cpu, long wl, long wg)
|
2008-06-27 13:41:27 +02:00
|
|
|
{
|
2008-06-27 13:41:30 +02:00
|
|
|
struct sched_entity *se = tg->se[cpu];
|
2008-06-27 13:41:38 +02:00
|
|
|
|
2011-10-13 16:52:28 +02:00
|
|
|
if (!tg->parent) /* the trivial, non-cgroup case */
|
2008-06-27 13:41:38 +02:00
|
|
|
return wl;
|
|
|
|
|
2008-06-27 13:41:30 +02:00
|
|
|
for_each_sched_entity(se) {
|
2011-10-13 16:52:28 +02:00
|
|
|
long w, W;
|
2008-06-27 13:41:30 +02:00
|
|
|
|
2011-01-15 02:57:50 +01:00
|
|
|
tg = se->my_q->tg;
|
2008-06-27 13:41:27 +02:00
|
|
|
|
2011-10-13 16:52:28 +02:00
|
|
|
/*
|
|
|
|
* W = @wg + \Sum rw_j
|
|
|
|
*/
|
|
|
|
W = wg + calc_tg_weight(tg, se->my_q);
|
2008-06-27 13:41:30 +02:00
|
|
|
|
2011-10-13 16:52:28 +02:00
|
|
|
/*
|
|
|
|
* w = rw_i + @wl
|
|
|
|
*/
|
|
|
|
w = se->my_q->load.weight + wl;
|
2008-09-23 15:33:42 +02:00
|
|
|
|
2011-10-13 16:52:28 +02:00
|
|
|
/*
|
|
|
|
* wl = S * s'_i; see (2)
|
|
|
|
*/
|
|
|
|
if (W > 0 && w < W)
|
|
|
|
wl = (w * tg->shares) / W;
|
2011-01-15 02:57:50 +01:00
|
|
|
else
|
|
|
|
wl = tg->shares;
|
2008-09-23 15:33:42 +02:00
|
|
|
|
2011-10-13 16:52:28 +02:00
|
|
|
/*
|
|
|
|
* Per the above, wl is the new se->load.weight value; since
|
|
|
|
* those are clipped to [MIN_SHARES, ...) do so now. See
|
|
|
|
* calc_cfs_shares().
|
|
|
|
*/
|
2011-01-15 02:57:50 +01:00
|
|
|
if (wl < MIN_SHARES)
|
|
|
|
wl = MIN_SHARES;
|
2011-10-13 16:52:28 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* wl = dw_i = S * (s'_i - s_i); see (3)
|
|
|
|
*/
|
2011-01-15 02:57:50 +01:00
|
|
|
wl -= se->load.weight;
|
2011-10-13 16:52:28 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Recursively apply this logic to all parent groups to compute
|
|
|
|
* the final effective load change on the root group. Since
|
|
|
|
* only the @tg group gets extra weight, all parent groups can
|
|
|
|
* only redistribute existing shares. @wl is the shift in shares
|
|
|
|
* resulting from this level per the above.
|
|
|
|
*/
|
2008-06-27 13:41:30 +02:00
|
|
|
wg = 0;
|
|
|
|
}
|
2008-06-27 13:41:27 +02:00
|
|
|
|
2008-06-27 13:41:30 +02:00
|
|
|
return wl;
|
2008-06-27 13:41:27 +02:00
|
|
|
}
|
|
|
|
#else
|
2008-06-27 13:41:30 +02:00
|
|
|
|
sched: correct wakeup weight calculations
rw_i = {2, 4, 1, 0}
s_i = {2/7, 4/7, 1/7, 0}
wakeup on cpu0, weight=1
rw'_i = {3, 4, 1, 0}
s'_i = {3/8, 4/8, 1/8, 0}
s_0 = S * rw_0 / \Sum rw_j ->
\Sum rw_j = S*rw_0/s_0 = 1*2*7/2 = 7 (correct)
s'_0 = S * (rw_0 + 1) / (\Sum rw_j + 1) =
1 * (2+1) / (7+1) = 3/8 (correct
so we find that adding 1 to cpu0 gains 5/56 in weight
if say the other cpu were, cpu1, we'd also have to calculate its 4/56 loss
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com>
Cc: Mike Galbraith <efault@gmx.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-06-27 13:41:37 +02:00
|
|
|
static inline unsigned long effective_load(struct task_group *tg, int cpu,
|
|
|
|
unsigned long wl, unsigned long wg)
|
2008-06-27 13:41:30 +02:00
|
|
|
{
|
sched: correct wakeup weight calculations
rw_i = {2, 4, 1, 0}
s_i = {2/7, 4/7, 1/7, 0}
wakeup on cpu0, weight=1
rw'_i = {3, 4, 1, 0}
s'_i = {3/8, 4/8, 1/8, 0}
s_0 = S * rw_0 / \Sum rw_j ->
\Sum rw_j = S*rw_0/s_0 = 1*2*7/2 = 7 (correct)
s'_0 = S * (rw_0 + 1) / (\Sum rw_j + 1) =
1 * (2+1) / (7+1) = 3/8 (correct
so we find that adding 1 to cpu0 gains 5/56 in weight
if say the other cpu were, cpu1, we'd also have to calculate its 4/56 loss
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com>
Cc: Mike Galbraith <efault@gmx.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-06-27 13:41:37 +02:00
|
|
|
return wl;
|
2008-06-27 13:41:27 +02:00
|
|
|
}
|
2008-06-27 13:41:30 +02:00
|
|
|
|
2008-06-27 13:41:27 +02:00
|
|
|
#endif
|
|
|
|
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 13:50:02 +02:00
|
|
|
static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync)
|
2008-03-16 20:36:10 +01:00
|
|
|
{
|
2011-01-22 05:44:59 +01:00
|
|
|
s64 this_load, load;
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 13:50:02 +02:00
|
|
|
int idx, this_cpu, prev_cpu;
|
2008-03-16 20:36:10 +01:00
|
|
|
unsigned long tl_per_task;
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 13:50:02 +02:00
|
|
|
struct task_group *tg;
|
sched: correct wakeup weight calculations
rw_i = {2, 4, 1, 0}
s_i = {2/7, 4/7, 1/7, 0}
wakeup on cpu0, weight=1
rw'_i = {3, 4, 1, 0}
s'_i = {3/8, 4/8, 1/8, 0}
s_0 = S * rw_0 / \Sum rw_j ->
\Sum rw_j = S*rw_0/s_0 = 1*2*7/2 = 7 (correct)
s'_0 = S * (rw_0 + 1) / (\Sum rw_j + 1) =
1 * (2+1) / (7+1) = 3/8 (correct
so we find that adding 1 to cpu0 gains 5/56 in weight
if say the other cpu were, cpu1, we'd also have to calculate its 4/56 loss
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com>
Cc: Mike Galbraith <efault@gmx.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-06-27 13:41:37 +02:00
|
|
|
unsigned long weight;
|
2008-05-29 11:11:41 +02:00
|
|
|
int balanced;
|
2008-03-16 20:36:10 +01:00
|
|
|
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 13:50:02 +02:00
|
|
|
idx = sd->wake_idx;
|
|
|
|
this_cpu = smp_processor_id();
|
|
|
|
prev_cpu = task_cpu(p);
|
|
|
|
load = source_load(prev_cpu, idx);
|
|
|
|
this_load = target_load(this_cpu, idx);
|
2008-03-16 20:36:10 +01:00
|
|
|
|
2008-05-29 11:11:41 +02:00
|
|
|
/*
|
|
|
|
* If sync wakeup then subtract the (maximum possible)
|
|
|
|
* effect of the currently running task from the load
|
|
|
|
* of the current CPU:
|
|
|
|
*/
|
sched: correct wakeup weight calculations
rw_i = {2, 4, 1, 0}
s_i = {2/7, 4/7, 1/7, 0}
wakeup on cpu0, weight=1
rw'_i = {3, 4, 1, 0}
s'_i = {3/8, 4/8, 1/8, 0}
s_0 = S * rw_0 / \Sum rw_j ->
\Sum rw_j = S*rw_0/s_0 = 1*2*7/2 = 7 (correct)
s'_0 = S * (rw_0 + 1) / (\Sum rw_j + 1) =
1 * (2+1) / (7+1) = 3/8 (correct
so we find that adding 1 to cpu0 gains 5/56 in weight
if say the other cpu were, cpu1, we'd also have to calculate its 4/56 loss
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com>
Cc: Mike Galbraith <efault@gmx.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-06-27 13:41:37 +02:00
|
|
|
if (sync) {
|
|
|
|
tg = task_group(current);
|
|
|
|
weight = current->se.load.weight;
|
|
|
|
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 13:50:02 +02:00
|
|
|
this_load += effective_load(tg, this_cpu, -weight, -weight);
|
sched: correct wakeup weight calculations
rw_i = {2, 4, 1, 0}
s_i = {2/7, 4/7, 1/7, 0}
wakeup on cpu0, weight=1
rw'_i = {3, 4, 1, 0}
s'_i = {3/8, 4/8, 1/8, 0}
s_0 = S * rw_0 / \Sum rw_j ->
\Sum rw_j = S*rw_0/s_0 = 1*2*7/2 = 7 (correct)
s'_0 = S * (rw_0 + 1) / (\Sum rw_j + 1) =
1 * (2+1) / (7+1) = 3/8 (correct
so we find that adding 1 to cpu0 gains 5/56 in weight
if say the other cpu were, cpu1, we'd also have to calculate its 4/56 loss
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com>
Cc: Mike Galbraith <efault@gmx.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-06-27 13:41:37 +02:00
|
|
|
load += effective_load(tg, prev_cpu, 0, -weight);
|
|
|
|
}
|
2008-05-29 11:11:41 +02:00
|
|
|
|
sched: correct wakeup weight calculations
rw_i = {2, 4, 1, 0}
s_i = {2/7, 4/7, 1/7, 0}
wakeup on cpu0, weight=1
rw'_i = {3, 4, 1, 0}
s'_i = {3/8, 4/8, 1/8, 0}
s_0 = S * rw_0 / \Sum rw_j ->
\Sum rw_j = S*rw_0/s_0 = 1*2*7/2 = 7 (correct)
s'_0 = S * (rw_0 + 1) / (\Sum rw_j + 1) =
1 * (2+1) / (7+1) = 3/8 (correct
so we find that adding 1 to cpu0 gains 5/56 in weight
if say the other cpu were, cpu1, we'd also have to calculate its 4/56 loss
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com>
Cc: Mike Galbraith <efault@gmx.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-06-27 13:41:37 +02:00
|
|
|
tg = task_group(p);
|
|
|
|
weight = p->se.load.weight;
|
2008-05-29 11:11:41 +02:00
|
|
|
|
2009-09-07 18:28:05 +02:00
|
|
|
/*
|
|
|
|
* In low-load situations, where prev_cpu is idle and this_cpu is idle
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 13:50:02 +02:00
|
|
|
* due to the sync cause above having dropped this_load to 0, we'll
|
|
|
|
* always have an imbalance, but there's really nothing you can do
|
|
|
|
* about that, so that's good too.
|
2009-09-07 18:28:05 +02:00
|
|
|
*
|
|
|
|
* Otherwise check if either cpus are near enough in load to allow this
|
|
|
|
* task to be woken on this_cpu.
|
|
|
|
*/
|
2011-01-22 05:44:59 +01:00
|
|
|
if (this_load > 0) {
|
|
|
|
s64 this_eff_load, prev_eff_load;
|
2010-05-31 12:37:30 +02:00
|
|
|
|
|
|
|
this_eff_load = 100;
|
|
|
|
this_eff_load *= power_of(prev_cpu);
|
|
|
|
this_eff_load *= this_load +
|
|
|
|
effective_load(tg, this_cpu, weight, weight);
|
|
|
|
|
|
|
|
prev_eff_load = 100 + (sd->imbalance_pct - 100) / 2;
|
|
|
|
prev_eff_load *= power_of(this_cpu);
|
|
|
|
prev_eff_load *= load + effective_load(tg, prev_cpu, 0, weight);
|
|
|
|
|
|
|
|
balanced = this_eff_load <= prev_eff_load;
|
|
|
|
} else
|
|
|
|
balanced = true;
|
2008-05-29 11:11:41 +02:00
|
|
|
|
2008-03-16 20:36:10 +01:00
|
|
|
/*
|
2008-03-19 01:42:00 +01:00
|
|
|
* If the currently running task will sleep within
|
|
|
|
* a reasonable amount of time then attract this newly
|
|
|
|
* woken task:
|
2008-03-16 20:36:10 +01:00
|
|
|
*/
|
2008-10-08 09:16:04 +02:00
|
|
|
if (sync && balanced)
|
|
|
|
return 1;
|
2008-03-16 20:36:10 +01:00
|
|
|
|
2010-03-11 03:37:45 +01:00
|
|
|
schedstat_inc(p, se.statistics.nr_wakeups_affine_attempts);
|
2008-03-16 20:36:10 +01:00
|
|
|
tl_per_task = cpu_avg_load_per_task(this_cpu);
|
|
|
|
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 13:50:02 +02:00
|
|
|
if (balanced ||
|
|
|
|
(this_load <= load &&
|
|
|
|
this_load + target_load(prev_cpu, idx) <= tl_per_task)) {
|
2008-03-16 20:36:10 +01:00
|
|
|
/*
|
|
|
|
* This domain has SD_WAKE_AFFINE and
|
|
|
|
* p is cache cold in this domain, and
|
|
|
|
* there is no bad imbalance.
|
|
|
|
*/
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 13:50:02 +02:00
|
|
|
schedstat_inc(sd, ttwu_move_affine);
|
2010-03-11 03:37:45 +01:00
|
|
|
schedstat_inc(p, se.statistics.nr_wakeups_affine);
|
2008-03-16 20:36:10 +01:00
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2009-09-10 13:36:25 +02:00
|
|
|
/*
|
|
|
|
* find_idlest_group finds and returns the least busy CPU group within the
|
|
|
|
* domain.
|
|
|
|
*/
|
|
|
|
static struct sched_group *
|
2009-09-03 13:16:51 +02:00
|
|
|
find_idlest_group(struct sched_domain *sd, struct task_struct *p,
|
2009-09-16 13:46:59 +02:00
|
|
|
int this_cpu, int load_idx)
|
2008-01-25 21:08:09 +01:00
|
|
|
{
|
2010-08-10 23:17:51 +02:00
|
|
|
struct sched_group *idlest = NULL, *group = sd->groups;
|
2009-09-10 13:36:25 +02:00
|
|
|
unsigned long min_load = ULONG_MAX, this_load = 0;
|
|
|
|
int imbalance = 100 + (sd->imbalance_pct-100)/2;
|
2008-01-25 21:08:09 +01:00
|
|
|
|
2009-09-10 13:36:25 +02:00
|
|
|
do {
|
|
|
|
unsigned long load, avg_load;
|
|
|
|
int local_group;
|
|
|
|
int i;
|
2008-01-25 21:08:09 +01:00
|
|
|
|
2009-09-10 13:36:25 +02:00
|
|
|
/* Skip over this group if it has no CPUs allowed */
|
|
|
|
if (!cpumask_intersects(sched_group_cpus(group),
|
2011-06-16 12:23:22 +02:00
|
|
|
tsk_cpus_allowed(p)))
|
2009-09-10 13:36:25 +02:00
|
|
|
continue;
|
|
|
|
|
|
|
|
local_group = cpumask_test_cpu(this_cpu,
|
|
|
|
sched_group_cpus(group));
|
|
|
|
|
|
|
|
/* Tally up the load of all CPUs in the group */
|
|
|
|
avg_load = 0;
|
|
|
|
|
|
|
|
for_each_cpu(i, sched_group_cpus(group)) {
|
|
|
|
/* Bias balancing toward cpus of our domain */
|
|
|
|
if (local_group)
|
|
|
|
load = source_load(i, load_idx);
|
|
|
|
else
|
|
|
|
load = target_load(i, load_idx);
|
|
|
|
|
|
|
|
avg_load += load;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Adjust by relative CPU power of the group */
|
2011-07-14 13:00:06 +02:00
|
|
|
avg_load = (avg_load * SCHED_POWER_SCALE) / group->sgp->power;
|
2009-09-10 13:36:25 +02:00
|
|
|
|
|
|
|
if (local_group) {
|
|
|
|
this_load = avg_load;
|
|
|
|
} else if (avg_load < min_load) {
|
|
|
|
min_load = avg_load;
|
|
|
|
idlest = group;
|
|
|
|
}
|
|
|
|
} while (group = group->next, group != sd->groups);
|
|
|
|
|
|
|
|
if (!idlest || 100*this_load < imbalance*min_load)
|
|
|
|
return NULL;
|
|
|
|
return idlest;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* find_idlest_cpu - find the idlest cpu among the cpus in group.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
|
|
|
|
{
|
|
|
|
unsigned long load, min_load = ULONG_MAX;
|
|
|
|
int idlest = -1;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
/* Traverse only the allowed CPUs */
|
2011-06-16 12:23:22 +02:00
|
|
|
for_each_cpu_and(i, sched_group_cpus(group), tsk_cpus_allowed(p)) {
|
2009-09-10 13:36:25 +02:00
|
|
|
load = weighted_cpuload(i);
|
|
|
|
|
|
|
|
if (load < min_load || (load == min_load && i == this_cpu)) {
|
|
|
|
min_load = load;
|
|
|
|
idlest = i;
|
2008-01-25 21:08:09 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2009-09-10 13:36:25 +02:00
|
|
|
return idlest;
|
|
|
|
}
|
2008-01-25 21:08:09 +01:00
|
|
|
|
2009-11-12 15:55:28 +01:00
|
|
|
/*
|
|
|
|
* Try and locate an idle CPU in the sched_domain.
|
|
|
|
*/
|
2010-04-01 01:47:45 +02:00
|
|
|
static int select_idle_sibling(struct task_struct *p, int target)
|
2009-11-12 15:55:28 +01:00
|
|
|
{
|
|
|
|
int cpu = smp_processor_id();
|
|
|
|
int prev_cpu = task_cpu(p);
|
2010-04-01 01:47:45 +02:00
|
|
|
struct sched_domain *sd;
|
2011-11-10 13:01:10 +01:00
|
|
|
struct sched_group *sg;
|
2011-11-17 20:08:23 +01:00
|
|
|
int i;
|
2009-11-12 15:55:28 +01:00
|
|
|
|
|
|
|
/*
|
2010-04-01 01:47:45 +02:00
|
|
|
* If the task is going to be woken-up on this cpu and if it is
|
|
|
|
* already idle, then it is the right target.
|
2009-11-12 15:55:28 +01:00
|
|
|
*/
|
2010-04-01 01:47:45 +02:00
|
|
|
if (target == cpu && idle_cpu(cpu))
|
|
|
|
return cpu;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If the task is going to be woken-up on the cpu where it previously
|
|
|
|
* ran and if it is currently idle, then it the right target.
|
|
|
|
*/
|
|
|
|
if (target == prev_cpu && idle_cpu(prev_cpu))
|
2009-11-12 15:55:29 +01:00
|
|
|
return prev_cpu;
|
2009-11-12 15:55:28 +01:00
|
|
|
|
|
|
|
/*
|
2010-04-01 01:47:45 +02:00
|
|
|
* Otherwise, iterate the domains and find an elegible idle cpu.
|
2009-11-12 15:55:28 +01:00
|
|
|
*/
|
2011-04-07 14:09:50 +02:00
|
|
|
rcu_read_lock();
|
2010-04-01 01:47:45 +02:00
|
|
|
|
2011-12-07 15:07:31 +01:00
|
|
|
sd = rcu_dereference(per_cpu(sd_llc, target));
|
2011-11-17 20:08:23 +01:00
|
|
|
for_each_lower_domain(sd) {
|
2011-11-10 13:01:10 +01:00
|
|
|
sg = sd->groups;
|
|
|
|
do {
|
|
|
|
if (!cpumask_intersects(sched_group_cpus(sg),
|
|
|
|
tsk_cpus_allowed(p)))
|
|
|
|
goto next;
|
|
|
|
|
|
|
|
for_each_cpu(i, sched_group_cpus(sg)) {
|
|
|
|
if (!idle_cpu(i))
|
|
|
|
goto next;
|
|
|
|
}
|
|
|
|
|
|
|
|
target = cpumask_first_and(sched_group_cpus(sg),
|
|
|
|
tsk_cpus_allowed(p));
|
|
|
|
goto done;
|
|
|
|
next:
|
|
|
|
sg = sg->next;
|
|
|
|
} while (sg != sd->groups);
|
2009-11-12 15:55:28 +01:00
|
|
|
}
|
2011-11-10 13:01:10 +01:00
|
|
|
done:
|
2011-04-07 14:09:50 +02:00
|
|
|
rcu_read_unlock();
|
2009-11-12 15:55:28 +01:00
|
|
|
|
|
|
|
return target;
|
|
|
|
}
|
|
|
|
|
2009-09-10 13:36:25 +02:00
|
|
|
/*
|
|
|
|
* sched_balance_self: balance the current task (running on cpu) in domains
|
|
|
|
* that have the 'flag' flag set. In practice, this is SD_BALANCE_FORK and
|
|
|
|
* SD_BALANCE_EXEC.
|
|
|
|
*
|
|
|
|
* Balance, ie. select the least loaded group.
|
|
|
|
*
|
|
|
|
* Returns the target CPU number, or the same CPU if no balancing is needed.
|
|
|
|
*
|
|
|
|
* preempt must be disabled.
|
|
|
|
*/
|
2010-03-24 18:34:10 +01:00
|
|
|
static int
|
2011-04-05 17:23:46 +02:00
|
|
|
select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flags)
|
2009-09-10 13:36:25 +02:00
|
|
|
{
|
2009-09-17 09:01:14 +02:00
|
|
|
struct sched_domain *tmp, *affine_sd = NULL, *sd = NULL;
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 13:50:02 +02:00
|
|
|
int cpu = smp_processor_id();
|
|
|
|
int prev_cpu = task_cpu(p);
|
|
|
|
int new_cpu = cpu;
|
2010-04-01 01:47:45 +02:00
|
|
|
int want_affine = 0;
|
2009-09-17 09:01:14 +02:00
|
|
|
int want_sd = 1;
|
2009-09-16 13:46:59 +02:00
|
|
|
int sync = wake_flags & WF_SYNC;
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 13:50:02 +02:00
|
|
|
|
2011-11-22 15:18:24 +01:00
|
|
|
if (p->rt.nr_cpus_allowed == 1)
|
|
|
|
return prev_cpu;
|
|
|
|
|
2009-09-14 19:37:39 +02:00
|
|
|
if (sd_flag & SD_BALANCE_WAKE) {
|
2011-06-16 12:23:22 +02:00
|
|
|
if (cpumask_test_cpu(cpu, tsk_cpus_allowed(p)))
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 13:50:02 +02:00
|
|
|
want_affine = 1;
|
|
|
|
new_cpu = prev_cpu;
|
|
|
|
}
|
2009-09-10 13:36:25 +02:00
|
|
|
|
2011-04-07 14:09:50 +02:00
|
|
|
rcu_read_lock();
|
2009-09-10 13:36:25 +02:00
|
|
|
for_each_domain(cpu, tmp) {
|
2009-12-16 18:04:34 +01:00
|
|
|
if (!(tmp->flags & SD_LOAD_BALANCE))
|
|
|
|
continue;
|
|
|
|
|
2009-09-10 13:36:25 +02:00
|
|
|
/*
|
2009-09-10 14:40:57 +02:00
|
|
|
* If power savings logic is enabled for a domain, see if we
|
|
|
|
* are not overloaded, if so, don't balance wider.
|
2009-09-10 13:36:25 +02:00
|
|
|
*/
|
2009-09-16 08:28:30 +02:00
|
|
|
if (tmp->flags & (SD_POWERSAVINGS_BALANCE|SD_PREFER_LOCAL)) {
|
2009-09-10 14:40:57 +02:00
|
|
|
unsigned long power = 0;
|
|
|
|
unsigned long nr_running = 0;
|
|
|
|
unsigned long capacity;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for_each_cpu(i, sched_domain_span(tmp)) {
|
|
|
|
power += power_of(i);
|
|
|
|
nr_running += cpu_rq(i)->cfs.nr_running;
|
|
|
|
}
|
|
|
|
|
2011-05-18 19:09:39 +02:00
|
|
|
capacity = DIV_ROUND_CLOSEST(power, SCHED_POWER_SCALE);
|
2009-09-10 14:40:57 +02:00
|
|
|
|
2009-09-16 08:28:30 +02:00
|
|
|
if (tmp->flags & SD_POWERSAVINGS_BALANCE)
|
|
|
|
nr_running /= 2;
|
|
|
|
|
|
|
|
if (nr_running < capacity)
|
2009-09-17 09:01:14 +02:00
|
|
|
want_sd = 0;
|
2009-09-10 14:40:57 +02:00
|
|
|
}
|
2009-09-10 13:36:25 +02:00
|
|
|
|
2009-11-12 15:55:29 +01:00
|
|
|
/*
|
2010-04-01 01:47:45 +02:00
|
|
|
* If both cpu and prev_cpu are part of this domain,
|
|
|
|
* cpu is a valid SD_WAKE_AFFINE target.
|
2009-11-12 15:55:29 +01:00
|
|
|
*/
|
2010-04-01 01:47:45 +02:00
|
|
|
if (want_affine && (tmp->flags & SD_WAKE_AFFINE) &&
|
|
|
|
cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) {
|
|
|
|
affine_sd = tmp;
|
|
|
|
want_affine = 0;
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 13:50:02 +02:00
|
|
|
}
|
|
|
|
|
2009-09-17 09:01:14 +02:00
|
|
|
if (!want_sd && !want_affine)
|
|
|
|
break;
|
|
|
|
|
2009-09-14 19:37:39 +02:00
|
|
|
if (!(tmp->flags & sd_flag))
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 13:50:02 +02:00
|
|
|
continue;
|
|
|
|
|
2009-09-17 09:01:14 +02:00
|
|
|
if (want_sd)
|
|
|
|
sd = tmp;
|
|
|
|
}
|
|
|
|
|
2010-03-11 17:17:16 +01:00
|
|
|
if (affine_sd) {
|
2010-04-01 01:47:45 +02:00
|
|
|
if (cpu == prev_cpu || wake_affine(affine_sd, p, sync))
|
2011-04-07 14:09:50 +02:00
|
|
|
prev_cpu = cpu;
|
|
|
|
|
|
|
|
new_cpu = select_idle_sibling(p, prev_cpu);
|
|
|
|
goto unlock;
|
2010-03-11 17:17:16 +01:00
|
|
|
}
|
2008-01-25 21:08:09 +01:00
|
|
|
|
2009-09-10 13:36:25 +02:00
|
|
|
while (sd) {
|
2009-09-16 13:46:59 +02:00
|
|
|
int load_idx = sd->forkexec_idx;
|
2009-09-10 13:36:25 +02:00
|
|
|
struct sched_group *group;
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 13:50:02 +02:00
|
|
|
int weight;
|
2008-03-16 20:36:10 +01:00
|
|
|
|
2009-09-14 19:37:39 +02:00
|
|
|
if (!(sd->flags & sd_flag)) {
|
2009-09-10 13:36:25 +02:00
|
|
|
sd = sd->child;
|
|
|
|
continue;
|
|
|
|
}
|
2008-03-16 20:36:10 +01:00
|
|
|
|
2009-09-16 13:46:59 +02:00
|
|
|
if (sd_flag & SD_BALANCE_WAKE)
|
|
|
|
load_idx = sd->wake_idx;
|
2008-03-16 20:36:10 +01:00
|
|
|
|
2009-09-16 13:46:59 +02:00
|
|
|
group = find_idlest_group(sd, p, cpu, load_idx);
|
2009-09-10 13:36:25 +02:00
|
|
|
if (!group) {
|
|
|
|
sd = sd->child;
|
|
|
|
continue;
|
|
|
|
}
|
2008-03-19 01:42:00 +01:00
|
|
|
|
2009-09-11 12:45:38 +02:00
|
|
|
new_cpu = find_idlest_cpu(group, p, cpu);
|
2009-09-10 13:36:25 +02:00
|
|
|
if (new_cpu == -1 || new_cpu == cpu) {
|
|
|
|
/* Now try balancing at a lower domain level of cpu */
|
|
|
|
sd = sd->child;
|
|
|
|
continue;
|
2008-01-25 21:08:09 +01:00
|
|
|
}
|
2009-09-10 13:36:25 +02:00
|
|
|
|
|
|
|
/* Now try balancing at a lower domain level of new_cpu */
|
|
|
|
cpu = new_cpu;
|
2010-04-16 14:59:29 +02:00
|
|
|
weight = sd->span_weight;
|
2009-09-10 13:36:25 +02:00
|
|
|
sd = NULL;
|
|
|
|
for_each_domain(cpu, tmp) {
|
2010-04-16 14:59:29 +02:00
|
|
|
if (weight <= tmp->span_weight)
|
2009-09-10 13:36:25 +02:00
|
|
|
break;
|
2009-09-14 19:37:39 +02:00
|
|
|
if (tmp->flags & sd_flag)
|
2009-09-10 13:36:25 +02:00
|
|
|
sd = tmp;
|
|
|
|
}
|
|
|
|
/* while loop will break here if sd == NULL */
|
2008-01-25 21:08:09 +01:00
|
|
|
}
|
2011-04-07 14:09:50 +02:00
|
|
|
unlock:
|
|
|
|
rcu_read_unlock();
|
2008-01-25 21:08:09 +01:00
|
|
|
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 13:50:02 +02:00
|
|
|
return new_cpu;
|
2008-01-25 21:08:09 +01:00
|
|
|
}
|
|
|
|
#endif /* CONFIG_SMP */
|
|
|
|
|
2009-01-14 12:39:19 +01:00
|
|
|
static unsigned long
|
|
|
|
wakeup_gran(struct sched_entity *curr, struct sched_entity *se)
|
2008-04-19 19:44:57 +02:00
|
|
|
{
|
|
|
|
unsigned long gran = sysctl_sched_wakeup_granularity;
|
|
|
|
|
|
|
|
/*
|
2009-01-14 12:39:19 +01:00
|
|
|
* Since its curr running now, convert the gran from real-time
|
|
|
|
* to virtual-time in his units.
|
2010-03-11 17:17:04 +01:00
|
|
|
*
|
|
|
|
* By using 'se' instead of 'curr' we penalize light tasks, so
|
|
|
|
* they get preempted easier. That is, if 'se' < 'curr' then
|
|
|
|
* the resulting gran will be larger, therefore penalizing the
|
|
|
|
* lighter, if otoh 'se' > 'curr' then the resulting gran will
|
|
|
|
* be smaller, again penalizing the lighter task.
|
|
|
|
*
|
|
|
|
* This is especially important for buddies when the leftmost
|
|
|
|
* task is higher priority than the buddy.
|
2008-04-19 19:44:57 +02:00
|
|
|
*/
|
2011-04-08 06:53:09 +02:00
|
|
|
return calc_delta_fair(gran, se);
|
2008-04-19 19:44:57 +02:00
|
|
|
}
|
|
|
|
|
2008-10-24 11:06:15 +02:00
|
|
|
/*
|
|
|
|
* Should 'se' preempt 'curr'.
|
|
|
|
*
|
|
|
|
* |s1
|
|
|
|
* |s2
|
|
|
|
* |s3
|
|
|
|
* g
|
|
|
|
* |<--->|c
|
|
|
|
*
|
|
|
|
* w(c, s1) = -1
|
|
|
|
* w(c, s2) = 0
|
|
|
|
* w(c, s3) = 1
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se)
|
|
|
|
{
|
|
|
|
s64 gran, vdiff = curr->vruntime - se->vruntime;
|
|
|
|
|
|
|
|
if (vdiff <= 0)
|
|
|
|
return -1;
|
|
|
|
|
2009-01-14 12:39:19 +01:00
|
|
|
gran = wakeup_gran(curr, se);
|
2008-10-24 11:06:15 +02:00
|
|
|
if (vdiff > gran)
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2008-11-04 21:25:10 +01:00
|
|
|
static void set_last_buddy(struct sched_entity *se)
|
|
|
|
{
|
2011-04-14 03:21:09 +02:00
|
|
|
if (entity_is_task(se) && unlikely(task_of(se)->policy == SCHED_IDLE))
|
|
|
|
return;
|
|
|
|
|
|
|
|
for_each_sched_entity(se)
|
|
|
|
cfs_rq_of(se)->last = se;
|
2008-11-04 21:25:10 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
static void set_next_buddy(struct sched_entity *se)
|
|
|
|
{
|
2011-04-14 03:21:09 +02:00
|
|
|
if (entity_is_task(se) && unlikely(task_of(se)->policy == SCHED_IDLE))
|
|
|
|
return;
|
|
|
|
|
|
|
|
for_each_sched_entity(se)
|
|
|
|
cfs_rq_of(se)->next = se;
|
2008-11-04 21:25:10 +01:00
|
|
|
}
|
|
|
|
|
2011-02-01 15:51:03 +01:00
|
|
|
static void set_skip_buddy(struct sched_entity *se)
|
|
|
|
{
|
2011-04-14 03:21:09 +02:00
|
|
|
for_each_sched_entity(se)
|
|
|
|
cfs_rq_of(se)->skip = se;
|
2011-02-01 15:51:03 +01:00
|
|
|
}
|
|
|
|
|
2007-07-09 18:51:58 +02:00
|
|
|
/*
|
|
|
|
* Preempt the current task with a newly woken task if needed:
|
|
|
|
*/
|
2009-09-16 13:47:58 +02:00
|
|
|
static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_flags)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
|
|
|
struct task_struct *curr = rq->curr;
|
2007-10-15 17:00:12 +02:00
|
|
|
struct sched_entity *se = &curr->se, *pse = &p->se;
|
2008-12-16 08:45:30 +01:00
|
|
|
struct cfs_rq *cfs_rq = task_cfs_rq(curr);
|
sched: Strengthen buddies and mitigate buddy induced latencies
This patch restores the effectiveness of LAST_BUDDY in preventing
pgsql+oltp from collapsing due to wakeup preemption. It also
switches LAST_BUDDY to exclusively do what it does best, namely
mitigate the effects of aggressive wakeup preemption, which
improves vmark throughput markedly, and restores mysql+oltp
scalability.
Since buddies are about scalability, enable them beginning at the
point where we begin expanding sched_latency, namely
sched_nr_latency. Previously, buddies were cleared aggressively,
which seriously reduced their effectiveness. Not clearing
aggressively however, produces a small drop in mysql+oltp
throughput immediately after peak, indicating that LAST_BUDDY is
actually doing some harm. This is right at the point where X on the
desktop in competition with another load wants low latency service.
Ergo, do not enable until we need to scale.
To mitigate latency induced by buddies, or by a task just missing
wakeup preemption, check latency at tick time.
Last hunk prevents buddies from stymieing BALANCE_NEWIDLE via
CACHE_HOT_BUDDY.
Supporting performance tests:
tip = v2.6.32-rc5-1497-ga525b32
tipx = NO_GENTLE_FAIR_SLEEPERS NEXT_BUDDY granularity knobs = 31 knobs + 31 buddies
tip+x = NO_GENTLE_FAIR_SLEEPERS granularity knobs = 31 knobs
(Three run averages except where noted.)
vmark:
------
tip 108466 messages per second
tip+ 125307 messages per second
tip+x 125335 messages per second
tipx 117781 messages per second
2.6.31.3 122729 messages per second
mysql+oltp:
-----------
clients 1 2 4 8 16 32 64 128 256
..........................................................................................
tip 9949.89 18690.20 34801.24 34460.04 32682.88 30765.97 28305.27 25059.64 19548.08
tip+ 10013.90 18526.84 34900.38 34420.14 33069.83 32083.40 30578.30 28010.71 25605.47
tipx 9698.71 18002.70 34477.56 33420.01 32634.30 31657.27 29932.67 26827.52 21487.18
2.6.31.3 8243.11 18784.20 34404.83 33148.38 31900.32 31161.90 29663.81 25995.94 18058.86
pgsql+oltp:
-----------
clients 1 2 4 8 16 32 64 128 256
..........................................................................................
tip 13686.37 26609.25 51934.28 51347.81 49479.51 45312.65 36691.91 26851.57 24145.35
tip+ (1x) 13907.85 27135.87 52951.98 52514.04 51742.52 50705.43 49947.97 48374.19 46227.94
tip+x 13906.78 27065.81 52951.19 52542.59 52176.11 51815.94 50838.90 49439.46 46891.00
tipx 13742.46 26769.81 52351.99 51891.73 51320.79 50938.98 50248.65 48908.70 46553.84
2.6.31.3 13815.35 26906.46 52683.34 52061.31 51937.10 51376.80 50474.28 49394.47 47003.25
Signed-off-by: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-10-23 23:09:22 +02:00
|
|
|
int scale = cfs_rq->nr_running >= sched_nr_latency;
|
2011-04-14 19:30:53 +02:00
|
|
|
int next_buddy_marked = 0;
|
2007-07-09 18:51:58 +02:00
|
|
|
|
2008-03-19 01:42:00 +01:00
|
|
|
if (unlikely(se == pse))
|
|
|
|
return;
|
|
|
|
|
2011-07-21 18:43:37 +02:00
|
|
|
/*
|
|
|
|
* This is possible from callers such as pull_task(), in which we
|
|
|
|
* unconditionally check_prempt_curr() after an enqueue (which may have
|
|
|
|
* lead to a throttle). This both saves work and prevents false
|
|
|
|
* next-buddy nomination below.
|
|
|
|
*/
|
|
|
|
if (unlikely(throttled_hierarchy(cfs_rq_of(pse))))
|
|
|
|
return;
|
|
|
|
|
2011-04-14 19:30:53 +02:00
|
|
|
if (sched_feat(NEXT_BUDDY) && scale && !(wake_flags & WF_FORK)) {
|
2009-09-11 12:01:17 +02:00
|
|
|
set_next_buddy(pse);
|
2011-04-14 19:30:53 +02:00
|
|
|
next_buddy_marked = 1;
|
|
|
|
}
|
2008-09-23 15:33:45 +02:00
|
|
|
|
2008-08-28 11:12:49 +02:00
|
|
|
/*
|
|
|
|
* We can come here with TIF_NEED_RESCHED already set from new task
|
|
|
|
* wake up path.
|
2011-07-21 18:43:37 +02:00
|
|
|
*
|
|
|
|
* Note: this also catches the edge-case of curr being in a throttled
|
|
|
|
* group (e.g. via set_curr_task), since update_curr() (in the
|
|
|
|
* enqueue of curr) will have resulted in resched being set. This
|
|
|
|
* prevents us from potentially nominating it as a false LAST_BUDDY
|
|
|
|
* below.
|
2008-08-28 11:12:49 +02:00
|
|
|
*/
|
|
|
|
if (test_tsk_need_resched(curr))
|
|
|
|
return;
|
|
|
|
|
2011-02-22 22:04:33 +01:00
|
|
|
/* Idle tasks are by definition preempted by non-idle tasks. */
|
|
|
|
if (unlikely(curr->policy == SCHED_IDLE) &&
|
|
|
|
likely(p->policy != SCHED_IDLE))
|
|
|
|
goto preempt;
|
|
|
|
|
2007-10-15 17:00:18 +02:00
|
|
|
/*
|
2011-02-22 22:04:33 +01:00
|
|
|
* Batch and idle tasks do not preempt non-idle tasks (their preemption
|
|
|
|
* is driven by the tick):
|
2007-10-15 17:00:18 +02:00
|
|
|
*/
|
2009-01-15 14:53:38 +01:00
|
|
|
if (unlikely(p->policy != SCHED_NORMAL))
|
2007-10-15 17:00:18 +02:00
|
|
|
return;
|
2007-07-09 18:51:58 +02:00
|
|
|
|
2008-10-24 11:06:15 +02:00
|
|
|
find_matching_se(&se, &pse);
|
2011-07-06 04:07:21 +02:00
|
|
|
update_curr(cfs_rq_of(se));
|
2009-04-09 00:29:43 +02:00
|
|
|
BUG_ON(!pse);
|
2011-04-14 19:30:53 +02:00
|
|
|
if (wakeup_preempt_entity(se, pse) == 1) {
|
|
|
|
/*
|
|
|
|
* Bias pick_next to pick the sched entity that is
|
|
|
|
* triggering this preemption.
|
|
|
|
*/
|
|
|
|
if (!next_buddy_marked)
|
|
|
|
set_next_buddy(pse);
|
2009-11-28 18:51:02 +01:00
|
|
|
goto preempt;
|
2011-04-14 19:30:53 +02:00
|
|
|
}
|
2008-10-24 11:06:15 +02:00
|
|
|
|
2009-11-28 18:51:02 +01:00
|
|
|
return;
|
2009-11-17 10:51:40 +01:00
|
|
|
|
2009-11-28 18:51:02 +01:00
|
|
|
preempt:
|
|
|
|
resched_task(curr);
|
|
|
|
/*
|
|
|
|
* Only set the backward buddy when the current task is still
|
|
|
|
* on the rq. This can happen when a wakeup gets interleaved
|
|
|
|
* with schedule on the ->pre_schedule() or idle_balance()
|
|
|
|
* point, either of which can * drop the rq lock.
|
|
|
|
*
|
|
|
|
* Also, during early boot the idle thread is in the fair class,
|
|
|
|
* for obvious reasons its a bad idea to schedule back to it.
|
|
|
|
*/
|
|
|
|
if (unlikely(!se->on_rq || curr == rq->idle))
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (sched_feat(LAST_BUDDY) && scale && entity_is_task(se))
|
|
|
|
set_last_buddy(se);
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
2007-08-09 11:16:48 +02:00
|
|
|
static struct task_struct *pick_next_task_fair(struct rq *rq)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
2008-01-25 21:08:29 +01:00
|
|
|
struct task_struct *p;
|
2007-07-09 18:51:58 +02:00
|
|
|
struct cfs_rq *cfs_rq = &rq->cfs;
|
|
|
|
struct sched_entity *se;
|
|
|
|
|
2009-11-24 11:55:45 +01:00
|
|
|
if (!cfs_rq->nr_running)
|
2007-07-09 18:51:58 +02:00
|
|
|
return NULL;
|
|
|
|
|
|
|
|
do {
|
2007-08-09 11:16:48 +02:00
|
|
|
se = pick_next_entity(cfs_rq);
|
2008-11-04 21:25:07 +01:00
|
|
|
set_next_entity(cfs_rq, se);
|
2007-07-09 18:51:58 +02:00
|
|
|
cfs_rq = group_cfs_rq(se);
|
|
|
|
} while (cfs_rq);
|
|
|
|
|
2008-01-25 21:08:29 +01:00
|
|
|
p = task_of(se);
|
2011-11-22 15:20:07 +01:00
|
|
|
if (hrtick_enabled(rq))
|
|
|
|
hrtick_start_fair(rq, p);
|
2008-01-25 21:08:29 +01:00
|
|
|
|
|
|
|
return p;
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Account for a descheduled task:
|
|
|
|
*/
|
2007-08-09 11:16:49 +02:00
|
|
|
static void put_prev_task_fair(struct rq *rq, struct task_struct *prev)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
|
|
|
struct sched_entity *se = &prev->se;
|
|
|
|
struct cfs_rq *cfs_rq;
|
|
|
|
|
|
|
|
for_each_sched_entity(se) {
|
|
|
|
cfs_rq = cfs_rq_of(se);
|
2007-08-09 11:16:48 +02:00
|
|
|
put_prev_entity(cfs_rq, se);
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-02-01 15:51:03 +01:00
|
|
|
/*
|
|
|
|
* sched_yield() is very simple
|
|
|
|
*
|
|
|
|
* The magic of dealing with the ->skip buddy is in pick_next_entity.
|
|
|
|
*/
|
|
|
|
static void yield_task_fair(struct rq *rq)
|
|
|
|
{
|
|
|
|
struct task_struct *curr = rq->curr;
|
|
|
|
struct cfs_rq *cfs_rq = task_cfs_rq(curr);
|
|
|
|
struct sched_entity *se = &curr->se;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Are we the only task in the tree?
|
|
|
|
*/
|
|
|
|
if (unlikely(rq->nr_running == 1))
|
|
|
|
return;
|
|
|
|
|
|
|
|
clear_buddies(cfs_rq, se);
|
|
|
|
|
|
|
|
if (curr->policy != SCHED_BATCH) {
|
|
|
|
update_rq_clock(rq);
|
|
|
|
/*
|
|
|
|
* Update run-time statistics of the 'current'.
|
|
|
|
*/
|
|
|
|
update_curr(cfs_rq);
|
2011-11-22 15:21:26 +01:00
|
|
|
/*
|
|
|
|
* Tell update_rq_clock() that we've just updated,
|
|
|
|
* so we don't do microscopic update in schedule()
|
|
|
|
* and double the fastpath cost.
|
|
|
|
*/
|
|
|
|
rq->skip_clock_update = 1;
|
2011-02-01 15:51:03 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
set_skip_buddy(se);
|
|
|
|
}
|
|
|
|
|
2011-02-01 15:50:51 +01:00
|
|
|
static bool yield_to_task_fair(struct rq *rq, struct task_struct *p, bool preempt)
|
|
|
|
{
|
|
|
|
struct sched_entity *se = &p->se;
|
|
|
|
|
2011-07-21 18:43:37 +02:00
|
|
|
/* throttled hierarchies are not runnable */
|
|
|
|
if (!se->on_rq || throttled_hierarchy(cfs_rq_of(se)))
|
2011-02-01 15:50:51 +01:00
|
|
|
return false;
|
|
|
|
|
|
|
|
/* Tell the scheduler that we'd really like pse to run next. */
|
|
|
|
set_next_buddy(se);
|
|
|
|
|
|
|
|
yield_task_fair(rq);
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2007-10-24 18:23:51 +02:00
|
|
|
#ifdef CONFIG_SMP
|
2007-07-09 18:51:58 +02:00
|
|
|
/**************************************************
|
|
|
|
* Fair scheduling class load-balancing methods:
|
|
|
|
*/
|
|
|
|
|
2009-12-17 17:00:43 +01:00
|
|
|
/*
|
|
|
|
* pull_task - move a task from a remote runqueue to the local runqueue.
|
|
|
|
* Both runqueues must be locked.
|
|
|
|
*/
|
|
|
|
static void pull_task(struct rq *src_rq, struct task_struct *p,
|
|
|
|
struct rq *this_rq, int this_cpu)
|
|
|
|
{
|
|
|
|
deactivate_task(src_rq, p, 0);
|
|
|
|
set_task_cpu(p, this_cpu);
|
|
|
|
activate_task(this_rq, p, 0);
|
|
|
|
check_preempt_curr(this_rq, p, 0);
|
|
|
|
}
|
|
|
|
|
2011-10-25 10:00:11 +02:00
|
|
|
/*
|
|
|
|
* Is this task likely cache-hot:
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
|
|
|
|
{
|
|
|
|
s64 delta;
|
|
|
|
|
|
|
|
if (p->sched_class != &fair_sched_class)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (unlikely(p->policy == SCHED_IDLE))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Buddy candidates are cache hot:
|
|
|
|
*/
|
|
|
|
if (sched_feat(CACHE_HOT_BUDDY) && this_rq()->nr_running &&
|
|
|
|
(&p->se == cfs_rq_of(&p->se)->next ||
|
|
|
|
&p->se == cfs_rq_of(&p->se)->last))
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
if (sysctl_sched_migration_cost == -1)
|
|
|
|
return 1;
|
|
|
|
if (sysctl_sched_migration_cost == 0)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
delta = now - p->se.exec_start;
|
|
|
|
|
|
|
|
return delta < (s64)sysctl_sched_migration_cost;
|
|
|
|
}
|
|
|
|
|
2011-09-22 15:23:13 +02:00
|
|
|
#define LBF_ALL_PINNED 0x01
|
2011-09-22 15:30:18 +02:00
|
|
|
#define LBF_NEED_BREAK 0x02
|
|
|
|
#define LBF_ABORT 0x04
|
2011-09-22 15:23:13 +02:00
|
|
|
|
2009-12-17 17:00:43 +01:00
|
|
|
/*
|
|
|
|
* can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
|
|
|
|
*/
|
|
|
|
static
|
|
|
|
int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
|
|
|
|
struct sched_domain *sd, enum cpu_idle_type idle,
|
2011-09-22 15:23:13 +02:00
|
|
|
int *lb_flags)
|
2009-12-17 17:00:43 +01:00
|
|
|
{
|
|
|
|
int tsk_cache_hot = 0;
|
|
|
|
/*
|
|
|
|
* We do not migrate tasks that are:
|
|
|
|
* 1) running (obviously), or
|
|
|
|
* 2) cannot be migrated to this CPU due to cpus_allowed, or
|
|
|
|
* 3) are cache-hot on their current CPU.
|
|
|
|
*/
|
2011-06-16 12:23:22 +02:00
|
|
|
if (!cpumask_test_cpu(this_cpu, tsk_cpus_allowed(p))) {
|
2010-03-11 03:37:45 +01:00
|
|
|
schedstat_inc(p, se.statistics.nr_failed_migrations_affine);
|
2009-12-17 17:00:43 +01:00
|
|
|
return 0;
|
|
|
|
}
|
2011-09-22 15:23:13 +02:00
|
|
|
*lb_flags &= ~LBF_ALL_PINNED;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
|
|
|
if (task_running(rq, p)) {
|
2010-03-11 03:37:45 +01:00
|
|
|
schedstat_inc(p, se.statistics.nr_failed_migrations_running);
|
2009-12-17 17:00:43 +01:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Aggressive migration if:
|
|
|
|
* 1) task is cache cold, or
|
|
|
|
* 2) too many balance attempts have failed.
|
|
|
|
*/
|
|
|
|
|
2010-10-05 02:03:21 +02:00
|
|
|
tsk_cache_hot = task_hot(p, rq->clock_task, sd);
|
2009-12-17 17:00:43 +01:00
|
|
|
if (!tsk_cache_hot ||
|
|
|
|
sd->nr_balance_failed > sd->cache_nice_tries) {
|
|
|
|
#ifdef CONFIG_SCHEDSTATS
|
|
|
|
if (tsk_cache_hot) {
|
|
|
|
schedstat_inc(sd, lb_hot_gained[idle]);
|
2010-03-11 03:37:45 +01:00
|
|
|
schedstat_inc(p, se.statistics.nr_forced_migrations);
|
2009-12-17 17:00:43 +01:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (tsk_cache_hot) {
|
2010-03-11 03:37:45 +01:00
|
|
|
schedstat_inc(p, se.statistics.nr_failed_migrations_hot);
|
2009-12-17 17:00:43 +01:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2009-12-17 17:45:42 +01:00
|
|
|
/*
|
|
|
|
* move_one_task tries to move exactly one task from busiest to this_rq, as
|
|
|
|
* part of active balancing operations within "domain".
|
|
|
|
* Returns 1 if successful and 0 otherwise.
|
|
|
|
*
|
|
|
|
* Called with both runqueues locked.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
|
|
|
|
struct sched_domain *sd, enum cpu_idle_type idle)
|
|
|
|
{
|
|
|
|
struct task_struct *p, *n;
|
|
|
|
struct cfs_rq *cfs_rq;
|
|
|
|
int pinned = 0;
|
|
|
|
|
|
|
|
for_each_leaf_cfs_rq(busiest, cfs_rq) {
|
|
|
|
list_for_each_entry_safe(p, n, &cfs_rq->tasks, se.group_node) {
|
2011-07-21 18:43:36 +02:00
|
|
|
if (throttled_lb_pair(task_group(p),
|
|
|
|
busiest->cpu, this_cpu))
|
|
|
|
break;
|
2009-12-17 17:45:42 +01:00
|
|
|
|
|
|
|
if (!can_migrate_task(p, busiest, this_cpu,
|
|
|
|
sd, idle, &pinned))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
pull_task(busiest, p, this_rq, this_cpu);
|
|
|
|
/*
|
|
|
|
* Right now, this is only the second place pull_task()
|
|
|
|
* is called, so we can safely collect pull_task()
|
|
|
|
* stats here rather than inside pull_task().
|
|
|
|
*/
|
|
|
|
schedstat_inc(sd, lb_gained[idle]);
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2009-12-17 17:00:43 +01:00
|
|
|
static unsigned long
|
|
|
|
balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
|
|
|
|
unsigned long max_load_move, struct sched_domain *sd,
|
2011-09-22 15:23:13 +02:00
|
|
|
enum cpu_idle_type idle, int *lb_flags,
|
2011-05-03 20:31:07 +02:00
|
|
|
struct cfs_rq *busiest_cfs_rq)
|
2009-12-17 17:00:43 +01:00
|
|
|
{
|
sched: Fix erroneous all_pinned logic
The scheduler load balancer has specific code to deal with cases of
unbalanced system due to lots of unmovable tasks (for example because of
hard CPU affinity). In those situation, it excludes the busiest CPU that
has pinned tasks for load balance consideration such that it can perform
second 2nd load balance pass on the rest of the system.
This all works as designed if there is only one cgroup in the system.
However, when we have multiple cgroups, this logic has false positives and
triggers multiple load balance passes despite there are actually no pinned
tasks at all.
The reason it has false positives is that the all pinned logic is deep in
the lowest function of can_migrate_task() and is too low level:
load_balance_fair() iterates each task group and calls balance_tasks() to
migrate target load. Along the way, balance_tasks() will also set a
all_pinned variable. Given that task-groups are iterated, this all_pinned
variable is essentially the status of last group in the scanning process.
Task group can have number of reasons that no load being migrated, none
due to cpu affinity. However, this status bit is being propagated back up
to the higher level load_balance(), which incorrectly think that no tasks
were moved. It kick off the all pinned logic and start multiple passes
attempt to move load onto puller CPU.
To fix this, move the all_pinned aggregation up at the iterator level.
This ensures that the status is aggregated over all task-groups, not just
last one in the list.
Signed-off-by: Ken Chen <kenchen@google.com>
Cc: stable@kernel.org
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Link: http://lkml.kernel.org/r/BANLkTi=ernzNawaR5tJZEsV_QVnfxqXmsQ@mail.gmail.com
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2011-04-08 21:20:16 +02:00
|
|
|
int loops = 0, pulled = 0;
|
2009-12-17 17:00:43 +01:00
|
|
|
long rem_load_move = max_load_move;
|
2009-12-17 17:25:20 +01:00
|
|
|
struct task_struct *p, *n;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
|
|
|
if (max_load_move == 0)
|
|
|
|
goto out;
|
|
|
|
|
2009-12-17 17:25:20 +01:00
|
|
|
list_for_each_entry_safe(p, n, &busiest_cfs_rq->tasks, se.group_node) {
|
2011-09-22 15:30:18 +02:00
|
|
|
if (loops++ > sysctl_sched_nr_migrate) {
|
|
|
|
*lb_flags |= LBF_NEED_BREAK;
|
2009-12-17 17:25:20 +01:00
|
|
|
break;
|
2011-09-22 15:30:18 +02:00
|
|
|
}
|
2009-12-17 17:00:43 +01:00
|
|
|
|
2009-12-17 17:25:20 +01:00
|
|
|
if ((p->se.load.weight >> 1) > rem_load_move ||
|
sched: Fix erroneous all_pinned logic
The scheduler load balancer has specific code to deal with cases of
unbalanced system due to lots of unmovable tasks (for example because of
hard CPU affinity). In those situation, it excludes the busiest CPU that
has pinned tasks for load balance consideration such that it can perform
second 2nd load balance pass on the rest of the system.
This all works as designed if there is only one cgroup in the system.
However, when we have multiple cgroups, this logic has false positives and
triggers multiple load balance passes despite there are actually no pinned
tasks at all.
The reason it has false positives is that the all pinned logic is deep in
the lowest function of can_migrate_task() and is too low level:
load_balance_fair() iterates each task group and calls balance_tasks() to
migrate target load. Along the way, balance_tasks() will also set a
all_pinned variable. Given that task-groups are iterated, this all_pinned
variable is essentially the status of last group in the scanning process.
Task group can have number of reasons that no load being migrated, none
due to cpu affinity. However, this status bit is being propagated back up
to the higher level load_balance(), which incorrectly think that no tasks
were moved. It kick off the all pinned logic and start multiple passes
attempt to move load onto puller CPU.
To fix this, move the all_pinned aggregation up at the iterator level.
This ensures that the status is aggregated over all task-groups, not just
last one in the list.
Signed-off-by: Ken Chen <kenchen@google.com>
Cc: stable@kernel.org
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Link: http://lkml.kernel.org/r/BANLkTi=ernzNawaR5tJZEsV_QVnfxqXmsQ@mail.gmail.com
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2011-04-08 21:20:16 +02:00
|
|
|
!can_migrate_task(p, busiest, this_cpu, sd, idle,
|
2011-09-22 15:23:13 +02:00
|
|
|
lb_flags))
|
2009-12-17 17:25:20 +01:00
|
|
|
continue;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
2009-12-17 17:25:20 +01:00
|
|
|
pull_task(busiest, p, this_rq, this_cpu);
|
|
|
|
pulled++;
|
|
|
|
rem_load_move -= p->se.load.weight;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
|
|
|
#ifdef CONFIG_PREEMPT
|
2009-12-17 17:25:20 +01:00
|
|
|
/*
|
|
|
|
* NEWIDLE balancing is a source of latency, so preemptible
|
|
|
|
* kernels will stop after the first task is pulled to minimize
|
|
|
|
* the critical section.
|
|
|
|
*/
|
2011-09-22 15:30:18 +02:00
|
|
|
if (idle == CPU_NEWLY_IDLE) {
|
|
|
|
*lb_flags |= LBF_ABORT;
|
2009-12-17 17:25:20 +01:00
|
|
|
break;
|
2011-09-22 15:30:18 +02:00
|
|
|
}
|
2009-12-17 17:00:43 +01:00
|
|
|
#endif
|
|
|
|
|
2009-12-17 17:25:20 +01:00
|
|
|
/*
|
|
|
|
* We only want to steal up to the prescribed amount of
|
|
|
|
* weighted load.
|
|
|
|
*/
|
|
|
|
if (rem_load_move <= 0)
|
|
|
|
break;
|
2009-12-17 17:00:43 +01:00
|
|
|
}
|
|
|
|
out:
|
|
|
|
/*
|
|
|
|
* Right now, this is one of only two places pull_task() is called,
|
|
|
|
* so we can safely collect pull_task() stats here rather than
|
|
|
|
* inside pull_task().
|
|
|
|
*/
|
|
|
|
schedstat_add(sd, lb_gained[idle], pulled);
|
|
|
|
|
|
|
|
return max_load_move - rem_load_move;
|
|
|
|
}
|
|
|
|
|
2009-12-17 17:47:12 +01:00
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
2010-11-16 00:47:02 +01:00
|
|
|
/*
|
|
|
|
* update tg->load_weight by folding this cpu's load_avg
|
|
|
|
*/
|
2010-11-16 00:47:05 +01:00
|
|
|
static int update_shares_cpu(struct task_group *tg, int cpu)
|
2010-11-16 00:47:02 +01:00
|
|
|
{
|
|
|
|
struct cfs_rq *cfs_rq;
|
|
|
|
unsigned long flags;
|
|
|
|
struct rq *rq;
|
|
|
|
|
|
|
|
if (!tg->se[cpu])
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
rq = cpu_rq(cpu);
|
|
|
|
cfs_rq = tg->cfs_rq[cpu];
|
|
|
|
|
|
|
|
raw_spin_lock_irqsave(&rq->lock, flags);
|
|
|
|
|
|
|
|
update_rq_clock(rq);
|
2010-11-16 00:47:09 +01:00
|
|
|
update_cfs_load(cfs_rq, 1);
|
2010-11-16 00:47:02 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* We need to update shares after updating tg->load_weight in
|
|
|
|
* order to adjust the weight of groups with long running tasks.
|
|
|
|
*/
|
2011-01-22 05:45:01 +01:00
|
|
|
update_cfs_shares(cfs_rq);
|
2010-11-16 00:47:02 +01:00
|
|
|
|
|
|
|
raw_spin_unlock_irqrestore(&rq->lock, flags);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void update_shares(int cpu)
|
|
|
|
{
|
|
|
|
struct cfs_rq *cfs_rq;
|
|
|
|
struct rq *rq = cpu_rq(cpu);
|
|
|
|
|
|
|
|
rcu_read_lock();
|
2011-07-13 13:09:25 +02:00
|
|
|
/*
|
|
|
|
* Iterates the task_group tree in a bottom up fashion, see
|
|
|
|
* list_add_leaf_cfs_rq() for details.
|
|
|
|
*/
|
2011-07-21 18:43:36 +02:00
|
|
|
for_each_leaf_cfs_rq(rq, cfs_rq) {
|
|
|
|
/* throttled entities do not contribute to load */
|
|
|
|
if (throttled_hierarchy(cfs_rq))
|
|
|
|
continue;
|
|
|
|
|
2010-11-16 00:47:05 +01:00
|
|
|
update_shares_cpu(cfs_rq->tg, cpu);
|
2011-07-21 18:43:36 +02:00
|
|
|
}
|
2010-11-16 00:47:02 +01:00
|
|
|
rcu_read_unlock();
|
|
|
|
}
|
|
|
|
|
2011-07-13 13:09:25 +02:00
|
|
|
/*
|
|
|
|
* Compute the cpu's hierarchical load factor for each task group.
|
|
|
|
* This needs to be done in a top-down fashion because the load of a child
|
|
|
|
* group is a fraction of its parents load.
|
|
|
|
*/
|
|
|
|
static int tg_load_down(struct task_group *tg, void *data)
|
|
|
|
{
|
|
|
|
unsigned long load;
|
|
|
|
long cpu = (long)data;
|
|
|
|
|
|
|
|
if (!tg->parent) {
|
|
|
|
load = cpu_rq(cpu)->load.weight;
|
|
|
|
} else {
|
|
|
|
load = tg->parent->cfs_rq[cpu]->h_load;
|
|
|
|
load *= tg->se[cpu]->load.weight;
|
|
|
|
load /= tg->parent->cfs_rq[cpu]->load.weight + 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
tg->cfs_rq[cpu]->h_load = load;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void update_h_load(long cpu)
|
|
|
|
{
|
|
|
|
walk_tg_tree(tg_load_down, tg_nop, (void *)cpu);
|
|
|
|
}
|
|
|
|
|
2009-12-17 17:47:12 +01:00
|
|
|
static unsigned long
|
|
|
|
load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
|
|
|
|
unsigned long max_load_move,
|
|
|
|
struct sched_domain *sd, enum cpu_idle_type idle,
|
2011-09-22 15:23:13 +02:00
|
|
|
int *lb_flags)
|
2009-12-17 17:47:12 +01:00
|
|
|
{
|
|
|
|
long rem_load_move = max_load_move;
|
2011-07-13 13:09:25 +02:00
|
|
|
struct cfs_rq *busiest_cfs_rq;
|
2009-12-17 17:47:12 +01:00
|
|
|
|
|
|
|
rcu_read_lock();
|
2011-07-13 13:09:25 +02:00
|
|
|
update_h_load(cpu_of(busiest));
|
2009-12-17 17:47:12 +01:00
|
|
|
|
2011-07-13 13:09:25 +02:00
|
|
|
for_each_leaf_cfs_rq(busiest, busiest_cfs_rq) {
|
2009-12-17 17:47:12 +01:00
|
|
|
unsigned long busiest_h_load = busiest_cfs_rq->h_load;
|
|
|
|
unsigned long busiest_weight = busiest_cfs_rq->load.weight;
|
|
|
|
u64 rem_load, moved_load;
|
|
|
|
|
2011-09-22 15:30:18 +02:00
|
|
|
if (*lb_flags & (LBF_NEED_BREAK|LBF_ABORT))
|
|
|
|
break;
|
|
|
|
|
2009-12-17 17:47:12 +01:00
|
|
|
/*
|
2011-07-21 18:43:36 +02:00
|
|
|
* empty group or part of a throttled hierarchy
|
2009-12-17 17:47:12 +01:00
|
|
|
*/
|
2011-07-21 18:43:36 +02:00
|
|
|
if (!busiest_cfs_rq->task_weight ||
|
|
|
|
throttled_lb_pair(busiest_cfs_rq->tg, cpu_of(busiest), this_cpu))
|
2009-12-17 17:47:12 +01:00
|
|
|
continue;
|
|
|
|
|
|
|
|
rem_load = (u64)rem_load_move * busiest_weight;
|
|
|
|
rem_load = div_u64(rem_load, busiest_h_load + 1);
|
|
|
|
|
|
|
|
moved_load = balance_tasks(this_rq, this_cpu, busiest,
|
2011-09-22 15:23:13 +02:00
|
|
|
rem_load, sd, idle, lb_flags,
|
2009-12-17 17:47:12 +01:00
|
|
|
busiest_cfs_rq);
|
|
|
|
|
|
|
|
if (!moved_load)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
moved_load *= busiest_h_load;
|
|
|
|
moved_load = div_u64(moved_load, busiest_weight + 1);
|
|
|
|
|
|
|
|
rem_load_move -= moved_load;
|
|
|
|
if (rem_load_move < 0)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
|
|
|
return max_load_move - rem_load_move;
|
|
|
|
}
|
|
|
|
#else
|
2010-11-16 00:47:02 +01:00
|
|
|
static inline void update_shares(int cpu)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2009-12-17 17:47:12 +01:00
|
|
|
static unsigned long
|
|
|
|
load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
|
|
|
|
unsigned long max_load_move,
|
|
|
|
struct sched_domain *sd, enum cpu_idle_type idle,
|
2011-09-22 15:23:13 +02:00
|
|
|
int *lb_flags)
|
2009-12-17 17:47:12 +01:00
|
|
|
{
|
|
|
|
return balance_tasks(this_rq, this_cpu, busiest,
|
2011-09-22 15:23:13 +02:00
|
|
|
max_load_move, sd, idle, lb_flags,
|
2011-05-03 20:31:07 +02:00
|
|
|
&busiest->cfs);
|
2009-12-17 17:47:12 +01:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2009-12-17 17:00:43 +01:00
|
|
|
/*
|
|
|
|
* move_tasks tries to move up to max_load_move weighted load from busiest to
|
|
|
|
* this_rq, as part of a balancing operation within domain "sd".
|
|
|
|
* Returns 1 if successful and 0 otherwise.
|
|
|
|
*
|
|
|
|
* Called with both runqueues locked.
|
|
|
|
*/
|
|
|
|
static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
|
|
|
|
unsigned long max_load_move,
|
|
|
|
struct sched_domain *sd, enum cpu_idle_type idle,
|
2011-09-22 15:23:13 +02:00
|
|
|
int *lb_flags)
|
2009-12-17 17:00:43 +01:00
|
|
|
{
|
2009-12-17 17:12:46 +01:00
|
|
|
unsigned long total_load_moved = 0, load_moved;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
|
|
|
do {
|
2009-12-17 17:12:46 +01:00
|
|
|
load_moved = load_balance_fair(this_rq, this_cpu, busiest,
|
2009-12-17 17:00:43 +01:00
|
|
|
max_load_move - total_load_moved,
|
2011-09-22 15:23:13 +02:00
|
|
|
sd, idle, lb_flags);
|
2009-12-17 17:12:46 +01:00
|
|
|
|
|
|
|
total_load_moved += load_moved;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
2011-09-22 15:30:18 +02:00
|
|
|
if (*lb_flags & (LBF_NEED_BREAK|LBF_ABORT))
|
|
|
|
break;
|
|
|
|
|
2009-12-17 17:00:43 +01:00
|
|
|
#ifdef CONFIG_PREEMPT
|
|
|
|
/*
|
|
|
|
* NEWIDLE balancing is a source of latency, so preemptible
|
|
|
|
* kernels will stop after the first task is pulled to minimize
|
|
|
|
* the critical section.
|
|
|
|
*/
|
2011-09-22 15:30:18 +02:00
|
|
|
if (idle == CPU_NEWLY_IDLE && this_rq->nr_running) {
|
|
|
|
*lb_flags |= LBF_ABORT;
|
2009-12-17 18:10:09 +01:00
|
|
|
break;
|
2011-09-22 15:30:18 +02:00
|
|
|
}
|
2009-12-17 17:00:43 +01:00
|
|
|
#endif
|
2009-12-17 17:12:46 +01:00
|
|
|
} while (load_moved && max_load_move > total_load_moved);
|
2009-12-17 17:00:43 +01:00
|
|
|
|
|
|
|
return total_load_moved > 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/********** Helpers for find_busiest_group ************************/
|
|
|
|
/*
|
|
|
|
* sd_lb_stats - Structure to store the statistics of a sched_domain
|
|
|
|
* during load balancing.
|
|
|
|
*/
|
|
|
|
struct sd_lb_stats {
|
|
|
|
struct sched_group *busiest; /* Busiest group in this sd */
|
|
|
|
struct sched_group *this; /* Local group in this sd */
|
|
|
|
unsigned long total_load; /* Total load of all groups in sd */
|
|
|
|
unsigned long total_pwr; /* Total power of all groups in sd */
|
|
|
|
unsigned long avg_load; /* Average load across all groups in sd */
|
|
|
|
|
|
|
|
/** Statistics of this group */
|
|
|
|
unsigned long this_load;
|
|
|
|
unsigned long this_load_per_task;
|
|
|
|
unsigned long this_nr_running;
|
2010-10-15 22:12:29 +02:00
|
|
|
unsigned long this_has_capacity;
|
2010-09-18 00:02:32 +02:00
|
|
|
unsigned int this_idle_cpus;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
|
|
|
/* Statistics of the busiest group */
|
2010-09-18 00:02:32 +02:00
|
|
|
unsigned int busiest_idle_cpus;
|
2009-12-17 17:00:43 +01:00
|
|
|
unsigned long max_load;
|
|
|
|
unsigned long busiest_load_per_task;
|
|
|
|
unsigned long busiest_nr_running;
|
2010-02-24 01:13:52 +01:00
|
|
|
unsigned long busiest_group_capacity;
|
2010-10-15 22:12:29 +02:00
|
|
|
unsigned long busiest_has_capacity;
|
2010-09-18 00:02:32 +02:00
|
|
|
unsigned int busiest_group_weight;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
|
|
|
int group_imb; /* Is there imbalance in this sd */
|
|
|
|
#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
|
|
|
|
int power_savings_balance; /* Is powersave balance needed for this sd */
|
|
|
|
struct sched_group *group_min; /* Least loaded group in sd */
|
|
|
|
struct sched_group *group_leader; /* Group which relieves group_min */
|
|
|
|
unsigned long min_load_per_task; /* load_per_task in group_min */
|
|
|
|
unsigned long leader_nr_running; /* Nr running of group_leader */
|
|
|
|
unsigned long min_nr_running; /* Nr running of group_min */
|
|
|
|
#endif
|
|
|
|
};
|
|
|
|
|
|
|
|
/*
|
|
|
|
* sg_lb_stats - stats of a sched_group required for load_balancing
|
|
|
|
*/
|
|
|
|
struct sg_lb_stats {
|
|
|
|
unsigned long avg_load; /*Avg load across the CPUs of the group */
|
|
|
|
unsigned long group_load; /* Total load over the CPUs of the group */
|
|
|
|
unsigned long sum_nr_running; /* Nr tasks running in the group */
|
|
|
|
unsigned long sum_weighted_load; /* Weighted load of group's tasks */
|
|
|
|
unsigned long group_capacity;
|
2010-09-18 00:02:32 +02:00
|
|
|
unsigned long idle_cpus;
|
|
|
|
unsigned long group_weight;
|
2009-12-17 17:00:43 +01:00
|
|
|
int group_imb; /* Is there an imbalance in the group ? */
|
2010-10-15 22:12:29 +02:00
|
|
|
int group_has_capacity; /* Is there extra capacity in the group? */
|
2009-12-17 17:00:43 +01:00
|
|
|
};
|
|
|
|
|
|
|
|
/**
|
|
|
|
* get_sd_load_idx - Obtain the load index for a given sched domain.
|
|
|
|
* @sd: The sched_domain whose load_idx is to be obtained.
|
|
|
|
* @idle: The Idle status of the CPU for whose sd load_icx is obtained.
|
|
|
|
*/
|
|
|
|
static inline int get_sd_load_idx(struct sched_domain *sd,
|
|
|
|
enum cpu_idle_type idle)
|
|
|
|
{
|
|
|
|
int load_idx;
|
|
|
|
|
|
|
|
switch (idle) {
|
|
|
|
case CPU_NOT_IDLE:
|
|
|
|
load_idx = sd->busy_idx;
|
|
|
|
break;
|
|
|
|
|
|
|
|
case CPU_NEWLY_IDLE:
|
|
|
|
load_idx = sd->newidle_idx;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
load_idx = sd->idle_idx;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return load_idx;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
|
|
|
|
/**
|
|
|
|
* init_sd_power_savings_stats - Initialize power savings statistics for
|
|
|
|
* the given sched_domain, during load balancing.
|
|
|
|
*
|
|
|
|
* @sd: Sched domain whose power-savings statistics are to be initialized.
|
|
|
|
* @sds: Variable containing the statistics for sd.
|
|
|
|
* @idle: Idle status of the CPU at which we're performing load-balancing.
|
|
|
|
*/
|
|
|
|
static inline void init_sd_power_savings_stats(struct sched_domain *sd,
|
|
|
|
struct sd_lb_stats *sds, enum cpu_idle_type idle)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Busy processors will not participate in power savings
|
|
|
|
* balance.
|
|
|
|
*/
|
|
|
|
if (idle == CPU_NOT_IDLE || !(sd->flags & SD_POWERSAVINGS_BALANCE))
|
|
|
|
sds->power_savings_balance = 0;
|
|
|
|
else {
|
|
|
|
sds->power_savings_balance = 1;
|
|
|
|
sds->min_nr_running = ULONG_MAX;
|
|
|
|
sds->leader_nr_running = 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* update_sd_power_savings_stats - Update the power saving stats for a
|
|
|
|
* sched_domain while performing load balancing.
|
|
|
|
*
|
|
|
|
* @group: sched_group belonging to the sched_domain under consideration.
|
|
|
|
* @sds: Variable containing the statistics of the sched_domain
|
|
|
|
* @local_group: Does group contain the CPU for which we're performing
|
|
|
|
* load balancing ?
|
|
|
|
* @sgs: Variable containing the statistics of the group.
|
|
|
|
*/
|
|
|
|
static inline void update_sd_power_savings_stats(struct sched_group *group,
|
|
|
|
struct sd_lb_stats *sds, int local_group, struct sg_lb_stats *sgs)
|
|
|
|
{
|
|
|
|
|
|
|
|
if (!sds->power_savings_balance)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If the local group is idle or completely loaded
|
|
|
|
* no need to do power savings balance at this domain
|
|
|
|
*/
|
|
|
|
if (local_group && (sds->this_nr_running >= sgs->group_capacity ||
|
|
|
|
!sds->this_nr_running))
|
|
|
|
sds->power_savings_balance = 0;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If a group is already running at full capacity or idle,
|
|
|
|
* don't include that group in power savings calculations
|
|
|
|
*/
|
|
|
|
if (!sds->power_savings_balance ||
|
|
|
|
sgs->sum_nr_running >= sgs->group_capacity ||
|
|
|
|
!sgs->sum_nr_running)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Calculate the group which has the least non-idle load.
|
|
|
|
* This is the group from where we need to pick up the load
|
|
|
|
* for saving power
|
|
|
|
*/
|
|
|
|
if ((sgs->sum_nr_running < sds->min_nr_running) ||
|
|
|
|
(sgs->sum_nr_running == sds->min_nr_running &&
|
|
|
|
group_first_cpu(group) > group_first_cpu(sds->group_min))) {
|
|
|
|
sds->group_min = group;
|
|
|
|
sds->min_nr_running = sgs->sum_nr_running;
|
|
|
|
sds->min_load_per_task = sgs->sum_weighted_load /
|
|
|
|
sgs->sum_nr_running;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Calculate the group which is almost near its
|
|
|
|
* capacity but still has some space to pick up some load
|
|
|
|
* from other group and save more power
|
|
|
|
*/
|
|
|
|
if (sgs->sum_nr_running + 1 > sgs->group_capacity)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (sgs->sum_nr_running > sds->leader_nr_running ||
|
|
|
|
(sgs->sum_nr_running == sds->leader_nr_running &&
|
|
|
|
group_first_cpu(group) < group_first_cpu(sds->group_leader))) {
|
|
|
|
sds->group_leader = group;
|
|
|
|
sds->leader_nr_running = sgs->sum_nr_running;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* check_power_save_busiest_group - see if there is potential for some power-savings balance
|
|
|
|
* @sds: Variable containing the statistics of the sched_domain
|
|
|
|
* under consideration.
|
|
|
|
* @this_cpu: Cpu at which we're currently performing load-balancing.
|
|
|
|
* @imbalance: Variable to store the imbalance.
|
|
|
|
*
|
|
|
|
* Description:
|
|
|
|
* Check if we have potential to perform some power-savings balance.
|
|
|
|
* If yes, set the busiest group to be the least loaded group in the
|
|
|
|
* sched_domain, so that it's CPUs can be put to idle.
|
|
|
|
*
|
|
|
|
* Returns 1 if there is potential to perform power-savings balance.
|
|
|
|
* Else returns 0.
|
|
|
|
*/
|
|
|
|
static inline int check_power_save_busiest_group(struct sd_lb_stats *sds,
|
|
|
|
int this_cpu, unsigned long *imbalance)
|
|
|
|
{
|
|
|
|
if (!sds->power_savings_balance)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (sds->this != sds->group_leader ||
|
|
|
|
sds->group_leader == sds->group_min)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
*imbalance = sds->min_load_per_task;
|
|
|
|
sds->busiest = sds->group_min;
|
|
|
|
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
}
|
|
|
|
#else /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */
|
|
|
|
static inline void init_sd_power_savings_stats(struct sched_domain *sd,
|
|
|
|
struct sd_lb_stats *sds, enum cpu_idle_type idle)
|
|
|
|
{
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void update_sd_power_savings_stats(struct sched_group *group,
|
|
|
|
struct sd_lb_stats *sds, int local_group, struct sg_lb_stats *sgs)
|
|
|
|
{
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int check_power_save_busiest_group(struct sd_lb_stats *sds,
|
|
|
|
int this_cpu, unsigned long *imbalance)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
#endif /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */
|
|
|
|
|
|
|
|
|
|
|
|
unsigned long default_scale_freq_power(struct sched_domain *sd, int cpu)
|
|
|
|
{
|
2011-05-18 19:09:39 +02:00
|
|
|
return SCHED_POWER_SCALE;
|
2009-12-17 17:00:43 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
unsigned long __weak arch_scale_freq_power(struct sched_domain *sd, int cpu)
|
|
|
|
{
|
|
|
|
return default_scale_freq_power(sd, cpu);
|
|
|
|
}
|
|
|
|
|
|
|
|
unsigned long default_scale_smt_power(struct sched_domain *sd, int cpu)
|
|
|
|
{
|
2010-04-16 14:59:29 +02:00
|
|
|
unsigned long weight = sd->span_weight;
|
2009-12-17 17:00:43 +01:00
|
|
|
unsigned long smt_gain = sd->smt_gain;
|
|
|
|
|
|
|
|
smt_gain /= weight;
|
|
|
|
|
|
|
|
return smt_gain;
|
|
|
|
}
|
|
|
|
|
|
|
|
unsigned long __weak arch_scale_smt_power(struct sched_domain *sd, int cpu)
|
|
|
|
{
|
|
|
|
return default_scale_smt_power(sd, cpu);
|
|
|
|
}
|
|
|
|
|
|
|
|
unsigned long scale_rt_power(int cpu)
|
|
|
|
{
|
|
|
|
struct rq *rq = cpu_rq(cpu);
|
|
|
|
u64 total, available;
|
|
|
|
|
|
|
|
total = sched_avg_period() + (rq->clock - rq->age_stamp);
|
2010-10-05 02:03:22 +02:00
|
|
|
|
|
|
|
if (unlikely(total < rq->rt_avg)) {
|
|
|
|
/* Ensures that power won't end up being negative */
|
|
|
|
available = 0;
|
|
|
|
} else {
|
|
|
|
available = total - rq->rt_avg;
|
|
|
|
}
|
2009-12-17 17:00:43 +01:00
|
|
|
|
2011-05-18 19:09:39 +02:00
|
|
|
if (unlikely((s64)total < SCHED_POWER_SCALE))
|
|
|
|
total = SCHED_POWER_SCALE;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
2011-05-18 19:09:39 +02:00
|
|
|
total >>= SCHED_POWER_SHIFT;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
|
|
|
return div_u64(available, total);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void update_cpu_power(struct sched_domain *sd, int cpu)
|
|
|
|
{
|
2010-04-16 14:59:29 +02:00
|
|
|
unsigned long weight = sd->span_weight;
|
2011-05-18 19:09:39 +02:00
|
|
|
unsigned long power = SCHED_POWER_SCALE;
|
2009-12-17 17:00:43 +01:00
|
|
|
struct sched_group *sdg = sd->groups;
|
|
|
|
|
|
|
|
if ((sd->flags & SD_SHARE_CPUPOWER) && weight > 1) {
|
|
|
|
if (sched_feat(ARCH_POWER))
|
|
|
|
power *= arch_scale_smt_power(sd, cpu);
|
|
|
|
else
|
|
|
|
power *= default_scale_smt_power(sd, cpu);
|
|
|
|
|
2011-05-18 19:09:39 +02:00
|
|
|
power >>= SCHED_POWER_SHIFT;
|
2009-12-17 17:00:43 +01:00
|
|
|
}
|
|
|
|
|
2011-07-14 13:00:06 +02:00
|
|
|
sdg->sgp->power_orig = power;
|
2010-06-08 06:57:02 +02:00
|
|
|
|
|
|
|
if (sched_feat(ARCH_POWER))
|
|
|
|
power *= arch_scale_freq_power(sd, cpu);
|
|
|
|
else
|
|
|
|
power *= default_scale_freq_power(sd, cpu);
|
|
|
|
|
2011-05-18 19:09:39 +02:00
|
|
|
power >>= SCHED_POWER_SHIFT;
|
2010-06-08 06:57:02 +02:00
|
|
|
|
2009-12-17 17:00:43 +01:00
|
|
|
power *= scale_rt_power(cpu);
|
2011-05-18 19:09:39 +02:00
|
|
|
power >>= SCHED_POWER_SHIFT;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
|
|
|
if (!power)
|
|
|
|
power = 1;
|
|
|
|
|
2010-05-31 12:37:30 +02:00
|
|
|
cpu_rq(cpu)->cpu_power = power;
|
2011-07-14 13:00:06 +02:00
|
|
|
sdg->sgp->power = power;
|
2009-12-17 17:00:43 +01:00
|
|
|
}
|
|
|
|
|
2011-10-25 10:00:11 +02:00
|
|
|
void update_group_power(struct sched_domain *sd, int cpu)
|
2009-12-17 17:00:43 +01:00
|
|
|
{
|
|
|
|
struct sched_domain *child = sd->child;
|
|
|
|
struct sched_group *group, *sdg = sd->groups;
|
|
|
|
unsigned long power;
|
|
|
|
|
|
|
|
if (!child) {
|
|
|
|
update_cpu_power(sd, cpu);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
power = 0;
|
|
|
|
|
|
|
|
group = child->groups;
|
|
|
|
do {
|
2011-07-14 13:00:06 +02:00
|
|
|
power += group->sgp->power;
|
2009-12-17 17:00:43 +01:00
|
|
|
group = group->next;
|
|
|
|
} while (group != child->groups);
|
|
|
|
|
2011-07-14 13:00:06 +02:00
|
|
|
sdg->sgp->power = power;
|
2009-12-17 17:00:43 +01:00
|
|
|
}
|
|
|
|
|
2010-06-08 06:57:02 +02:00
|
|
|
/*
|
|
|
|
* Try and fix up capacity for tiny siblings, this is needed when
|
|
|
|
* things like SD_ASYM_PACKING need f_b_g to select another sibling
|
|
|
|
* which on its own isn't powerful enough.
|
|
|
|
*
|
|
|
|
* See update_sd_pick_busiest() and check_asym_packing().
|
|
|
|
*/
|
|
|
|
static inline int
|
|
|
|
fix_small_capacity(struct sched_domain *sd, struct sched_group *group)
|
|
|
|
{
|
|
|
|
/*
|
2011-05-18 19:09:39 +02:00
|
|
|
* Only siblings can have significantly less than SCHED_POWER_SCALE
|
2010-06-08 06:57:02 +02:00
|
|
|
*/
|
2011-04-07 14:09:52 +02:00
|
|
|
if (!(sd->flags & SD_SHARE_CPUPOWER))
|
2010-06-08 06:57:02 +02:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If ~90% of the cpu_power is still there, we're good.
|
|
|
|
*/
|
2011-07-14 13:00:06 +02:00
|
|
|
if (group->sgp->power * 32 > group->sgp->power_orig * 29)
|
2010-06-08 06:57:02 +02:00
|
|
|
return 1;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2009-12-17 17:00:43 +01:00
|
|
|
/**
|
|
|
|
* update_sg_lb_stats - Update sched_group's statistics for load balancing.
|
|
|
|
* @sd: The sched_domain whose statistics are to be updated.
|
|
|
|
* @group: sched_group whose statistics are to be updated.
|
|
|
|
* @this_cpu: Cpu for which load balance is currently performed.
|
|
|
|
* @idle: Idle status of this_cpu
|
|
|
|
* @load_idx: Load index of sched_domain of this_cpu for load calc.
|
|
|
|
* @local_group: Does group contain this_cpu.
|
|
|
|
* @cpus: Set of cpus considered for load balancing.
|
|
|
|
* @balance: Should we balance.
|
|
|
|
* @sgs: variable to hold the statistics for this group.
|
|
|
|
*/
|
|
|
|
static inline void update_sg_lb_stats(struct sched_domain *sd,
|
|
|
|
struct sched_group *group, int this_cpu,
|
2011-02-14 23:38:50 +01:00
|
|
|
enum cpu_idle_type idle, int load_idx,
|
2009-12-17 17:00:43 +01:00
|
|
|
int local_group, const struct cpumask *cpus,
|
|
|
|
int *balance, struct sg_lb_stats *sgs)
|
|
|
|
{
|
2010-10-13 21:09:36 +02:00
|
|
|
unsigned long load, max_cpu_load, min_cpu_load, max_nr_running;
|
2009-12-17 17:00:43 +01:00
|
|
|
int i;
|
|
|
|
unsigned int balance_cpu = -1, first_idle_cpu = 0;
|
2010-02-24 01:13:52 +01:00
|
|
|
unsigned long avg_load_per_task = 0;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
2010-01-20 21:02:44 +01:00
|
|
|
if (local_group)
|
2009-12-17 17:00:43 +01:00
|
|
|
balance_cpu = group_first_cpu(group);
|
|
|
|
|
|
|
|
/* Tally up the load of all CPUs in the group */
|
|
|
|
max_cpu_load = 0;
|
|
|
|
min_cpu_load = ~0UL;
|
2010-10-13 21:09:36 +02:00
|
|
|
max_nr_running = 0;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
|
|
|
for_each_cpu_and(i, sched_group_cpus(group), cpus) {
|
|
|
|
struct rq *rq = cpu_rq(i);
|
|
|
|
|
|
|
|
/* Bias balancing toward cpus of our domain */
|
|
|
|
if (local_group) {
|
|
|
|
if (idle_cpu(i) && !first_idle_cpu) {
|
|
|
|
first_idle_cpu = 1;
|
|
|
|
balance_cpu = i;
|
|
|
|
}
|
|
|
|
|
|
|
|
load = target_load(i, load_idx);
|
|
|
|
} else {
|
|
|
|
load = source_load(i, load_idx);
|
2010-10-13 21:09:36 +02:00
|
|
|
if (load > max_cpu_load) {
|
2009-12-17 17:00:43 +01:00
|
|
|
max_cpu_load = load;
|
2010-10-13 21:09:36 +02:00
|
|
|
max_nr_running = rq->nr_running;
|
|
|
|
}
|
2009-12-17 17:00:43 +01:00
|
|
|
if (min_cpu_load > load)
|
|
|
|
min_cpu_load = load;
|
|
|
|
}
|
|
|
|
|
|
|
|
sgs->group_load += load;
|
|
|
|
sgs->sum_nr_running += rq->nr_running;
|
|
|
|
sgs->sum_weighted_load += weighted_cpuload(i);
|
2010-09-18 00:02:32 +02:00
|
|
|
if (idle_cpu(i))
|
|
|
|
sgs->idle_cpus++;
|
2009-12-17 17:00:43 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* First idle cpu or the first cpu(busiest) in this sched group
|
|
|
|
* is eligible for doing load balancing at this and above
|
|
|
|
* domains. In the newly idle case, we will allow all the cpu's
|
|
|
|
* to do the newly idle load balance.
|
|
|
|
*/
|
2010-07-09 15:15:43 +02:00
|
|
|
if (idle != CPU_NEWLY_IDLE && local_group) {
|
|
|
|
if (balance_cpu != this_cpu) {
|
|
|
|
*balance = 0;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
update_group_power(sd, this_cpu);
|
2009-12-17 17:00:43 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Adjust by relative CPU power of the group */
|
2011-07-14 13:00:06 +02:00
|
|
|
sgs->avg_load = (sgs->group_load*SCHED_POWER_SCALE) / group->sgp->power;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Consider the group unbalanced when the imbalance is larger
|
2011-02-21 18:56:47 +01:00
|
|
|
* than the average weight of a task.
|
2009-12-17 17:00:43 +01:00
|
|
|
*
|
|
|
|
* APZ: with cgroup the avg task weight can vary wildly and
|
|
|
|
* might not be a suitable number - should we keep a
|
|
|
|
* normalized nr_running number somewhere that negates
|
|
|
|
* the hierarchy?
|
|
|
|
*/
|
2010-02-24 01:13:52 +01:00
|
|
|
if (sgs->sum_nr_running)
|
|
|
|
avg_load_per_task = sgs->sum_weighted_load / sgs->sum_nr_running;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
2011-02-21 18:56:47 +01:00
|
|
|
if ((max_cpu_load - min_cpu_load) >= avg_load_per_task && max_nr_running > 1)
|
2009-12-17 17:00:43 +01:00
|
|
|
sgs->group_imb = 1;
|
|
|
|
|
2011-07-14 13:00:06 +02:00
|
|
|
sgs->group_capacity = DIV_ROUND_CLOSEST(group->sgp->power,
|
2011-05-18 19:09:39 +02:00
|
|
|
SCHED_POWER_SCALE);
|
2010-06-08 06:57:02 +02:00
|
|
|
if (!sgs->group_capacity)
|
|
|
|
sgs->group_capacity = fix_small_capacity(sd, group);
|
2010-09-18 00:02:32 +02:00
|
|
|
sgs->group_weight = group->group_weight;
|
2010-10-15 22:12:29 +02:00
|
|
|
|
|
|
|
if (sgs->group_capacity > sgs->sum_nr_running)
|
|
|
|
sgs->group_has_capacity = 1;
|
2009-12-17 17:00:43 +01:00
|
|
|
}
|
|
|
|
|
2010-06-08 06:57:02 +02:00
|
|
|
/**
|
|
|
|
* update_sd_pick_busiest - return 1 on busiest group
|
|
|
|
* @sd: sched_domain whose statistics are to be checked
|
|
|
|
* @sds: sched_domain statistics
|
|
|
|
* @sg: sched_group candidate to be checked for being the busiest
|
2010-06-10 04:06:21 +02:00
|
|
|
* @sgs: sched_group statistics
|
|
|
|
* @this_cpu: the current cpu
|
2010-06-08 06:57:02 +02:00
|
|
|
*
|
|
|
|
* Determine if @sg is a busier group than the previously selected
|
|
|
|
* busiest group.
|
|
|
|
*/
|
|
|
|
static bool update_sd_pick_busiest(struct sched_domain *sd,
|
|
|
|
struct sd_lb_stats *sds,
|
|
|
|
struct sched_group *sg,
|
|
|
|
struct sg_lb_stats *sgs,
|
|
|
|
int this_cpu)
|
|
|
|
{
|
|
|
|
if (sgs->avg_load <= sds->max_load)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
if (sgs->sum_nr_running > sgs->group_capacity)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
if (sgs->group_imb)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* ASYM_PACKING needs to move all the work to the lowest
|
|
|
|
* numbered CPUs in the group, therefore mark all groups
|
|
|
|
* higher than ourself as busy.
|
|
|
|
*/
|
|
|
|
if ((sd->flags & SD_ASYM_PACKING) && sgs->sum_nr_running &&
|
|
|
|
this_cpu < group_first_cpu(sg)) {
|
|
|
|
if (!sds->busiest)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
if (group_first_cpu(sds->busiest) > group_first_cpu(sg))
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2009-12-17 17:00:43 +01:00
|
|
|
/**
|
2011-10-12 05:00:59 +02:00
|
|
|
* update_sd_lb_stats - Update sched_domain's statistics for load balancing.
|
2009-12-17 17:00:43 +01:00
|
|
|
* @sd: sched_domain whose statistics are to be updated.
|
|
|
|
* @this_cpu: Cpu for which load balance is currently performed.
|
|
|
|
* @idle: Idle status of this_cpu
|
|
|
|
* @cpus: Set of cpus considered for load balancing.
|
|
|
|
* @balance: Should we balance.
|
|
|
|
* @sds: variable to hold the statistics for this sched_domain.
|
|
|
|
*/
|
|
|
|
static inline void update_sd_lb_stats(struct sched_domain *sd, int this_cpu,
|
2011-02-14 23:38:50 +01:00
|
|
|
enum cpu_idle_type idle, const struct cpumask *cpus,
|
|
|
|
int *balance, struct sd_lb_stats *sds)
|
2009-12-17 17:00:43 +01:00
|
|
|
{
|
|
|
|
struct sched_domain *child = sd->child;
|
2010-06-08 06:57:02 +02:00
|
|
|
struct sched_group *sg = sd->groups;
|
2009-12-17 17:00:43 +01:00
|
|
|
struct sg_lb_stats sgs;
|
|
|
|
int load_idx, prefer_sibling = 0;
|
|
|
|
|
|
|
|
if (child && child->flags & SD_PREFER_SIBLING)
|
|
|
|
prefer_sibling = 1;
|
|
|
|
|
|
|
|
init_sd_power_savings_stats(sd, sds, idle);
|
|
|
|
load_idx = get_sd_load_idx(sd, idle);
|
|
|
|
|
|
|
|
do {
|
|
|
|
int local_group;
|
|
|
|
|
2010-06-08 06:57:02 +02:00
|
|
|
local_group = cpumask_test_cpu(this_cpu, sched_group_cpus(sg));
|
2009-12-17 17:00:43 +01:00
|
|
|
memset(&sgs, 0, sizeof(sgs));
|
2011-02-14 23:38:50 +01:00
|
|
|
update_sg_lb_stats(sd, sg, this_cpu, idle, load_idx,
|
2009-12-17 17:00:43 +01:00
|
|
|
local_group, cpus, balance, &sgs);
|
|
|
|
|
2009-12-24 14:18:21 +01:00
|
|
|
if (local_group && !(*balance))
|
2009-12-17 17:00:43 +01:00
|
|
|
return;
|
|
|
|
|
|
|
|
sds->total_load += sgs.group_load;
|
2011-07-14 13:00:06 +02:00
|
|
|
sds->total_pwr += sg->sgp->power;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* In case the child domain prefers tasks go to siblings
|
2010-06-08 06:57:02 +02:00
|
|
|
* first, lower the sg capacity to one so that we'll try
|
sched: Drop group_capacity to 1 only if local group has extra capacity
When SD_PREFER_SIBLING is set on a sched domain, drop group_capacity to 1
only if the local group has extra capacity. The extra check prevents the case
where you always pull from the heaviest group when it is already under-utilized
(possible with a large weight task outweighs the tasks on the system).
For example, consider a 16-cpu quad-core quad-socket machine with MC and NUMA
scheduling domains. Let's say we spawn 15 nice0 tasks and one nice-15 task,
and each task is running on one core. In this case, we observe the following
events when balancing at the NUMA domain:
- find_busiest_group() will always pick the sched group containing the niced
task to be the busiest group.
- find_busiest_queue() will then always pick one of the cpus running the
nice0 task (never picks the cpu with the nice -15 task since
weighted_cpuload > imbalance).
- The load balancer fails to migrate the task since it is the running task
and increments sd->nr_balance_failed.
- It repeats the above steps a few more times until sd->nr_balance_failed > 5,
at which point it kicks off the active load balancer, wakes up the migration
thread and kicks the nice 0 task off the cpu.
The load balancer doesn't stop until we kick out all nice 0 tasks from
the sched group, leaving you with 3 idle cpus and one cpu running the
nice -15 task.
When balancing at the NUMA domain, we drop sgs.group_capacity to 1 if the child
domain (in this case MC) has SD_PREFER_SIBLING set. Subsequent load checks are
not relevant because the niced task has a very large weight.
In this patch, we add an extra condition to the "if(prefer_sibling)" check in
update_sd_lb_stats(). We drop the capacity of a group only if the local group
has extra capacity, ie. nr_running < group_capacity. This patch preserves the
original intent of the prefer_siblings check (to spread tasks across the system
in low utilization scenarios) and fixes the case above.
It helps in the following ways:
- In low utilization cases (where nr_tasks << nr_cpus), we still drop
group_capacity down to 1 if we prefer siblings.
- On very busy systems (where nr_tasks >> nr_cpus), sgs.nr_running will most
likely be > sgs.group_capacity.
- When balancing large weight tasks, if the local group does not have extra
capacity, we do not pick the group with the niced task as the busiest group.
This prevents failed balances, active migration and the under-utilization
described above.
Signed-off-by: Nikhil Rao <ncrao@google.com>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <1287173550-30365-5-git-send-email-ncrao@google.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-10-15 22:12:30 +02:00
|
|
|
* and move all the excess tasks away. We lower the capacity
|
|
|
|
* of a group only if the local group has the capacity to fit
|
|
|
|
* these excess tasks, i.e. nr_running < group_capacity. The
|
|
|
|
* extra check prevents the case where you always pull from the
|
|
|
|
* heaviest group when it is already under-utilized (possible
|
|
|
|
* with a large weight task outweighs the tasks on the system).
|
2009-12-17 17:00:43 +01:00
|
|
|
*/
|
sched: Drop group_capacity to 1 only if local group has extra capacity
When SD_PREFER_SIBLING is set on a sched domain, drop group_capacity to 1
only if the local group has extra capacity. The extra check prevents the case
where you always pull from the heaviest group when it is already under-utilized
(possible with a large weight task outweighs the tasks on the system).
For example, consider a 16-cpu quad-core quad-socket machine with MC and NUMA
scheduling domains. Let's say we spawn 15 nice0 tasks and one nice-15 task,
and each task is running on one core. In this case, we observe the following
events when balancing at the NUMA domain:
- find_busiest_group() will always pick the sched group containing the niced
task to be the busiest group.
- find_busiest_queue() will then always pick one of the cpus running the
nice0 task (never picks the cpu with the nice -15 task since
weighted_cpuload > imbalance).
- The load balancer fails to migrate the task since it is the running task
and increments sd->nr_balance_failed.
- It repeats the above steps a few more times until sd->nr_balance_failed > 5,
at which point it kicks off the active load balancer, wakes up the migration
thread and kicks the nice 0 task off the cpu.
The load balancer doesn't stop until we kick out all nice 0 tasks from
the sched group, leaving you with 3 idle cpus and one cpu running the
nice -15 task.
When balancing at the NUMA domain, we drop sgs.group_capacity to 1 if the child
domain (in this case MC) has SD_PREFER_SIBLING set. Subsequent load checks are
not relevant because the niced task has a very large weight.
In this patch, we add an extra condition to the "if(prefer_sibling)" check in
update_sd_lb_stats(). We drop the capacity of a group only if the local group
has extra capacity, ie. nr_running < group_capacity. This patch preserves the
original intent of the prefer_siblings check (to spread tasks across the system
in low utilization scenarios) and fixes the case above.
It helps in the following ways:
- In low utilization cases (where nr_tasks << nr_cpus), we still drop
group_capacity down to 1 if we prefer siblings.
- On very busy systems (where nr_tasks >> nr_cpus), sgs.nr_running will most
likely be > sgs.group_capacity.
- When balancing large weight tasks, if the local group does not have extra
capacity, we do not pick the group with the niced task as the busiest group.
This prevents failed balances, active migration and the under-utilization
described above.
Signed-off-by: Nikhil Rao <ncrao@google.com>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <1287173550-30365-5-git-send-email-ncrao@google.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-10-15 22:12:30 +02:00
|
|
|
if (prefer_sibling && !local_group && sds->this_has_capacity)
|
2009-12-17 17:00:43 +01:00
|
|
|
sgs.group_capacity = min(sgs.group_capacity, 1UL);
|
|
|
|
|
|
|
|
if (local_group) {
|
|
|
|
sds->this_load = sgs.avg_load;
|
2010-06-08 06:57:02 +02:00
|
|
|
sds->this = sg;
|
2009-12-17 17:00:43 +01:00
|
|
|
sds->this_nr_running = sgs.sum_nr_running;
|
|
|
|
sds->this_load_per_task = sgs.sum_weighted_load;
|
2010-10-15 22:12:29 +02:00
|
|
|
sds->this_has_capacity = sgs.group_has_capacity;
|
2010-09-18 00:02:32 +02:00
|
|
|
sds->this_idle_cpus = sgs.idle_cpus;
|
2010-06-08 06:57:02 +02:00
|
|
|
} else if (update_sd_pick_busiest(sd, sds, sg, &sgs, this_cpu)) {
|
2009-12-17 17:00:43 +01:00
|
|
|
sds->max_load = sgs.avg_load;
|
2010-06-08 06:57:02 +02:00
|
|
|
sds->busiest = sg;
|
2009-12-17 17:00:43 +01:00
|
|
|
sds->busiest_nr_running = sgs.sum_nr_running;
|
2010-09-18 00:02:32 +02:00
|
|
|
sds->busiest_idle_cpus = sgs.idle_cpus;
|
2010-02-24 01:13:52 +01:00
|
|
|
sds->busiest_group_capacity = sgs.group_capacity;
|
2009-12-17 17:00:43 +01:00
|
|
|
sds->busiest_load_per_task = sgs.sum_weighted_load;
|
2010-10-15 22:12:29 +02:00
|
|
|
sds->busiest_has_capacity = sgs.group_has_capacity;
|
2010-09-18 00:02:32 +02:00
|
|
|
sds->busiest_group_weight = sgs.group_weight;
|
2009-12-17 17:00:43 +01:00
|
|
|
sds->group_imb = sgs.group_imb;
|
|
|
|
}
|
|
|
|
|
2010-06-08 06:57:02 +02:00
|
|
|
update_sd_power_savings_stats(sg, sds, local_group, &sgs);
|
|
|
|
sg = sg->next;
|
|
|
|
} while (sg != sd->groups);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* check_asym_packing - Check to see if the group is packed into the
|
|
|
|
* sched doman.
|
|
|
|
*
|
|
|
|
* This is primarily intended to used at the sibling level. Some
|
|
|
|
* cores like POWER7 prefer to use lower numbered SMT threads. In the
|
|
|
|
* case of POWER7, it can move to lower SMT modes only when higher
|
|
|
|
* threads are idle. When in lower SMT modes, the threads will
|
|
|
|
* perform better since they share less core resources. Hence when we
|
|
|
|
* have idle threads, we want them to be the higher ones.
|
|
|
|
*
|
|
|
|
* This packing function is run on idle threads. It checks to see if
|
|
|
|
* the busiest CPU in this domain (core in the P7 case) has a higher
|
|
|
|
* CPU number than the packing function is being run on. Here we are
|
|
|
|
* assuming lower CPU number will be equivalent to lower a SMT thread
|
|
|
|
* number.
|
|
|
|
*
|
2010-06-10 04:06:21 +02:00
|
|
|
* Returns 1 when packing is required and a task should be moved to
|
|
|
|
* this CPU. The amount of the imbalance is returned in *imbalance.
|
|
|
|
*
|
2010-06-08 06:57:02 +02:00
|
|
|
* @sd: The sched_domain whose packing is to be checked.
|
|
|
|
* @sds: Statistics of the sched_domain which is to be packed
|
|
|
|
* @this_cpu: The cpu at whose sched_domain we're performing load-balance.
|
|
|
|
* @imbalance: returns amount of imbalanced due to packing.
|
|
|
|
*/
|
|
|
|
static int check_asym_packing(struct sched_domain *sd,
|
|
|
|
struct sd_lb_stats *sds,
|
|
|
|
int this_cpu, unsigned long *imbalance)
|
|
|
|
{
|
|
|
|
int busiest_cpu;
|
|
|
|
|
|
|
|
if (!(sd->flags & SD_ASYM_PACKING))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (!sds->busiest)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
busiest_cpu = group_first_cpu(sds->busiest);
|
|
|
|
if (this_cpu > busiest_cpu)
|
|
|
|
return 0;
|
|
|
|
|
2011-07-14 13:00:06 +02:00
|
|
|
*imbalance = DIV_ROUND_CLOSEST(sds->max_load * sds->busiest->sgp->power,
|
2011-05-18 19:09:39 +02:00
|
|
|
SCHED_POWER_SCALE);
|
2010-06-08 06:57:02 +02:00
|
|
|
return 1;
|
2009-12-17 17:00:43 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* fix_small_imbalance - Calculate the minor imbalance that exists
|
|
|
|
* amongst the groups of a sched_domain, during
|
|
|
|
* load balancing.
|
|
|
|
* @sds: Statistics of the sched_domain whose imbalance is to be calculated.
|
|
|
|
* @this_cpu: The cpu at whose sched_domain we're performing load-balance.
|
|
|
|
* @imbalance: Variable to store the imbalance.
|
|
|
|
*/
|
|
|
|
static inline void fix_small_imbalance(struct sd_lb_stats *sds,
|
|
|
|
int this_cpu, unsigned long *imbalance)
|
|
|
|
{
|
|
|
|
unsigned long tmp, pwr_now = 0, pwr_move = 0;
|
|
|
|
unsigned int imbn = 2;
|
2010-02-24 01:13:52 +01:00
|
|
|
unsigned long scaled_busy_load_per_task;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
|
|
|
if (sds->this_nr_running) {
|
|
|
|
sds->this_load_per_task /= sds->this_nr_running;
|
|
|
|
if (sds->busiest_load_per_task >
|
|
|
|
sds->this_load_per_task)
|
|
|
|
imbn = 1;
|
|
|
|
} else
|
|
|
|
sds->this_load_per_task =
|
|
|
|
cpu_avg_load_per_task(this_cpu);
|
|
|
|
|
2010-02-24 01:13:52 +01:00
|
|
|
scaled_busy_load_per_task = sds->busiest_load_per_task
|
2011-05-18 19:09:39 +02:00
|
|
|
* SCHED_POWER_SCALE;
|
2011-07-14 13:00:06 +02:00
|
|
|
scaled_busy_load_per_task /= sds->busiest->sgp->power;
|
2010-02-24 01:13:52 +01:00
|
|
|
|
|
|
|
if (sds->max_load - sds->this_load + scaled_busy_load_per_task >=
|
|
|
|
(scaled_busy_load_per_task * imbn)) {
|
2009-12-17 17:00:43 +01:00
|
|
|
*imbalance = sds->busiest_load_per_task;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* OK, we don't have enough imbalance to justify moving tasks,
|
|
|
|
* however we may be able to increase total CPU power used by
|
|
|
|
* moving them.
|
|
|
|
*/
|
|
|
|
|
2011-07-14 13:00:06 +02:00
|
|
|
pwr_now += sds->busiest->sgp->power *
|
2009-12-17 17:00:43 +01:00
|
|
|
min(sds->busiest_load_per_task, sds->max_load);
|
2011-07-14 13:00:06 +02:00
|
|
|
pwr_now += sds->this->sgp->power *
|
2009-12-17 17:00:43 +01:00
|
|
|
min(sds->this_load_per_task, sds->this_load);
|
2011-05-18 19:09:39 +02:00
|
|
|
pwr_now /= SCHED_POWER_SCALE;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
|
|
|
/* Amount of load we'd subtract */
|
2011-05-18 19:09:39 +02:00
|
|
|
tmp = (sds->busiest_load_per_task * SCHED_POWER_SCALE) /
|
2011-07-14 13:00:06 +02:00
|
|
|
sds->busiest->sgp->power;
|
2009-12-17 17:00:43 +01:00
|
|
|
if (sds->max_load > tmp)
|
2011-07-14 13:00:06 +02:00
|
|
|
pwr_move += sds->busiest->sgp->power *
|
2009-12-17 17:00:43 +01:00
|
|
|
min(sds->busiest_load_per_task, sds->max_load - tmp);
|
|
|
|
|
|
|
|
/* Amount of load we'd add */
|
2011-07-14 13:00:06 +02:00
|
|
|
if (sds->max_load * sds->busiest->sgp->power <
|
2011-05-18 19:09:39 +02:00
|
|
|
sds->busiest_load_per_task * SCHED_POWER_SCALE)
|
2011-07-14 13:00:06 +02:00
|
|
|
tmp = (sds->max_load * sds->busiest->sgp->power) /
|
|
|
|
sds->this->sgp->power;
|
2009-12-17 17:00:43 +01:00
|
|
|
else
|
2011-05-18 19:09:39 +02:00
|
|
|
tmp = (sds->busiest_load_per_task * SCHED_POWER_SCALE) /
|
2011-07-14 13:00:06 +02:00
|
|
|
sds->this->sgp->power;
|
|
|
|
pwr_move += sds->this->sgp->power *
|
2009-12-17 17:00:43 +01:00
|
|
|
min(sds->this_load_per_task, sds->this_load + tmp);
|
2011-05-18 19:09:39 +02:00
|
|
|
pwr_move /= SCHED_POWER_SCALE;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
|
|
|
/* Move if we gain throughput */
|
|
|
|
if (pwr_move > pwr_now)
|
|
|
|
*imbalance = sds->busiest_load_per_task;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* calculate_imbalance - Calculate the amount of imbalance present within the
|
|
|
|
* groups of a given sched_domain during load balance.
|
|
|
|
* @sds: statistics of the sched_domain whose imbalance is to be calculated.
|
|
|
|
* @this_cpu: Cpu for which currently load balance is being performed.
|
|
|
|
* @imbalance: The variable to store the imbalance.
|
|
|
|
*/
|
|
|
|
static inline void calculate_imbalance(struct sd_lb_stats *sds, int this_cpu,
|
|
|
|
unsigned long *imbalance)
|
|
|
|
{
|
2010-02-24 01:13:52 +01:00
|
|
|
unsigned long max_pull, load_above_capacity = ~0UL;
|
|
|
|
|
|
|
|
sds->busiest_load_per_task /= sds->busiest_nr_running;
|
|
|
|
if (sds->group_imb) {
|
|
|
|
sds->busiest_load_per_task =
|
|
|
|
min(sds->busiest_load_per_task, sds->avg_load);
|
|
|
|
}
|
|
|
|
|
2009-12-17 17:00:43 +01:00
|
|
|
/*
|
|
|
|
* In the presence of smp nice balancing, certain scenarios can have
|
|
|
|
* max load less than avg load(as we skip the groups at or below
|
|
|
|
* its cpu_power, while calculating max_load..)
|
|
|
|
*/
|
|
|
|
if (sds->max_load < sds->avg_load) {
|
|
|
|
*imbalance = 0;
|
|
|
|
return fix_small_imbalance(sds, this_cpu, imbalance);
|
|
|
|
}
|
|
|
|
|
2010-02-24 01:13:52 +01:00
|
|
|
if (!sds->group_imb) {
|
|
|
|
/*
|
|
|
|
* Don't want to pull so many tasks that a group would go idle.
|
|
|
|
*/
|
|
|
|
load_above_capacity = (sds->busiest_nr_running -
|
|
|
|
sds->busiest_group_capacity);
|
|
|
|
|
2011-05-18 19:09:39 +02:00
|
|
|
load_above_capacity *= (SCHED_LOAD_SCALE * SCHED_POWER_SCALE);
|
2010-02-24 01:13:52 +01:00
|
|
|
|
2011-07-14 13:00:06 +02:00
|
|
|
load_above_capacity /= sds->busiest->sgp->power;
|
2010-02-24 01:13:52 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We're trying to get all the cpus to the average_load, so we don't
|
|
|
|
* want to push ourselves above the average load, nor do we wish to
|
|
|
|
* reduce the max loaded cpu below the average load. At the same time,
|
|
|
|
* we also don't want to reduce the group load below the group capacity
|
|
|
|
* (so that we can implement power-savings policies etc). Thus we look
|
|
|
|
* for the minimum possible imbalance.
|
|
|
|
* Be careful of negative numbers as they'll appear as very large values
|
|
|
|
* with unsigned longs.
|
|
|
|
*/
|
|
|
|
max_pull = min(sds->max_load - sds->avg_load, load_above_capacity);
|
2009-12-17 17:00:43 +01:00
|
|
|
|
|
|
|
/* How much load to actually move to equalise the imbalance */
|
2011-07-14 13:00:06 +02:00
|
|
|
*imbalance = min(max_pull * sds->busiest->sgp->power,
|
|
|
|
(sds->avg_load - sds->this_load) * sds->this->sgp->power)
|
2011-05-18 19:09:39 +02:00
|
|
|
/ SCHED_POWER_SCALE;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* if *imbalance is less than the average load per runnable task
|
2011-03-31 03:57:33 +02:00
|
|
|
* there is no guarantee that any tasks will be moved so we'll have
|
2009-12-17 17:00:43 +01:00
|
|
|
* a think about bumping its value to force at least one task to be
|
|
|
|
* moved
|
|
|
|
*/
|
|
|
|
if (*imbalance < sds->busiest_load_per_task)
|
|
|
|
return fix_small_imbalance(sds, this_cpu, imbalance);
|
|
|
|
|
|
|
|
}
|
2010-10-15 22:12:29 +02:00
|
|
|
|
2009-12-17 17:00:43 +01:00
|
|
|
/******* find_busiest_group() helpers end here *********************/
|
|
|
|
|
|
|
|
/**
|
|
|
|
* find_busiest_group - Returns the busiest group within the sched_domain
|
|
|
|
* if there is an imbalance. If there isn't an imbalance, and
|
|
|
|
* the user has opted for power-savings, it returns a group whose
|
|
|
|
* CPUs can be put to idle by rebalancing those tasks elsewhere, if
|
|
|
|
* such a group exists.
|
|
|
|
*
|
|
|
|
* Also calculates the amount of weighted load which should be moved
|
|
|
|
* to restore balance.
|
|
|
|
*
|
|
|
|
* @sd: The sched_domain whose busiest group is to be returned.
|
|
|
|
* @this_cpu: The cpu for which load balancing is currently being performed.
|
|
|
|
* @imbalance: Variable which stores amount of weighted load which should
|
|
|
|
* be moved to restore balance/put a group to idle.
|
|
|
|
* @idle: The idle status of this_cpu.
|
|
|
|
* @cpus: The set of CPUs under consideration for load-balancing.
|
|
|
|
* @balance: Pointer to a variable indicating if this_cpu
|
|
|
|
* is the appropriate cpu to perform load balancing at this_level.
|
|
|
|
*
|
|
|
|
* Returns: - the busiest group if imbalance exists.
|
|
|
|
* - If no imbalance and user has opted for power-savings balance,
|
|
|
|
* return the least loaded group whose CPUs can be
|
|
|
|
* put to idle by rebalancing its tasks onto our group.
|
|
|
|
*/
|
|
|
|
static struct sched_group *
|
|
|
|
find_busiest_group(struct sched_domain *sd, int this_cpu,
|
|
|
|
unsigned long *imbalance, enum cpu_idle_type idle,
|
2011-02-14 23:38:50 +01:00
|
|
|
const struct cpumask *cpus, int *balance)
|
2009-12-17 17:00:43 +01:00
|
|
|
{
|
|
|
|
struct sd_lb_stats sds;
|
|
|
|
|
|
|
|
memset(&sds, 0, sizeof(sds));
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Compute the various statistics relavent for load balancing at
|
|
|
|
* this level.
|
|
|
|
*/
|
2011-02-14 23:38:50 +01:00
|
|
|
update_sd_lb_stats(sd, this_cpu, idle, cpus, balance, &sds);
|
2009-12-17 17:00:43 +01:00
|
|
|
|
2011-02-21 18:55:32 +01:00
|
|
|
/*
|
|
|
|
* this_cpu is not the appropriate cpu to perform load balancing at
|
|
|
|
* this level.
|
2009-12-17 17:00:43 +01:00
|
|
|
*/
|
2009-12-24 14:18:21 +01:00
|
|
|
if (!(*balance))
|
2009-12-17 17:00:43 +01:00
|
|
|
goto ret;
|
|
|
|
|
2010-06-08 06:57:02 +02:00
|
|
|
if ((idle == CPU_IDLE || idle == CPU_NEWLY_IDLE) &&
|
|
|
|
check_asym_packing(sd, &sds, this_cpu, imbalance))
|
|
|
|
return sds.busiest;
|
|
|
|
|
2011-02-21 18:55:32 +01:00
|
|
|
/* There is no busy sibling group to pull tasks from */
|
2009-12-17 17:00:43 +01:00
|
|
|
if (!sds.busiest || sds.busiest_nr_running == 0)
|
|
|
|
goto out_balanced;
|
|
|
|
|
2011-05-18 19:09:39 +02:00
|
|
|
sds.avg_load = (SCHED_POWER_SCALE * sds.total_load) / sds.total_pwr;
|
2011-04-08 02:23:22 +02:00
|
|
|
|
2011-02-21 18:56:47 +01:00
|
|
|
/*
|
|
|
|
* If the busiest group is imbalanced the below checks don't
|
|
|
|
* work because they assumes all things are equal, which typically
|
|
|
|
* isn't true due to cpus_allowed constraints and the like.
|
|
|
|
*/
|
|
|
|
if (sds.group_imb)
|
|
|
|
goto force_balance;
|
|
|
|
|
2011-02-21 18:55:32 +01:00
|
|
|
/* SD_BALANCE_NEWIDLE trumps SMP nice when underutilized */
|
2010-10-15 22:12:29 +02:00
|
|
|
if (idle == CPU_NEWLY_IDLE && sds.this_has_capacity &&
|
|
|
|
!sds.busiest_has_capacity)
|
|
|
|
goto force_balance;
|
|
|
|
|
2011-02-21 18:55:32 +01:00
|
|
|
/*
|
|
|
|
* If the local group is more busy than the selected busiest group
|
|
|
|
* don't try and pull any tasks.
|
|
|
|
*/
|
2009-12-17 17:00:43 +01:00
|
|
|
if (sds.this_load >= sds.max_load)
|
|
|
|
goto out_balanced;
|
|
|
|
|
2011-02-21 18:55:32 +01:00
|
|
|
/*
|
|
|
|
* Don't pull any tasks if this group is already above the domain
|
|
|
|
* average load.
|
|
|
|
*/
|
2009-12-17 17:00:43 +01:00
|
|
|
if (sds.this_load >= sds.avg_load)
|
|
|
|
goto out_balanced;
|
|
|
|
|
2011-02-21 18:52:53 +01:00
|
|
|
if (idle == CPU_IDLE) {
|
2010-09-18 00:02:32 +02:00
|
|
|
/*
|
|
|
|
* This cpu is idle. If the busiest group load doesn't
|
|
|
|
* have more tasks than the number of available cpu's and
|
|
|
|
* there is no imbalance between this and busiest group
|
|
|
|
* wrt to idle cpu's, it is balanced.
|
|
|
|
*/
|
2011-02-21 18:52:53 +01:00
|
|
|
if ((sds.this_idle_cpus <= sds.busiest_idle_cpus + 1) &&
|
2010-09-18 00:02:32 +02:00
|
|
|
sds.busiest_nr_running <= sds.busiest_group_weight)
|
|
|
|
goto out_balanced;
|
2011-02-21 18:52:53 +01:00
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* In the CPU_NEWLY_IDLE, CPU_NOT_IDLE cases, use
|
|
|
|
* imbalance_pct to be conservative.
|
|
|
|
*/
|
|
|
|
if (100 * sds.max_load <= sd->imbalance_pct * sds.this_load)
|
|
|
|
goto out_balanced;
|
2010-09-18 00:02:32 +02:00
|
|
|
}
|
2009-12-17 17:00:43 +01:00
|
|
|
|
2010-10-15 22:12:29 +02:00
|
|
|
force_balance:
|
2009-12-17 17:00:43 +01:00
|
|
|
/* Looks like there is an imbalance. Compute it */
|
|
|
|
calculate_imbalance(&sds, this_cpu, imbalance);
|
|
|
|
return sds.busiest;
|
|
|
|
|
|
|
|
out_balanced:
|
|
|
|
/*
|
|
|
|
* There is no obvious imbalance. But check if we can do some balancing
|
|
|
|
* to save power.
|
|
|
|
*/
|
|
|
|
if (check_power_save_busiest_group(&sds, this_cpu, imbalance))
|
|
|
|
return sds.busiest;
|
|
|
|
ret:
|
|
|
|
*imbalance = 0;
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* find_busiest_queue - find the busiest runqueue among the cpus in group.
|
|
|
|
*/
|
|
|
|
static struct rq *
|
2010-06-08 06:57:02 +02:00
|
|
|
find_busiest_queue(struct sched_domain *sd, struct sched_group *group,
|
|
|
|
enum cpu_idle_type idle, unsigned long imbalance,
|
|
|
|
const struct cpumask *cpus)
|
2009-12-17 17:00:43 +01:00
|
|
|
{
|
|
|
|
struct rq *busiest = NULL, *rq;
|
|
|
|
unsigned long max_load = 0;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for_each_cpu(i, sched_group_cpus(group)) {
|
|
|
|
unsigned long power = power_of(i);
|
2011-05-18 19:09:39 +02:00
|
|
|
unsigned long capacity = DIV_ROUND_CLOSEST(power,
|
|
|
|
SCHED_POWER_SCALE);
|
2009-12-17 17:00:43 +01:00
|
|
|
unsigned long wl;
|
|
|
|
|
2010-06-08 06:57:02 +02:00
|
|
|
if (!capacity)
|
|
|
|
capacity = fix_small_capacity(sd, group);
|
|
|
|
|
2009-12-17 17:00:43 +01:00
|
|
|
if (!cpumask_test_cpu(i, cpus))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
rq = cpu_rq(i);
|
2010-02-16 16:48:56 +01:00
|
|
|
wl = weighted_cpuload(i);
|
2009-12-17 17:00:43 +01:00
|
|
|
|
2010-02-16 16:48:56 +01:00
|
|
|
/*
|
|
|
|
* When comparing with imbalance, use weighted_cpuload()
|
|
|
|
* which is not scaled with the cpu power.
|
|
|
|
*/
|
2009-12-17 17:00:43 +01:00
|
|
|
if (capacity && rq->nr_running == 1 && wl > imbalance)
|
|
|
|
continue;
|
|
|
|
|
2010-02-16 16:48:56 +01:00
|
|
|
/*
|
|
|
|
* For the load comparisons with the other cpu's, consider
|
|
|
|
* the weighted_cpuload() scaled with the cpu power, so that
|
|
|
|
* the load can be moved away from the cpu that is potentially
|
|
|
|
* running at a lower capacity.
|
|
|
|
*/
|
2011-05-18 19:09:39 +02:00
|
|
|
wl = (wl * SCHED_POWER_SCALE) / power;
|
2010-02-16 16:48:56 +01:00
|
|
|
|
2009-12-17 17:00:43 +01:00
|
|
|
if (wl > max_load) {
|
|
|
|
max_load = wl;
|
|
|
|
busiest = rq;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return busiest;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Max backoff if we encounter pinned tasks. Pretty arbitrary value, but
|
|
|
|
* so long as it is large enough.
|
|
|
|
*/
|
|
|
|
#define MAX_PINNED_INTERVAL 512
|
|
|
|
|
|
|
|
/* Working cpumask for load_balance and load_balance_newidle. */
|
2011-10-25 10:00:11 +02:00
|
|
|
DEFINE_PER_CPU(cpumask_var_t, load_balance_tmpmask);
|
2009-12-17 17:00:43 +01:00
|
|
|
|
2011-02-14 23:38:50 +01:00
|
|
|
static int need_active_balance(struct sched_domain *sd, int idle,
|
2010-06-08 06:57:02 +02:00
|
|
|
int busiest_cpu, int this_cpu)
|
2009-12-23 15:10:31 +01:00
|
|
|
{
|
|
|
|
if (idle == CPU_NEWLY_IDLE) {
|
2010-06-08 06:57:02 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* ASYM_PACKING needs to force migrate tasks from busy but
|
|
|
|
* higher numbered CPUs in order to pack all tasks in the
|
|
|
|
* lowest numbered CPUs.
|
|
|
|
*/
|
|
|
|
if ((sd->flags & SD_ASYM_PACKING) && busiest_cpu > this_cpu)
|
|
|
|
return 1;
|
|
|
|
|
2009-12-23 15:10:31 +01:00
|
|
|
/*
|
|
|
|
* The only task running in a non-idle cpu can be moved to this
|
|
|
|
* cpu in an attempt to completely freeup the other CPU
|
|
|
|
* package.
|
|
|
|
*
|
|
|
|
* The package power saving logic comes from
|
|
|
|
* find_busiest_group(). If there are no imbalance, then
|
|
|
|
* f_b_g() will return NULL. However when sched_mc={1,2} then
|
|
|
|
* f_b_g() will select a group from which a running task may be
|
|
|
|
* pulled to this cpu in order to make the other package idle.
|
|
|
|
* If there is no opportunity to make a package idle and if
|
|
|
|
* there are no imbalance, then f_b_g() will return NULL and no
|
|
|
|
* action will be taken in load_balance_newidle().
|
|
|
|
*
|
|
|
|
* Under normal task pull operation due to imbalance, there
|
|
|
|
* will be more than one task in the source run queue and
|
|
|
|
* move_tasks() will succeed. ld_moved will be true and this
|
|
|
|
* active balance code will not be triggered.
|
|
|
|
*/
|
|
|
|
if (sched_mc_power_savings < POWERSAVINGS_BALANCE_WAKEUP)
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
return unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2);
|
|
|
|
}
|
|
|
|
|
2010-05-06 18:49:21 +02:00
|
|
|
static int active_load_balance_cpu_stop(void *data);
|
|
|
|
|
2009-12-17 17:00:43 +01:00
|
|
|
/*
|
|
|
|
* Check this_cpu to ensure it is balanced within domain. Attempt to move
|
|
|
|
* tasks if there is an imbalance.
|
|
|
|
*/
|
|
|
|
static int load_balance(int this_cpu, struct rq *this_rq,
|
|
|
|
struct sched_domain *sd, enum cpu_idle_type idle,
|
|
|
|
int *balance)
|
|
|
|
{
|
2011-09-22 15:23:13 +02:00
|
|
|
int ld_moved, lb_flags = 0, active_balance = 0;
|
2009-12-17 17:00:43 +01:00
|
|
|
struct sched_group *group;
|
|
|
|
unsigned long imbalance;
|
|
|
|
struct rq *busiest;
|
|
|
|
unsigned long flags;
|
|
|
|
struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask);
|
|
|
|
|
|
|
|
cpumask_copy(cpus, cpu_active_mask);
|
|
|
|
|
|
|
|
schedstat_inc(sd, lb_count[idle]);
|
|
|
|
|
|
|
|
redo:
|
2011-02-14 23:38:50 +01:00
|
|
|
group = find_busiest_group(sd, this_cpu, &imbalance, idle,
|
2009-12-17 17:00:43 +01:00
|
|
|
cpus, balance);
|
|
|
|
|
|
|
|
if (*balance == 0)
|
|
|
|
goto out_balanced;
|
|
|
|
|
|
|
|
if (!group) {
|
|
|
|
schedstat_inc(sd, lb_nobusyg[idle]);
|
|
|
|
goto out_balanced;
|
|
|
|
}
|
|
|
|
|
2010-06-08 06:57:02 +02:00
|
|
|
busiest = find_busiest_queue(sd, group, idle, imbalance, cpus);
|
2009-12-17 17:00:43 +01:00
|
|
|
if (!busiest) {
|
|
|
|
schedstat_inc(sd, lb_nobusyq[idle]);
|
|
|
|
goto out_balanced;
|
|
|
|
}
|
|
|
|
|
|
|
|
BUG_ON(busiest == this_rq);
|
|
|
|
|
|
|
|
schedstat_add(sd, lb_imbalance[idle], imbalance);
|
|
|
|
|
|
|
|
ld_moved = 0;
|
|
|
|
if (busiest->nr_running > 1) {
|
|
|
|
/*
|
|
|
|
* Attempt to move tasks. If find_busiest_group has found
|
|
|
|
* an imbalance but busiest->nr_running <= 1, the group is
|
|
|
|
* still unbalanced. ld_moved simply stays zero, so it is
|
|
|
|
* correctly treated as an imbalance.
|
|
|
|
*/
|
2011-09-22 15:23:13 +02:00
|
|
|
lb_flags |= LBF_ALL_PINNED;
|
2009-12-17 17:00:43 +01:00
|
|
|
local_irq_save(flags);
|
|
|
|
double_rq_lock(this_rq, busiest);
|
|
|
|
ld_moved = move_tasks(this_rq, this_cpu, busiest,
|
2011-09-22 15:23:13 +02:00
|
|
|
imbalance, sd, idle, &lb_flags);
|
2009-12-17 17:00:43 +01:00
|
|
|
double_rq_unlock(this_rq, busiest);
|
|
|
|
local_irq_restore(flags);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* some other cpu did the load balance for us.
|
|
|
|
*/
|
|
|
|
if (ld_moved && this_cpu != smp_processor_id())
|
|
|
|
resched_cpu(this_cpu);
|
|
|
|
|
2011-09-22 15:30:18 +02:00
|
|
|
if (lb_flags & LBF_ABORT)
|
|
|
|
goto out_balanced;
|
|
|
|
|
|
|
|
if (lb_flags & LBF_NEED_BREAK) {
|
|
|
|
lb_flags &= ~LBF_NEED_BREAK;
|
|
|
|
goto redo;
|
|
|
|
}
|
|
|
|
|
2009-12-17 17:00:43 +01:00
|
|
|
/* All tasks on this runqueue were pinned by CPU affinity */
|
2011-09-22 15:23:13 +02:00
|
|
|
if (unlikely(lb_flags & LBF_ALL_PINNED)) {
|
2009-12-17 17:00:43 +01:00
|
|
|
cpumask_clear_cpu(cpu_of(busiest), cpus);
|
|
|
|
if (!cpumask_empty(cpus))
|
|
|
|
goto redo;
|
|
|
|
goto out_balanced;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!ld_moved) {
|
|
|
|
schedstat_inc(sd, lb_failed[idle]);
|
2010-09-11 03:19:17 +02:00
|
|
|
/*
|
|
|
|
* Increment the failure counter only on periodic balance.
|
|
|
|
* We do not want newidle balance, which can be very
|
|
|
|
* frequent, pollute the failure counter causing
|
|
|
|
* excessive cache_hot migrations and active balances.
|
|
|
|
*/
|
|
|
|
if (idle != CPU_NEWLY_IDLE)
|
|
|
|
sd->nr_balance_failed++;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
2011-02-14 23:38:50 +01:00
|
|
|
if (need_active_balance(sd, idle, cpu_of(busiest), this_cpu)) {
|
2009-12-17 17:00:43 +01:00
|
|
|
raw_spin_lock_irqsave(&busiest->lock, flags);
|
|
|
|
|
2010-05-06 18:49:21 +02:00
|
|
|
/* don't kick the active_load_balance_cpu_stop,
|
|
|
|
* if the curr task on busiest cpu can't be
|
|
|
|
* moved to this_cpu
|
2009-12-17 17:00:43 +01:00
|
|
|
*/
|
|
|
|
if (!cpumask_test_cpu(this_cpu,
|
2011-06-16 12:23:22 +02:00
|
|
|
tsk_cpus_allowed(busiest->curr))) {
|
2009-12-17 17:00:43 +01:00
|
|
|
raw_spin_unlock_irqrestore(&busiest->lock,
|
|
|
|
flags);
|
2011-09-22 15:23:13 +02:00
|
|
|
lb_flags |= LBF_ALL_PINNED;
|
2009-12-17 17:00:43 +01:00
|
|
|
goto out_one_pinned;
|
|
|
|
}
|
|
|
|
|
2010-05-06 18:49:21 +02:00
|
|
|
/*
|
|
|
|
* ->active_balance synchronizes accesses to
|
|
|
|
* ->active_balance_work. Once set, it's cleared
|
|
|
|
* only after active load balance is finished.
|
|
|
|
*/
|
2009-12-17 17:00:43 +01:00
|
|
|
if (!busiest->active_balance) {
|
|
|
|
busiest->active_balance = 1;
|
|
|
|
busiest->push_cpu = this_cpu;
|
|
|
|
active_balance = 1;
|
|
|
|
}
|
|
|
|
raw_spin_unlock_irqrestore(&busiest->lock, flags);
|
2010-05-06 18:49:21 +02:00
|
|
|
|
2009-12-17 17:00:43 +01:00
|
|
|
if (active_balance)
|
2010-05-06 18:49:21 +02:00
|
|
|
stop_one_cpu_nowait(cpu_of(busiest),
|
|
|
|
active_load_balance_cpu_stop, busiest,
|
|
|
|
&busiest->active_balance_work);
|
2009-12-17 17:00:43 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* We've kicked active balancing, reset the failure
|
|
|
|
* counter.
|
|
|
|
*/
|
|
|
|
sd->nr_balance_failed = sd->cache_nice_tries+1;
|
|
|
|
}
|
|
|
|
} else
|
|
|
|
sd->nr_balance_failed = 0;
|
|
|
|
|
|
|
|
if (likely(!active_balance)) {
|
|
|
|
/* We were unbalanced, so reset the balancing interval */
|
|
|
|
sd->balance_interval = sd->min_interval;
|
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* If we've begun active balancing, start to back off. This
|
|
|
|
* case may not be covered by the all_pinned logic if there
|
|
|
|
* is only 1 task on the busy runqueue (because we don't call
|
|
|
|
* move_tasks).
|
|
|
|
*/
|
|
|
|
if (sd->balance_interval < sd->max_interval)
|
|
|
|
sd->balance_interval *= 2;
|
|
|
|
}
|
|
|
|
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
out_balanced:
|
|
|
|
schedstat_inc(sd, lb_balanced[idle]);
|
|
|
|
|
|
|
|
sd->nr_balance_failed = 0;
|
|
|
|
|
|
|
|
out_one_pinned:
|
|
|
|
/* tune up the balancing interval */
|
2011-09-22 15:23:13 +02:00
|
|
|
if (((lb_flags & LBF_ALL_PINNED) &&
|
|
|
|
sd->balance_interval < MAX_PINNED_INTERVAL) ||
|
2009-12-17 17:00:43 +01:00
|
|
|
(sd->balance_interval < sd->max_interval))
|
|
|
|
sd->balance_interval *= 2;
|
|
|
|
|
2011-02-14 23:38:50 +01:00
|
|
|
ld_moved = 0;
|
2009-12-17 17:00:43 +01:00
|
|
|
out:
|
|
|
|
return ld_moved;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* idle_balance is called by schedule() if this_cpu is about to become
|
|
|
|
* idle. Attempts to pull tasks from other CPUs.
|
|
|
|
*/
|
2011-10-25 10:00:11 +02:00
|
|
|
void idle_balance(int this_cpu, struct rq *this_rq)
|
2009-12-17 17:00:43 +01:00
|
|
|
{
|
|
|
|
struct sched_domain *sd;
|
|
|
|
int pulled_task = 0;
|
|
|
|
unsigned long next_balance = jiffies + HZ;
|
|
|
|
|
|
|
|
this_rq->idle_stamp = this_rq->clock;
|
|
|
|
|
|
|
|
if (this_rq->avg_idle < sysctl_sched_migration_cost)
|
|
|
|
return;
|
|
|
|
|
2009-12-23 15:29:42 +01:00
|
|
|
/*
|
|
|
|
* Drop the rq->lock, but keep IRQ/preempt disabled.
|
|
|
|
*/
|
|
|
|
raw_spin_unlock(&this_rq->lock);
|
|
|
|
|
2010-11-16 00:47:07 +01:00
|
|
|
update_shares(this_cpu);
|
2011-04-07 14:09:50 +02:00
|
|
|
rcu_read_lock();
|
2009-12-17 17:00:43 +01:00
|
|
|
for_each_domain(this_cpu, sd) {
|
|
|
|
unsigned long interval;
|
2009-12-23 15:29:42 +01:00
|
|
|
int balance = 1;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
|
|
|
if (!(sd->flags & SD_LOAD_BALANCE))
|
|
|
|
continue;
|
|
|
|
|
2009-12-23 15:29:42 +01:00
|
|
|
if (sd->flags & SD_BALANCE_NEWIDLE) {
|
2009-12-17 17:00:43 +01:00
|
|
|
/* If we've pulled tasks over stop searching: */
|
2009-12-23 15:29:42 +01:00
|
|
|
pulled_task = load_balance(this_cpu, this_rq,
|
|
|
|
sd, CPU_NEWLY_IDLE, &balance);
|
|
|
|
}
|
2009-12-17 17:00:43 +01:00
|
|
|
|
|
|
|
interval = msecs_to_jiffies(sd->balance_interval);
|
|
|
|
if (time_after(next_balance, sd->last_balance + interval))
|
|
|
|
next_balance = sd->last_balance + interval;
|
2010-11-17 20:42:04 +01:00
|
|
|
if (pulled_task) {
|
|
|
|
this_rq->idle_stamp = 0;
|
2009-12-17 17:00:43 +01:00
|
|
|
break;
|
2010-11-17 20:42:04 +01:00
|
|
|
}
|
2009-12-17 17:00:43 +01:00
|
|
|
}
|
2011-04-07 14:09:50 +02:00
|
|
|
rcu_read_unlock();
|
2009-12-23 15:29:42 +01:00
|
|
|
|
|
|
|
raw_spin_lock(&this_rq->lock);
|
|
|
|
|
2009-12-17 17:00:43 +01:00
|
|
|
if (pulled_task || time_after(jiffies, this_rq->next_balance)) {
|
|
|
|
/*
|
|
|
|
* We are going idle. next_balance may be set based on
|
|
|
|
* a busy processor. So reset next_balance.
|
|
|
|
*/
|
|
|
|
this_rq->next_balance = next_balance;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2010-05-06 18:49:21 +02:00
|
|
|
* active_load_balance_cpu_stop is run by cpu stopper. It pushes
|
|
|
|
* running tasks off the busiest CPU onto idle CPUs. It requires at
|
|
|
|
* least 1 task to be running on each physical CPU where possible, and
|
|
|
|
* avoids physical / logical imbalances.
|
2009-12-17 17:00:43 +01:00
|
|
|
*/
|
2010-05-06 18:49:21 +02:00
|
|
|
static int active_load_balance_cpu_stop(void *data)
|
2009-12-17 17:00:43 +01:00
|
|
|
{
|
2010-05-06 18:49:21 +02:00
|
|
|
struct rq *busiest_rq = data;
|
|
|
|
int busiest_cpu = cpu_of(busiest_rq);
|
2009-12-17 17:00:43 +01:00
|
|
|
int target_cpu = busiest_rq->push_cpu;
|
2010-05-06 18:49:21 +02:00
|
|
|
struct rq *target_rq = cpu_rq(target_cpu);
|
2009-12-17 17:00:43 +01:00
|
|
|
struct sched_domain *sd;
|
2010-05-06 18:49:21 +02:00
|
|
|
|
|
|
|
raw_spin_lock_irq(&busiest_rq->lock);
|
|
|
|
|
|
|
|
/* make sure the requested cpu hasn't gone down in the meantime */
|
|
|
|
if (unlikely(busiest_cpu != smp_processor_id() ||
|
|
|
|
!busiest_rq->active_balance))
|
|
|
|
goto out_unlock;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
|
|
|
/* Is there any task to move? */
|
|
|
|
if (busiest_rq->nr_running <= 1)
|
2010-05-06 18:49:21 +02:00
|
|
|
goto out_unlock;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* This condition is "impossible", if it occurs
|
|
|
|
* we need to fix it. Originally reported by
|
|
|
|
* Bjorn Helgaas on a 128-cpu setup.
|
|
|
|
*/
|
|
|
|
BUG_ON(busiest_rq == target_rq);
|
|
|
|
|
|
|
|
/* move a task from busiest_rq to target_rq */
|
|
|
|
double_lock_balance(busiest_rq, target_rq);
|
|
|
|
|
|
|
|
/* Search for an sd spanning us and the target CPU. */
|
2011-04-07 14:09:50 +02:00
|
|
|
rcu_read_lock();
|
2009-12-17 17:00:43 +01:00
|
|
|
for_each_domain(target_cpu, sd) {
|
|
|
|
if ((sd->flags & SD_LOAD_BALANCE) &&
|
|
|
|
cpumask_test_cpu(busiest_cpu, sched_domain_span(sd)))
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (likely(sd)) {
|
|
|
|
schedstat_inc(sd, alb_count);
|
|
|
|
|
|
|
|
if (move_one_task(target_rq, target_cpu, busiest_rq,
|
|
|
|
sd, CPU_IDLE))
|
|
|
|
schedstat_inc(sd, alb_pushed);
|
|
|
|
else
|
|
|
|
schedstat_inc(sd, alb_failed);
|
|
|
|
}
|
2011-04-07 14:09:50 +02:00
|
|
|
rcu_read_unlock();
|
2009-12-17 17:00:43 +01:00
|
|
|
double_unlock_balance(busiest_rq, target_rq);
|
2010-05-06 18:49:21 +02:00
|
|
|
out_unlock:
|
|
|
|
busiest_rq->active_balance = 0;
|
|
|
|
raw_spin_unlock_irq(&busiest_rq->lock);
|
|
|
|
return 0;
|
2009-12-17 17:00:43 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_NO_HZ
|
2010-05-22 02:09:41 +02:00
|
|
|
/*
|
|
|
|
* idle load balancing details
|
|
|
|
* - When one of the busy CPUs notice that there may be an idle rebalancing
|
|
|
|
* needed, they will kick the idle load balancer, which then does idle
|
|
|
|
* load balancing for all the idle CPUs.
|
|
|
|
*/
|
2009-12-17 17:00:43 +01:00
|
|
|
static struct {
|
2010-05-22 02:09:41 +02:00
|
|
|
cpumask_var_t idle_cpus_mask;
|
2011-12-02 02:07:34 +01:00
|
|
|
atomic_t nr_cpus;
|
2010-05-22 02:09:41 +02:00
|
|
|
unsigned long next_balance; /* in jiffy units */
|
|
|
|
} nohz ____cacheline_aligned;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
|
|
|
#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
|
|
|
|
/**
|
|
|
|
* lowest_flag_domain - Return lowest sched_domain containing flag.
|
|
|
|
* @cpu: The cpu whose lowest level of sched domain is to
|
|
|
|
* be returned.
|
|
|
|
* @flag: The flag to check for the lowest sched_domain
|
|
|
|
* for the given cpu.
|
|
|
|
*
|
|
|
|
* Returns the lowest sched_domain of a cpu which contains the given flag.
|
|
|
|
*/
|
|
|
|
static inline struct sched_domain *lowest_flag_domain(int cpu, int flag)
|
|
|
|
{
|
|
|
|
struct sched_domain *sd;
|
|
|
|
|
|
|
|
for_each_domain(cpu, sd)
|
2011-06-17 03:55:19 +02:00
|
|
|
if (sd->flags & flag)
|
2009-12-17 17:00:43 +01:00
|
|
|
break;
|
|
|
|
|
|
|
|
return sd;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* for_each_flag_domain - Iterates over sched_domains containing the flag.
|
|
|
|
* @cpu: The cpu whose domains we're iterating over.
|
|
|
|
* @sd: variable holding the value of the power_savings_sd
|
|
|
|
* for cpu.
|
|
|
|
* @flag: The flag to filter the sched_domains to be iterated.
|
|
|
|
*
|
|
|
|
* Iterates over all the scheduler domains for a given cpu that has the 'flag'
|
|
|
|
* set, starting from the lowest sched_domain to the highest.
|
|
|
|
*/
|
|
|
|
#define for_each_flag_domain(cpu, sd, flag) \
|
|
|
|
for (sd = lowest_flag_domain(cpu, flag); \
|
|
|
|
(sd && (sd->flags & flag)); sd = sd->parent)
|
|
|
|
|
|
|
|
/**
|
|
|
|
* find_new_ilb - Finds the optimum idle load balancer for nomination.
|
|
|
|
* @cpu: The cpu which is nominating a new idle_load_balancer.
|
|
|
|
*
|
|
|
|
* Returns: Returns the id of the idle load balancer if it exists,
|
|
|
|
* Else, returns >= nr_cpu_ids.
|
|
|
|
*
|
|
|
|
* This algorithm picks the idle load balancer such that it belongs to a
|
|
|
|
* semi-idle powersavings sched_domain. The idea is to try and avoid
|
|
|
|
* completely idle packages/cores just for the purpose of idle load balancing
|
|
|
|
* when there are other idle cpu's which are better suited for that job.
|
|
|
|
*/
|
|
|
|
static int find_new_ilb(int cpu)
|
|
|
|
{
|
2011-12-02 02:07:34 +01:00
|
|
|
int ilb = cpumask_first(nohz.idle_cpus_mask);
|
2011-12-02 02:07:35 +01:00
|
|
|
struct sched_group *ilbg;
|
2009-12-17 17:00:43 +01:00
|
|
|
struct sched_domain *sd;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Have idle load balancer selection from semi-idle packages only
|
|
|
|
* when power-aware load balancing is enabled
|
|
|
|
*/
|
|
|
|
if (!(sched_smt_power_savings || sched_mc_power_savings))
|
|
|
|
goto out_done;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Optimize for the case when we have no idle CPUs or only one
|
|
|
|
* idle CPU. Don't walk the sched_domain hierarchy in such cases
|
|
|
|
*/
|
2010-05-22 02:09:41 +02:00
|
|
|
if (cpumask_weight(nohz.idle_cpus_mask) < 2)
|
2009-12-17 17:00:43 +01:00
|
|
|
goto out_done;
|
|
|
|
|
2011-04-07 14:09:50 +02:00
|
|
|
rcu_read_lock();
|
2009-12-17 17:00:43 +01:00
|
|
|
for_each_flag_domain(cpu, sd, SD_POWERSAVINGS_BALANCE) {
|
2011-12-02 02:07:35 +01:00
|
|
|
ilbg = sd->groups;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
|
|
|
do {
|
2011-12-02 02:07:35 +01:00
|
|
|
if (ilbg->group_weight !=
|
|
|
|
atomic_read(&ilbg->sgp->nr_busy_cpus)) {
|
|
|
|
ilb = cpumask_first_and(nohz.idle_cpus_mask,
|
|
|
|
sched_group_cpus(ilbg));
|
2011-04-07 14:09:50 +02:00
|
|
|
goto unlock;
|
|
|
|
}
|
2009-12-17 17:00:43 +01:00
|
|
|
|
2011-12-02 02:07:35 +01:00
|
|
|
ilbg = ilbg->next;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
2011-12-02 02:07:35 +01:00
|
|
|
} while (ilbg != sd->groups);
|
2009-12-17 17:00:43 +01:00
|
|
|
}
|
2011-04-07 14:09:50 +02:00
|
|
|
unlock:
|
|
|
|
rcu_read_unlock();
|
2009-12-17 17:00:43 +01:00
|
|
|
|
|
|
|
out_done:
|
2011-12-02 02:07:35 +01:00
|
|
|
if (ilb < nr_cpu_ids && idle_cpu(ilb))
|
|
|
|
return ilb;
|
|
|
|
|
|
|
|
return nr_cpu_ids;
|
2009-12-17 17:00:43 +01:00
|
|
|
}
|
|
|
|
#else /* (CONFIG_SCHED_MC || CONFIG_SCHED_SMT) */
|
|
|
|
static inline int find_new_ilb(int call_cpu)
|
|
|
|
{
|
2010-05-22 02:09:41 +02:00
|
|
|
return nr_cpu_ids;
|
2009-12-17 17:00:43 +01:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2010-05-22 02:09:41 +02:00
|
|
|
/*
|
|
|
|
* Kick a CPU to do the nohz balancing, if it is time for it. We pick the
|
|
|
|
* nohz_load_balancer CPU (if there is one) otherwise fallback to any idle
|
|
|
|
* CPU (if there is one).
|
|
|
|
*/
|
|
|
|
static void nohz_balancer_kick(int cpu)
|
|
|
|
{
|
|
|
|
int ilb_cpu;
|
|
|
|
|
|
|
|
nohz.next_balance++;
|
|
|
|
|
2011-12-02 02:07:34 +01:00
|
|
|
ilb_cpu = find_new_ilb(cpu);
|
2010-05-22 02:09:41 +02:00
|
|
|
|
2011-12-02 02:07:34 +01:00
|
|
|
if (ilb_cpu >= nr_cpu_ids)
|
|
|
|
return;
|
2010-05-22 02:09:41 +02:00
|
|
|
|
2011-12-06 20:26:34 +01:00
|
|
|
if (test_and_set_bit(NOHZ_BALANCE_KICK, nohz_flags(ilb_cpu)))
|
2011-12-02 02:07:32 +01:00
|
|
|
return;
|
|
|
|
/*
|
|
|
|
* Use smp_send_reschedule() instead of resched_cpu().
|
|
|
|
* This way we generate a sched IPI on the target cpu which
|
|
|
|
* is idle. And the softirq performing nohz idle load balance
|
|
|
|
* will be run before returning from the IPI.
|
|
|
|
*/
|
|
|
|
smp_send_reschedule(ilb_cpu);
|
2010-05-22 02:09:41 +02:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2011-12-02 02:07:33 +01:00
|
|
|
static inline void set_cpu_sd_state_busy(void)
|
|
|
|
{
|
|
|
|
struct sched_domain *sd;
|
|
|
|
int cpu = smp_processor_id();
|
|
|
|
|
|
|
|
if (!test_bit(NOHZ_IDLE, nohz_flags(cpu)))
|
|
|
|
return;
|
|
|
|
clear_bit(NOHZ_IDLE, nohz_flags(cpu));
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
for_each_domain(cpu, sd)
|
|
|
|
atomic_inc(&sd->groups->sgp->nr_busy_cpus);
|
|
|
|
rcu_read_unlock();
|
|
|
|
}
|
|
|
|
|
|
|
|
void set_cpu_sd_state_idle(void)
|
|
|
|
{
|
|
|
|
struct sched_domain *sd;
|
|
|
|
int cpu = smp_processor_id();
|
|
|
|
|
|
|
|
if (test_bit(NOHZ_IDLE, nohz_flags(cpu)))
|
|
|
|
return;
|
|
|
|
set_bit(NOHZ_IDLE, nohz_flags(cpu));
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
for_each_domain(cpu, sd)
|
|
|
|
atomic_dec(&sd->groups->sgp->nr_busy_cpus);
|
|
|
|
rcu_read_unlock();
|
|
|
|
}
|
|
|
|
|
2009-12-17 17:00:43 +01:00
|
|
|
/*
|
2011-12-02 02:07:34 +01:00
|
|
|
* This routine will record that this cpu is going idle with tick stopped.
|
|
|
|
* This info will be used in performing idle load balancing in the future.
|
2009-12-17 17:00:43 +01:00
|
|
|
*/
|
2010-05-22 02:09:41 +02:00
|
|
|
void select_nohz_load_balancer(int stop_tick)
|
2009-12-17 17:00:43 +01:00
|
|
|
{
|
|
|
|
int cpu = smp_processor_id();
|
|
|
|
|
|
|
|
if (stop_tick) {
|
2011-12-02 02:07:34 +01:00
|
|
|
if (test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)))
|
2010-05-22 02:09:41 +02:00
|
|
|
return;
|
2009-12-17 17:00:43 +01:00
|
|
|
|
2010-05-22 02:09:41 +02:00
|
|
|
cpumask_set_cpu(cpu, nohz.idle_cpus_mask);
|
2011-12-02 02:07:34 +01:00
|
|
|
atomic_inc(&nohz.nr_cpus);
|
2011-12-02 02:07:32 +01:00
|
|
|
set_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu));
|
2009-12-17 17:00:43 +01:00
|
|
|
}
|
2010-05-22 02:09:41 +02:00
|
|
|
return;
|
2009-12-17 17:00:43 +01:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
static DEFINE_SPINLOCK(balancing);
|
|
|
|
|
2011-04-05 10:14:25 +02:00
|
|
|
static unsigned long __read_mostly max_load_balance_interval = HZ/10;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Scale the max load_balance interval with the number of CPUs in the system.
|
|
|
|
* This trades load-balance latency on larger machines for less cross talk.
|
|
|
|
*/
|
2011-10-25 10:00:11 +02:00
|
|
|
void update_max_interval(void)
|
2011-04-05 10:14:25 +02:00
|
|
|
{
|
|
|
|
max_load_balance_interval = HZ*num_online_cpus()/10;
|
|
|
|
}
|
|
|
|
|
2009-12-17 17:00:43 +01:00
|
|
|
/*
|
|
|
|
* It checks each scheduling domain to see if it is due to be balanced,
|
|
|
|
* and initiates a balancing operation if so.
|
|
|
|
*
|
|
|
|
* Balancing parameters are set up in arch_init_sched_domains.
|
|
|
|
*/
|
|
|
|
static void rebalance_domains(int cpu, enum cpu_idle_type idle)
|
|
|
|
{
|
|
|
|
int balance = 1;
|
|
|
|
struct rq *rq = cpu_rq(cpu);
|
|
|
|
unsigned long interval;
|
|
|
|
struct sched_domain *sd;
|
|
|
|
/* Earliest time when we have to do rebalance again */
|
|
|
|
unsigned long next_balance = jiffies + 60*HZ;
|
|
|
|
int update_next_balance = 0;
|
|
|
|
int need_serialize;
|
|
|
|
|
2010-11-16 00:47:00 +01:00
|
|
|
update_shares(cpu);
|
|
|
|
|
2011-04-07 14:09:50 +02:00
|
|
|
rcu_read_lock();
|
2009-12-17 17:00:43 +01:00
|
|
|
for_each_domain(cpu, sd) {
|
|
|
|
if (!(sd->flags & SD_LOAD_BALANCE))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
interval = sd->balance_interval;
|
|
|
|
if (idle != CPU_IDLE)
|
|
|
|
interval *= sd->busy_factor;
|
|
|
|
|
|
|
|
/* scale ms to jiffies */
|
|
|
|
interval = msecs_to_jiffies(interval);
|
2011-04-05 10:14:25 +02:00
|
|
|
interval = clamp(interval, 1UL, max_load_balance_interval);
|
2009-12-17 17:00:43 +01:00
|
|
|
|
|
|
|
need_serialize = sd->flags & SD_SERIALIZE;
|
|
|
|
|
|
|
|
if (need_serialize) {
|
|
|
|
if (!spin_trylock(&balancing))
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (time_after_eq(jiffies, sd->last_balance + interval)) {
|
|
|
|
if (load_balance(cpu, rq, sd, idle, &balance)) {
|
|
|
|
/*
|
|
|
|
* We've pulled tasks over so either we're no
|
2011-02-21 18:52:53 +01:00
|
|
|
* longer idle.
|
2009-12-17 17:00:43 +01:00
|
|
|
*/
|
|
|
|
idle = CPU_NOT_IDLE;
|
|
|
|
}
|
|
|
|
sd->last_balance = jiffies;
|
|
|
|
}
|
|
|
|
if (need_serialize)
|
|
|
|
spin_unlock(&balancing);
|
|
|
|
out:
|
|
|
|
if (time_after(next_balance, sd->last_balance + interval)) {
|
|
|
|
next_balance = sd->last_balance + interval;
|
|
|
|
update_next_balance = 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Stop the load balance at this level. There is another
|
|
|
|
* CPU in our sched group which is doing load balancing more
|
|
|
|
* actively.
|
|
|
|
*/
|
|
|
|
if (!balance)
|
|
|
|
break;
|
|
|
|
}
|
2011-04-07 14:09:50 +02:00
|
|
|
rcu_read_unlock();
|
2009-12-17 17:00:43 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* next_balance will be updated only when there is a need.
|
|
|
|
* When the cpu is attached to null domain for ex, it will not be
|
|
|
|
* updated.
|
|
|
|
*/
|
|
|
|
if (likely(update_next_balance))
|
|
|
|
rq->next_balance = next_balance;
|
|
|
|
}
|
|
|
|
|
2010-05-22 02:09:41 +02:00
|
|
|
#ifdef CONFIG_NO_HZ
|
2009-12-17 17:00:43 +01:00
|
|
|
/*
|
2010-05-22 02:09:41 +02:00
|
|
|
* In CONFIG_NO_HZ case, the idle balance kickee will do the
|
2009-12-17 17:00:43 +01:00
|
|
|
* rebalancing for all the cpus for whom scheduler ticks are stopped.
|
|
|
|
*/
|
2010-05-22 02:09:41 +02:00
|
|
|
static void nohz_idle_balance(int this_cpu, enum cpu_idle_type idle)
|
|
|
|
{
|
|
|
|
struct rq *this_rq = cpu_rq(this_cpu);
|
|
|
|
struct rq *rq;
|
|
|
|
int balance_cpu;
|
|
|
|
|
2011-12-02 02:07:32 +01:00
|
|
|
if (idle != CPU_IDLE ||
|
|
|
|
!test_bit(NOHZ_BALANCE_KICK, nohz_flags(this_cpu)))
|
|
|
|
goto end;
|
2010-05-22 02:09:41 +02:00
|
|
|
|
|
|
|
for_each_cpu(balance_cpu, nohz.idle_cpus_mask) {
|
2011-12-06 20:19:37 +01:00
|
|
|
if (balance_cpu == this_cpu || !idle_cpu(balance_cpu))
|
2010-05-22 02:09:41 +02:00
|
|
|
continue;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If this cpu gets work to do, stop the load balancing
|
|
|
|
* work being done for other cpus. Next load
|
|
|
|
* balancing owner will pick it up.
|
|
|
|
*/
|
2011-12-02 02:07:32 +01:00
|
|
|
if (need_resched())
|
2010-05-22 02:09:41 +02:00
|
|
|
break;
|
|
|
|
|
|
|
|
raw_spin_lock_irq(&this_rq->lock);
|
2010-07-09 15:19:54 +02:00
|
|
|
update_rq_clock(this_rq);
|
2010-05-22 02:09:41 +02:00
|
|
|
update_cpu_load(this_rq);
|
|
|
|
raw_spin_unlock_irq(&this_rq->lock);
|
|
|
|
|
|
|
|
rebalance_domains(balance_cpu, CPU_IDLE);
|
|
|
|
|
|
|
|
rq = cpu_rq(balance_cpu);
|
|
|
|
if (time_after(this_rq->next_balance, rq->next_balance))
|
|
|
|
this_rq->next_balance = rq->next_balance;
|
|
|
|
}
|
|
|
|
nohz.next_balance = this_rq->next_balance;
|
2011-12-02 02:07:32 +01:00
|
|
|
end:
|
|
|
|
clear_bit(NOHZ_BALANCE_KICK, nohz_flags(this_cpu));
|
2010-05-22 02:09:41 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2011-12-02 02:07:34 +01:00
|
|
|
* Current heuristic for kicking the idle load balancer in the presence
|
|
|
|
* of an idle cpu is the system.
|
|
|
|
* - This rq has more than one task.
|
|
|
|
* - At any scheduler domain level, this cpu's scheduler group has multiple
|
|
|
|
* busy cpu's exceeding the group's power.
|
|
|
|
* - For SD_ASYM_PACKING, if the lower numbered cpu's in the scheduler
|
|
|
|
* domain span are idle.
|
2010-05-22 02:09:41 +02:00
|
|
|
*/
|
|
|
|
static inline int nohz_kick_needed(struct rq *rq, int cpu)
|
|
|
|
{
|
|
|
|
unsigned long now = jiffies;
|
2011-12-02 02:07:34 +01:00
|
|
|
struct sched_domain *sd;
|
2010-05-22 02:09:41 +02:00
|
|
|
|
2011-12-02 02:07:32 +01:00
|
|
|
if (unlikely(idle_cpu(cpu)))
|
2010-05-22 02:09:41 +02:00
|
|
|
return 0;
|
|
|
|
|
2011-12-02 02:07:32 +01:00
|
|
|
/*
|
|
|
|
* We may be recently in ticked or tickless idle mode. At the first
|
|
|
|
* busy tick after returning from idle, we will update the busy stats.
|
|
|
|
*/
|
2011-12-02 02:07:33 +01:00
|
|
|
set_cpu_sd_state_busy();
|
2011-12-02 02:07:34 +01:00
|
|
|
if (unlikely(test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)))) {
|
2011-12-02 02:07:32 +01:00
|
|
|
clear_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu));
|
2011-12-02 02:07:34 +01:00
|
|
|
cpumask_clear_cpu(cpu, nohz.idle_cpus_mask);
|
|
|
|
atomic_dec(&nohz.nr_cpus);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* None are in tickless mode and hence no need for NOHZ idle load
|
|
|
|
* balancing.
|
|
|
|
*/
|
|
|
|
if (likely(!atomic_read(&nohz.nr_cpus)))
|
|
|
|
return 0;
|
2011-12-02 02:07:32 +01:00
|
|
|
|
|
|
|
if (time_before(now, nohz.next_balance))
|
2010-05-22 02:09:41 +02:00
|
|
|
return 0;
|
|
|
|
|
2011-12-02 02:07:34 +01:00
|
|
|
if (rq->nr_running >= 2)
|
|
|
|
goto need_kick;
|
2010-05-22 02:09:41 +02:00
|
|
|
|
2011-12-07 14:32:08 +01:00
|
|
|
rcu_read_lock();
|
2011-12-02 02:07:34 +01:00
|
|
|
for_each_domain(cpu, sd) {
|
|
|
|
struct sched_group *sg = sd->groups;
|
|
|
|
struct sched_group_power *sgp = sg->sgp;
|
|
|
|
int nr_busy = atomic_read(&sgp->nr_busy_cpus);
|
2010-05-22 02:09:41 +02:00
|
|
|
|
2011-12-02 02:07:34 +01:00
|
|
|
if (sd->flags & SD_SHARE_PKG_RESOURCES && nr_busy > 1)
|
2011-12-07 14:32:08 +01:00
|
|
|
goto need_kick_unlock;
|
2011-12-02 02:07:34 +01:00
|
|
|
|
|
|
|
if (sd->flags & SD_ASYM_PACKING && nr_busy != sg->group_weight
|
|
|
|
&& (cpumask_first_and(nohz.idle_cpus_mask,
|
|
|
|
sched_domain_span(sd)) < cpu))
|
2011-12-07 14:32:08 +01:00
|
|
|
goto need_kick_unlock;
|
2011-12-02 02:07:34 +01:00
|
|
|
|
|
|
|
if (!(sd->flags & (SD_SHARE_PKG_RESOURCES | SD_ASYM_PACKING)))
|
|
|
|
break;
|
2010-05-22 02:09:41 +02:00
|
|
|
}
|
2011-12-07 14:32:08 +01:00
|
|
|
rcu_read_unlock();
|
2010-05-22 02:09:41 +02:00
|
|
|
return 0;
|
2011-12-07 14:32:08 +01:00
|
|
|
|
|
|
|
need_kick_unlock:
|
|
|
|
rcu_read_unlock();
|
2011-12-02 02:07:34 +01:00
|
|
|
need_kick:
|
|
|
|
return 1;
|
2010-05-22 02:09:41 +02:00
|
|
|
}
|
|
|
|
#else
|
|
|
|
static void nohz_idle_balance(int this_cpu, enum cpu_idle_type idle) { }
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/*
|
|
|
|
* run_rebalance_domains is triggered when needed from the scheduler tick.
|
|
|
|
* Also triggered for nohz idle balancing (with nohz_balancing_kick set).
|
|
|
|
*/
|
2009-12-17 17:00:43 +01:00
|
|
|
static void run_rebalance_domains(struct softirq_action *h)
|
|
|
|
{
|
|
|
|
int this_cpu = smp_processor_id();
|
|
|
|
struct rq *this_rq = cpu_rq(this_cpu);
|
2011-10-04 00:09:01 +02:00
|
|
|
enum cpu_idle_type idle = this_rq->idle_balance ?
|
2009-12-17 17:00:43 +01:00
|
|
|
CPU_IDLE : CPU_NOT_IDLE;
|
|
|
|
|
|
|
|
rebalance_domains(this_cpu, idle);
|
|
|
|
|
|
|
|
/*
|
2010-05-22 02:09:41 +02:00
|
|
|
* If this cpu has a pending nohz_balance_kick, then do the
|
2009-12-17 17:00:43 +01:00
|
|
|
* balancing on behalf of the other idle cpus whose ticks are
|
|
|
|
* stopped.
|
|
|
|
*/
|
2010-05-22 02:09:41 +02:00
|
|
|
nohz_idle_balance(this_cpu, idle);
|
2009-12-17 17:00:43 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline int on_null_domain(int cpu)
|
|
|
|
{
|
2010-02-28 17:32:18 +01:00
|
|
|
return !rcu_dereference_sched(cpu_rq(cpu)->sd);
|
2009-12-17 17:00:43 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing.
|
|
|
|
*/
|
2011-10-25 10:00:11 +02:00
|
|
|
void trigger_load_balance(struct rq *rq, int cpu)
|
2009-12-17 17:00:43 +01:00
|
|
|
{
|
|
|
|
/* Don't need to rebalance while attached to NULL domain */
|
|
|
|
if (time_after_eq(jiffies, rq->next_balance) &&
|
|
|
|
likely(!on_null_domain(cpu)))
|
|
|
|
raise_softirq(SCHED_SOFTIRQ);
|
2010-05-22 02:09:41 +02:00
|
|
|
#ifdef CONFIG_NO_HZ
|
2011-12-02 02:07:32 +01:00
|
|
|
if (nohz_kick_needed(rq, cpu) && likely(!on_null_domain(cpu)))
|
2010-05-22 02:09:41 +02:00
|
|
|
nohz_balancer_kick(cpu);
|
|
|
|
#endif
|
2009-12-17 17:00:43 +01:00
|
|
|
}
|
|
|
|
|
2009-11-30 12:16:46 +01:00
|
|
|
static void rq_online_fair(struct rq *rq)
|
|
|
|
{
|
|
|
|
update_sysctl();
|
|
|
|
}
|
|
|
|
|
|
|
|
static void rq_offline_fair(struct rq *rq)
|
|
|
|
{
|
|
|
|
update_sysctl();
|
|
|
|
}
|
|
|
|
|
2008-06-24 20:09:43 +02:00
|
|
|
#endif /* CONFIG_SMP */
|
2007-10-24 18:23:51 +02:00
|
|
|
|
2007-07-09 18:51:58 +02:00
|
|
|
/*
|
|
|
|
* scheduler tick hitting a task of our scheduling class:
|
|
|
|
*/
|
2008-01-25 21:08:29 +01:00
|
|
|
static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
|
|
|
struct cfs_rq *cfs_rq;
|
|
|
|
struct sched_entity *se = &curr->se;
|
|
|
|
|
|
|
|
for_each_sched_entity(se) {
|
|
|
|
cfs_rq = cfs_rq_of(se);
|
2008-01-25 21:08:29 +01:00
|
|
|
entity_tick(cfs_rq, se, queued);
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2009-11-27 17:32:46 +01:00
|
|
|
* called on fork with the child task as argument from the parent's context
|
|
|
|
* - child not yet on the tasklist
|
|
|
|
* - preemption disabled
|
2007-07-09 18:51:58 +02:00
|
|
|
*/
|
2009-11-27 17:32:46 +01:00
|
|
|
static void task_fork_fair(struct task_struct *p)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
2011-12-15 06:36:55 +01:00
|
|
|
struct cfs_rq *cfs_rq;
|
|
|
|
struct sched_entity *se = &p->se, *curr;
|
2007-10-15 17:00:14 +02:00
|
|
|
int this_cpu = smp_processor_id();
|
2009-11-27 17:32:46 +01:00
|
|
|
struct rq *rq = this_rq();
|
|
|
|
unsigned long flags;
|
|
|
|
|
2009-11-17 14:28:38 +01:00
|
|
|
raw_spin_lock_irqsave(&rq->lock, flags);
|
2007-07-09 18:51:58 +02:00
|
|
|
|
2010-08-19 13:31:43 +02:00
|
|
|
update_rq_clock(rq);
|
|
|
|
|
2011-12-15 06:36:55 +01:00
|
|
|
cfs_rq = task_cfs_rq(current);
|
|
|
|
curr = cfs_rq->curr;
|
|
|
|
|
2010-10-07 02:32:51 +02:00
|
|
|
if (unlikely(task_cpu(p) != this_cpu)) {
|
|
|
|
rcu_read_lock();
|
2009-11-27 17:32:46 +01:00
|
|
|
__set_task_cpu(p, this_cpu);
|
2010-10-07 02:32:51 +02:00
|
|
|
rcu_read_unlock();
|
|
|
|
}
|
2007-07-09 18:51:58 +02:00
|
|
|
|
2007-08-28 12:53:24 +02:00
|
|
|
update_curr(cfs_rq);
|
2009-11-27 17:32:46 +01:00
|
|
|
|
sched: Ensure that a child can't gain time over it's parent after fork()
A fork/exec load is usually "pass the baton", so the child
should never be placed behind the parent. With START_DEBIT we
make room for the new task, but with child_runs_first, that
room comes out of the _parent's_ hide. There's nothing to say
that the parent wasn't ahead of min_vruntime at fork() time,
which means that the "baton carrier", who is essentially the
parent in drag, can gain time and increase scheduling latencies
for waiters.
With NEW_FAIR_SLEEPERS + START_DEBIT + child_runs_first
enabled, we essentially pass the sleeper fairness off to the
child, which is fine, but if we don't base placement on the
parent's updated vruntime, we can end up compounding latency
woes if the child itself then does fork/exec. The debit
incurred at fork doesn't hurt the parent who is then going to
sleep and maybe exit, but the child who acquires the error
harms all comers.
This improves latencies of make -j<n> kernel build workloads.
Reported-by: Jens Axboe <jens.axboe@oracle.com>
Signed-off-by: Mike Galbraith <efault@gmx.de>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-08 11:12:28 +02:00
|
|
|
if (curr)
|
|
|
|
se->vruntime = curr->vruntime;
|
2007-10-15 17:00:05 +02:00
|
|
|
place_entity(cfs_rq, se, 1);
|
2007-10-15 17:00:04 +02:00
|
|
|
|
2009-11-27 17:32:46 +01:00
|
|
|
if (sysctl_sched_child_runs_first && curr && entity_before(curr, se)) {
|
2007-10-15 17:00:08 +02:00
|
|
|
/*
|
2007-10-15 17:00:08 +02:00
|
|
|
* Upon rescheduling, sched_class::put_prev_task() will place
|
|
|
|
* 'current' within the tree based on its new key value.
|
|
|
|
*/
|
2007-10-15 17:00:04 +02:00
|
|
|
swap(curr->vruntime, se->vruntime);
|
2008-08-28 11:12:49 +02:00
|
|
|
resched_task(rq->curr);
|
2007-10-15 17:00:04 +02:00
|
|
|
}
|
2007-07-09 18:51:58 +02:00
|
|
|
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 18:04:41 +01:00
|
|
|
se->vruntime -= cfs_rq->min_vruntime;
|
|
|
|
|
2009-11-17 14:28:38 +01:00
|
|
|
raw_spin_unlock_irqrestore(&rq->lock, flags);
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
2008-01-25 21:08:22 +01:00
|
|
|
/*
|
|
|
|
* Priority of the task has changed. Check to see if we preempt
|
|
|
|
* the current task.
|
|
|
|
*/
|
2011-01-17 17:03:27 +01:00
|
|
|
static void
|
|
|
|
prio_changed_fair(struct rq *rq, struct task_struct *p, int oldprio)
|
2008-01-25 21:08:22 +01:00
|
|
|
{
|
2011-01-17 17:03:27 +01:00
|
|
|
if (!p->se.on_rq)
|
|
|
|
return;
|
|
|
|
|
2008-01-25 21:08:22 +01:00
|
|
|
/*
|
|
|
|
* Reschedule if we are currently running on this runqueue and
|
|
|
|
* our priority decreased, or if we are not currently running on
|
|
|
|
* this runqueue and our priority is higher than the current's
|
|
|
|
*/
|
2011-01-17 17:03:27 +01:00
|
|
|
if (rq->curr == p) {
|
2008-01-25 21:08:22 +01:00
|
|
|
if (p->prio > oldprio)
|
|
|
|
resched_task(rq->curr);
|
|
|
|
} else
|
2008-09-20 23:38:02 +02:00
|
|
|
check_preempt_curr(rq, p, 0);
|
2008-01-25 21:08:22 +01:00
|
|
|
}
|
|
|
|
|
2011-01-17 17:03:27 +01:00
|
|
|
static void switched_from_fair(struct rq *rq, struct task_struct *p)
|
|
|
|
{
|
|
|
|
struct sched_entity *se = &p->se;
|
|
|
|
struct cfs_rq *cfs_rq = cfs_rq_of(se);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Ensure the task's vruntime is normalized, so that when its
|
|
|
|
* switched back to the fair class the enqueue_entity(.flags=0) will
|
|
|
|
* do the right thing.
|
|
|
|
*
|
|
|
|
* If it was on_rq, then the dequeue_entity(.flags=0) will already
|
|
|
|
* have normalized the vruntime, if it was !on_rq, then only when
|
|
|
|
* the task is sleeping will it still have non-normalized vruntime.
|
|
|
|
*/
|
|
|
|
if (!se->on_rq && p->state != TASK_RUNNING) {
|
|
|
|
/*
|
|
|
|
* Fix up our vruntime so that the current sleep doesn't
|
|
|
|
* cause 'unlimited' sleep bonus.
|
|
|
|
*/
|
|
|
|
place_entity(cfs_rq, se, 0);
|
|
|
|
se->vruntime -= cfs_rq->min_vruntime;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2008-01-25 21:08:22 +01:00
|
|
|
/*
|
|
|
|
* We switched to the sched_fair class.
|
|
|
|
*/
|
2011-01-17 17:03:27 +01:00
|
|
|
static void switched_to_fair(struct rq *rq, struct task_struct *p)
|
2008-01-25 21:08:22 +01:00
|
|
|
{
|
2011-01-17 17:03:27 +01:00
|
|
|
if (!p->se.on_rq)
|
|
|
|
return;
|
|
|
|
|
2008-01-25 21:08:22 +01:00
|
|
|
/*
|
|
|
|
* We were most likely switched from sched_rt, so
|
|
|
|
* kick off the schedule if running, otherwise just see
|
|
|
|
* if we can still preempt the current task.
|
|
|
|
*/
|
2011-01-17 17:03:27 +01:00
|
|
|
if (rq->curr == p)
|
2008-01-25 21:08:22 +01:00
|
|
|
resched_task(rq->curr);
|
|
|
|
else
|
2008-09-20 23:38:02 +02:00
|
|
|
check_preempt_curr(rq, p, 0);
|
2008-01-25 21:08:22 +01:00
|
|
|
}
|
|
|
|
|
2007-10-15 17:00:08 +02:00
|
|
|
/* Account for a task changing its policy or group.
|
|
|
|
*
|
|
|
|
* This routine is mostly called to set cfs_rq->curr field when a task
|
|
|
|
* migrates between groups/classes.
|
|
|
|
*/
|
|
|
|
static void set_curr_task_fair(struct rq *rq)
|
|
|
|
{
|
|
|
|
struct sched_entity *se = &rq->curr->se;
|
|
|
|
|
2011-07-21 18:43:30 +02:00
|
|
|
for_each_sched_entity(se) {
|
|
|
|
struct cfs_rq *cfs_rq = cfs_rq_of(se);
|
|
|
|
|
|
|
|
set_next_entity(cfs_rq, se);
|
|
|
|
/* ensure bandwidth has been allocated on our new cfs_rq */
|
|
|
|
account_cfs_rq_runtime(cfs_rq, 0);
|
|
|
|
}
|
2007-10-15 17:00:08 +02:00
|
|
|
}
|
|
|
|
|
2011-10-25 10:00:11 +02:00
|
|
|
void init_cfs_rq(struct cfs_rq *cfs_rq)
|
|
|
|
{
|
|
|
|
cfs_rq->tasks_timeline = RB_ROOT;
|
|
|
|
INIT_LIST_HEAD(&cfs_rq->tasks);
|
|
|
|
cfs_rq->min_vruntime = (u64)(-(1LL << 20));
|
|
|
|
#ifndef CONFIG_64BIT
|
|
|
|
cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2008-02-29 21:21:01 +01:00
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
2010-10-15 15:24:15 +02:00
|
|
|
static void task_move_group_fair(struct task_struct *p, int on_rq)
|
2008-02-29 21:21:01 +01:00
|
|
|
{
|
2010-10-15 15:24:15 +02:00
|
|
|
/*
|
|
|
|
* If the task was not on the rq at the time of this cgroup movement
|
|
|
|
* it must have been asleep, sleeping tasks keep their ->vruntime
|
|
|
|
* absolute on their old rq until wakeup (needed for the fair sleeper
|
|
|
|
* bonus in place_entity()).
|
|
|
|
*
|
|
|
|
* If it was on the rq, we've just 'preempted' it, which does convert
|
|
|
|
* ->vruntime to a relative base.
|
|
|
|
*
|
|
|
|
* Make sure both cases convert their relative position when migrating
|
|
|
|
* to another cgroup's rq. This does somewhat interfere with the
|
|
|
|
* fair sleeper stuff for the first placement, but who cares.
|
|
|
|
*/
|
2011-12-15 06:36:07 +01:00
|
|
|
/*
|
|
|
|
* When !on_rq, vruntime of the task has usually NOT been normalized.
|
|
|
|
* But there are some cases where it has already been normalized:
|
|
|
|
*
|
|
|
|
* - Moving a forked child which is waiting for being woken up by
|
|
|
|
* wake_up_new_task().
|
|
|
|
*
|
|
|
|
* To prevent boost or penalty in the new cfs_rq caused by delta
|
|
|
|
* min_vruntime between the two cfs_rqs, we skip vruntime adjustment.
|
|
|
|
*/
|
|
|
|
if (!on_rq && !p->se.sum_exec_runtime)
|
|
|
|
on_rq = 1;
|
|
|
|
|
2010-10-15 15:24:15 +02:00
|
|
|
if (!on_rq)
|
|
|
|
p->se.vruntime -= cfs_rq_of(&p->se)->min_vruntime;
|
|
|
|
set_task_rq(p, task_cpu(p));
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 18:04:41 +01:00
|
|
|
if (!on_rq)
|
2010-10-15 15:24:15 +02:00
|
|
|
p->se.vruntime += cfs_rq_of(&p->se)->min_vruntime;
|
2008-02-29 21:21:01 +01:00
|
|
|
}
|
2011-10-25 10:00:11 +02:00
|
|
|
|
|
|
|
void free_fair_sched_group(struct task_group *tg)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
destroy_cfs_bandwidth(tg_cfs_bandwidth(tg));
|
|
|
|
|
|
|
|
for_each_possible_cpu(i) {
|
|
|
|
if (tg->cfs_rq)
|
|
|
|
kfree(tg->cfs_rq[i]);
|
|
|
|
if (tg->se)
|
|
|
|
kfree(tg->se[i]);
|
|
|
|
}
|
|
|
|
|
|
|
|
kfree(tg->cfs_rq);
|
|
|
|
kfree(tg->se);
|
|
|
|
}
|
|
|
|
|
|
|
|
int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
|
|
|
|
{
|
|
|
|
struct cfs_rq *cfs_rq;
|
|
|
|
struct sched_entity *se;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
tg->cfs_rq = kzalloc(sizeof(cfs_rq) * nr_cpu_ids, GFP_KERNEL);
|
|
|
|
if (!tg->cfs_rq)
|
|
|
|
goto err;
|
|
|
|
tg->se = kzalloc(sizeof(se) * nr_cpu_ids, GFP_KERNEL);
|
|
|
|
if (!tg->se)
|
|
|
|
goto err;
|
|
|
|
|
|
|
|
tg->shares = NICE_0_LOAD;
|
|
|
|
|
|
|
|
init_cfs_bandwidth(tg_cfs_bandwidth(tg));
|
|
|
|
|
|
|
|
for_each_possible_cpu(i) {
|
|
|
|
cfs_rq = kzalloc_node(sizeof(struct cfs_rq),
|
|
|
|
GFP_KERNEL, cpu_to_node(i));
|
|
|
|
if (!cfs_rq)
|
|
|
|
goto err;
|
|
|
|
|
|
|
|
se = kzalloc_node(sizeof(struct sched_entity),
|
|
|
|
GFP_KERNEL, cpu_to_node(i));
|
|
|
|
if (!se)
|
|
|
|
goto err_free_rq;
|
|
|
|
|
|
|
|
init_cfs_rq(cfs_rq);
|
|
|
|
init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]);
|
|
|
|
}
|
|
|
|
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
err_free_rq:
|
|
|
|
kfree(cfs_rq);
|
|
|
|
err:
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
void unregister_fair_sched_group(struct task_group *tg, int cpu)
|
|
|
|
{
|
|
|
|
struct rq *rq = cpu_rq(cpu);
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Only empty task groups can be destroyed; so we can speculatively
|
|
|
|
* check on_list without danger of it being re-added.
|
|
|
|
*/
|
|
|
|
if (!tg->cfs_rq[cpu]->on_list)
|
|
|
|
return;
|
|
|
|
|
|
|
|
raw_spin_lock_irqsave(&rq->lock, flags);
|
|
|
|
list_del_leaf_cfs_rq(tg->cfs_rq[cpu]);
|
|
|
|
raw_spin_unlock_irqrestore(&rq->lock, flags);
|
|
|
|
}
|
|
|
|
|
|
|
|
void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
|
|
|
|
struct sched_entity *se, int cpu,
|
|
|
|
struct sched_entity *parent)
|
|
|
|
{
|
|
|
|
struct rq *rq = cpu_rq(cpu);
|
|
|
|
|
|
|
|
cfs_rq->tg = tg;
|
|
|
|
cfs_rq->rq = rq;
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
/* allow initial update_cfs_load() to truncate */
|
|
|
|
cfs_rq->load_stamp = 1;
|
2008-02-29 21:21:01 +01:00
|
|
|
#endif
|
2011-10-25 10:00:11 +02:00
|
|
|
init_cfs_rq_runtime(cfs_rq);
|
|
|
|
|
|
|
|
tg->cfs_rq[cpu] = cfs_rq;
|
|
|
|
tg->se[cpu] = se;
|
|
|
|
|
|
|
|
/* se could be NULL for root_task_group */
|
|
|
|
if (!se)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (!parent)
|
|
|
|
se->cfs_rq = &rq->cfs;
|
|
|
|
else
|
|
|
|
se->cfs_rq = parent->my_q;
|
|
|
|
|
|
|
|
se->my_q = cfs_rq;
|
|
|
|
update_load_set(&se->load, 0);
|
|
|
|
se->parent = parent;
|
|
|
|
}
|
|
|
|
|
|
|
|
static DEFINE_MUTEX(shares_mutex);
|
|
|
|
|
|
|
|
int sched_group_set_shares(struct task_group *tg, unsigned long shares)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We can't change the weight of the root cgroup.
|
|
|
|
*/
|
|
|
|
if (!tg->se[0])
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES));
|
|
|
|
|
|
|
|
mutex_lock(&shares_mutex);
|
|
|
|
if (tg->shares == shares)
|
|
|
|
goto done;
|
|
|
|
|
|
|
|
tg->shares = shares;
|
|
|
|
for_each_possible_cpu(i) {
|
|
|
|
struct rq *rq = cpu_rq(i);
|
|
|
|
struct sched_entity *se;
|
|
|
|
|
|
|
|
se = tg->se[i];
|
|
|
|
/* Propagate contribution to hierarchy */
|
|
|
|
raw_spin_lock_irqsave(&rq->lock, flags);
|
|
|
|
for_each_sched_entity(se)
|
|
|
|
update_cfs_shares(group_cfs_rq(se));
|
|
|
|
raw_spin_unlock_irqrestore(&rq->lock, flags);
|
|
|
|
}
|
|
|
|
|
|
|
|
done:
|
|
|
|
mutex_unlock(&shares_mutex);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
#else /* CONFIG_FAIR_GROUP_SCHED */
|
|
|
|
|
|
|
|
void free_fair_sched_group(struct task_group *tg) { }
|
|
|
|
|
|
|
|
int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
|
|
|
|
{
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
void unregister_fair_sched_group(struct task_group *tg, int cpu) { }
|
|
|
|
|
|
|
|
#endif /* CONFIG_FAIR_GROUP_SCHED */
|
|
|
|
|
2008-02-29 21:21:01 +01:00
|
|
|
|
2010-01-14 04:21:52 +01:00
|
|
|
static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task)
|
2009-09-21 03:31:53 +02:00
|
|
|
{
|
|
|
|
struct sched_entity *se = &task->se;
|
|
|
|
unsigned int rr_interval = 0;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Time slice is 0 for SCHED_OTHER tasks that are on an otherwise
|
|
|
|
* idle runqueue:
|
|
|
|
*/
|
|
|
|
if (rq->cfs.load.weight)
|
|
|
|
rr_interval = NS_TO_JIFFIES(sched_slice(&rq->cfs, se));
|
|
|
|
|
|
|
|
return rr_interval;
|
|
|
|
}
|
|
|
|
|
2007-07-09 18:51:58 +02:00
|
|
|
/*
|
|
|
|
* All the scheduling class methods:
|
|
|
|
*/
|
2011-10-25 10:00:11 +02:00
|
|
|
const struct sched_class fair_sched_class = {
|
2007-10-15 17:00:12 +02:00
|
|
|
.next = &idle_sched_class,
|
2007-07-09 18:51:58 +02:00
|
|
|
.enqueue_task = enqueue_task_fair,
|
|
|
|
.dequeue_task = dequeue_task_fair,
|
|
|
|
.yield_task = yield_task_fair,
|
2011-02-01 15:50:51 +01:00
|
|
|
.yield_to_task = yield_to_task_fair,
|
2007-07-09 18:51:58 +02:00
|
|
|
|
2007-10-15 17:00:05 +02:00
|
|
|
.check_preempt_curr = check_preempt_wakeup,
|
2007-07-09 18:51:58 +02:00
|
|
|
|
|
|
|
.pick_next_task = pick_next_task_fair,
|
|
|
|
.put_prev_task = put_prev_task_fair,
|
|
|
|
|
2007-10-24 18:23:51 +02:00
|
|
|
#ifdef CONFIG_SMP
|
2008-10-22 09:25:26 +02:00
|
|
|
.select_task_rq = select_task_rq_fair,
|
|
|
|
|
2009-11-30 12:16:46 +01:00
|
|
|
.rq_online = rq_online_fair,
|
|
|
|
.rq_offline = rq_offline_fair,
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 18:04:41 +01:00
|
|
|
|
|
|
|
.task_waking = task_waking_fair,
|
2007-10-24 18:23:51 +02:00
|
|
|
#endif
|
2007-07-09 18:51:58 +02:00
|
|
|
|
2007-10-15 17:00:08 +02:00
|
|
|
.set_curr_task = set_curr_task_fair,
|
2007-07-09 18:51:58 +02:00
|
|
|
.task_tick = task_tick_fair,
|
2009-11-27 17:32:46 +01:00
|
|
|
.task_fork = task_fork_fair,
|
2008-01-25 21:08:22 +01:00
|
|
|
|
|
|
|
.prio_changed = prio_changed_fair,
|
2011-01-17 17:03:27 +01:00
|
|
|
.switched_from = switched_from_fair,
|
2008-01-25 21:08:22 +01:00
|
|
|
.switched_to = switched_to_fair,
|
2008-02-29 21:21:01 +01:00
|
|
|
|
2009-09-21 03:31:53 +02:00
|
|
|
.get_rr_interval = get_rr_interval_fair,
|
|
|
|
|
2008-02-29 21:21:01 +01:00
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
2010-10-15 15:24:15 +02:00
|
|
|
.task_move_group = task_move_group_fair,
|
2008-02-29 21:21:01 +01:00
|
|
|
#endif
|
2007-07-09 18:51:58 +02:00
|
|
|
};
|
|
|
|
|
|
|
|
#ifdef CONFIG_SCHED_DEBUG
|
2011-10-25 10:00:11 +02:00
|
|
|
void print_cfs_stats(struct seq_file *m, int cpu)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
|
|
|
struct cfs_rq *cfs_rq;
|
|
|
|
|
2008-01-25 21:08:34 +01:00
|
|
|
rcu_read_lock();
|
2007-08-09 11:16:51 +02:00
|
|
|
for_each_leaf_cfs_rq(cpu_rq(cpu), cfs_rq)
|
2007-08-09 11:16:47 +02:00
|
|
|
print_cfs_rq(m, cpu, cfs_rq);
|
2008-01-25 21:08:34 +01:00
|
|
|
rcu_read_unlock();
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
#endif
|
2011-10-25 10:00:11 +02:00
|
|
|
|
|
|
|
__init void init_sched_fair_class(void)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
open_softirq(SCHED_SOFTIRQ, run_rebalance_domains);
|
|
|
|
|
|
|
|
#ifdef CONFIG_NO_HZ
|
|
|
|
zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT);
|
|
|
|
#endif
|
|
|
|
#endif /* SMP */
|
|
|
|
|
|
|
|
}
|