2007-07-09 18:51:58 +02:00
|
|
|
/*
|
|
|
|
* Real-Time Scheduling Class (mapped to the SCHED_FIFO and SCHED_RR
|
|
|
|
* policies)
|
|
|
|
*/
|
|
|
|
|
2011-10-25 10:00:11 +02:00
|
|
|
#include "sched.h"
|
|
|
|
|
|
|
|
#include <linux/slab.h>
|
|
|
|
|
2013-02-07 16:47:04 +01:00
|
|
|
int sched_rr_timeslice = RR_TIMESLICE;
|
|
|
|
|
2011-10-25 10:00:11 +02:00
|
|
|
static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);
|
|
|
|
|
|
|
|
struct rt_bandwidth def_rt_bandwidth;
|
|
|
|
|
|
|
|
static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)
|
|
|
|
{
|
|
|
|
struct rt_bandwidth *rt_b =
|
|
|
|
container_of(timer, struct rt_bandwidth, rt_period_timer);
|
|
|
|
ktime_t now;
|
|
|
|
int overrun;
|
|
|
|
int idle = 0;
|
|
|
|
|
|
|
|
for (;;) {
|
|
|
|
now = hrtimer_cb_get_time(timer);
|
|
|
|
overrun = hrtimer_forward(timer, now, rt_b->rt_period);
|
|
|
|
|
|
|
|
if (!overrun)
|
|
|
|
break;
|
|
|
|
|
|
|
|
idle = do_sched_rt_period_timer(rt_b, overrun);
|
|
|
|
}
|
|
|
|
|
|
|
|
return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
|
|
|
|
}
|
|
|
|
|
|
|
|
void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
|
|
|
|
{
|
|
|
|
rt_b->rt_period = ns_to_ktime(period);
|
|
|
|
rt_b->rt_runtime = runtime;
|
|
|
|
|
|
|
|
raw_spin_lock_init(&rt_b->rt_runtime_lock);
|
|
|
|
|
|
|
|
hrtimer_init(&rt_b->rt_period_timer,
|
|
|
|
CLOCK_MONOTONIC, HRTIMER_MODE_REL);
|
|
|
|
rt_b->rt_period_timer.function = sched_rt_period_timer;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
|
|
|
|
{
|
|
|
|
if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (hrtimer_active(&rt_b->rt_period_timer))
|
|
|
|
return;
|
|
|
|
|
|
|
|
raw_spin_lock(&rt_b->rt_runtime_lock);
|
|
|
|
start_bandwidth_timer(&rt_b->rt_period_timer, rt_b->rt_period);
|
|
|
|
raw_spin_unlock(&rt_b->rt_runtime_lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq)
|
|
|
|
{
|
|
|
|
struct rt_prio_array *array;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
array = &rt_rq->active;
|
|
|
|
for (i = 0; i < MAX_RT_PRIO; i++) {
|
|
|
|
INIT_LIST_HEAD(array->queue + i);
|
|
|
|
__clear_bit(i, array->bitmap);
|
|
|
|
}
|
|
|
|
/* delimiter for bitsearch: */
|
|
|
|
__set_bit(MAX_RT_PRIO, array->bitmap);
|
|
|
|
|
|
|
|
#if defined CONFIG_SMP
|
|
|
|
rt_rq->highest_prio.curr = MAX_RT_PRIO;
|
|
|
|
rt_rq->highest_prio.next = MAX_RT_PRIO;
|
|
|
|
rt_rq->rt_nr_migratory = 0;
|
|
|
|
rt_rq->overloaded = 0;
|
|
|
|
plist_head_init(&rt_rq->pushable_tasks);
|
|
|
|
#endif
|
|
|
|
|
|
|
|
rt_rq->rt_time = 0;
|
|
|
|
rt_rq->rt_throttled = 0;
|
|
|
|
rt_rq->rt_runtime = 0;
|
|
|
|
raw_spin_lock_init(&rt_rq->rt_runtime_lock);
|
|
|
|
}
|
|
|
|
|
2009-07-24 12:25:30 +02:00
|
|
|
#ifdef CONFIG_RT_GROUP_SCHED
|
2011-10-25 10:00:11 +02:00
|
|
|
static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b)
|
|
|
|
{
|
|
|
|
hrtimer_cancel(&rt_b->rt_period_timer);
|
|
|
|
}
|
2009-07-24 12:25:30 +02:00
|
|
|
|
|
|
|
#define rt_entity_is_task(rt_se) (!(rt_se)->my_q)
|
|
|
|
|
2009-01-14 15:10:04 +01:00
|
|
|
static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)
|
|
|
|
{
|
2009-07-24 12:25:30 +02:00
|
|
|
#ifdef CONFIG_SCHED_DEBUG
|
|
|
|
WARN_ON_ONCE(!rt_entity_is_task(rt_se));
|
|
|
|
#endif
|
2009-01-14 15:10:04 +01:00
|
|
|
return container_of(rt_se, struct task_struct, rt);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)
|
|
|
|
{
|
|
|
|
return rt_rq->rq;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)
|
|
|
|
{
|
|
|
|
return rt_se->rt_rq;
|
|
|
|
}
|
|
|
|
|
2011-10-25 10:00:11 +02:00
|
|
|
void free_rt_sched_group(struct task_group *tg)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
if (tg->rt_se)
|
|
|
|
destroy_rt_bandwidth(&tg->rt_bandwidth);
|
|
|
|
|
|
|
|
for_each_possible_cpu(i) {
|
|
|
|
if (tg->rt_rq)
|
|
|
|
kfree(tg->rt_rq[i]);
|
|
|
|
if (tg->rt_se)
|
|
|
|
kfree(tg->rt_se[i]);
|
|
|
|
}
|
|
|
|
|
|
|
|
kfree(tg->rt_rq);
|
|
|
|
kfree(tg->rt_se);
|
|
|
|
}
|
|
|
|
|
|
|
|
void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
|
|
|
|
struct sched_rt_entity *rt_se, int cpu,
|
|
|
|
struct sched_rt_entity *parent)
|
|
|
|
{
|
|
|
|
struct rq *rq = cpu_rq(cpu);
|
|
|
|
|
|
|
|
rt_rq->highest_prio.curr = MAX_RT_PRIO;
|
|
|
|
rt_rq->rt_nr_boosted = 0;
|
|
|
|
rt_rq->rq = rq;
|
|
|
|
rt_rq->tg = tg;
|
|
|
|
|
|
|
|
tg->rt_rq[cpu] = rt_rq;
|
|
|
|
tg->rt_se[cpu] = rt_se;
|
|
|
|
|
|
|
|
if (!rt_se)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (!parent)
|
|
|
|
rt_se->rt_rq = &rq->rt;
|
|
|
|
else
|
|
|
|
rt_se->rt_rq = parent->my_q;
|
|
|
|
|
|
|
|
rt_se->my_q = rt_rq;
|
|
|
|
rt_se->parent = parent;
|
|
|
|
INIT_LIST_HEAD(&rt_se->run_list);
|
|
|
|
}
|
|
|
|
|
|
|
|
int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
|
|
|
|
{
|
|
|
|
struct rt_rq *rt_rq;
|
|
|
|
struct sched_rt_entity *rt_se;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
tg->rt_rq = kzalloc(sizeof(rt_rq) * nr_cpu_ids, GFP_KERNEL);
|
|
|
|
if (!tg->rt_rq)
|
|
|
|
goto err;
|
|
|
|
tg->rt_se = kzalloc(sizeof(rt_se) * nr_cpu_ids, GFP_KERNEL);
|
|
|
|
if (!tg->rt_se)
|
|
|
|
goto err;
|
|
|
|
|
|
|
|
init_rt_bandwidth(&tg->rt_bandwidth,
|
|
|
|
ktime_to_ns(def_rt_bandwidth.rt_period), 0);
|
|
|
|
|
|
|
|
for_each_possible_cpu(i) {
|
|
|
|
rt_rq = kzalloc_node(sizeof(struct rt_rq),
|
|
|
|
GFP_KERNEL, cpu_to_node(i));
|
|
|
|
if (!rt_rq)
|
|
|
|
goto err;
|
|
|
|
|
|
|
|
rt_se = kzalloc_node(sizeof(struct sched_rt_entity),
|
|
|
|
GFP_KERNEL, cpu_to_node(i));
|
|
|
|
if (!rt_se)
|
|
|
|
goto err_free_rq;
|
|
|
|
|
|
|
|
init_rt_rq(rt_rq, cpu_rq(i));
|
|
|
|
rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;
|
|
|
|
init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]);
|
|
|
|
}
|
|
|
|
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
err_free_rq:
|
|
|
|
kfree(rt_rq);
|
|
|
|
err:
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2009-01-14 15:10:04 +01:00
|
|
|
#else /* CONFIG_RT_GROUP_SCHED */
|
|
|
|
|
2009-04-01 18:40:15 +02:00
|
|
|
#define rt_entity_is_task(rt_se) (1)
|
|
|
|
|
2009-07-24 12:25:30 +02:00
|
|
|
static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)
|
|
|
|
{
|
|
|
|
return container_of(rt_se, struct task_struct, rt);
|
|
|
|
}
|
|
|
|
|
2009-01-14 15:10:04 +01:00
|
|
|
static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)
|
|
|
|
{
|
|
|
|
return container_of(rt_rq, struct rq, rt);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)
|
|
|
|
{
|
|
|
|
struct task_struct *p = rt_task_of(rt_se);
|
|
|
|
struct rq *rq = task_rq(p);
|
|
|
|
|
|
|
|
return &rq->rt;
|
|
|
|
}
|
|
|
|
|
2011-10-25 10:00:11 +02:00
|
|
|
void free_rt_sched_group(struct task_group *tg) { }
|
|
|
|
|
|
|
|
int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
|
|
|
|
{
|
|
|
|
return 1;
|
|
|
|
}
|
2009-01-14 15:10:04 +01:00
|
|
|
#endif /* CONFIG_RT_GROUP_SCHED */
|
|
|
|
|
2008-01-25 21:08:06 +01:00
|
|
|
#ifdef CONFIG_SMP
|
2008-01-25 21:08:15 +01:00
|
|
|
|
2008-01-25 21:08:18 +01:00
|
|
|
static inline int rt_overloaded(struct rq *rq)
|
2008-01-25 21:08:06 +01:00
|
|
|
{
|
2008-01-25 21:08:18 +01:00
|
|
|
return atomic_read(&rq->rd->rto_count);
|
2008-01-25 21:08:06 +01:00
|
|
|
}
|
2008-01-25 21:08:15 +01:00
|
|
|
|
2008-01-25 21:08:06 +01:00
|
|
|
static inline void rt_set_overload(struct rq *rq)
|
|
|
|
{
|
2008-06-04 21:04:05 +02:00
|
|
|
if (!rq->online)
|
|
|
|
return;
|
|
|
|
|
2008-11-24 17:05:05 +01:00
|
|
|
cpumask_set_cpu(rq->cpu, rq->rd->rto_mask);
|
2008-01-25 21:08:06 +01:00
|
|
|
/*
|
|
|
|
* Make sure the mask is visible before we set
|
|
|
|
* the overload count. That is checked to determine
|
|
|
|
* if we should look at the mask. It would be a shame
|
|
|
|
* if we looked at the mask, but the mask was not
|
|
|
|
* updated yet.
|
2013-10-15 12:35:07 +02:00
|
|
|
*
|
|
|
|
* Matched by the barrier in pull_rt_task().
|
2008-01-25 21:08:06 +01:00
|
|
|
*/
|
2013-10-15 12:35:07 +02:00
|
|
|
smp_wmb();
|
2008-01-25 21:08:18 +01:00
|
|
|
atomic_inc(&rq->rd->rto_count);
|
2008-01-25 21:08:06 +01:00
|
|
|
}
|
2008-01-25 21:08:15 +01:00
|
|
|
|
2008-01-25 21:08:06 +01:00
|
|
|
static inline void rt_clear_overload(struct rq *rq)
|
|
|
|
{
|
2008-06-04 21:04:05 +02:00
|
|
|
if (!rq->online)
|
|
|
|
return;
|
|
|
|
|
2008-01-25 21:08:06 +01:00
|
|
|
/* the order here really doesn't matter */
|
2008-01-25 21:08:18 +01:00
|
|
|
atomic_dec(&rq->rd->rto_count);
|
2008-11-24 17:05:05 +01:00
|
|
|
cpumask_clear_cpu(rq->cpu, rq->rd->rto_mask);
|
2008-01-25 21:08:06 +01:00
|
|
|
}
|
2008-01-25 21:08:07 +01:00
|
|
|
|
2009-01-14 15:10:04 +01:00
|
|
|
static void update_rt_migration(struct rt_rq *rt_rq)
|
2008-01-25 21:08:07 +01:00
|
|
|
{
|
2009-04-01 18:40:15 +02:00
|
|
|
if (rt_rq->rt_nr_migratory && rt_rq->rt_nr_total > 1) {
|
2009-01-14 15:10:04 +01:00
|
|
|
if (!rt_rq->overloaded) {
|
|
|
|
rt_set_overload(rq_of_rt_rq(rt_rq));
|
|
|
|
rt_rq->overloaded = 1;
|
2008-01-25 21:08:23 +01:00
|
|
|
}
|
2009-01-14 15:10:04 +01:00
|
|
|
} else if (rt_rq->overloaded) {
|
|
|
|
rt_clear_overload(rq_of_rt_rq(rt_rq));
|
|
|
|
rt_rq->overloaded = 0;
|
2008-01-25 21:08:18 +01:00
|
|
|
}
|
2008-01-25 21:08:07 +01:00
|
|
|
}
|
2008-01-25 21:08:06 +01:00
|
|
|
|
2009-01-14 15:10:04 +01:00
|
|
|
static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
|
|
|
|
{
|
2012-04-23 12:11:21 +02:00
|
|
|
struct task_struct *p;
|
|
|
|
|
2009-04-01 18:40:15 +02:00
|
|
|
if (!rt_entity_is_task(rt_se))
|
|
|
|
return;
|
|
|
|
|
2012-04-23 12:11:21 +02:00
|
|
|
p = rt_task_of(rt_se);
|
2009-04-01 18:40:15 +02:00
|
|
|
rt_rq = &rq_of_rt_rq(rt_rq)->rt;
|
|
|
|
|
|
|
|
rt_rq->rt_nr_total++;
|
2012-04-23 12:11:21 +02:00
|
|
|
if (p->nr_cpus_allowed > 1)
|
2009-01-14 15:10:04 +01:00
|
|
|
rt_rq->rt_nr_migratory++;
|
|
|
|
|
|
|
|
update_rt_migration(rt_rq);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
|
|
|
|
{
|
2012-04-23 12:11:21 +02:00
|
|
|
struct task_struct *p;
|
|
|
|
|
2009-04-01 18:40:15 +02:00
|
|
|
if (!rt_entity_is_task(rt_se))
|
|
|
|
return;
|
|
|
|
|
2012-04-23 12:11:21 +02:00
|
|
|
p = rt_task_of(rt_se);
|
2009-04-01 18:40:15 +02:00
|
|
|
rt_rq = &rq_of_rt_rq(rt_rq)->rt;
|
|
|
|
|
|
|
|
rt_rq->rt_nr_total--;
|
2012-04-23 12:11:21 +02:00
|
|
|
if (p->nr_cpus_allowed > 1)
|
2009-01-14 15:10:04 +01:00
|
|
|
rt_rq->rt_nr_migratory--;
|
|
|
|
|
|
|
|
update_rt_migration(rt_rq);
|
|
|
|
}
|
|
|
|
|
sched: Use pushable_tasks to determine next highest prio
Hillf Danton proposed a patch (see link) that cleaned up the
sched_rt code that calculates the priority of the next highest priority
task to be used in finding run queues to pull from.
His patch removed the calculating of the next prio to just use the current
prio when deteriming if we should examine a run queue to pull from. The problem
with his patch was that it caused more false checks. Because we check a run
queue for pushable tasks if the current priority of that run queue is higher
in priority than the task about to run on our run queue. But after grabbing
the locks and doing the real check, we find that there may not be a task
that has a higher prio task to pull. Thus the locks were taken with nothing to
do.
I added some trace_printks() to record when and how many times the run queue
locks were taken to check for pullable tasks, compared to how many times we
pulled a task.
With the current method, it was:
3806 locks taken vs 2812 pulled tasks
With Hillf's patch:
6728 locks taken vs 2804 pulled tasks
The number of times locks were taken to pull a task went up almost double with
no more success rate.
But his patch did get me thinking. When we look at the priority of the highest
task to consider taking the locks to do a pull, a failure to pull can be one
of the following: (in order of most likely)
o RT task was pushed off already between the check and taking the lock
o Waiting RT task can not be migrated
o RT task's CPU affinity does not include the target run queue's CPU
o RT task's priority changed between the check and taking the lock
And with Hillf's patch, the thing that caused most of the failures, is
the RT task to pull was not at the right priority to pull (not greater than
the current RT task priority on the target run queue).
Most of the above cases we can't help. But the current method does not check
if the next highest prio RT task can be migrated or not, and if it can not,
we still grab the locks to do the test (we don't find out about this fact until
after we have the locks). I thought about this case, and realized that the
pushable task plist that is maintained only holds RT tasks that can migrate.
If we move the calculating of the next highest prio task from the inc/dec_rt_task()
functions into the queuing of the pushable tasks, then we only measure the
priorities of those tasks that we push, and we get this basically for free.
Not only does this patch make the code a little more efficient, it cleans it
up and makes it a little simpler.
Thanks to Hillf Danton for inspiring me on this patch.
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Hillf Danton <dhillf@gmail.com>
Cc: Gregory Haskins <ghaskins@novell.com>
Link: http://lkml.kernel.org/r/BANLkTimQ67180HxCx5vgMqumqw1EkFh3qg@mail.gmail.com
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2011-06-17 03:55:23 +02:00
|
|
|
static inline int has_pushable_tasks(struct rq *rq)
|
|
|
|
{
|
|
|
|
return !plist_head_empty(&rq->rt.pushable_tasks);
|
|
|
|
}
|
|
|
|
|
sched: create "pushable_tasks" list to limit pushing to one attempt
The RT scheduler employs a "push/pull" design to actively balance tasks
within the system (on a per disjoint cpuset basis). When a task is
awoken, it is immediately determined if there are any lower priority
cpus which should be preempted. This is opposed to the way normal
SCHED_OTHER tasks behave, which will wait for a periodic rebalancing
operation to occur before spreading out load.
When a particular RQ has more than 1 active RT task, it is said to
be in an "overloaded" state. Once this occurs, the system enters
the active balancing mode, where it will try to push the task away,
or persuade a different cpu to pull it over. The system will stay
in this state until the system falls back below the <= 1 queued RT
task per RQ.
However, the current implementation suffers from a limitation in the
push logic. Once overloaded, all tasks (other than current) on the
RQ are analyzed on every push operation, even if it was previously
unpushable (due to affinity, etc). Whats more, the operation stops
at the first task that is unpushable and will not look at items
lower in the queue. This causes two problems:
1) We can have the same tasks analyzed over and over again during each
push, which extends out the fast path in the scheduler for no
gain. Consider a RQ that has dozens of tasks that are bound to a
core. Each one of those tasks will be encountered and skipped
for each push operation while they are queued.
2) There may be lower-priority tasks under the unpushable task that
could have been successfully pushed, but will never be considered
until either the unpushable task is cleared, or a pull operation
succeeds. The net result is a potential latency source for mid
priority tasks.
This patch aims to rectify these two conditions by introducing a new
priority sorted list: "pushable_tasks". A task is added to the list
each time a task is activated or preempted. It is removed from the
list any time it is deactivated, made current, or fails to push.
This works because a task only needs to be attempted to push once.
After an initial failure to push, the other cpus will eventually try to
pull the task when the conditions are proper. This also solves the
problem that we don't completely analyze all tasks due to encountering
an unpushable tasks. Now every task will have a push attempted (when
appropriate).
This reduces latency both by shorting the critical section of the
rq->lock for certain workloads, and by making sure the algorithm
considers all eligible tasks in the system.
[ rostedt: added a couple more BUG_ONs ]
Signed-off-by: Gregory Haskins <ghaskins@novell.com>
Acked-by: Steven Rostedt <srostedt@redhat.com>
2008-12-29 15:39:53 +01:00
|
|
|
static void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
|
|
|
|
{
|
|
|
|
plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);
|
|
|
|
plist_node_init(&p->pushable_tasks, p->prio);
|
|
|
|
plist_add(&p->pushable_tasks, &rq->rt.pushable_tasks);
|
sched: Use pushable_tasks to determine next highest prio
Hillf Danton proposed a patch (see link) that cleaned up the
sched_rt code that calculates the priority of the next highest priority
task to be used in finding run queues to pull from.
His patch removed the calculating of the next prio to just use the current
prio when deteriming if we should examine a run queue to pull from. The problem
with his patch was that it caused more false checks. Because we check a run
queue for pushable tasks if the current priority of that run queue is higher
in priority than the task about to run on our run queue. But after grabbing
the locks and doing the real check, we find that there may not be a task
that has a higher prio task to pull. Thus the locks were taken with nothing to
do.
I added some trace_printks() to record when and how many times the run queue
locks were taken to check for pullable tasks, compared to how many times we
pulled a task.
With the current method, it was:
3806 locks taken vs 2812 pulled tasks
With Hillf's patch:
6728 locks taken vs 2804 pulled tasks
The number of times locks were taken to pull a task went up almost double with
no more success rate.
But his patch did get me thinking. When we look at the priority of the highest
task to consider taking the locks to do a pull, a failure to pull can be one
of the following: (in order of most likely)
o RT task was pushed off already between the check and taking the lock
o Waiting RT task can not be migrated
o RT task's CPU affinity does not include the target run queue's CPU
o RT task's priority changed between the check and taking the lock
And with Hillf's patch, the thing that caused most of the failures, is
the RT task to pull was not at the right priority to pull (not greater than
the current RT task priority on the target run queue).
Most of the above cases we can't help. But the current method does not check
if the next highest prio RT task can be migrated or not, and if it can not,
we still grab the locks to do the test (we don't find out about this fact until
after we have the locks). I thought about this case, and realized that the
pushable task plist that is maintained only holds RT tasks that can migrate.
If we move the calculating of the next highest prio task from the inc/dec_rt_task()
functions into the queuing of the pushable tasks, then we only measure the
priorities of those tasks that we push, and we get this basically for free.
Not only does this patch make the code a little more efficient, it cleans it
up and makes it a little simpler.
Thanks to Hillf Danton for inspiring me on this patch.
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Hillf Danton <dhillf@gmail.com>
Cc: Gregory Haskins <ghaskins@novell.com>
Link: http://lkml.kernel.org/r/BANLkTimQ67180HxCx5vgMqumqw1EkFh3qg@mail.gmail.com
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2011-06-17 03:55:23 +02:00
|
|
|
|
|
|
|
/* Update the highest prio pushable task */
|
|
|
|
if (p->prio < rq->rt.highest_prio.next)
|
|
|
|
rq->rt.highest_prio.next = p->prio;
|
sched: create "pushable_tasks" list to limit pushing to one attempt
The RT scheduler employs a "push/pull" design to actively balance tasks
within the system (on a per disjoint cpuset basis). When a task is
awoken, it is immediately determined if there are any lower priority
cpus which should be preempted. This is opposed to the way normal
SCHED_OTHER tasks behave, which will wait for a periodic rebalancing
operation to occur before spreading out load.
When a particular RQ has more than 1 active RT task, it is said to
be in an "overloaded" state. Once this occurs, the system enters
the active balancing mode, where it will try to push the task away,
or persuade a different cpu to pull it over. The system will stay
in this state until the system falls back below the <= 1 queued RT
task per RQ.
However, the current implementation suffers from a limitation in the
push logic. Once overloaded, all tasks (other than current) on the
RQ are analyzed on every push operation, even if it was previously
unpushable (due to affinity, etc). Whats more, the operation stops
at the first task that is unpushable and will not look at items
lower in the queue. This causes two problems:
1) We can have the same tasks analyzed over and over again during each
push, which extends out the fast path in the scheduler for no
gain. Consider a RQ that has dozens of tasks that are bound to a
core. Each one of those tasks will be encountered and skipped
for each push operation while they are queued.
2) There may be lower-priority tasks under the unpushable task that
could have been successfully pushed, but will never be considered
until either the unpushable task is cleared, or a pull operation
succeeds. The net result is a potential latency source for mid
priority tasks.
This patch aims to rectify these two conditions by introducing a new
priority sorted list: "pushable_tasks". A task is added to the list
each time a task is activated or preempted. It is removed from the
list any time it is deactivated, made current, or fails to push.
This works because a task only needs to be attempted to push once.
After an initial failure to push, the other cpus will eventually try to
pull the task when the conditions are proper. This also solves the
problem that we don't completely analyze all tasks due to encountering
an unpushable tasks. Now every task will have a push attempted (when
appropriate).
This reduces latency both by shorting the critical section of the
rq->lock for certain workloads, and by making sure the algorithm
considers all eligible tasks in the system.
[ rostedt: added a couple more BUG_ONs ]
Signed-off-by: Gregory Haskins <ghaskins@novell.com>
Acked-by: Steven Rostedt <srostedt@redhat.com>
2008-12-29 15:39:53 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
static void dequeue_pushable_task(struct rq *rq, struct task_struct *p)
|
|
|
|
{
|
|
|
|
plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);
|
|
|
|
|
sched: Use pushable_tasks to determine next highest prio
Hillf Danton proposed a patch (see link) that cleaned up the
sched_rt code that calculates the priority of the next highest priority
task to be used in finding run queues to pull from.
His patch removed the calculating of the next prio to just use the current
prio when deteriming if we should examine a run queue to pull from. The problem
with his patch was that it caused more false checks. Because we check a run
queue for pushable tasks if the current priority of that run queue is higher
in priority than the task about to run on our run queue. But after grabbing
the locks and doing the real check, we find that there may not be a task
that has a higher prio task to pull. Thus the locks were taken with nothing to
do.
I added some trace_printks() to record when and how many times the run queue
locks were taken to check for pullable tasks, compared to how many times we
pulled a task.
With the current method, it was:
3806 locks taken vs 2812 pulled tasks
With Hillf's patch:
6728 locks taken vs 2804 pulled tasks
The number of times locks were taken to pull a task went up almost double with
no more success rate.
But his patch did get me thinking. When we look at the priority of the highest
task to consider taking the locks to do a pull, a failure to pull can be one
of the following: (in order of most likely)
o RT task was pushed off already between the check and taking the lock
o Waiting RT task can not be migrated
o RT task's CPU affinity does not include the target run queue's CPU
o RT task's priority changed between the check and taking the lock
And with Hillf's patch, the thing that caused most of the failures, is
the RT task to pull was not at the right priority to pull (not greater than
the current RT task priority on the target run queue).
Most of the above cases we can't help. But the current method does not check
if the next highest prio RT task can be migrated or not, and if it can not,
we still grab the locks to do the test (we don't find out about this fact until
after we have the locks). I thought about this case, and realized that the
pushable task plist that is maintained only holds RT tasks that can migrate.
If we move the calculating of the next highest prio task from the inc/dec_rt_task()
functions into the queuing of the pushable tasks, then we only measure the
priorities of those tasks that we push, and we get this basically for free.
Not only does this patch make the code a little more efficient, it cleans it
up and makes it a little simpler.
Thanks to Hillf Danton for inspiring me on this patch.
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Hillf Danton <dhillf@gmail.com>
Cc: Gregory Haskins <ghaskins@novell.com>
Link: http://lkml.kernel.org/r/BANLkTimQ67180HxCx5vgMqumqw1EkFh3qg@mail.gmail.com
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2011-06-17 03:55:23 +02:00
|
|
|
/* Update the new highest prio pushable task */
|
|
|
|
if (has_pushable_tasks(rq)) {
|
|
|
|
p = plist_first_entry(&rq->rt.pushable_tasks,
|
|
|
|
struct task_struct, pushable_tasks);
|
|
|
|
rq->rt.highest_prio.next = p->prio;
|
|
|
|
} else
|
|
|
|
rq->rt.highest_prio.next = MAX_RT_PRIO;
|
2008-04-19 12:11:10 +02:00
|
|
|
}
|
|
|
|
|
sched: create "pushable_tasks" list to limit pushing to one attempt
The RT scheduler employs a "push/pull" design to actively balance tasks
within the system (on a per disjoint cpuset basis). When a task is
awoken, it is immediately determined if there are any lower priority
cpus which should be preempted. This is opposed to the way normal
SCHED_OTHER tasks behave, which will wait for a periodic rebalancing
operation to occur before spreading out load.
When a particular RQ has more than 1 active RT task, it is said to
be in an "overloaded" state. Once this occurs, the system enters
the active balancing mode, where it will try to push the task away,
or persuade a different cpu to pull it over. The system will stay
in this state until the system falls back below the <= 1 queued RT
task per RQ.
However, the current implementation suffers from a limitation in the
push logic. Once overloaded, all tasks (other than current) on the
RQ are analyzed on every push operation, even if it was previously
unpushable (due to affinity, etc). Whats more, the operation stops
at the first task that is unpushable and will not look at items
lower in the queue. This causes two problems:
1) We can have the same tasks analyzed over and over again during each
push, which extends out the fast path in the scheduler for no
gain. Consider a RQ that has dozens of tasks that are bound to a
core. Each one of those tasks will be encountered and skipped
for each push operation while they are queued.
2) There may be lower-priority tasks under the unpushable task that
could have been successfully pushed, but will never be considered
until either the unpushable task is cleared, or a pull operation
succeeds. The net result is a potential latency source for mid
priority tasks.
This patch aims to rectify these two conditions by introducing a new
priority sorted list: "pushable_tasks". A task is added to the list
each time a task is activated or preempted. It is removed from the
list any time it is deactivated, made current, or fails to push.
This works because a task only needs to be attempted to push once.
After an initial failure to push, the other cpus will eventually try to
pull the task when the conditions are proper. This also solves the
problem that we don't completely analyze all tasks due to encountering
an unpushable tasks. Now every task will have a push attempted (when
appropriate).
This reduces latency both by shorting the critical section of the
rq->lock for certain workloads, and by making sure the algorithm
considers all eligible tasks in the system.
[ rostedt: added a couple more BUG_ONs ]
Signed-off-by: Gregory Haskins <ghaskins@novell.com>
Acked-by: Steven Rostedt <srostedt@redhat.com>
2008-12-29 15:39:53 +01:00
|
|
|
#else
|
|
|
|
|
2009-01-16 14:46:40 +01:00
|
|
|
static inline void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
|
2008-01-25 21:08:29 +01:00
|
|
|
{
|
2008-01-25 21:08:30 +01:00
|
|
|
}
|
|
|
|
|
2009-01-16 14:46:40 +01:00
|
|
|
static inline void dequeue_pushable_task(struct rq *rq, struct task_struct *p)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2009-01-14 14:55:39 +01:00
|
|
|
static inline
|
2009-01-16 14:46:40 +01:00
|
|
|
void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2009-01-14 15:10:04 +01:00
|
|
|
static inline
|
2009-01-16 14:46:40 +01:00
|
|
|
void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
|
|
|
|
{
|
|
|
|
}
|
sched: create "pushable_tasks" list to limit pushing to one attempt
The RT scheduler employs a "push/pull" design to actively balance tasks
within the system (on a per disjoint cpuset basis). When a task is
awoken, it is immediately determined if there are any lower priority
cpus which should be preempted. This is opposed to the way normal
SCHED_OTHER tasks behave, which will wait for a periodic rebalancing
operation to occur before spreading out load.
When a particular RQ has more than 1 active RT task, it is said to
be in an "overloaded" state. Once this occurs, the system enters
the active balancing mode, where it will try to push the task away,
or persuade a different cpu to pull it over. The system will stay
in this state until the system falls back below the <= 1 queued RT
task per RQ.
However, the current implementation suffers from a limitation in the
push logic. Once overloaded, all tasks (other than current) on the
RQ are analyzed on every push operation, even if it was previously
unpushable (due to affinity, etc). Whats more, the operation stops
at the first task that is unpushable and will not look at items
lower in the queue. This causes two problems:
1) We can have the same tasks analyzed over and over again during each
push, which extends out the fast path in the scheduler for no
gain. Consider a RQ that has dozens of tasks that are bound to a
core. Each one of those tasks will be encountered and skipped
for each push operation while they are queued.
2) There may be lower-priority tasks under the unpushable task that
could have been successfully pushed, but will never be considered
until either the unpushable task is cleared, or a pull operation
succeeds. The net result is a potential latency source for mid
priority tasks.
This patch aims to rectify these two conditions by introducing a new
priority sorted list: "pushable_tasks". A task is added to the list
each time a task is activated or preempted. It is removed from the
list any time it is deactivated, made current, or fails to push.
This works because a task only needs to be attempted to push once.
After an initial failure to push, the other cpus will eventually try to
pull the task when the conditions are proper. This also solves the
problem that we don't completely analyze all tasks due to encountering
an unpushable tasks. Now every task will have a push attempted (when
appropriate).
This reduces latency both by shorting the critical section of the
rq->lock for certain workloads, and by making sure the algorithm
considers all eligible tasks in the system.
[ rostedt: added a couple more BUG_ONs ]
Signed-off-by: Gregory Haskins <ghaskins@novell.com>
Acked-by: Steven Rostedt <srostedt@redhat.com>
2008-12-29 15:39:53 +01:00
|
|
|
|
2008-01-25 21:08:06 +01:00
|
|
|
#endif /* CONFIG_SMP */
|
|
|
|
|
2008-01-25 21:08:30 +01:00
|
|
|
static inline int on_rt_rq(struct sched_rt_entity *rt_se)
|
|
|
|
{
|
|
|
|
return !list_empty(&rt_se->run_list);
|
|
|
|
}
|
|
|
|
|
2008-02-13 15:45:40 +01:00
|
|
|
#ifdef CONFIG_RT_GROUP_SCHED
|
2008-01-25 21:08:30 +01:00
|
|
|
|
2008-02-13 15:45:39 +01:00
|
|
|
static inline u64 sched_rt_runtime(struct rt_rq *rt_rq)
|
2008-01-25 21:08:30 +01:00
|
|
|
{
|
|
|
|
if (!rt_rq->tg)
|
2008-02-13 15:45:39 +01:00
|
|
|
return RUNTIME_INF;
|
2008-01-25 21:08:30 +01:00
|
|
|
|
2008-04-19 19:44:58 +02:00
|
|
|
return rt_rq->rt_runtime;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline u64 sched_rt_period(struct rt_rq *rt_rq)
|
|
|
|
{
|
|
|
|
return ktime_to_ns(rt_rq->tg->rt_bandwidth.rt_period);
|
2008-01-25 21:08:30 +01:00
|
|
|
}
|
|
|
|
|
2011-05-14 08:20:02 +02:00
|
|
|
typedef struct task_group *rt_rq_iter_t;
|
|
|
|
|
2011-06-28 04:51:31 +02:00
|
|
|
static inline struct task_group *next_task_group(struct task_group *tg)
|
|
|
|
{
|
|
|
|
do {
|
|
|
|
tg = list_entry_rcu(tg->list.next,
|
|
|
|
typeof(struct task_group), list);
|
|
|
|
} while (&tg->list != &task_groups && task_group_is_autogroup(tg));
|
|
|
|
|
|
|
|
if (&tg->list == &task_groups)
|
|
|
|
tg = NULL;
|
|
|
|
|
|
|
|
return tg;
|
|
|
|
}
|
|
|
|
|
|
|
|
#define for_each_rt_rq(rt_rq, iter, rq) \
|
|
|
|
for (iter = container_of(&task_groups, typeof(*iter), list); \
|
|
|
|
(iter = next_task_group(iter)) && \
|
|
|
|
(rt_rq = iter->rt_rq[cpu_of(rq)]);)
|
2011-05-14 08:20:02 +02:00
|
|
|
|
2008-01-25 21:08:30 +01:00
|
|
|
#define for_each_sched_rt_entity(rt_se) \
|
|
|
|
for (; rt_se; rt_se = rt_se->parent)
|
|
|
|
|
|
|
|
static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)
|
|
|
|
{
|
|
|
|
return rt_se->my_q;
|
|
|
|
}
|
|
|
|
|
2010-01-20 21:59:01 +01:00
|
|
|
static void enqueue_rt_entity(struct sched_rt_entity *rt_se, bool head);
|
2008-01-25 21:08:30 +01:00
|
|
|
static void dequeue_rt_entity(struct sched_rt_entity *rt_se);
|
|
|
|
|
2008-02-13 15:45:39 +01:00
|
|
|
static void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
|
2008-01-25 21:08:30 +01:00
|
|
|
{
|
sched_rt.c: resch needed in rt_rq_enqueue() for the root rt_rq
While working on the new version of the code for SCHED_SPORADIC I
noticed something strange in the present throttling mechanism. More
specifically in the throttling timer handler in sched_rt.c
(do_sched_rt_period_timer()) and in rt_rq_enqueue().
The problem is that, when unthrottling a runqueue, rt_rq_enqueue() only
asks for rescheduling if the runqueue has a sched_entity associated to
it (i.e., rt_rq->rt_se != NULL).
Now, if the runqueue is the root rq (which has a rt_se = NULL)
rescheduling does not take place, and it is delayed to some undefined
instant in the future.
This imply some random bandwidth usage by the RT tasks under throttling.
For instance, setting rt_runtime_us/rt_period_us = 950ms/1000ms an RT
task will get less than 95%. In our tests we got something varying
between 70% to 95%.
Using smaller time values, e.g., 95ms/100ms, things are even worse, and
I can see values also going down to 20-25%!!
The tests we performed are simply running 'yes' as a SCHED_FIFO task,
and checking the CPU usage with top, but we can investigate thoroughly
if you think it is needed.
Things go much better, for us, with the attached patch... Don't know if
it is the best approach, but it solved the issue for us.
Signed-off-by: Dario Faggioli <raistlin@linux.it>
Signed-off-by: Michael Trimarchi <trimarchimichael@yahoo.it>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: <stable@kernel.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-10-03 17:40:46 +02:00
|
|
|
struct task_struct *curr = rq_of_rt_rq(rt_rq)->curr;
|
2010-01-29 07:57:52 +01:00
|
|
|
struct sched_rt_entity *rt_se;
|
|
|
|
|
sched: Fix sched rt group scheduling when hierachy is enabled
The current sched rt code is broken when it comes to hierarchical
scheduling, this patch fixes two problems
1. It adds redundant enqueuing (harmless) when it finds a queue
has tasks enqueued, but it has no run time and it is not
throttled.
2. The most important change is in sched_rt_rq_enqueue/dequeue.
The code just picks the rt_rq belonging to the current cpu
on which the period timer runs, the patch fixes it, so that
the correct rt_se is enqueued/dequeued.
Tested with a simple hierarchy
/c/d, c and d assigned similar runtimes of 50,000 and a while
1 loop runs within "d". Both c and d get throttled, without
the patch, the task just stops running and never runs (depends
on where the sched_rt b/w timer runs). With the patch, the
task is throttled and runs as expected.
[ bharata, suggestions on how to pick the rt_se belong to the
rt_rq and correct cpu ]
Signed-off-by: Balbir Singh <balbir@linux.vnet.ibm.com>
Acked-by: Bharata B Rao <bharata@linux.vnet.ibm.com>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: stable@kernel.org
LKML-Reference: <20110303113435.GA2868@balbir.in.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2011-03-03 12:34:35 +01:00
|
|
|
int cpu = cpu_of(rq_of_rt_rq(rt_rq));
|
|
|
|
|
|
|
|
rt_se = rt_rq->tg->rt_se[cpu];
|
2008-01-25 21:08:30 +01:00
|
|
|
|
sched_rt.c: resch needed in rt_rq_enqueue() for the root rt_rq
While working on the new version of the code for SCHED_SPORADIC I
noticed something strange in the present throttling mechanism. More
specifically in the throttling timer handler in sched_rt.c
(do_sched_rt_period_timer()) and in rt_rq_enqueue().
The problem is that, when unthrottling a runqueue, rt_rq_enqueue() only
asks for rescheduling if the runqueue has a sched_entity associated to
it (i.e., rt_rq->rt_se != NULL).
Now, if the runqueue is the root rq (which has a rt_se = NULL)
rescheduling does not take place, and it is delayed to some undefined
instant in the future.
This imply some random bandwidth usage by the RT tasks under throttling.
For instance, setting rt_runtime_us/rt_period_us = 950ms/1000ms an RT
task will get less than 95%. In our tests we got something varying
between 70% to 95%.
Using smaller time values, e.g., 95ms/100ms, things are even worse, and
I can see values also going down to 20-25%!!
The tests we performed are simply running 'yes' as a SCHED_FIFO task,
and checking the CPU usage with top, but we can investigate thoroughly
if you think it is needed.
Things go much better, for us, with the attached patch... Don't know if
it is the best approach, but it solved the issue for us.
Signed-off-by: Dario Faggioli <raistlin@linux.it>
Signed-off-by: Michael Trimarchi <trimarchimichael@yahoo.it>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: <stable@kernel.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-10-03 17:40:46 +02:00
|
|
|
if (rt_rq->rt_nr_running) {
|
|
|
|
if (rt_se && !on_rt_rq(rt_se))
|
2010-01-20 21:59:01 +01:00
|
|
|
enqueue_rt_entity(rt_se, false);
|
2008-12-29 15:39:49 +01:00
|
|
|
if (rt_rq->highest_prio.curr < curr->prio)
|
2008-01-25 21:08:32 +01:00
|
|
|
resched_task(curr);
|
2008-01-25 21:08:30 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2008-02-13 15:45:39 +01:00
|
|
|
static void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
|
2008-01-25 21:08:30 +01:00
|
|
|
{
|
2010-01-29 07:57:52 +01:00
|
|
|
struct sched_rt_entity *rt_se;
|
sched: Fix sched rt group scheduling when hierachy is enabled
The current sched rt code is broken when it comes to hierarchical
scheduling, this patch fixes two problems
1. It adds redundant enqueuing (harmless) when it finds a queue
has tasks enqueued, but it has no run time and it is not
throttled.
2. The most important change is in sched_rt_rq_enqueue/dequeue.
The code just picks the rt_rq belonging to the current cpu
on which the period timer runs, the patch fixes it, so that
the correct rt_se is enqueued/dequeued.
Tested with a simple hierarchy
/c/d, c and d assigned similar runtimes of 50,000 and a while
1 loop runs within "d". Both c and d get throttled, without
the patch, the task just stops running and never runs (depends
on where the sched_rt b/w timer runs). With the patch, the
task is throttled and runs as expected.
[ bharata, suggestions on how to pick the rt_se belong to the
rt_rq and correct cpu ]
Signed-off-by: Balbir Singh <balbir@linux.vnet.ibm.com>
Acked-by: Bharata B Rao <bharata@linux.vnet.ibm.com>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: stable@kernel.org
LKML-Reference: <20110303113435.GA2868@balbir.in.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2011-03-03 12:34:35 +01:00
|
|
|
int cpu = cpu_of(rq_of_rt_rq(rt_rq));
|
2010-01-29 07:57:52 +01:00
|
|
|
|
sched: Fix sched rt group scheduling when hierachy is enabled
The current sched rt code is broken when it comes to hierarchical
scheduling, this patch fixes two problems
1. It adds redundant enqueuing (harmless) when it finds a queue
has tasks enqueued, but it has no run time and it is not
throttled.
2. The most important change is in sched_rt_rq_enqueue/dequeue.
The code just picks the rt_rq belonging to the current cpu
on which the period timer runs, the patch fixes it, so that
the correct rt_se is enqueued/dequeued.
Tested with a simple hierarchy
/c/d, c and d assigned similar runtimes of 50,000 and a while
1 loop runs within "d". Both c and d get throttled, without
the patch, the task just stops running and never runs (depends
on where the sched_rt b/w timer runs). With the patch, the
task is throttled and runs as expected.
[ bharata, suggestions on how to pick the rt_se belong to the
rt_rq and correct cpu ]
Signed-off-by: Balbir Singh <balbir@linux.vnet.ibm.com>
Acked-by: Bharata B Rao <bharata@linux.vnet.ibm.com>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: stable@kernel.org
LKML-Reference: <20110303113435.GA2868@balbir.in.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2011-03-03 12:34:35 +01:00
|
|
|
rt_se = rt_rq->tg->rt_se[cpu];
|
2008-01-25 21:08:30 +01:00
|
|
|
|
|
|
|
if (rt_se && on_rt_rq(rt_se))
|
|
|
|
dequeue_rt_entity(rt_se);
|
|
|
|
}
|
|
|
|
|
2008-02-13 15:45:39 +01:00
|
|
|
static inline int rt_rq_throttled(struct rt_rq *rt_rq)
|
|
|
|
{
|
|
|
|
return rt_rq->rt_throttled && !rt_rq->rt_nr_boosted;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int rt_se_boosted(struct sched_rt_entity *rt_se)
|
|
|
|
{
|
|
|
|
struct rt_rq *rt_rq = group_rt_rq(rt_se);
|
|
|
|
struct task_struct *p;
|
|
|
|
|
|
|
|
if (rt_rq)
|
|
|
|
return !!rt_rq->rt_nr_boosted;
|
|
|
|
|
|
|
|
p = rt_task_of(rt_se);
|
|
|
|
return p->prio != p->normal_prio;
|
|
|
|
}
|
|
|
|
|
2008-04-19 19:44:57 +02:00
|
|
|
#ifdef CONFIG_SMP
|
2008-11-24 17:05:05 +01:00
|
|
|
static inline const struct cpumask *sched_rt_period_mask(void)
|
2008-04-19 19:44:57 +02:00
|
|
|
{
|
2013-05-09 18:24:03 +02:00
|
|
|
return this_rq()->rd->span;
|
2008-04-19 19:44:57 +02:00
|
|
|
}
|
2008-01-25 21:08:30 +01:00
|
|
|
#else
|
2008-11-24 17:05:05 +01:00
|
|
|
static inline const struct cpumask *sched_rt_period_mask(void)
|
2008-04-19 19:44:57 +02:00
|
|
|
{
|
2008-11-24 17:05:05 +01:00
|
|
|
return cpu_online_mask;
|
2008-04-19 19:44:57 +02:00
|
|
|
}
|
|
|
|
#endif
|
2008-01-25 21:08:30 +01:00
|
|
|
|
2008-04-19 19:44:57 +02:00
|
|
|
static inline
|
|
|
|
struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu)
|
2008-01-25 21:08:30 +01:00
|
|
|
{
|
2008-04-19 19:44:57 +02:00
|
|
|
return container_of(rt_b, struct task_group, rt_bandwidth)->rt_rq[cpu];
|
|
|
|
}
|
2008-02-13 15:45:39 +01:00
|
|
|
|
2008-04-19 19:44:58 +02:00
|
|
|
static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq)
|
|
|
|
{
|
|
|
|
return &rt_rq->tg->rt_bandwidth;
|
|
|
|
}
|
|
|
|
|
2008-06-24 20:09:43 +02:00
|
|
|
#else /* !CONFIG_RT_GROUP_SCHED */
|
2008-04-19 19:44:57 +02:00
|
|
|
|
|
|
|
static inline u64 sched_rt_runtime(struct rt_rq *rt_rq)
|
|
|
|
{
|
2008-04-19 19:44:58 +02:00
|
|
|
return rt_rq->rt_runtime;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline u64 sched_rt_period(struct rt_rq *rt_rq)
|
|
|
|
{
|
|
|
|
return ktime_to_ns(def_rt_bandwidth.rt_period);
|
2008-01-25 21:08:30 +01:00
|
|
|
}
|
|
|
|
|
2011-05-14 08:20:02 +02:00
|
|
|
typedef struct rt_rq *rt_rq_iter_t;
|
|
|
|
|
|
|
|
#define for_each_rt_rq(rt_rq, iter, rq) \
|
|
|
|
for ((void) iter, rt_rq = &rq->rt; rt_rq; rt_rq = NULL)
|
|
|
|
|
2008-01-25 21:08:30 +01:00
|
|
|
#define for_each_sched_rt_entity(rt_se) \
|
|
|
|
for (; rt_se; rt_se = NULL)
|
|
|
|
|
|
|
|
static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)
|
|
|
|
{
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2008-02-13 15:45:39 +01:00
|
|
|
static inline void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
|
2008-01-25 21:08:30 +01:00
|
|
|
{
|
2008-08-26 21:09:43 +02:00
|
|
|
if (rt_rq->rt_nr_running)
|
|
|
|
resched_task(rq_of_rt_rq(rt_rq)->curr);
|
2008-01-25 21:08:30 +01:00
|
|
|
}
|
|
|
|
|
2008-02-13 15:45:39 +01:00
|
|
|
static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
|
2008-01-25 21:08:30 +01:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2008-02-13 15:45:39 +01:00
|
|
|
static inline int rt_rq_throttled(struct rt_rq *rt_rq)
|
|
|
|
{
|
|
|
|
return rt_rq->rt_throttled;
|
|
|
|
}
|
2008-04-19 19:44:57 +02:00
|
|
|
|
2008-11-24 17:05:05 +01:00
|
|
|
static inline const struct cpumask *sched_rt_period_mask(void)
|
2008-04-19 19:44:57 +02:00
|
|
|
{
|
2008-11-24 17:05:05 +01:00
|
|
|
return cpu_online_mask;
|
2008-04-19 19:44:57 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline
|
|
|
|
struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu)
|
|
|
|
{
|
|
|
|
return &cpu_rq(cpu)->rt;
|
|
|
|
}
|
|
|
|
|
2008-04-19 19:44:58 +02:00
|
|
|
static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq)
|
|
|
|
{
|
|
|
|
return &def_rt_bandwidth;
|
|
|
|
}
|
|
|
|
|
2008-06-24 20:09:43 +02:00
|
|
|
#endif /* CONFIG_RT_GROUP_SCHED */
|
2008-04-19 19:44:57 +02:00
|
|
|
|
2008-04-19 19:44:58 +02:00
|
|
|
#ifdef CONFIG_SMP
|
2008-09-23 15:33:43 +02:00
|
|
|
/*
|
|
|
|
* We ran out of runtime, see if we can borrow some from our neighbours.
|
|
|
|
*/
|
2008-06-19 14:22:25 +02:00
|
|
|
static int do_balance_runtime(struct rt_rq *rt_rq)
|
2008-04-19 19:44:58 +02:00
|
|
|
{
|
|
|
|
struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
|
2013-01-14 18:55:31 +01:00
|
|
|
struct root_domain *rd = rq_of_rt_rq(rt_rq)->rd;
|
2008-04-19 19:44:58 +02:00
|
|
|
int i, weight, more = 0;
|
|
|
|
u64 rt_period;
|
|
|
|
|
2008-11-24 17:05:05 +01:00
|
|
|
weight = cpumask_weight(rd->span);
|
2008-04-19 19:44:58 +02:00
|
|
|
|
2009-11-17 15:32:06 +01:00
|
|
|
raw_spin_lock(&rt_b->rt_runtime_lock);
|
2008-04-19 19:44:58 +02:00
|
|
|
rt_period = ktime_to_ns(rt_b->rt_period);
|
2008-11-24 17:05:05 +01:00
|
|
|
for_each_cpu(i, rd->span) {
|
2008-04-19 19:44:58 +02:00
|
|
|
struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);
|
|
|
|
s64 diff;
|
|
|
|
|
|
|
|
if (iter == rt_rq)
|
|
|
|
continue;
|
|
|
|
|
2009-11-17 15:32:06 +01:00
|
|
|
raw_spin_lock(&iter->rt_runtime_lock);
|
2008-09-23 15:33:43 +02:00
|
|
|
/*
|
|
|
|
* Either all rqs have inf runtime and there's nothing to steal
|
|
|
|
* or __disable_runtime() below sets a specific rq to inf to
|
|
|
|
* indicate its been disabled and disalow stealing.
|
|
|
|
*/
|
2008-06-05 14:49:58 +02:00
|
|
|
if (iter->rt_runtime == RUNTIME_INF)
|
|
|
|
goto next;
|
|
|
|
|
2008-09-23 15:33:43 +02:00
|
|
|
/*
|
|
|
|
* From runqueues with spare time, take 1/n part of their
|
|
|
|
* spare time, but no more than our period.
|
|
|
|
*/
|
2008-04-19 19:44:58 +02:00
|
|
|
diff = iter->rt_runtime - iter->rt_time;
|
|
|
|
if (diff > 0) {
|
2008-07-24 12:43:13 +02:00
|
|
|
diff = div_u64((u64)diff, weight);
|
2008-04-19 19:44:58 +02:00
|
|
|
if (rt_rq->rt_runtime + diff > rt_period)
|
|
|
|
diff = rt_period - rt_rq->rt_runtime;
|
|
|
|
iter->rt_runtime -= diff;
|
|
|
|
rt_rq->rt_runtime += diff;
|
|
|
|
more = 1;
|
|
|
|
if (rt_rq->rt_runtime == rt_period) {
|
2009-11-17 15:32:06 +01:00
|
|
|
raw_spin_unlock(&iter->rt_runtime_lock);
|
2008-04-19 19:44:58 +02:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2008-06-05 14:49:58 +02:00
|
|
|
next:
|
2009-11-17 15:32:06 +01:00
|
|
|
raw_spin_unlock(&iter->rt_runtime_lock);
|
2008-04-19 19:44:58 +02:00
|
|
|
}
|
2009-11-17 15:32:06 +01:00
|
|
|
raw_spin_unlock(&rt_b->rt_runtime_lock);
|
2008-04-19 19:44:58 +02:00
|
|
|
|
|
|
|
return more;
|
|
|
|
}
|
2008-06-05 14:49:58 +02:00
|
|
|
|
2008-09-23 15:33:43 +02:00
|
|
|
/*
|
|
|
|
* Ensure this RQ takes back all the runtime it lend to its neighbours.
|
|
|
|
*/
|
2008-06-05 14:49:58 +02:00
|
|
|
static void __disable_runtime(struct rq *rq)
|
|
|
|
{
|
|
|
|
struct root_domain *rd = rq->rd;
|
2011-05-14 08:20:02 +02:00
|
|
|
rt_rq_iter_t iter;
|
2008-06-05 14:49:58 +02:00
|
|
|
struct rt_rq *rt_rq;
|
|
|
|
|
|
|
|
if (unlikely(!scheduler_running))
|
|
|
|
return;
|
|
|
|
|
2011-05-14 08:20:02 +02:00
|
|
|
for_each_rt_rq(rt_rq, iter, rq) {
|
2008-06-05 14:49:58 +02:00
|
|
|
struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
|
|
|
|
s64 want;
|
|
|
|
int i;
|
|
|
|
|
2009-11-17 15:32:06 +01:00
|
|
|
raw_spin_lock(&rt_b->rt_runtime_lock);
|
|
|
|
raw_spin_lock(&rt_rq->rt_runtime_lock);
|
2008-09-23 15:33:43 +02:00
|
|
|
/*
|
|
|
|
* Either we're all inf and nobody needs to borrow, or we're
|
|
|
|
* already disabled and thus have nothing to do, or we have
|
|
|
|
* exactly the right amount of runtime to take out.
|
|
|
|
*/
|
2008-06-05 14:49:58 +02:00
|
|
|
if (rt_rq->rt_runtime == RUNTIME_INF ||
|
|
|
|
rt_rq->rt_runtime == rt_b->rt_runtime)
|
|
|
|
goto balanced;
|
2009-11-17 15:32:06 +01:00
|
|
|
raw_spin_unlock(&rt_rq->rt_runtime_lock);
|
2008-06-05 14:49:58 +02:00
|
|
|
|
2008-09-23 15:33:43 +02:00
|
|
|
/*
|
|
|
|
* Calculate the difference between what we started out with
|
|
|
|
* and what we current have, that's the amount of runtime
|
|
|
|
* we lend and now have to reclaim.
|
|
|
|
*/
|
2008-06-05 14:49:58 +02:00
|
|
|
want = rt_b->rt_runtime - rt_rq->rt_runtime;
|
|
|
|
|
2008-09-23 15:33:43 +02:00
|
|
|
/*
|
|
|
|
* Greedy reclaim, take back as much as we can.
|
|
|
|
*/
|
2008-11-24 17:05:05 +01:00
|
|
|
for_each_cpu(i, rd->span) {
|
2008-06-05 14:49:58 +02:00
|
|
|
struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);
|
|
|
|
s64 diff;
|
|
|
|
|
2008-09-23 15:33:43 +02:00
|
|
|
/*
|
|
|
|
* Can't reclaim from ourselves or disabled runqueues.
|
|
|
|
*/
|
2008-08-14 15:49:00 +02:00
|
|
|
if (iter == rt_rq || iter->rt_runtime == RUNTIME_INF)
|
2008-06-05 14:49:58 +02:00
|
|
|
continue;
|
|
|
|
|
2009-11-17 15:32:06 +01:00
|
|
|
raw_spin_lock(&iter->rt_runtime_lock);
|
2008-06-05 14:49:58 +02:00
|
|
|
if (want > 0) {
|
|
|
|
diff = min_t(s64, iter->rt_runtime, want);
|
|
|
|
iter->rt_runtime -= diff;
|
|
|
|
want -= diff;
|
|
|
|
} else {
|
|
|
|
iter->rt_runtime -= want;
|
|
|
|
want -= want;
|
|
|
|
}
|
2009-11-17 15:32:06 +01:00
|
|
|
raw_spin_unlock(&iter->rt_runtime_lock);
|
2008-06-05 14:49:58 +02:00
|
|
|
|
|
|
|
if (!want)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2009-11-17 15:32:06 +01:00
|
|
|
raw_spin_lock(&rt_rq->rt_runtime_lock);
|
2008-09-23 15:33:43 +02:00
|
|
|
/*
|
|
|
|
* We cannot be left wanting - that would mean some runtime
|
|
|
|
* leaked out of the system.
|
|
|
|
*/
|
2008-06-05 14:49:58 +02:00
|
|
|
BUG_ON(want);
|
|
|
|
balanced:
|
2008-09-23 15:33:43 +02:00
|
|
|
/*
|
|
|
|
* Disable all the borrow logic by pretending we have inf
|
|
|
|
* runtime - in which case borrowing doesn't make sense.
|
|
|
|
*/
|
2008-06-05 14:49:58 +02:00
|
|
|
rt_rq->rt_runtime = RUNTIME_INF;
|
2012-08-10 00:34:47 +02:00
|
|
|
rt_rq->rt_throttled = 0;
|
2009-11-17 15:32:06 +01:00
|
|
|
raw_spin_unlock(&rt_rq->rt_runtime_lock);
|
|
|
|
raw_spin_unlock(&rt_b->rt_runtime_lock);
|
2008-06-05 14:49:58 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __enable_runtime(struct rq *rq)
|
|
|
|
{
|
2011-05-14 08:20:02 +02:00
|
|
|
rt_rq_iter_t iter;
|
2008-06-05 14:49:58 +02:00
|
|
|
struct rt_rq *rt_rq;
|
|
|
|
|
|
|
|
if (unlikely(!scheduler_running))
|
|
|
|
return;
|
|
|
|
|
2008-09-23 15:33:43 +02:00
|
|
|
/*
|
|
|
|
* Reset each runqueue's bandwidth settings
|
|
|
|
*/
|
2011-05-14 08:20:02 +02:00
|
|
|
for_each_rt_rq(rt_rq, iter, rq) {
|
2008-06-05 14:49:58 +02:00
|
|
|
struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
|
|
|
|
|
2009-11-17 15:32:06 +01:00
|
|
|
raw_spin_lock(&rt_b->rt_runtime_lock);
|
|
|
|
raw_spin_lock(&rt_rq->rt_runtime_lock);
|
2008-06-05 14:49:58 +02:00
|
|
|
rt_rq->rt_runtime = rt_b->rt_runtime;
|
|
|
|
rt_rq->rt_time = 0;
|
2008-09-09 05:26:33 +02:00
|
|
|
rt_rq->rt_throttled = 0;
|
2009-11-17 15:32:06 +01:00
|
|
|
raw_spin_unlock(&rt_rq->rt_runtime_lock);
|
|
|
|
raw_spin_unlock(&rt_b->rt_runtime_lock);
|
2008-06-05 14:49:58 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2008-06-19 14:22:26 +02:00
|
|
|
static int balance_runtime(struct rt_rq *rt_rq)
|
|
|
|
{
|
|
|
|
int more = 0;
|
|
|
|
|
2011-10-06 22:39:14 +02:00
|
|
|
if (!sched_feat(RT_RUNTIME_SHARE))
|
|
|
|
return more;
|
|
|
|
|
2008-06-19 14:22:26 +02:00
|
|
|
if (rt_rq->rt_time > rt_rq->rt_runtime) {
|
2009-11-17 15:32:06 +01:00
|
|
|
raw_spin_unlock(&rt_rq->rt_runtime_lock);
|
2008-06-19 14:22:26 +02:00
|
|
|
more = do_balance_runtime(rt_rq);
|
2009-11-17 15:32:06 +01:00
|
|
|
raw_spin_lock(&rt_rq->rt_runtime_lock);
|
2008-06-19 14:22:26 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
return more;
|
|
|
|
}
|
2008-06-24 20:09:43 +02:00
|
|
|
#else /* !CONFIG_SMP */
|
2008-06-19 14:22:26 +02:00
|
|
|
static inline int balance_runtime(struct rt_rq *rt_rq)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
2008-06-24 20:09:43 +02:00
|
|
|
#endif /* CONFIG_SMP */
|
2008-04-19 19:44:58 +02:00
|
|
|
|
2008-06-19 14:22:26 +02:00
|
|
|
static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun)
|
|
|
|
{
|
2011-10-18 22:03:48 +02:00
|
|
|
int i, idle = 1, throttled = 0;
|
2008-11-24 17:05:05 +01:00
|
|
|
const struct cpumask *span;
|
2008-06-19 14:22:26 +02:00
|
|
|
|
|
|
|
span = sched_rt_period_mask();
|
2012-08-07 10:02:38 +02:00
|
|
|
#ifdef CONFIG_RT_GROUP_SCHED
|
|
|
|
/*
|
|
|
|
* FIXME: isolated CPUs should really leave the root task group,
|
|
|
|
* whether they are isolcpus or were isolated via cpusets, lest
|
|
|
|
* the timer run on a CPU which does not service all runqueues,
|
|
|
|
* potentially leaving other CPUs indefinitely throttled. If
|
|
|
|
* isolation is really required, the user will turn the throttle
|
|
|
|
* off to kill the perturbations it causes anyway. Meanwhile,
|
|
|
|
* this maintains functionality for boot and/or troubleshooting.
|
|
|
|
*/
|
|
|
|
if (rt_b == &root_task_group.rt_bandwidth)
|
|
|
|
span = cpu_online_mask;
|
|
|
|
#endif
|
2008-11-24 17:05:05 +01:00
|
|
|
for_each_cpu(i, span) {
|
2008-06-19 14:22:26 +02:00
|
|
|
int enqueue = 0;
|
|
|
|
struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i);
|
|
|
|
struct rq *rq = rq_of_rt_rq(rt_rq);
|
|
|
|
|
2009-11-17 14:28:38 +01:00
|
|
|
raw_spin_lock(&rq->lock);
|
2008-06-19 14:22:26 +02:00
|
|
|
if (rt_rq->rt_time) {
|
|
|
|
u64 runtime;
|
|
|
|
|
2009-11-17 15:32:06 +01:00
|
|
|
raw_spin_lock(&rt_rq->rt_runtime_lock);
|
2008-06-19 14:22:26 +02:00
|
|
|
if (rt_rq->rt_throttled)
|
|
|
|
balance_runtime(rt_rq);
|
|
|
|
runtime = rt_rq->rt_runtime;
|
|
|
|
rt_rq->rt_time -= min(rt_rq->rt_time, overrun*runtime);
|
|
|
|
if (rt_rq->rt_throttled && rt_rq->rt_time < runtime) {
|
|
|
|
rt_rq->rt_throttled = 0;
|
|
|
|
enqueue = 1;
|
2011-04-29 08:36:50 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Force a clock update if the CPU was idle,
|
|
|
|
* lest wakeup -> unthrottle time accumulate.
|
|
|
|
*/
|
|
|
|
if (rt_rq->rt_nr_running && rq->curr == rq->idle)
|
|
|
|
rq->skip_clock_update = -1;
|
2008-06-19 14:22:26 +02:00
|
|
|
}
|
|
|
|
if (rt_rq->rt_time || rt_rq->rt_nr_running)
|
|
|
|
idle = 0;
|
2009-11-17 15:32:06 +01:00
|
|
|
raw_spin_unlock(&rt_rq->rt_runtime_lock);
|
sched: Fix sched rt group scheduling when hierachy is enabled
The current sched rt code is broken when it comes to hierarchical
scheduling, this patch fixes two problems
1. It adds redundant enqueuing (harmless) when it finds a queue
has tasks enqueued, but it has no run time and it is not
throttled.
2. The most important change is in sched_rt_rq_enqueue/dequeue.
The code just picks the rt_rq belonging to the current cpu
on which the period timer runs, the patch fixes it, so that
the correct rt_se is enqueued/dequeued.
Tested with a simple hierarchy
/c/d, c and d assigned similar runtimes of 50,000 and a while
1 loop runs within "d". Both c and d get throttled, without
the patch, the task just stops running and never runs (depends
on where the sched_rt b/w timer runs). With the patch, the
task is throttled and runs as expected.
[ bharata, suggestions on how to pick the rt_se belong to the
rt_rq and correct cpu ]
Signed-off-by: Balbir Singh <balbir@linux.vnet.ibm.com>
Acked-by: Bharata B Rao <bharata@linux.vnet.ibm.com>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: stable@kernel.org
LKML-Reference: <20110303113435.GA2868@balbir.in.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2011-03-03 12:34:35 +01:00
|
|
|
} else if (rt_rq->rt_nr_running) {
|
2008-06-19 14:22:28 +02:00
|
|
|
idle = 0;
|
sched: Fix sched rt group scheduling when hierachy is enabled
The current sched rt code is broken when it comes to hierarchical
scheduling, this patch fixes two problems
1. It adds redundant enqueuing (harmless) when it finds a queue
has tasks enqueued, but it has no run time and it is not
throttled.
2. The most important change is in sched_rt_rq_enqueue/dequeue.
The code just picks the rt_rq belonging to the current cpu
on which the period timer runs, the patch fixes it, so that
the correct rt_se is enqueued/dequeued.
Tested with a simple hierarchy
/c/d, c and d assigned similar runtimes of 50,000 and a while
1 loop runs within "d". Both c and d get throttled, without
the patch, the task just stops running and never runs (depends
on where the sched_rt b/w timer runs). With the patch, the
task is throttled and runs as expected.
[ bharata, suggestions on how to pick the rt_se belong to the
rt_rq and correct cpu ]
Signed-off-by: Balbir Singh <balbir@linux.vnet.ibm.com>
Acked-by: Bharata B Rao <bharata@linux.vnet.ibm.com>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: stable@kernel.org
LKML-Reference: <20110303113435.GA2868@balbir.in.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2011-03-03 12:34:35 +01:00
|
|
|
if (!rt_rq_throttled(rt_rq))
|
|
|
|
enqueue = 1;
|
|
|
|
}
|
2011-10-18 22:03:48 +02:00
|
|
|
if (rt_rq->rt_throttled)
|
|
|
|
throttled = 1;
|
2008-06-19 14:22:26 +02:00
|
|
|
|
|
|
|
if (enqueue)
|
|
|
|
sched_rt_rq_enqueue(rt_rq);
|
2009-11-17 14:28:38 +01:00
|
|
|
raw_spin_unlock(&rq->lock);
|
2008-06-19 14:22:26 +02:00
|
|
|
}
|
|
|
|
|
2011-10-18 22:03:48 +02:00
|
|
|
if (!throttled && (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF))
|
|
|
|
return 1;
|
|
|
|
|
2008-06-19 14:22:26 +02:00
|
|
|
return idle;
|
|
|
|
}
|
2008-04-19 19:44:58 +02:00
|
|
|
|
2008-01-25 21:08:30 +01:00
|
|
|
static inline int rt_se_prio(struct sched_rt_entity *rt_se)
|
|
|
|
{
|
2008-02-13 15:45:40 +01:00
|
|
|
#ifdef CONFIG_RT_GROUP_SCHED
|
2008-01-25 21:08:30 +01:00
|
|
|
struct rt_rq *rt_rq = group_rt_rq(rt_se);
|
|
|
|
|
|
|
|
if (rt_rq)
|
2008-12-29 15:39:49 +01:00
|
|
|
return rt_rq->highest_prio.curr;
|
2008-01-25 21:08:30 +01:00
|
|
|
#endif
|
|
|
|
|
|
|
|
return rt_task_of(rt_se)->prio;
|
|
|
|
}
|
|
|
|
|
2008-02-13 15:45:39 +01:00
|
|
|
static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq)
|
2008-01-25 21:08:30 +01:00
|
|
|
{
|
2008-02-13 15:45:39 +01:00
|
|
|
u64 runtime = sched_rt_runtime(rt_rq);
|
2008-01-25 21:08:29 +01:00
|
|
|
|
|
|
|
if (rt_rq->rt_throttled)
|
2008-02-13 15:45:39 +01:00
|
|
|
return rt_rq_throttled(rt_rq);
|
2008-01-25 21:08:29 +01:00
|
|
|
|
2011-11-29 04:03:56 +01:00
|
|
|
if (runtime >= sched_rt_period(rt_rq))
|
2008-04-19 19:44:58 +02:00
|
|
|
return 0;
|
|
|
|
|
2008-06-19 14:22:25 +02:00
|
|
|
balance_runtime(rt_rq);
|
|
|
|
runtime = sched_rt_runtime(rt_rq);
|
|
|
|
if (runtime == RUNTIME_INF)
|
|
|
|
return 0;
|
2008-04-19 19:44:58 +02:00
|
|
|
|
2008-02-13 15:45:39 +01:00
|
|
|
if (rt_rq->rt_time > runtime) {
|
2011-10-18 22:03:48 +02:00
|
|
|
struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Don't actually throttle groups that have no runtime assigned
|
|
|
|
* but accrue some time due to boosting.
|
|
|
|
*/
|
|
|
|
if (likely(rt_b->rt_runtime)) {
|
2012-02-27 10:47:00 +01:00
|
|
|
static bool once = false;
|
|
|
|
|
2011-10-18 22:03:48 +02:00
|
|
|
rt_rq->rt_throttled = 1;
|
2012-02-27 10:47:00 +01:00
|
|
|
|
|
|
|
if (!once) {
|
|
|
|
once = true;
|
|
|
|
printk_sched("sched: RT throttling activated\n");
|
|
|
|
}
|
2011-10-18 22:03:48 +02:00
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* In case we did anyway, make it go away,
|
|
|
|
* replenishment is a joke, since it will replenish us
|
|
|
|
* with exactly 0 ns.
|
|
|
|
*/
|
|
|
|
rt_rq->rt_time = 0;
|
|
|
|
}
|
|
|
|
|
2008-02-13 15:45:39 +01:00
|
|
|
if (rt_rq_throttled(rt_rq)) {
|
2008-02-13 15:45:39 +01:00
|
|
|
sched_rt_rq_dequeue(rt_rq);
|
2008-02-13 15:45:39 +01:00
|
|
|
return 1;
|
|
|
|
}
|
2008-01-25 21:08:29 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2007-07-09 18:51:58 +02:00
|
|
|
/*
|
|
|
|
* Update the current task's runtime statistics. Skip current tasks that
|
|
|
|
* are not in our scheduling class.
|
|
|
|
*/
|
2007-10-15 17:00:13 +02:00
|
|
|
static void update_curr_rt(struct rq *rq)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
|
|
|
struct task_struct *curr = rq->curr;
|
2008-01-25 21:08:30 +01:00
|
|
|
struct sched_rt_entity *rt_se = &curr->rt;
|
|
|
|
struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
|
2007-07-09 18:51:58 +02:00
|
|
|
u64 delta_exec;
|
|
|
|
|
2011-02-02 13:19:48 +01:00
|
|
|
if (curr->sched_class != &rt_sched_class)
|
2007-07-09 18:51:58 +02:00
|
|
|
return;
|
|
|
|
|
2013-04-12 01:51:02 +02:00
|
|
|
delta_exec = rq_clock_task(rq) - curr->se.exec_start;
|
2013-01-30 13:50:36 +01:00
|
|
|
if (unlikely((s64)delta_exec <= 0))
|
|
|
|
return;
|
2007-08-02 17:41:40 +02:00
|
|
|
|
2011-10-18 22:03:48 +02:00
|
|
|
schedstat_set(curr->se.statistics.exec_max,
|
|
|
|
max(curr->se.statistics.exec_max, delta_exec));
|
2007-07-09 18:51:58 +02:00
|
|
|
|
|
|
|
curr->se.sum_exec_runtime += delta_exec;
|
timers: fix itimer/many thread hang
Overview
This patch reworks the handling of POSIX CPU timers, including the
ITIMER_PROF, ITIMER_VIRT timers and rlimit handling. It was put together
with the help of Roland McGrath, the owner and original writer of this code.
The problem we ran into, and the reason for this rework, has to do with using
a profiling timer in a process with a large number of threads. It appears
that the performance of the old implementation of run_posix_cpu_timers() was
at least O(n*3) (where "n" is the number of threads in a process) or worse.
Everything is fine with an increasing number of threads until the time taken
for that routine to run becomes the same as or greater than the tick time, at
which point things degrade rather quickly.
This patch fixes bug 9906, "Weird hang with NPTL and SIGPROF."
Code Changes
This rework corrects the implementation of run_posix_cpu_timers() to make it
run in constant time for a particular machine. (Performance may vary between
one machine and another depending upon whether the kernel is built as single-
or multiprocessor and, in the latter case, depending upon the number of
running processors.) To do this, at each tick we now update fields in
signal_struct as well as task_struct. The run_posix_cpu_timers() function
uses those fields to make its decisions.
We define a new structure, "task_cputime," to contain user, system and
scheduler times and use these in appropriate places:
struct task_cputime {
cputime_t utime;
cputime_t stime;
unsigned long long sum_exec_runtime;
};
This is included in the structure "thread_group_cputime," which is a new
substructure of signal_struct and which varies for uniprocessor versus
multiprocessor kernels. For uniprocessor kernels, it uses "task_cputime" as
a simple substructure, while for multiprocessor kernels it is a pointer:
struct thread_group_cputime {
struct task_cputime totals;
};
struct thread_group_cputime {
struct task_cputime *totals;
};
We also add a new task_cputime substructure directly to signal_struct, to
cache the earliest expiration of process-wide timers, and task_cputime also
replaces the it_*_expires fields of task_struct (used for earliest expiration
of thread timers). The "thread_group_cputime" structure contains process-wide
timers that are updated via account_user_time() and friends. In the non-SMP
case the structure is a simple aggregator; unfortunately in the SMP case that
simplicity was not achievable due to cache-line contention between CPUs (in
one measured case performance was actually _worse_ on a 16-cpu system than
the same test on a 4-cpu system, due to this contention). For SMP, the
thread_group_cputime counters are maintained as a per-cpu structure allocated
using alloc_percpu(). The timer functions update only the timer field in
the structure corresponding to the running CPU, obtained using per_cpu_ptr().
We define a set of inline functions in sched.h that we use to maintain the
thread_group_cputime structure and hide the differences between UP and SMP
implementations from the rest of the kernel. The thread_group_cputime_init()
function initializes the thread_group_cputime structure for the given task.
The thread_group_cputime_alloc() is a no-op for UP; for SMP it calls the
out-of-line function thread_group_cputime_alloc_smp() to allocate and fill
in the per-cpu structures and fields. The thread_group_cputime_free()
function, also a no-op for UP, in SMP frees the per-cpu structures. The
thread_group_cputime_clone_thread() function (also a UP no-op) for SMP calls
thread_group_cputime_alloc() if the per-cpu structures haven't yet been
allocated. The thread_group_cputime() function fills the task_cputime
structure it is passed with the contents of the thread_group_cputime fields;
in UP it's that simple but in SMP it must also safely check that tsk->signal
is non-NULL (if it is it just uses the appropriate fields of task_struct) and,
if so, sums the per-cpu values for each online CPU. Finally, the three
functions account_group_user_time(), account_group_system_time() and
account_group_exec_runtime() are used by timer functions to update the
respective fields of the thread_group_cputime structure.
Non-SMP operation is trivial and will not be mentioned further.
The per-cpu structure is always allocated when a task creates its first new
thread, via a call to thread_group_cputime_clone_thread() from copy_signal().
It is freed at process exit via a call to thread_group_cputime_free() from
cleanup_signal().
All functions that formerly summed utime/stime/sum_sched_runtime values from
from all threads in the thread group now use thread_group_cputime() to
snapshot the values in the thread_group_cputime structure or the values in
the task structure itself if the per-cpu structure hasn't been allocated.
Finally, the code in kernel/posix-cpu-timers.c has changed quite a bit.
The run_posix_cpu_timers() function has been split into a fast path and a
slow path; the former safely checks whether there are any expired thread
timers and, if not, just returns, while the slow path does the heavy lifting.
With the dedicated thread group fields, timers are no longer "rebalanced" and
the process_timer_rebalance() function and related code has gone away. All
summing loops are gone and all code that used them now uses the
thread_group_cputime() inline. When process-wide timers are set, the new
task_cputime structure in signal_struct is used to cache the earliest
expiration; this is checked in the fast path.
Performance
The fix appears not to add significant overhead to existing operations. It
generally performs the same as the current code except in two cases, one in
which it performs slightly worse (Case 5 below) and one in which it performs
very significantly better (Case 2 below). Overall it's a wash except in those
two cases.
I've since done somewhat more involved testing on a dual-core Opteron system.
Case 1: With no itimer running, for a test with 100,000 threads, the fixed
kernel took 1428.5 seconds, 513 seconds more than the unfixed system,
all of which was spent in the system. There were twice as many
voluntary context switches with the fix as without it.
Case 2: With an itimer running at .01 second ticks and 4000 threads (the most
an unmodified kernel can handle), the fixed kernel ran the test in
eight percent of the time (5.8 seconds as opposed to 70 seconds) and
had better tick accuracy (.012 seconds per tick as opposed to .023
seconds per tick).
Case 3: A 4000-thread test with an initial timer tick of .01 second and an
interval of 10,000 seconds (i.e. a timer that ticks only once) had
very nearly the same performance in both cases: 6.3 seconds elapsed
for the fixed kernel versus 5.5 seconds for the unfixed kernel.
With fewer threads (eight in these tests), the Case 1 test ran in essentially
the same time on both the modified and unmodified kernels (5.2 seconds versus
5.8 seconds). The Case 2 test ran in about the same time as well, 5.9 seconds
versus 5.4 seconds but again with much better tick accuracy, .013 seconds per
tick versus .025 seconds per tick for the unmodified kernel.
Since the fix affected the rlimit code, I also tested soft and hard CPU limits.
Case 4: With a hard CPU limit of 20 seconds and eight threads (and an itimer
running), the modified kernel was very slightly favored in that while
it killed the process in 19.997 seconds of CPU time (5.002 seconds of
wall time), only .003 seconds of that was system time, the rest was
user time. The unmodified kernel killed the process in 20.001 seconds
of CPU (5.014 seconds of wall time) of which .016 seconds was system
time. Really, though, the results were too close to call. The results
were essentially the same with no itimer running.
Case 5: With a soft limit of 20 seconds and a hard limit of 2000 seconds
(where the hard limit would never be reached) and an itimer running,
the modified kernel exhibited worse tick accuracy than the unmodified
kernel: .050 seconds/tick versus .028 seconds/tick. Otherwise,
performance was almost indistinguishable. With no itimer running this
test exhibited virtually identical behavior and times in both cases.
In times past I did some limited performance testing. those results are below.
On a four-cpu Opteron system without this fix, a sixteen-thread test executed
in 3569.991 seconds, of which user was 3568.435s and system was 1.556s. On
the same system with the fix, user and elapsed time were about the same, but
system time dropped to 0.007 seconds. Performance with eight, four and one
thread were comparable. Interestingly, the timer ticks with the fix seemed
more accurate: The sixteen-thread test with the fix received 149543 ticks
for 0.024 seconds per tick, while the same test without the fix received 58720
for 0.061 seconds per tick. Both cases were configured for an interval of
0.01 seconds. Again, the other tests were comparable. Each thread in this
test computed the primes up to 25,000,000.
I also did a test with a large number of threads, 100,000 threads, which is
impossible without the fix. In this case each thread computed the primes only
up to 10,000 (to make the runtime manageable). System time dominated, at
1546.968 seconds out of a total 2176.906 seconds (giving a user time of
629.938s). It received 147651 ticks for 0.015 seconds per tick, still quite
accurate. There is obviously no comparable test without the fix.
Signed-off-by: Frank Mayhar <fmayhar@google.com>
Cc: Roland McGrath <roland@redhat.com>
Cc: Alexey Dobriyan <adobriyan@gmail.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-09-12 18:54:39 +02:00
|
|
|
account_group_exec_runtime(curr, delta_exec);
|
|
|
|
|
2013-04-12 01:51:02 +02:00
|
|
|
curr->se.exec_start = rq_clock_task(rq);
|
2007-12-02 20:04:49 +01:00
|
|
|
cpuacct_charge(curr, delta_exec);
|
2008-01-25 21:08:29 +01:00
|
|
|
|
2009-09-01 10:34:37 +02:00
|
|
|
sched_rt_avg_update(rq, delta_exec);
|
|
|
|
|
2008-08-19 12:33:04 +02:00
|
|
|
if (!rt_bandwidth_enabled())
|
|
|
|
return;
|
|
|
|
|
2008-04-19 19:44:59 +02:00
|
|
|
for_each_sched_rt_entity(rt_se) {
|
|
|
|
rt_rq = rt_rq_of_se(rt_se);
|
|
|
|
|
2008-08-19 12:33:03 +02:00
|
|
|
if (sched_rt_runtime(rt_rq) != RUNTIME_INF) {
|
2009-11-17 15:32:06 +01:00
|
|
|
raw_spin_lock(&rt_rq->rt_runtime_lock);
|
2008-08-19 12:33:03 +02:00
|
|
|
rt_rq->rt_time += delta_exec;
|
|
|
|
if (sched_rt_runtime_exceeded(rt_rq))
|
|
|
|
resched_task(curr);
|
2009-11-17 15:32:06 +01:00
|
|
|
raw_spin_unlock(&rt_rq->rt_runtime_lock);
|
2008-08-19 12:33:03 +02:00
|
|
|
}
|
2008-04-19 19:44:59 +02:00
|
|
|
}
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
2009-01-14 15:10:04 +01:00
|
|
|
#if defined CONFIG_SMP
|
2008-12-29 15:39:49 +01:00
|
|
|
|
2009-01-14 15:10:04 +01:00
|
|
|
static void
|
|
|
|
inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio)
|
2008-01-25 21:08:03 +01:00
|
|
|
{
|
2008-12-29 15:39:49 +01:00
|
|
|
struct rq *rq = rq_of_rt_rq(rt_rq);
|
2008-06-04 21:04:05 +02:00
|
|
|
|
2013-11-27 16:59:13 +01:00
|
|
|
#ifdef CONFIG_RT_GROUP_SCHED
|
|
|
|
/*
|
|
|
|
* Change rq's cpupri only if rt_rq is the top queue.
|
|
|
|
*/
|
|
|
|
if (&rq->rt != rt_rq)
|
|
|
|
return;
|
|
|
|
#endif
|
sched: Use pushable_tasks to determine next highest prio
Hillf Danton proposed a patch (see link) that cleaned up the
sched_rt code that calculates the priority of the next highest priority
task to be used in finding run queues to pull from.
His patch removed the calculating of the next prio to just use the current
prio when deteriming if we should examine a run queue to pull from. The problem
with his patch was that it caused more false checks. Because we check a run
queue for pushable tasks if the current priority of that run queue is higher
in priority than the task about to run on our run queue. But after grabbing
the locks and doing the real check, we find that there may not be a task
that has a higher prio task to pull. Thus the locks were taken with nothing to
do.
I added some trace_printks() to record when and how many times the run queue
locks were taken to check for pullable tasks, compared to how many times we
pulled a task.
With the current method, it was:
3806 locks taken vs 2812 pulled tasks
With Hillf's patch:
6728 locks taken vs 2804 pulled tasks
The number of times locks were taken to pull a task went up almost double with
no more success rate.
But his patch did get me thinking. When we look at the priority of the highest
task to consider taking the locks to do a pull, a failure to pull can be one
of the following: (in order of most likely)
o RT task was pushed off already between the check and taking the lock
o Waiting RT task can not be migrated
o RT task's CPU affinity does not include the target run queue's CPU
o RT task's priority changed between the check and taking the lock
And with Hillf's patch, the thing that caused most of the failures, is
the RT task to pull was not at the right priority to pull (not greater than
the current RT task priority on the target run queue).
Most of the above cases we can't help. But the current method does not check
if the next highest prio RT task can be migrated or not, and if it can not,
we still grab the locks to do the test (we don't find out about this fact until
after we have the locks). I thought about this case, and realized that the
pushable task plist that is maintained only holds RT tasks that can migrate.
If we move the calculating of the next highest prio task from the inc/dec_rt_task()
functions into the queuing of the pushable tasks, then we only measure the
priorities of those tasks that we push, and we get this basically for free.
Not only does this patch make the code a little more efficient, it cleans it
up and makes it a little simpler.
Thanks to Hillf Danton for inspiring me on this patch.
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Hillf Danton <dhillf@gmail.com>
Cc: Gregory Haskins <ghaskins@novell.com>
Link: http://lkml.kernel.org/r/BANLkTimQ67180HxCx5vgMqumqw1EkFh3qg@mail.gmail.com
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2011-06-17 03:55:23 +02:00
|
|
|
if (rq->online && prio < prev_prio)
|
|
|
|
cpupri_set(&rq->rd->cpupri, rq->cpu, prio);
|
2009-01-14 15:10:04 +01:00
|
|
|
}
|
2008-01-25 21:08:07 +01:00
|
|
|
|
2009-01-14 15:10:04 +01:00
|
|
|
static void
|
|
|
|
dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio)
|
|
|
|
{
|
|
|
|
struct rq *rq = rq_of_rt_rq(rt_rq);
|
2008-04-19 19:44:57 +02:00
|
|
|
|
2013-11-27 16:59:13 +01:00
|
|
|
#ifdef CONFIG_RT_GROUP_SCHED
|
|
|
|
/*
|
|
|
|
* Change rq's cpupri only if rt_rq is the top queue.
|
|
|
|
*/
|
|
|
|
if (&rq->rt != rt_rq)
|
|
|
|
return;
|
|
|
|
#endif
|
2009-01-14 15:10:04 +01:00
|
|
|
if (rq->online && rt_rq->highest_prio.curr != prev_prio)
|
|
|
|
cpupri_set(&rq->rd->cpupri, rq->cpu, rt_rq->highest_prio.curr);
|
2008-01-25 21:08:03 +01:00
|
|
|
}
|
|
|
|
|
2009-01-14 15:10:04 +01:00
|
|
|
#else /* CONFIG_SMP */
|
|
|
|
|
2008-01-25 21:08:30 +01:00
|
|
|
static inline
|
2009-01-14 15:10:04 +01:00
|
|
|
void inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) {}
|
|
|
|
static inline
|
|
|
|
void dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) {}
|
|
|
|
|
|
|
|
#endif /* CONFIG_SMP */
|
2008-05-12 21:21:01 +02:00
|
|
|
|
2008-02-13 15:45:40 +01:00
|
|
|
#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
|
2009-01-14 15:10:04 +01:00
|
|
|
static void
|
|
|
|
inc_rt_prio(struct rt_rq *rt_rq, int prio)
|
|
|
|
{
|
|
|
|
int prev_prio = rt_rq->highest_prio.curr;
|
|
|
|
|
|
|
|
if (prio < prev_prio)
|
|
|
|
rt_rq->highest_prio.curr = prio;
|
|
|
|
|
|
|
|
inc_rt_prio_smp(rt_rq, prio, prev_prio);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
dec_rt_prio(struct rt_rq *rt_rq, int prio)
|
|
|
|
{
|
|
|
|
int prev_prio = rt_rq->highest_prio.curr;
|
|
|
|
|
2008-01-25 21:08:30 +01:00
|
|
|
if (rt_rq->rt_nr_running) {
|
2008-01-25 21:08:04 +01:00
|
|
|
|
2009-01-14 15:10:04 +01:00
|
|
|
WARN_ON(prio < prev_prio);
|
2008-01-25 21:08:04 +01:00
|
|
|
|
2008-12-29 15:39:49 +01:00
|
|
|
/*
|
2009-01-14 15:10:04 +01:00
|
|
|
* This may have been our highest task, and therefore
|
|
|
|
* we may have some recomputation to do
|
2008-12-29 15:39:49 +01:00
|
|
|
*/
|
2009-01-14 15:10:04 +01:00
|
|
|
if (prio == prev_prio) {
|
2008-12-29 15:39:49 +01:00
|
|
|
struct rt_prio_array *array = &rt_rq->active;
|
|
|
|
|
|
|
|
rt_rq->highest_prio.curr =
|
2008-01-25 21:08:04 +01:00
|
|
|
sched_find_first_bit(array->bitmap);
|
2008-12-29 15:39:49 +01:00
|
|
|
}
|
|
|
|
|
2008-01-25 21:08:04 +01:00
|
|
|
} else
|
2008-12-29 15:39:49 +01:00
|
|
|
rt_rq->highest_prio.curr = MAX_RT_PRIO;
|
2008-01-25 21:08:07 +01:00
|
|
|
|
2009-01-14 15:10:04 +01:00
|
|
|
dec_rt_prio_smp(rt_rq, prio, prev_prio);
|
|
|
|
}
|
2008-06-04 21:04:05 +02:00
|
|
|
|
2009-01-14 15:10:04 +01:00
|
|
|
#else
|
|
|
|
|
|
|
|
static inline void inc_rt_prio(struct rt_rq *rt_rq, int prio) {}
|
|
|
|
static inline void dec_rt_prio(struct rt_rq *rt_rq, int prio) {}
|
|
|
|
|
|
|
|
#endif /* CONFIG_SMP || CONFIG_RT_GROUP_SCHED */
|
2008-05-12 21:21:01 +02:00
|
|
|
|
2008-02-13 15:45:40 +01:00
|
|
|
#ifdef CONFIG_RT_GROUP_SCHED
|
2009-01-14 15:10:04 +01:00
|
|
|
|
|
|
|
static void
|
|
|
|
inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
|
|
|
|
{
|
|
|
|
if (rt_se_boosted(rt_se))
|
|
|
|
rt_rq->rt_nr_boosted++;
|
|
|
|
|
|
|
|
if (rt_rq->tg)
|
|
|
|
start_rt_bandwidth(&rt_rq->tg->rt_bandwidth);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
|
|
|
|
{
|
2008-02-13 15:45:39 +01:00
|
|
|
if (rt_se_boosted(rt_se))
|
|
|
|
rt_rq->rt_nr_boosted--;
|
|
|
|
|
|
|
|
WARN_ON(!rt_rq->rt_nr_running && rt_rq->rt_nr_boosted);
|
2009-01-14 15:10:04 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
#else /* CONFIG_RT_GROUP_SCHED */
|
|
|
|
|
|
|
|
static void
|
|
|
|
inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
|
|
|
|
{
|
|
|
|
start_rt_bandwidth(&def_rt_bandwidth);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline
|
|
|
|
void dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) {}
|
|
|
|
|
|
|
|
#endif /* CONFIG_RT_GROUP_SCHED */
|
|
|
|
|
|
|
|
static inline
|
|
|
|
void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
|
|
|
|
{
|
|
|
|
int prio = rt_se_prio(rt_se);
|
|
|
|
|
|
|
|
WARN_ON(!rt_prio(prio));
|
|
|
|
rt_rq->rt_nr_running++;
|
|
|
|
|
|
|
|
inc_rt_prio(rt_rq, prio);
|
|
|
|
inc_rt_migration(rt_se, rt_rq);
|
|
|
|
inc_rt_group(rt_se, rt_rq);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline
|
|
|
|
void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
|
|
|
|
{
|
|
|
|
WARN_ON(!rt_prio(rt_se_prio(rt_se)));
|
|
|
|
WARN_ON(!rt_rq->rt_nr_running);
|
|
|
|
rt_rq->rt_nr_running--;
|
|
|
|
|
|
|
|
dec_rt_prio(rt_rq, rt_se_prio(rt_se));
|
|
|
|
dec_rt_migration(rt_se, rt_rq);
|
|
|
|
dec_rt_group(rt_se, rt_rq);
|
2008-01-25 21:08:03 +01:00
|
|
|
}
|
|
|
|
|
2010-01-20 21:59:01 +01:00
|
|
|
static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, bool head)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
2008-01-25 21:08:30 +01:00
|
|
|
struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
|
|
|
|
struct rt_prio_array *array = &rt_rq->active;
|
|
|
|
struct rt_rq *group_rq = group_rt_rq(rt_se);
|
sched: rework of "prioritize non-migratable tasks over migratable ones"
regarding this commit: 45c01e824991b2dd0a332e19efc4901acb31209f
I think we can do it simpler. Please take a look at the patch below.
Instead of having 2 separate arrays (which is + ~800 bytes on x86_32 and
twice so on x86_64), let's add "exclusive" (the ones that are bound to
this CPU) tasks to the head of the queue and "shared" ones -- to the
end.
In case of a few newly woken up "exclusive" tasks, they are 'stacked'
(not queued as now), meaning that a task {i+1} is being placed in front
of the previously woken up task {i}. But I don't think that this
behavior may cause any realistic problems.
There are a couple of changes on top of this one.
(1) in check_preempt_curr_rt()
I don't think there is a need for the "pick_next_rt_entity(rq, &rq->rt)
!= &rq->curr->rt" check.
enqueue_task_rt(p) and check_preempt_curr_rt() are always called one
after another with rq->lock being held so the following check
"p->rt.nr_cpus_allowed == 1 && rq->curr->rt.nr_cpus_allowed != 1" should
be enough (well, just its left part) to guarantee that 'p' has been
queued in front of the 'curr'.
(2) in set_cpus_allowed_rt()
I don't thinks there is a need for requeue_task_rt() here.
Perhaps, the only case when 'requeue' (+ reschedule) might be useful is
as follows:
i) weight == 1 && cpu_isset(task_cpu(p), *new_mask)
i.e. a task is being bound to this CPU);
ii) 'p' != rq->curr
but here, 'p' has already been on this CPU for a while and was not
migrated. i.e. it's possible that 'rq->curr' would not have high chances
to be migrated right at this particular moment (although, has chance in
a bit longer term), should we allow it to be preempted.
Anyway, I think we should not perhaps make it more complex trying to
address some rare corner cases. For instance, that's why a single queue
approach would be preferable. Unless I'm missing something obvious, this
approach gives us similar functionality at lower cost.
Verified only compilation-wise.
(Almost)-Signed-off-by: Dmitry Adamushko <dmitry.adamushko@gmail.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-06-11 00:58:30 +02:00
|
|
|
struct list_head *queue = array->queue + rt_se_prio(rt_se);
|
2007-07-09 18:51:58 +02:00
|
|
|
|
2008-06-19 09:06:57 +02:00
|
|
|
/*
|
|
|
|
* Don't enqueue the group if its throttled, or when empty.
|
|
|
|
* The latter is a consequence of the former when a child group
|
|
|
|
* get throttled and the current group doesn't have any other
|
|
|
|
* active members.
|
|
|
|
*/
|
|
|
|
if (group_rq && (rt_rq_throttled(group_rq) || !group_rq->rt_nr_running))
|
2008-01-25 21:08:30 +01:00
|
|
|
return;
|
2008-01-25 21:08:03 +01:00
|
|
|
|
2010-01-20 21:59:01 +01:00
|
|
|
if (head)
|
|
|
|
list_add(&rt_se->run_list, queue);
|
|
|
|
else
|
|
|
|
list_add_tail(&rt_se->run_list, queue);
|
2008-01-25 21:08:30 +01:00
|
|
|
__set_bit(rt_se_prio(rt_se), array->bitmap);
|
2008-01-25 21:08:27 +01:00
|
|
|
|
2008-01-25 21:08:30 +01:00
|
|
|
inc_rt_tasks(rt_se, rt_rq);
|
|
|
|
}
|
|
|
|
|
2008-06-19 09:06:57 +02:00
|
|
|
static void __dequeue_rt_entity(struct sched_rt_entity *rt_se)
|
2008-01-25 21:08:30 +01:00
|
|
|
{
|
|
|
|
struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
|
|
|
|
struct rt_prio_array *array = &rt_rq->active;
|
|
|
|
|
|
|
|
list_del_init(&rt_se->run_list);
|
|
|
|
if (list_empty(array->queue + rt_se_prio(rt_se)))
|
|
|
|
__clear_bit(rt_se_prio(rt_se), array->bitmap);
|
|
|
|
|
|
|
|
dec_rt_tasks(rt_se, rt_rq);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Because the prio of an upper entry depends on the lower
|
|
|
|
* entries, we must remove entries top - down.
|
|
|
|
*/
|
2008-06-19 09:06:57 +02:00
|
|
|
static void dequeue_rt_stack(struct sched_rt_entity *rt_se)
|
2008-01-25 21:08:30 +01:00
|
|
|
{
|
2008-06-19 09:06:57 +02:00
|
|
|
struct sched_rt_entity *back = NULL;
|
2008-01-25 21:08:30 +01:00
|
|
|
|
2008-04-19 19:45:00 +02:00
|
|
|
for_each_sched_rt_entity(rt_se) {
|
|
|
|
rt_se->back = back;
|
|
|
|
back = rt_se;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (rt_se = back; rt_se; rt_se = rt_se->back) {
|
|
|
|
if (on_rt_rq(rt_se))
|
2008-06-19 09:06:57 +02:00
|
|
|
__dequeue_rt_entity(rt_se);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-01-20 21:59:01 +01:00
|
|
|
static void enqueue_rt_entity(struct sched_rt_entity *rt_se, bool head)
|
2008-06-19 09:06:57 +02:00
|
|
|
{
|
|
|
|
dequeue_rt_stack(rt_se);
|
|
|
|
for_each_sched_rt_entity(rt_se)
|
2010-01-20 21:59:01 +01:00
|
|
|
__enqueue_rt_entity(rt_se, head);
|
2008-06-19 09:06:57 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static void dequeue_rt_entity(struct sched_rt_entity *rt_se)
|
|
|
|
{
|
|
|
|
dequeue_rt_stack(rt_se);
|
|
|
|
|
|
|
|
for_each_sched_rt_entity(rt_se) {
|
|
|
|
struct rt_rq *rt_rq = group_rt_rq(rt_se);
|
|
|
|
|
|
|
|
if (rt_rq && rt_rq->rt_nr_running)
|
2010-01-20 21:59:01 +01:00
|
|
|
__enqueue_rt_entity(rt_se, false);
|
2008-04-19 19:45:00 +02:00
|
|
|
}
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Adding/removing a task to/from a priority array:
|
|
|
|
*/
|
2010-01-20 21:58:57 +01:00
|
|
|
static void
|
2010-03-24 16:38:48 +01:00
|
|
|
enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags)
|
2008-01-25 21:08:30 +01:00
|
|
|
{
|
|
|
|
struct sched_rt_entity *rt_se = &p->rt;
|
|
|
|
|
2010-03-24 16:38:48 +01:00
|
|
|
if (flags & ENQUEUE_WAKEUP)
|
2008-01-25 21:08:30 +01:00
|
|
|
rt_se->timeout = 0;
|
|
|
|
|
2010-03-24 16:38:48 +01:00
|
|
|
enqueue_rt_entity(rt_se, flags & ENQUEUE_HEAD);
|
2008-06-27 13:41:14 +02:00
|
|
|
|
2012-04-23 12:11:21 +02:00
|
|
|
if (!task_current(rq, p) && p->nr_cpus_allowed > 1)
|
sched: create "pushable_tasks" list to limit pushing to one attempt
The RT scheduler employs a "push/pull" design to actively balance tasks
within the system (on a per disjoint cpuset basis). When a task is
awoken, it is immediately determined if there are any lower priority
cpus which should be preempted. This is opposed to the way normal
SCHED_OTHER tasks behave, which will wait for a periodic rebalancing
operation to occur before spreading out load.
When a particular RQ has more than 1 active RT task, it is said to
be in an "overloaded" state. Once this occurs, the system enters
the active balancing mode, where it will try to push the task away,
or persuade a different cpu to pull it over. The system will stay
in this state until the system falls back below the <= 1 queued RT
task per RQ.
However, the current implementation suffers from a limitation in the
push logic. Once overloaded, all tasks (other than current) on the
RQ are analyzed on every push operation, even if it was previously
unpushable (due to affinity, etc). Whats more, the operation stops
at the first task that is unpushable and will not look at items
lower in the queue. This causes two problems:
1) We can have the same tasks analyzed over and over again during each
push, which extends out the fast path in the scheduler for no
gain. Consider a RQ that has dozens of tasks that are bound to a
core. Each one of those tasks will be encountered and skipped
for each push operation while they are queued.
2) There may be lower-priority tasks under the unpushable task that
could have been successfully pushed, but will never be considered
until either the unpushable task is cleared, or a pull operation
succeeds. The net result is a potential latency source for mid
priority tasks.
This patch aims to rectify these two conditions by introducing a new
priority sorted list: "pushable_tasks". A task is added to the list
each time a task is activated or preempted. It is removed from the
list any time it is deactivated, made current, or fails to push.
This works because a task only needs to be attempted to push once.
After an initial failure to push, the other cpus will eventually try to
pull the task when the conditions are proper. This also solves the
problem that we don't completely analyze all tasks due to encountering
an unpushable tasks. Now every task will have a push attempted (when
appropriate).
This reduces latency both by shorting the critical section of the
rq->lock for certain workloads, and by making sure the algorithm
considers all eligible tasks in the system.
[ rostedt: added a couple more BUG_ONs ]
Signed-off-by: Gregory Haskins <ghaskins@novell.com>
Acked-by: Steven Rostedt <srostedt@redhat.com>
2008-12-29 15:39:53 +01:00
|
|
|
enqueue_pushable_task(rq, p);
|
2011-07-21 18:43:27 +02:00
|
|
|
|
|
|
|
inc_nr_running(rq);
|
2008-01-25 21:08:30 +01:00
|
|
|
}
|
|
|
|
|
2010-03-24 16:38:48 +01:00
|
|
|
static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
2008-01-25 21:08:30 +01:00
|
|
|
struct sched_rt_entity *rt_se = &p->rt;
|
2007-07-09 18:51:58 +02:00
|
|
|
|
2007-08-09 11:16:48 +02:00
|
|
|
update_curr_rt(rq);
|
2008-06-19 09:06:57 +02:00
|
|
|
dequeue_rt_entity(rt_se);
|
2008-06-27 13:41:14 +02:00
|
|
|
|
sched: create "pushable_tasks" list to limit pushing to one attempt
The RT scheduler employs a "push/pull" design to actively balance tasks
within the system (on a per disjoint cpuset basis). When a task is
awoken, it is immediately determined if there are any lower priority
cpus which should be preempted. This is opposed to the way normal
SCHED_OTHER tasks behave, which will wait for a periodic rebalancing
operation to occur before spreading out load.
When a particular RQ has more than 1 active RT task, it is said to
be in an "overloaded" state. Once this occurs, the system enters
the active balancing mode, where it will try to push the task away,
or persuade a different cpu to pull it over. The system will stay
in this state until the system falls back below the <= 1 queued RT
task per RQ.
However, the current implementation suffers from a limitation in the
push logic. Once overloaded, all tasks (other than current) on the
RQ are analyzed on every push operation, even if it was previously
unpushable (due to affinity, etc). Whats more, the operation stops
at the first task that is unpushable and will not look at items
lower in the queue. This causes two problems:
1) We can have the same tasks analyzed over and over again during each
push, which extends out the fast path in the scheduler for no
gain. Consider a RQ that has dozens of tasks that are bound to a
core. Each one of those tasks will be encountered and skipped
for each push operation while they are queued.
2) There may be lower-priority tasks under the unpushable task that
could have been successfully pushed, but will never be considered
until either the unpushable task is cleared, or a pull operation
succeeds. The net result is a potential latency source for mid
priority tasks.
This patch aims to rectify these two conditions by introducing a new
priority sorted list: "pushable_tasks". A task is added to the list
each time a task is activated or preempted. It is removed from the
list any time it is deactivated, made current, or fails to push.
This works because a task only needs to be attempted to push once.
After an initial failure to push, the other cpus will eventually try to
pull the task when the conditions are proper. This also solves the
problem that we don't completely analyze all tasks due to encountering
an unpushable tasks. Now every task will have a push attempted (when
appropriate).
This reduces latency both by shorting the critical section of the
rq->lock for certain workloads, and by making sure the algorithm
considers all eligible tasks in the system.
[ rostedt: added a couple more BUG_ONs ]
Signed-off-by: Gregory Haskins <ghaskins@novell.com>
Acked-by: Steven Rostedt <srostedt@redhat.com>
2008-12-29 15:39:53 +01:00
|
|
|
dequeue_pushable_task(rq, p);
|
2011-07-21 18:43:27 +02:00
|
|
|
|
|
|
|
dec_nr_running(rq);
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2011-11-12 18:07:57 +01:00
|
|
|
* Put task to the head or the end of the run list without the overhead of
|
|
|
|
* dequeue followed by enqueue.
|
2007-07-09 18:51:58 +02:00
|
|
|
*/
|
2008-07-01 23:32:15 +02:00
|
|
|
static void
|
|
|
|
requeue_rt_entity(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, int head)
|
2008-01-25 21:08:30 +01:00
|
|
|
{
|
2008-06-19 09:09:15 +02:00
|
|
|
if (on_rt_rq(rt_se)) {
|
2008-07-01 23:32:15 +02:00
|
|
|
struct rt_prio_array *array = &rt_rq->active;
|
|
|
|
struct list_head *queue = array->queue + rt_se_prio(rt_se);
|
|
|
|
|
|
|
|
if (head)
|
|
|
|
list_move(&rt_se->run_list, queue);
|
|
|
|
else
|
|
|
|
list_move_tail(&rt_se->run_list, queue);
|
2008-06-19 09:09:15 +02:00
|
|
|
}
|
2008-01-25 21:08:30 +01:00
|
|
|
}
|
|
|
|
|
2008-07-01 23:32:15 +02:00
|
|
|
static void requeue_task_rt(struct rq *rq, struct task_struct *p, int head)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
2008-01-25 21:08:30 +01:00
|
|
|
struct sched_rt_entity *rt_se = &p->rt;
|
|
|
|
struct rt_rq *rt_rq;
|
2007-07-09 18:51:58 +02:00
|
|
|
|
2008-01-25 21:08:30 +01:00
|
|
|
for_each_sched_rt_entity(rt_se) {
|
|
|
|
rt_rq = rt_rq_of_se(rt_se);
|
2008-07-01 23:32:15 +02:00
|
|
|
requeue_rt_entity(rt_rq, rt_se, head);
|
2008-01-25 21:08:30 +01:00
|
|
|
}
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
2008-01-25 21:08:30 +01:00
|
|
|
static void yield_task_rt(struct rq *rq)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
2008-07-01 23:32:15 +02:00
|
|
|
requeue_task_rt(rq, rq->curr, 0);
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
2008-01-25 21:08:09 +01:00
|
|
|
#ifdef CONFIG_SMP
|
2008-01-25 21:08:10 +01:00
|
|
|
static int find_lowest_rq(struct task_struct *task);
|
|
|
|
|
2010-03-24 18:34:10 +01:00
|
|
|
static int
|
2013-10-07 12:29:16 +02:00
|
|
|
select_task_rq_rt(struct task_struct *p, int cpu, int sd_flag, int flags)
|
2008-01-25 21:08:09 +01:00
|
|
|
{
|
2011-04-05 17:23:46 +02:00
|
|
|
struct task_struct *curr;
|
|
|
|
struct rq *rq;
|
2011-06-17 03:55:22 +02:00
|
|
|
|
2012-04-23 12:11:21 +02:00
|
|
|
if (p->nr_cpus_allowed == 1)
|
2011-11-22 15:18:24 +01:00
|
|
|
goto out;
|
|
|
|
|
2011-06-17 03:55:22 +02:00
|
|
|
/* For anything but wake ups, just return the task_cpu */
|
|
|
|
if (sd_flag != SD_BALANCE_WAKE && sd_flag != SD_BALANCE_FORK)
|
|
|
|
goto out;
|
|
|
|
|
2011-04-05 17:23:46 +02:00
|
|
|
rq = cpu_rq(cpu);
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
curr = ACCESS_ONCE(rq->curr); /* unlocked access */
|
|
|
|
|
2008-01-25 21:08:10 +01:00
|
|
|
/*
|
2011-04-05 17:23:46 +02:00
|
|
|
* If the current task on @p's runqueue is an RT task, then
|
2008-01-25 21:08:12 +01:00
|
|
|
* try to see if we can wake this RT task up on another
|
|
|
|
* runqueue. Otherwise simply start this RT task
|
|
|
|
* on its current runqueue.
|
|
|
|
*
|
sched: Try not to migrate higher priority RT tasks
When first working on the RT scheduler design, we concentrated on
keeping all CPUs running RT tasks instead of having multiple RT
tasks on a single CPU waiting for the migration thread to move
them. Instead we take a more proactive stance and push or pull RT
tasks from one CPU to another on wakeup or scheduling.
When an RT task wakes up on a CPU that is running another RT task,
instead of preempting it and killing the cache of the running RT
task, we look to see if we can migrate the RT task that is waking
up, even if the RT task waking up is of higher priority.
This may sound a bit odd, but RT tasks should be limited in
migration by the user anyway. But in practice, people do not do
this, which causes high prio RT tasks to bounce around the CPUs.
This becomes even worse when we have priority inheritance, because
a high prio task can block on a lower prio task and boost its
priority. When the lower prio task wakes up the high prio task, if
it happens to be on the same CPU it will migrate off of it.
But in reality, the above does not happen much either, because the
wake up of the lower prio task, which has already been boosted, if
it was on the same CPU as the higher prio task, it would then
migrate off of it. But anyway, we do not want to migrate them
either.
To examine the scheduling, I created a test program and examined it
under kernelshark. The test program created CPU * 2 threads, where
each thread had a different priority. The program takes different
options. The options used in this change log was to have priority
inheritance mutexes or not.
All threads did the following loop:
static void grab_lock(long id, int iter, int l)
{
ftrace_write("thread %ld iter %d, taking lock %d\n",
id, iter, l);
pthread_mutex_lock(&locks[l]);
ftrace_write("thread %ld iter %d, took lock %d\n",
id, iter, l);
busy_loop(nr_tasks - id);
ftrace_write("thread %ld iter %d, unlock lock %d\n",
id, iter, l);
pthread_mutex_unlock(&locks[l]);
}
void *start_task(void *id)
{
[...]
while (!done) {
for (l = 0; l < nr_locks; l++) {
grab_lock(id, i, l);
ftrace_write("thread %ld iter %d sleeping\n",
id, i);
ms_sleep(id);
}
i++;
}
[...]
}
The busy_loop(ms) keeps the CPU spinning for ms milliseconds. The
ms_sleep(ms) sleeps for ms milliseconds. The ftrace_write() writes
to the ftrace buffer to help analyze via ftrace.
The higher the id, the higher the prio, the shorter it does the
busy loop, but the longer it spins. This is usually the case with
RT tasks, the lower priority tasks usually run longer than higher
priority tasks.
At the end of the test, it records the number of loops each thread
took, as well as the number of voluntary preemptions, non-voluntary
preemptions, and number of migrations each thread took, taking the
information from /proc/$$/sched and /proc/$$/status.
Running this on a 4 CPU processor, the results without changes to
the kernel looked like this:
Task vol nonvol migrated iterations
---- --- ------ -------- ----------
0: 53 3220 1470 98
1: 562 773 724 98
2: 752 933 1375 98
3: 749 39 697 98
4: 758 5 515 98
5: 764 2 679 99
6: 761 2 535 99
7: 757 3 346 99
total: 5156 4977 6341 787
Each thread regardless of priority migrated a few hundred times.
The higher priority tasks, were a little better but still took
quite an impact.
By letting higher priority tasks bump the lower prio task from the
CPU, things changed a bit:
Task vol nonvol migrated iterations
---- --- ------ -------- ----------
0: 37 2835 1937 98
1: 666 1821 1865 98
2: 654 1003 1385 98
3: 664 635 973 99
4: 698 197 352 99
5: 703 101 159 99
6: 708 1 75 99
7: 713 1 2 99
total: 4843 6594 6748 789
The total # of migrations did not change (several runs showed the
difference all within the noise). But we now see a dramatic
improvement to the higher priority tasks. (kernelshark showed that
the watchdog timer bumped the highest priority task to give it the
2 count. This was actually consistent with every run).
Notice that the # of iterations did not change either.
The above was with priority inheritance mutexes. That is, when the
higher prority task blocked on a lower priority task, the lower
priority task would inherit the higher priority task (which shows
why task 6 was bumped so many times). When not using priority
inheritance mutexes, the current kernel shows this:
Task vol nonvol migrated iterations
---- --- ------ -------- ----------
0: 56 3101 1892 95
1: 594 713 937 95
2: 625 188 618 95
3: 628 4 491 96
4: 640 7 468 96
5: 631 2 501 96
6: 641 1 466 96
7: 643 2 497 96
total: 4458 4018 5870 765
Not much changed with or without priority inheritance mutexes. But
if we let the high priority task bump lower priority tasks on
wakeup we see:
Task vol nonvol migrated iterations
---- --- ------ -------- ----------
0: 115 3439 2782 98
1: 633 1354 1583 99
2: 652 919 1218 99
3: 645 713 934 99
4: 690 3 3 99
5: 694 1 4 99
6: 720 3 4 99
7: 747 0 1 100
Which shows a even bigger change. The big difference between task 3
and task 4 is because we have only 4 CPUs on the machine, causing
the 4 highest prio tasks to always have preference.
Although I did not measure cache misses, and I'm sure there would
be little to measure since the test was not data intensive, I could
imagine large improvements for higher priority tasks when dealing
with lower priority tasks. Thus, I'm satisfied with making the
change and agreeing with what Gregory Haskins argued a few years
ago when we first had this discussion.
One final note. All tasks in the above tests were RT tasks. Any RT
task will always preempt a non RT task that is running on the CPU
the RT task wants to run on.
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Gregory Haskins <ghaskins@novell.com>
LKML-Reference: <20100921024138.605460343@goodmis.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-09-21 04:40:03 +02:00
|
|
|
* We want to avoid overloading runqueues. If the woken
|
|
|
|
* task is a higher priority, then it will stay on this CPU
|
|
|
|
* and the lower prio task should be moved to another CPU.
|
|
|
|
* Even though this will probably make the lower prio task
|
|
|
|
* lose its cache, we do not want to bounce a higher task
|
|
|
|
* around just because it gave up its CPU, perhaps for a
|
|
|
|
* lock?
|
|
|
|
*
|
|
|
|
* For equal prio tasks, we just let the scheduler sort it out.
|
2011-04-05 17:23:46 +02:00
|
|
|
*
|
|
|
|
* Otherwise, just let it ride on the affined RQ and the
|
|
|
|
* post-schedule router will push the preempted task away
|
|
|
|
*
|
|
|
|
* This test is optimistic, if we get it wrong the load-balancer
|
|
|
|
* will have to sort it out.
|
2008-01-25 21:08:10 +01:00
|
|
|
*/
|
2011-04-05 17:23:46 +02:00
|
|
|
if (curr && unlikely(rt_task(curr)) &&
|
2012-04-23 12:11:21 +02:00
|
|
|
(curr->nr_cpus_allowed < 2 ||
|
2013-10-04 21:24:53 +02:00
|
|
|
curr->prio <= p->prio)) {
|
2011-04-05 17:23:46 +02:00
|
|
|
int target = find_lowest_rq(p);
|
2008-01-25 21:08:10 +01:00
|
|
|
|
2011-04-05 17:23:46 +02:00
|
|
|
if (target != -1)
|
|
|
|
cpu = target;
|
2008-01-25 21:08:10 +01:00
|
|
|
}
|
2011-04-05 17:23:46 +02:00
|
|
|
rcu_read_unlock();
|
2008-01-25 21:08:10 +01:00
|
|
|
|
2011-06-17 03:55:22 +02:00
|
|
|
out:
|
2011-04-05 17:23:46 +02:00
|
|
|
return cpu;
|
2008-01-25 21:08:09 +01:00
|
|
|
}
|
2008-07-01 23:32:15 +02:00
|
|
|
|
|
|
|
static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)
|
|
|
|
{
|
2012-04-23 12:11:21 +02:00
|
|
|
if (rq->curr->nr_cpus_allowed == 1)
|
2008-07-01 23:32:15 +02:00
|
|
|
return;
|
|
|
|
|
2012-04-23 12:11:21 +02:00
|
|
|
if (p->nr_cpus_allowed != 1
|
2009-03-25 05:31:22 +01:00
|
|
|
&& cpupri_find(&rq->rd->cpupri, p, NULL))
|
|
|
|
return;
|
2008-11-24 17:05:13 +01:00
|
|
|
|
2009-03-25 05:31:22 +01:00
|
|
|
if (!cpupri_find(&rq->rd->cpupri, rq->curr, NULL))
|
|
|
|
return;
|
2008-07-01 23:32:15 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* There appears to be other cpus that can accept
|
|
|
|
* current and none to run 'p', so lets reschedule
|
|
|
|
* to try and push current away:
|
|
|
|
*/
|
|
|
|
requeue_task_rt(rq, p, 1);
|
|
|
|
resched_task(rq->curr);
|
|
|
|
}
|
|
|
|
|
2008-01-25 21:08:09 +01:00
|
|
|
#endif /* CONFIG_SMP */
|
|
|
|
|
2007-07-09 18:51:58 +02:00
|
|
|
/*
|
|
|
|
* Preempt the current task with a newly woken task if needed:
|
|
|
|
*/
|
2009-09-14 19:55:44 +02:00
|
|
|
static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p, int flags)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
sched: prioritize non-migratable tasks over migratable ones
Dmitry Adamushko pointed out a known flaw in the rt-balancing algorithm
that could allow suboptimal balancing if a non-migratable task gets
queued behind a running migratable one. It is discussed in this thread:
http://lkml.org/lkml/2008/4/22/296
This issue has been further exacerbated by a recent checkin to
sched-devel (git-id 5eee63a5ebc19a870ac40055c0be49457f3a89a3).
>From a pure priority standpoint, the run-queue is doing the "right"
thing. Using Dmitry's nomenclature, if T0 is on cpu1 first, and T1
wakes up at equal or lower priority (affined only to cpu1) later, it
*should* wait for T0 to finish. However, in reality that is likely
suboptimal from a system perspective if there are other cores that
could allow T0 and T1 to run concurrently. Since T1 can not migrate,
the only choice for higher concurrency is to try to move T0. This is
not something we addessed in the recent rt-balancing re-work.
This patch tries to enhance the balancing algorithm by accomodating this
scenario. It accomplishes this by incorporating the migratability of a
task into its priority calculation. Within a numerical tsk->prio, a
non-migratable task is logically higher than a migratable one. We
maintain this by introducing a new per-priority queue (xqueue, or
exclusive-queue) for holding non-migratable tasks. The scheduler will
draw from the xqueue over the standard shared-queue (squeue) when
available.
There are several details for utilizing this properly.
1) During task-wake-up, we not only need to check if the priority
preempts the current task, but we also need to check for this
non-migratable condition. Therefore, if a non-migratable task wakes
up and sees an equal priority migratable task already running, it
will attempt to preempt it *if* there is a likelyhood that the
current task will find an immediate home.
2) Tasks only get this non-migratable "priority boost" on wake-up. Any
requeuing will result in the non-migratable task being queued to the
end of the shared queue. This is an attempt to prevent the system
from being completely unfair to migratable tasks during things like
SCHED_RR timeslicing.
I am sure this patch introduces potentially "odd" behavior if you
concoct a scenario where a bunch of non-migratable threads could starve
migratable ones given the right pattern. I am not yet convinced that
this is a problem since we are talking about tasks of equal RT priority
anyway, and there never is much in the way of guarantees against
starvation under that scenario anyway. (e.g. you could come up with a
similar scenario with a specific timing environment verses an affinity
environment). I can be convinced otherwise, but for now I think this is
"ok".
Signed-off-by: Gregory Haskins <ghaskins@novell.com>
CC: Dmitry Adamushko <dmitry.adamushko@gmail.com>
CC: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2008-05-12 21:20:41 +02:00
|
|
|
if (p->prio < rq->curr->prio) {
|
2007-07-09 18:51:58 +02:00
|
|
|
resched_task(rq->curr);
|
sched: prioritize non-migratable tasks over migratable ones
Dmitry Adamushko pointed out a known flaw in the rt-balancing algorithm
that could allow suboptimal balancing if a non-migratable task gets
queued behind a running migratable one. It is discussed in this thread:
http://lkml.org/lkml/2008/4/22/296
This issue has been further exacerbated by a recent checkin to
sched-devel (git-id 5eee63a5ebc19a870ac40055c0be49457f3a89a3).
>From a pure priority standpoint, the run-queue is doing the "right"
thing. Using Dmitry's nomenclature, if T0 is on cpu1 first, and T1
wakes up at equal or lower priority (affined only to cpu1) later, it
*should* wait for T0 to finish. However, in reality that is likely
suboptimal from a system perspective if there are other cores that
could allow T0 and T1 to run concurrently. Since T1 can not migrate,
the only choice for higher concurrency is to try to move T0. This is
not something we addessed in the recent rt-balancing re-work.
This patch tries to enhance the balancing algorithm by accomodating this
scenario. It accomplishes this by incorporating the migratability of a
task into its priority calculation. Within a numerical tsk->prio, a
non-migratable task is logically higher than a migratable one. We
maintain this by introducing a new per-priority queue (xqueue, or
exclusive-queue) for holding non-migratable tasks. The scheduler will
draw from the xqueue over the standard shared-queue (squeue) when
available.
There are several details for utilizing this properly.
1) During task-wake-up, we not only need to check if the priority
preempts the current task, but we also need to check for this
non-migratable condition. Therefore, if a non-migratable task wakes
up and sees an equal priority migratable task already running, it
will attempt to preempt it *if* there is a likelyhood that the
current task will find an immediate home.
2) Tasks only get this non-migratable "priority boost" on wake-up. Any
requeuing will result in the non-migratable task being queued to the
end of the shared queue. This is an attempt to prevent the system
from being completely unfair to migratable tasks during things like
SCHED_RR timeslicing.
I am sure this patch introduces potentially "odd" behavior if you
concoct a scenario where a bunch of non-migratable threads could starve
migratable ones given the right pattern. I am not yet convinced that
this is a problem since we are talking about tasks of equal RT priority
anyway, and there never is much in the way of guarantees against
starvation under that scenario anyway. (e.g. you could come up with a
similar scenario with a specific timing environment verses an affinity
environment). I can be convinced otherwise, but for now I think this is
"ok".
Signed-off-by: Gregory Haskins <ghaskins@novell.com>
CC: Dmitry Adamushko <dmitry.adamushko@gmail.com>
CC: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2008-05-12 21:20:41 +02:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
/*
|
|
|
|
* If:
|
|
|
|
*
|
|
|
|
* - the newly woken task is of equal priority to the current task
|
|
|
|
* - the newly woken task is non-migratable while current is migratable
|
|
|
|
* - current will be preempted on the next reschedule
|
|
|
|
*
|
|
|
|
* we should check to see if current can readily move to a different
|
|
|
|
* cpu. If so, we will reschedule to allow the push logic to try
|
|
|
|
* to move current somewhere else, making room for our non-migratable
|
|
|
|
* task.
|
|
|
|
*/
|
2011-06-15 00:36:24 +02:00
|
|
|
if (p->prio == rq->curr->prio && !test_tsk_need_resched(rq->curr))
|
2008-07-01 23:32:15 +02:00
|
|
|
check_preempt_equal_prio(rq, p);
|
sched: prioritize non-migratable tasks over migratable ones
Dmitry Adamushko pointed out a known flaw in the rt-balancing algorithm
that could allow suboptimal balancing if a non-migratable task gets
queued behind a running migratable one. It is discussed in this thread:
http://lkml.org/lkml/2008/4/22/296
This issue has been further exacerbated by a recent checkin to
sched-devel (git-id 5eee63a5ebc19a870ac40055c0be49457f3a89a3).
>From a pure priority standpoint, the run-queue is doing the "right"
thing. Using Dmitry's nomenclature, if T0 is on cpu1 first, and T1
wakes up at equal or lower priority (affined only to cpu1) later, it
*should* wait for T0 to finish. However, in reality that is likely
suboptimal from a system perspective if there are other cores that
could allow T0 and T1 to run concurrently. Since T1 can not migrate,
the only choice for higher concurrency is to try to move T0. This is
not something we addessed in the recent rt-balancing re-work.
This patch tries to enhance the balancing algorithm by accomodating this
scenario. It accomplishes this by incorporating the migratability of a
task into its priority calculation. Within a numerical tsk->prio, a
non-migratable task is logically higher than a migratable one. We
maintain this by introducing a new per-priority queue (xqueue, or
exclusive-queue) for holding non-migratable tasks. The scheduler will
draw from the xqueue over the standard shared-queue (squeue) when
available.
There are several details for utilizing this properly.
1) During task-wake-up, we not only need to check if the priority
preempts the current task, but we also need to check for this
non-migratable condition. Therefore, if a non-migratable task wakes
up and sees an equal priority migratable task already running, it
will attempt to preempt it *if* there is a likelyhood that the
current task will find an immediate home.
2) Tasks only get this non-migratable "priority boost" on wake-up. Any
requeuing will result in the non-migratable task being queued to the
end of the shared queue. This is an attempt to prevent the system
from being completely unfair to migratable tasks during things like
SCHED_RR timeslicing.
I am sure this patch introduces potentially "odd" behavior if you
concoct a scenario where a bunch of non-migratable threads could starve
migratable ones given the right pattern. I am not yet convinced that
this is a problem since we are talking about tasks of equal RT priority
anyway, and there never is much in the way of guarantees against
starvation under that scenario anyway. (e.g. you could come up with a
similar scenario with a specific timing environment verses an affinity
environment). I can be convinced otherwise, but for now I think this is
"ok".
Signed-off-by: Gregory Haskins <ghaskins@novell.com>
CC: Dmitry Adamushko <dmitry.adamushko@gmail.com>
CC: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2008-05-12 21:20:41 +02:00
|
|
|
#endif
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
2008-01-25 21:08:30 +01:00
|
|
|
static struct sched_rt_entity *pick_next_rt_entity(struct rq *rq,
|
|
|
|
struct rt_rq *rt_rq)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
2008-01-25 21:08:30 +01:00
|
|
|
struct rt_prio_array *array = &rt_rq->active;
|
|
|
|
struct sched_rt_entity *next = NULL;
|
2007-07-09 18:51:58 +02:00
|
|
|
struct list_head *queue;
|
|
|
|
int idx;
|
|
|
|
|
|
|
|
idx = sched_find_first_bit(array->bitmap);
|
2008-01-25 21:08:30 +01:00
|
|
|
BUG_ON(idx >= MAX_RT_PRIO);
|
2007-07-09 18:51:58 +02:00
|
|
|
|
|
|
|
queue = array->queue + idx;
|
2008-01-25 21:08:30 +01:00
|
|
|
next = list_entry(queue->next, struct sched_rt_entity, run_list);
|
2008-01-25 21:08:34 +01:00
|
|
|
|
2008-01-25 21:08:30 +01:00
|
|
|
return next;
|
|
|
|
}
|
2007-07-09 18:51:58 +02:00
|
|
|
|
sched: create "pushable_tasks" list to limit pushing to one attempt
The RT scheduler employs a "push/pull" design to actively balance tasks
within the system (on a per disjoint cpuset basis). When a task is
awoken, it is immediately determined if there are any lower priority
cpus which should be preempted. This is opposed to the way normal
SCHED_OTHER tasks behave, which will wait for a periodic rebalancing
operation to occur before spreading out load.
When a particular RQ has more than 1 active RT task, it is said to
be in an "overloaded" state. Once this occurs, the system enters
the active balancing mode, where it will try to push the task away,
or persuade a different cpu to pull it over. The system will stay
in this state until the system falls back below the <= 1 queued RT
task per RQ.
However, the current implementation suffers from a limitation in the
push logic. Once overloaded, all tasks (other than current) on the
RQ are analyzed on every push operation, even if it was previously
unpushable (due to affinity, etc). Whats more, the operation stops
at the first task that is unpushable and will not look at items
lower in the queue. This causes two problems:
1) We can have the same tasks analyzed over and over again during each
push, which extends out the fast path in the scheduler for no
gain. Consider a RQ that has dozens of tasks that are bound to a
core. Each one of those tasks will be encountered and skipped
for each push operation while they are queued.
2) There may be lower-priority tasks under the unpushable task that
could have been successfully pushed, but will never be considered
until either the unpushable task is cleared, or a pull operation
succeeds. The net result is a potential latency source for mid
priority tasks.
This patch aims to rectify these two conditions by introducing a new
priority sorted list: "pushable_tasks". A task is added to the list
each time a task is activated or preempted. It is removed from the
list any time it is deactivated, made current, or fails to push.
This works because a task only needs to be attempted to push once.
After an initial failure to push, the other cpus will eventually try to
pull the task when the conditions are proper. This also solves the
problem that we don't completely analyze all tasks due to encountering
an unpushable tasks. Now every task will have a push attempted (when
appropriate).
This reduces latency both by shorting the critical section of the
rq->lock for certain workloads, and by making sure the algorithm
considers all eligible tasks in the system.
[ rostedt: added a couple more BUG_ONs ]
Signed-off-by: Gregory Haskins <ghaskins@novell.com>
Acked-by: Steven Rostedt <srostedt@redhat.com>
2008-12-29 15:39:53 +01:00
|
|
|
static struct task_struct *_pick_next_task_rt(struct rq *rq)
|
2008-01-25 21:08:30 +01:00
|
|
|
{
|
|
|
|
struct sched_rt_entity *rt_se;
|
|
|
|
struct task_struct *p;
|
|
|
|
struct rt_rq *rt_rq;
|
2007-07-09 18:51:58 +02:00
|
|
|
|
2008-01-25 21:08:30 +01:00
|
|
|
rt_rq = &rq->rt;
|
|
|
|
|
2010-12-06 17:28:30 +01:00
|
|
|
if (!rt_rq->rt_nr_running)
|
2008-01-25 21:08:30 +01:00
|
|
|
return NULL;
|
|
|
|
|
2008-02-13 15:45:39 +01:00
|
|
|
if (rt_rq_throttled(rt_rq))
|
2008-01-25 21:08:30 +01:00
|
|
|
return NULL;
|
|
|
|
|
|
|
|
do {
|
|
|
|
rt_se = pick_next_rt_entity(rq, rt_rq);
|
2008-01-25 21:08:34 +01:00
|
|
|
BUG_ON(!rt_se);
|
2008-01-25 21:08:30 +01:00
|
|
|
rt_rq = group_rt_rq(rt_se);
|
|
|
|
} while (rt_rq);
|
|
|
|
|
|
|
|
p = rt_task_of(rt_se);
|
2013-04-12 01:51:02 +02:00
|
|
|
p->se.exec_start = rq_clock_task(rq);
|
sched: create "pushable_tasks" list to limit pushing to one attempt
The RT scheduler employs a "push/pull" design to actively balance tasks
within the system (on a per disjoint cpuset basis). When a task is
awoken, it is immediately determined if there are any lower priority
cpus which should be preempted. This is opposed to the way normal
SCHED_OTHER tasks behave, which will wait for a periodic rebalancing
operation to occur before spreading out load.
When a particular RQ has more than 1 active RT task, it is said to
be in an "overloaded" state. Once this occurs, the system enters
the active balancing mode, where it will try to push the task away,
or persuade a different cpu to pull it over. The system will stay
in this state until the system falls back below the <= 1 queued RT
task per RQ.
However, the current implementation suffers from a limitation in the
push logic. Once overloaded, all tasks (other than current) on the
RQ are analyzed on every push operation, even if it was previously
unpushable (due to affinity, etc). Whats more, the operation stops
at the first task that is unpushable and will not look at items
lower in the queue. This causes two problems:
1) We can have the same tasks analyzed over and over again during each
push, which extends out the fast path in the scheduler for no
gain. Consider a RQ that has dozens of tasks that are bound to a
core. Each one of those tasks will be encountered and skipped
for each push operation while they are queued.
2) There may be lower-priority tasks under the unpushable task that
could have been successfully pushed, but will never be considered
until either the unpushable task is cleared, or a pull operation
succeeds. The net result is a potential latency source for mid
priority tasks.
This patch aims to rectify these two conditions by introducing a new
priority sorted list: "pushable_tasks". A task is added to the list
each time a task is activated or preempted. It is removed from the
list any time it is deactivated, made current, or fails to push.
This works because a task only needs to be attempted to push once.
After an initial failure to push, the other cpus will eventually try to
pull the task when the conditions are proper. This also solves the
problem that we don't completely analyze all tasks due to encountering
an unpushable tasks. Now every task will have a push attempted (when
appropriate).
This reduces latency both by shorting the critical section of the
rq->lock for certain workloads, and by making sure the algorithm
considers all eligible tasks in the system.
[ rostedt: added a couple more BUG_ONs ]
Signed-off-by: Gregory Haskins <ghaskins@novell.com>
Acked-by: Steven Rostedt <srostedt@redhat.com>
2008-12-29 15:39:53 +01:00
|
|
|
|
|
|
|
return p;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct task_struct *pick_next_task_rt(struct rq *rq)
|
|
|
|
{
|
|
|
|
struct task_struct *p = _pick_next_task_rt(rq);
|
|
|
|
|
|
|
|
/* The running task is never eligible for pushing */
|
|
|
|
if (p)
|
|
|
|
dequeue_pushable_task(rq, p);
|
|
|
|
|
2008-04-19 12:11:10 +02:00
|
|
|
#ifdef CONFIG_SMP
|
2009-07-29 17:08:47 +02:00
|
|
|
/*
|
|
|
|
* We detect this state here so that we can avoid taking the RQ
|
|
|
|
* lock again later if there is no need to push
|
|
|
|
*/
|
|
|
|
rq->post_schedule = has_pushable_tasks(rq);
|
2008-04-19 12:11:10 +02:00
|
|
|
#endif
|
2009-07-29 17:08:47 +02:00
|
|
|
|
2008-01-25 21:08:30 +01:00
|
|
|
return p;
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
2007-08-09 11:16:49 +02:00
|
|
|
static void put_prev_task_rt(struct rq *rq, struct task_struct *p)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
2007-08-09 11:16:48 +02:00
|
|
|
update_curr_rt(rq);
|
sched: create "pushable_tasks" list to limit pushing to one attempt
The RT scheduler employs a "push/pull" design to actively balance tasks
within the system (on a per disjoint cpuset basis). When a task is
awoken, it is immediately determined if there are any lower priority
cpus which should be preempted. This is opposed to the way normal
SCHED_OTHER tasks behave, which will wait for a periodic rebalancing
operation to occur before spreading out load.
When a particular RQ has more than 1 active RT task, it is said to
be in an "overloaded" state. Once this occurs, the system enters
the active balancing mode, where it will try to push the task away,
or persuade a different cpu to pull it over. The system will stay
in this state until the system falls back below the <= 1 queued RT
task per RQ.
However, the current implementation suffers from a limitation in the
push logic. Once overloaded, all tasks (other than current) on the
RQ are analyzed on every push operation, even if it was previously
unpushable (due to affinity, etc). Whats more, the operation stops
at the first task that is unpushable and will not look at items
lower in the queue. This causes two problems:
1) We can have the same tasks analyzed over and over again during each
push, which extends out the fast path in the scheduler for no
gain. Consider a RQ that has dozens of tasks that are bound to a
core. Each one of those tasks will be encountered and skipped
for each push operation while they are queued.
2) There may be lower-priority tasks under the unpushable task that
could have been successfully pushed, but will never be considered
until either the unpushable task is cleared, or a pull operation
succeeds. The net result is a potential latency source for mid
priority tasks.
This patch aims to rectify these two conditions by introducing a new
priority sorted list: "pushable_tasks". A task is added to the list
each time a task is activated or preempted. It is removed from the
list any time it is deactivated, made current, or fails to push.
This works because a task only needs to be attempted to push once.
After an initial failure to push, the other cpus will eventually try to
pull the task when the conditions are proper. This also solves the
problem that we don't completely analyze all tasks due to encountering
an unpushable tasks. Now every task will have a push attempted (when
appropriate).
This reduces latency both by shorting the critical section of the
rq->lock for certain workloads, and by making sure the algorithm
considers all eligible tasks in the system.
[ rostedt: added a couple more BUG_ONs ]
Signed-off-by: Gregory Haskins <ghaskins@novell.com>
Acked-by: Steven Rostedt <srostedt@redhat.com>
2008-12-29 15:39:53 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The previous task needs to be made eligible for pushing
|
|
|
|
* if it is still active
|
|
|
|
*/
|
2012-04-23 12:11:21 +02:00
|
|
|
if (on_rt_rq(&p->rt) && p->nr_cpus_allowed > 1)
|
sched: create "pushable_tasks" list to limit pushing to one attempt
The RT scheduler employs a "push/pull" design to actively balance tasks
within the system (on a per disjoint cpuset basis). When a task is
awoken, it is immediately determined if there are any lower priority
cpus which should be preempted. This is opposed to the way normal
SCHED_OTHER tasks behave, which will wait for a periodic rebalancing
operation to occur before spreading out load.
When a particular RQ has more than 1 active RT task, it is said to
be in an "overloaded" state. Once this occurs, the system enters
the active balancing mode, where it will try to push the task away,
or persuade a different cpu to pull it over. The system will stay
in this state until the system falls back below the <= 1 queued RT
task per RQ.
However, the current implementation suffers from a limitation in the
push logic. Once overloaded, all tasks (other than current) on the
RQ are analyzed on every push operation, even if it was previously
unpushable (due to affinity, etc). Whats more, the operation stops
at the first task that is unpushable and will not look at items
lower in the queue. This causes two problems:
1) We can have the same tasks analyzed over and over again during each
push, which extends out the fast path in the scheduler for no
gain. Consider a RQ that has dozens of tasks that are bound to a
core. Each one of those tasks will be encountered and skipped
for each push operation while they are queued.
2) There may be lower-priority tasks under the unpushable task that
could have been successfully pushed, but will never be considered
until either the unpushable task is cleared, or a pull operation
succeeds. The net result is a potential latency source for mid
priority tasks.
This patch aims to rectify these two conditions by introducing a new
priority sorted list: "pushable_tasks". A task is added to the list
each time a task is activated or preempted. It is removed from the
list any time it is deactivated, made current, or fails to push.
This works because a task only needs to be attempted to push once.
After an initial failure to push, the other cpus will eventually try to
pull the task when the conditions are proper. This also solves the
problem that we don't completely analyze all tasks due to encountering
an unpushable tasks. Now every task will have a push attempted (when
appropriate).
This reduces latency both by shorting the critical section of the
rq->lock for certain workloads, and by making sure the algorithm
considers all eligible tasks in the system.
[ rostedt: added a couple more BUG_ONs ]
Signed-off-by: Gregory Haskins <ghaskins@novell.com>
Acked-by: Steven Rostedt <srostedt@redhat.com>
2008-12-29 15:39:53 +01:00
|
|
|
enqueue_pushable_task(rq, p);
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
2007-10-24 18:23:51 +02:00
|
|
|
#ifdef CONFIG_SMP
|
2008-01-25 21:08:30 +01:00
|
|
|
|
2008-01-25 21:08:05 +01:00
|
|
|
/* Only try algorithms three times */
|
|
|
|
#define RT_MAX_TRIES 3
|
|
|
|
|
2008-01-25 21:08:07 +01:00
|
|
|
static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu)
|
|
|
|
{
|
|
|
|
if (!task_running(rq, p) &&
|
2013-01-31 15:56:17 +01:00
|
|
|
cpumask_test_cpu(cpu, tsk_cpus_allowed(p)))
|
2008-01-25 21:08:07 +01:00
|
|
|
return 1;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2013-06-07 21:37:43 +02:00
|
|
|
/*
|
|
|
|
* Return the highest pushable rq's task, which is suitable to be executed
|
|
|
|
* on the cpu, NULL otherwise
|
|
|
|
*/
|
|
|
|
static struct task_struct *pick_highest_pushable_task(struct rq *rq, int cpu)
|
2008-01-25 21:08:05 +01:00
|
|
|
{
|
2013-06-07 21:37:43 +02:00
|
|
|
struct plist_head *head = &rq->rt.pushable_tasks;
|
|
|
|
struct task_struct *p;
|
2010-03-10 17:07:24 +01:00
|
|
|
|
2013-06-07 21:37:43 +02:00
|
|
|
if (!has_pushable_tasks(rq))
|
|
|
|
return NULL;
|
2010-03-10 17:07:24 +01:00
|
|
|
|
2013-06-07 21:37:43 +02:00
|
|
|
plist_for_each_entry(p, head, pushable_tasks) {
|
|
|
|
if (pick_rt_task(rq, p, cpu))
|
|
|
|
return p;
|
2008-01-25 21:08:07 +01:00
|
|
|
}
|
|
|
|
|
2013-06-07 21:37:43 +02:00
|
|
|
return NULL;
|
2008-01-25 21:08:05 +01:00
|
|
|
}
|
|
|
|
|
2008-11-24 17:05:13 +01:00
|
|
|
static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask);
|
2008-01-25 21:08:05 +01:00
|
|
|
|
2008-01-25 21:08:11 +01:00
|
|
|
static int find_lowest_rq(struct task_struct *task)
|
|
|
|
{
|
|
|
|
struct sched_domain *sd;
|
2008-11-24 17:05:14 +01:00
|
|
|
struct cpumask *lowest_mask = __get_cpu_var(local_cpu_mask);
|
2008-01-25 21:08:11 +01:00
|
|
|
int this_cpu = smp_processor_id();
|
|
|
|
int cpu = task_cpu(task);
|
2008-01-25 21:08:13 +01:00
|
|
|
|
2011-06-15 00:36:25 +02:00
|
|
|
/* Make sure the mask is initialized first */
|
|
|
|
if (unlikely(!lowest_mask))
|
|
|
|
return -1;
|
|
|
|
|
2012-04-23 12:11:21 +02:00
|
|
|
if (task->nr_cpus_allowed == 1)
|
2008-05-12 21:21:01 +02:00
|
|
|
return -1; /* No other targets possible */
|
2008-01-25 21:08:11 +01:00
|
|
|
|
2008-05-12 21:21:01 +02:00
|
|
|
if (!cpupri_find(&task_rq(task)->rd->cpupri, task, lowest_mask))
|
|
|
|
return -1; /* No targets found */
|
2008-01-25 21:08:11 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* At this point we have built a mask of cpus representing the
|
|
|
|
* lowest priority tasks in the system. Now we want to elect
|
|
|
|
* the best one based on our affinity and topology.
|
|
|
|
*
|
|
|
|
* We prioritize the last cpu that the task executed on since
|
|
|
|
* it is most likely cache-hot in that location.
|
|
|
|
*/
|
2008-11-24 17:05:14 +01:00
|
|
|
if (cpumask_test_cpu(cpu, lowest_mask))
|
2008-01-25 21:08:11 +01:00
|
|
|
return cpu;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Otherwise, we consult the sched_domains span maps to figure
|
|
|
|
* out which cpu is logically closest to our hot cache data.
|
|
|
|
*/
|
2009-11-03 05:23:15 +01:00
|
|
|
if (!cpumask_test_cpu(this_cpu, lowest_mask))
|
|
|
|
this_cpu = -1; /* Skip this_cpu opt if not among lowest */
|
2008-01-25 21:08:11 +01:00
|
|
|
|
2011-04-22 12:53:54 +02:00
|
|
|
rcu_read_lock();
|
2009-11-03 05:23:15 +01:00
|
|
|
for_each_domain(cpu, sd) {
|
|
|
|
if (sd->flags & SD_WAKE_AFFINE) {
|
|
|
|
int best_cpu;
|
2008-01-25 21:08:11 +01:00
|
|
|
|
2009-11-03 05:23:15 +01:00
|
|
|
/*
|
|
|
|
* "this_cpu" is cheaper to preempt than a
|
|
|
|
* remote processor.
|
|
|
|
*/
|
|
|
|
if (this_cpu != -1 &&
|
2011-04-22 12:53:54 +02:00
|
|
|
cpumask_test_cpu(this_cpu, sched_domain_span(sd))) {
|
|
|
|
rcu_read_unlock();
|
2009-11-03 05:23:15 +01:00
|
|
|
return this_cpu;
|
2011-04-22 12:53:54 +02:00
|
|
|
}
|
2009-11-03 05:23:15 +01:00
|
|
|
|
|
|
|
best_cpu = cpumask_first_and(lowest_mask,
|
|
|
|
sched_domain_span(sd));
|
2011-04-22 12:53:54 +02:00
|
|
|
if (best_cpu < nr_cpu_ids) {
|
|
|
|
rcu_read_unlock();
|
2009-11-03 05:23:15 +01:00
|
|
|
return best_cpu;
|
2011-04-22 12:53:54 +02:00
|
|
|
}
|
2008-01-25 21:08:11 +01:00
|
|
|
}
|
|
|
|
}
|
2011-04-22 12:53:54 +02:00
|
|
|
rcu_read_unlock();
|
2008-01-25 21:08:11 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* And finally, if there were no matches within the domains
|
|
|
|
* just give the caller *something* to work with from the compatible
|
|
|
|
* locations.
|
|
|
|
*/
|
2009-11-03 05:23:15 +01:00
|
|
|
if (this_cpu != -1)
|
|
|
|
return this_cpu;
|
|
|
|
|
|
|
|
cpu = cpumask_any(lowest_mask);
|
|
|
|
if (cpu < nr_cpu_ids)
|
|
|
|
return cpu;
|
|
|
|
return -1;
|
2008-01-25 21:08:10 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Will lock the rq it finds */
|
2008-01-25 21:08:15 +01:00
|
|
|
static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq)
|
2008-01-25 21:08:10 +01:00
|
|
|
{
|
|
|
|
struct rq *lowest_rq = NULL;
|
|
|
|
int tries;
|
2008-01-25 21:08:15 +01:00
|
|
|
int cpu;
|
2008-01-25 21:08:05 +01:00
|
|
|
|
2008-01-25 21:08:10 +01:00
|
|
|
for (tries = 0; tries < RT_MAX_TRIES; tries++) {
|
|
|
|
cpu = find_lowest_rq(task);
|
|
|
|
|
2008-01-25 21:08:10 +01:00
|
|
|
if ((cpu == -1) || (cpu == rq->cpu))
|
2008-01-25 21:08:05 +01:00
|
|
|
break;
|
|
|
|
|
2008-01-25 21:08:10 +01:00
|
|
|
lowest_rq = cpu_rq(cpu);
|
|
|
|
|
2008-01-25 21:08:05 +01:00
|
|
|
/* if the prio of this runqueue changed, try again */
|
2008-01-25 21:08:10 +01:00
|
|
|
if (double_lock_balance(rq, lowest_rq)) {
|
2008-01-25 21:08:05 +01:00
|
|
|
/*
|
|
|
|
* We had to unlock the run queue. In
|
|
|
|
* the mean time, task could have
|
|
|
|
* migrated already or had its affinity changed.
|
|
|
|
* Also make sure that it wasn't scheduled on its rq.
|
|
|
|
*/
|
2008-01-25 21:08:10 +01:00
|
|
|
if (unlikely(task_rq(task) != rq ||
|
2008-11-24 17:05:14 +01:00
|
|
|
!cpumask_test_cpu(lowest_rq->cpu,
|
2011-06-16 12:23:22 +02:00
|
|
|
tsk_cpus_allowed(task)) ||
|
2008-01-25 21:08:10 +01:00
|
|
|
task_running(rq, task) ||
|
2011-04-05 17:23:44 +02:00
|
|
|
!task->on_rq)) {
|
2008-01-25 21:08:15 +01:00
|
|
|
|
2012-05-17 21:19:46 +02:00
|
|
|
double_unlock_balance(rq, lowest_rq);
|
2008-01-25 21:08:05 +01:00
|
|
|
lowest_rq = NULL;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* If this rq is still suitable use it. */
|
2008-12-29 15:39:49 +01:00
|
|
|
if (lowest_rq->rt.highest_prio.curr > task->prio)
|
2008-01-25 21:08:05 +01:00
|
|
|
break;
|
|
|
|
|
|
|
|
/* try again */
|
2008-08-11 09:30:22 +02:00
|
|
|
double_unlock_balance(rq, lowest_rq);
|
2008-01-25 21:08:05 +01:00
|
|
|
lowest_rq = NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
return lowest_rq;
|
|
|
|
}
|
|
|
|
|
sched: create "pushable_tasks" list to limit pushing to one attempt
The RT scheduler employs a "push/pull" design to actively balance tasks
within the system (on a per disjoint cpuset basis). When a task is
awoken, it is immediately determined if there are any lower priority
cpus which should be preempted. This is opposed to the way normal
SCHED_OTHER tasks behave, which will wait for a periodic rebalancing
operation to occur before spreading out load.
When a particular RQ has more than 1 active RT task, it is said to
be in an "overloaded" state. Once this occurs, the system enters
the active balancing mode, where it will try to push the task away,
or persuade a different cpu to pull it over. The system will stay
in this state until the system falls back below the <= 1 queued RT
task per RQ.
However, the current implementation suffers from a limitation in the
push logic. Once overloaded, all tasks (other than current) on the
RQ are analyzed on every push operation, even if it was previously
unpushable (due to affinity, etc). Whats more, the operation stops
at the first task that is unpushable and will not look at items
lower in the queue. This causes two problems:
1) We can have the same tasks analyzed over and over again during each
push, which extends out the fast path in the scheduler for no
gain. Consider a RQ that has dozens of tasks that are bound to a
core. Each one of those tasks will be encountered and skipped
for each push operation while they are queued.
2) There may be lower-priority tasks under the unpushable task that
could have been successfully pushed, but will never be considered
until either the unpushable task is cleared, or a pull operation
succeeds. The net result is a potential latency source for mid
priority tasks.
This patch aims to rectify these two conditions by introducing a new
priority sorted list: "pushable_tasks". A task is added to the list
each time a task is activated or preempted. It is removed from the
list any time it is deactivated, made current, or fails to push.
This works because a task only needs to be attempted to push once.
After an initial failure to push, the other cpus will eventually try to
pull the task when the conditions are proper. This also solves the
problem that we don't completely analyze all tasks due to encountering
an unpushable tasks. Now every task will have a push attempted (when
appropriate).
This reduces latency both by shorting the critical section of the
rq->lock for certain workloads, and by making sure the algorithm
considers all eligible tasks in the system.
[ rostedt: added a couple more BUG_ONs ]
Signed-off-by: Gregory Haskins <ghaskins@novell.com>
Acked-by: Steven Rostedt <srostedt@redhat.com>
2008-12-29 15:39:53 +01:00
|
|
|
static struct task_struct *pick_next_pushable_task(struct rq *rq)
|
|
|
|
{
|
|
|
|
struct task_struct *p;
|
|
|
|
|
|
|
|
if (!has_pushable_tasks(rq))
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
p = plist_first_entry(&rq->rt.pushable_tasks,
|
|
|
|
struct task_struct, pushable_tasks);
|
|
|
|
|
|
|
|
BUG_ON(rq->cpu != task_cpu(p));
|
|
|
|
BUG_ON(task_current(rq, p));
|
2012-04-23 12:11:21 +02:00
|
|
|
BUG_ON(p->nr_cpus_allowed <= 1);
|
sched: create "pushable_tasks" list to limit pushing to one attempt
The RT scheduler employs a "push/pull" design to actively balance tasks
within the system (on a per disjoint cpuset basis). When a task is
awoken, it is immediately determined if there are any lower priority
cpus which should be preempted. This is opposed to the way normal
SCHED_OTHER tasks behave, which will wait for a periodic rebalancing
operation to occur before spreading out load.
When a particular RQ has more than 1 active RT task, it is said to
be in an "overloaded" state. Once this occurs, the system enters
the active balancing mode, where it will try to push the task away,
or persuade a different cpu to pull it over. The system will stay
in this state until the system falls back below the <= 1 queued RT
task per RQ.
However, the current implementation suffers from a limitation in the
push logic. Once overloaded, all tasks (other than current) on the
RQ are analyzed on every push operation, even if it was previously
unpushable (due to affinity, etc). Whats more, the operation stops
at the first task that is unpushable and will not look at items
lower in the queue. This causes two problems:
1) We can have the same tasks analyzed over and over again during each
push, which extends out the fast path in the scheduler for no
gain. Consider a RQ that has dozens of tasks that are bound to a
core. Each one of those tasks will be encountered and skipped
for each push operation while they are queued.
2) There may be lower-priority tasks under the unpushable task that
could have been successfully pushed, but will never be considered
until either the unpushable task is cleared, or a pull operation
succeeds. The net result is a potential latency source for mid
priority tasks.
This patch aims to rectify these two conditions by introducing a new
priority sorted list: "pushable_tasks". A task is added to the list
each time a task is activated or preempted. It is removed from the
list any time it is deactivated, made current, or fails to push.
This works because a task only needs to be attempted to push once.
After an initial failure to push, the other cpus will eventually try to
pull the task when the conditions are proper. This also solves the
problem that we don't completely analyze all tasks due to encountering
an unpushable tasks. Now every task will have a push attempted (when
appropriate).
This reduces latency both by shorting the critical section of the
rq->lock for certain workloads, and by making sure the algorithm
considers all eligible tasks in the system.
[ rostedt: added a couple more BUG_ONs ]
Signed-off-by: Gregory Haskins <ghaskins@novell.com>
Acked-by: Steven Rostedt <srostedt@redhat.com>
2008-12-29 15:39:53 +01:00
|
|
|
|
2011-04-05 17:23:44 +02:00
|
|
|
BUG_ON(!p->on_rq);
|
sched: create "pushable_tasks" list to limit pushing to one attempt
The RT scheduler employs a "push/pull" design to actively balance tasks
within the system (on a per disjoint cpuset basis). When a task is
awoken, it is immediately determined if there are any lower priority
cpus which should be preempted. This is opposed to the way normal
SCHED_OTHER tasks behave, which will wait for a periodic rebalancing
operation to occur before spreading out load.
When a particular RQ has more than 1 active RT task, it is said to
be in an "overloaded" state. Once this occurs, the system enters
the active balancing mode, where it will try to push the task away,
or persuade a different cpu to pull it over. The system will stay
in this state until the system falls back below the <= 1 queued RT
task per RQ.
However, the current implementation suffers from a limitation in the
push logic. Once overloaded, all tasks (other than current) on the
RQ are analyzed on every push operation, even if it was previously
unpushable (due to affinity, etc). Whats more, the operation stops
at the first task that is unpushable and will not look at items
lower in the queue. This causes two problems:
1) We can have the same tasks analyzed over and over again during each
push, which extends out the fast path in the scheduler for no
gain. Consider a RQ that has dozens of tasks that are bound to a
core. Each one of those tasks will be encountered and skipped
for each push operation while they are queued.
2) There may be lower-priority tasks under the unpushable task that
could have been successfully pushed, but will never be considered
until either the unpushable task is cleared, or a pull operation
succeeds. The net result is a potential latency source for mid
priority tasks.
This patch aims to rectify these two conditions by introducing a new
priority sorted list: "pushable_tasks". A task is added to the list
each time a task is activated or preempted. It is removed from the
list any time it is deactivated, made current, or fails to push.
This works because a task only needs to be attempted to push once.
After an initial failure to push, the other cpus will eventually try to
pull the task when the conditions are proper. This also solves the
problem that we don't completely analyze all tasks due to encountering
an unpushable tasks. Now every task will have a push attempted (when
appropriate).
This reduces latency both by shorting the critical section of the
rq->lock for certain workloads, and by making sure the algorithm
considers all eligible tasks in the system.
[ rostedt: added a couple more BUG_ONs ]
Signed-off-by: Gregory Haskins <ghaskins@novell.com>
Acked-by: Steven Rostedt <srostedt@redhat.com>
2008-12-29 15:39:53 +01:00
|
|
|
BUG_ON(!rt_task(p));
|
|
|
|
|
|
|
|
return p;
|
|
|
|
}
|
|
|
|
|
2008-01-25 21:08:05 +01:00
|
|
|
/*
|
|
|
|
* If the current CPU has more than one RT task, see if the non
|
|
|
|
* running task can migrate over to a CPU that is running a task
|
|
|
|
* of lesser priority.
|
|
|
|
*/
|
2008-01-25 21:08:09 +01:00
|
|
|
static int push_rt_task(struct rq *rq)
|
2008-01-25 21:08:05 +01:00
|
|
|
{
|
|
|
|
struct task_struct *next_task;
|
|
|
|
struct rq *lowest_rq;
|
2011-06-17 03:55:20 +02:00
|
|
|
int ret = 0;
|
2008-01-25 21:08:05 +01:00
|
|
|
|
2008-01-25 21:08:12 +01:00
|
|
|
if (!rq->rt.overloaded)
|
|
|
|
return 0;
|
|
|
|
|
sched: create "pushable_tasks" list to limit pushing to one attempt
The RT scheduler employs a "push/pull" design to actively balance tasks
within the system (on a per disjoint cpuset basis). When a task is
awoken, it is immediately determined if there are any lower priority
cpus which should be preempted. This is opposed to the way normal
SCHED_OTHER tasks behave, which will wait for a periodic rebalancing
operation to occur before spreading out load.
When a particular RQ has more than 1 active RT task, it is said to
be in an "overloaded" state. Once this occurs, the system enters
the active balancing mode, where it will try to push the task away,
or persuade a different cpu to pull it over. The system will stay
in this state until the system falls back below the <= 1 queued RT
task per RQ.
However, the current implementation suffers from a limitation in the
push logic. Once overloaded, all tasks (other than current) on the
RQ are analyzed on every push operation, even if it was previously
unpushable (due to affinity, etc). Whats more, the operation stops
at the first task that is unpushable and will not look at items
lower in the queue. This causes two problems:
1) We can have the same tasks analyzed over and over again during each
push, which extends out the fast path in the scheduler for no
gain. Consider a RQ that has dozens of tasks that are bound to a
core. Each one of those tasks will be encountered and skipped
for each push operation while they are queued.
2) There may be lower-priority tasks under the unpushable task that
could have been successfully pushed, but will never be considered
until either the unpushable task is cleared, or a pull operation
succeeds. The net result is a potential latency source for mid
priority tasks.
This patch aims to rectify these two conditions by introducing a new
priority sorted list: "pushable_tasks". A task is added to the list
each time a task is activated or preempted. It is removed from the
list any time it is deactivated, made current, or fails to push.
This works because a task only needs to be attempted to push once.
After an initial failure to push, the other cpus will eventually try to
pull the task when the conditions are proper. This also solves the
problem that we don't completely analyze all tasks due to encountering
an unpushable tasks. Now every task will have a push attempted (when
appropriate).
This reduces latency both by shorting the critical section of the
rq->lock for certain workloads, and by making sure the algorithm
considers all eligible tasks in the system.
[ rostedt: added a couple more BUG_ONs ]
Signed-off-by: Gregory Haskins <ghaskins@novell.com>
Acked-by: Steven Rostedt <srostedt@redhat.com>
2008-12-29 15:39:53 +01:00
|
|
|
next_task = pick_next_pushable_task(rq);
|
2008-01-25 21:08:05 +01:00
|
|
|
if (!next_task)
|
|
|
|
return 0;
|
|
|
|
|
2010-10-17 21:46:10 +02:00
|
|
|
retry:
|
2008-01-25 21:08:09 +01:00
|
|
|
if (unlikely(next_task == rq->curr)) {
|
2008-01-25 21:08:07 +01:00
|
|
|
WARN_ON(1);
|
2008-01-25 21:08:05 +01:00
|
|
|
return 0;
|
2008-01-25 21:08:07 +01:00
|
|
|
}
|
2008-01-25 21:08:05 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* It's possible that the next_task slipped in of
|
|
|
|
* higher priority than current. If that's the case
|
|
|
|
* just reschedule current.
|
|
|
|
*/
|
2008-01-25 21:08:09 +01:00
|
|
|
if (unlikely(next_task->prio < rq->curr->prio)) {
|
|
|
|
resched_task(rq->curr);
|
2008-01-25 21:08:05 +01:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2008-01-25 21:08:09 +01:00
|
|
|
/* We might release rq lock */
|
2008-01-25 21:08:05 +01:00
|
|
|
get_task_struct(next_task);
|
|
|
|
|
|
|
|
/* find_lock_lowest_rq locks the rq if found */
|
2008-01-25 21:08:09 +01:00
|
|
|
lowest_rq = find_lock_lowest_rq(next_task, rq);
|
2008-01-25 21:08:05 +01:00
|
|
|
if (!lowest_rq) {
|
|
|
|
struct task_struct *task;
|
|
|
|
/*
|
2011-06-17 03:55:20 +02:00
|
|
|
* find_lock_lowest_rq releases rq->lock
|
2008-12-29 15:39:53 +01:00
|
|
|
* so it is possible that next_task has migrated.
|
|
|
|
*
|
|
|
|
* We need to make sure that the task is still on the same
|
|
|
|
* run-queue and is also still the next task eligible for
|
|
|
|
* pushing.
|
2008-01-25 21:08:05 +01:00
|
|
|
*/
|
sched: create "pushable_tasks" list to limit pushing to one attempt
The RT scheduler employs a "push/pull" design to actively balance tasks
within the system (on a per disjoint cpuset basis). When a task is
awoken, it is immediately determined if there are any lower priority
cpus which should be preempted. This is opposed to the way normal
SCHED_OTHER tasks behave, which will wait for a periodic rebalancing
operation to occur before spreading out load.
When a particular RQ has more than 1 active RT task, it is said to
be in an "overloaded" state. Once this occurs, the system enters
the active balancing mode, where it will try to push the task away,
or persuade a different cpu to pull it over. The system will stay
in this state until the system falls back below the <= 1 queued RT
task per RQ.
However, the current implementation suffers from a limitation in the
push logic. Once overloaded, all tasks (other than current) on the
RQ are analyzed on every push operation, even if it was previously
unpushable (due to affinity, etc). Whats more, the operation stops
at the first task that is unpushable and will not look at items
lower in the queue. This causes two problems:
1) We can have the same tasks analyzed over and over again during each
push, which extends out the fast path in the scheduler for no
gain. Consider a RQ that has dozens of tasks that are bound to a
core. Each one of those tasks will be encountered and skipped
for each push operation while they are queued.
2) There may be lower-priority tasks under the unpushable task that
could have been successfully pushed, but will never be considered
until either the unpushable task is cleared, or a pull operation
succeeds. The net result is a potential latency source for mid
priority tasks.
This patch aims to rectify these two conditions by introducing a new
priority sorted list: "pushable_tasks". A task is added to the list
each time a task is activated or preempted. It is removed from the
list any time it is deactivated, made current, or fails to push.
This works because a task only needs to be attempted to push once.
After an initial failure to push, the other cpus will eventually try to
pull the task when the conditions are proper. This also solves the
problem that we don't completely analyze all tasks due to encountering
an unpushable tasks. Now every task will have a push attempted (when
appropriate).
This reduces latency both by shorting the critical section of the
rq->lock for certain workloads, and by making sure the algorithm
considers all eligible tasks in the system.
[ rostedt: added a couple more BUG_ONs ]
Signed-off-by: Gregory Haskins <ghaskins@novell.com>
Acked-by: Steven Rostedt <srostedt@redhat.com>
2008-12-29 15:39:53 +01:00
|
|
|
task = pick_next_pushable_task(rq);
|
2008-12-29 15:39:53 +01:00
|
|
|
if (task_cpu(next_task) == rq->cpu && task == next_task) {
|
|
|
|
/*
|
2011-06-17 03:55:20 +02:00
|
|
|
* The task hasn't migrated, and is still the next
|
|
|
|
* eligible task, but we failed to find a run-queue
|
|
|
|
* to push it to. Do not retry in this case, since
|
|
|
|
* other cpus will pull from us when ready.
|
2008-12-29 15:39:53 +01:00
|
|
|
*/
|
|
|
|
goto out;
|
2008-01-25 21:08:05 +01:00
|
|
|
}
|
sched: create "pushable_tasks" list to limit pushing to one attempt
The RT scheduler employs a "push/pull" design to actively balance tasks
within the system (on a per disjoint cpuset basis). When a task is
awoken, it is immediately determined if there are any lower priority
cpus which should be preempted. This is opposed to the way normal
SCHED_OTHER tasks behave, which will wait for a periodic rebalancing
operation to occur before spreading out load.
When a particular RQ has more than 1 active RT task, it is said to
be in an "overloaded" state. Once this occurs, the system enters
the active balancing mode, where it will try to push the task away,
or persuade a different cpu to pull it over. The system will stay
in this state until the system falls back below the <= 1 queued RT
task per RQ.
However, the current implementation suffers from a limitation in the
push logic. Once overloaded, all tasks (other than current) on the
RQ are analyzed on every push operation, even if it was previously
unpushable (due to affinity, etc). Whats more, the operation stops
at the first task that is unpushable and will not look at items
lower in the queue. This causes two problems:
1) We can have the same tasks analyzed over and over again during each
push, which extends out the fast path in the scheduler for no
gain. Consider a RQ that has dozens of tasks that are bound to a
core. Each one of those tasks will be encountered and skipped
for each push operation while they are queued.
2) There may be lower-priority tasks under the unpushable task that
could have been successfully pushed, but will never be considered
until either the unpushable task is cleared, or a pull operation
succeeds. The net result is a potential latency source for mid
priority tasks.
This patch aims to rectify these two conditions by introducing a new
priority sorted list: "pushable_tasks". A task is added to the list
each time a task is activated or preempted. It is removed from the
list any time it is deactivated, made current, or fails to push.
This works because a task only needs to be attempted to push once.
After an initial failure to push, the other cpus will eventually try to
pull the task when the conditions are proper. This also solves the
problem that we don't completely analyze all tasks due to encountering
an unpushable tasks. Now every task will have a push attempted (when
appropriate).
This reduces latency both by shorting the critical section of the
rq->lock for certain workloads, and by making sure the algorithm
considers all eligible tasks in the system.
[ rostedt: added a couple more BUG_ONs ]
Signed-off-by: Gregory Haskins <ghaskins@novell.com>
Acked-by: Steven Rostedt <srostedt@redhat.com>
2008-12-29 15:39:53 +01:00
|
|
|
|
2008-12-29 15:39:53 +01:00
|
|
|
if (!task)
|
|
|
|
/* No more tasks, just exit */
|
|
|
|
goto out;
|
|
|
|
|
sched: create "pushable_tasks" list to limit pushing to one attempt
The RT scheduler employs a "push/pull" design to actively balance tasks
within the system (on a per disjoint cpuset basis). When a task is
awoken, it is immediately determined if there are any lower priority
cpus which should be preempted. This is opposed to the way normal
SCHED_OTHER tasks behave, which will wait for a periodic rebalancing
operation to occur before spreading out load.
When a particular RQ has more than 1 active RT task, it is said to
be in an "overloaded" state. Once this occurs, the system enters
the active balancing mode, where it will try to push the task away,
or persuade a different cpu to pull it over. The system will stay
in this state until the system falls back below the <= 1 queued RT
task per RQ.
However, the current implementation suffers from a limitation in the
push logic. Once overloaded, all tasks (other than current) on the
RQ are analyzed on every push operation, even if it was previously
unpushable (due to affinity, etc). Whats more, the operation stops
at the first task that is unpushable and will not look at items
lower in the queue. This causes two problems:
1) We can have the same tasks analyzed over and over again during each
push, which extends out the fast path in the scheduler for no
gain. Consider a RQ that has dozens of tasks that are bound to a
core. Each one of those tasks will be encountered and skipped
for each push operation while they are queued.
2) There may be lower-priority tasks under the unpushable task that
could have been successfully pushed, but will never be considered
until either the unpushable task is cleared, or a pull operation
succeeds. The net result is a potential latency source for mid
priority tasks.
This patch aims to rectify these two conditions by introducing a new
priority sorted list: "pushable_tasks". A task is added to the list
each time a task is activated or preempted. It is removed from the
list any time it is deactivated, made current, or fails to push.
This works because a task only needs to be attempted to push once.
After an initial failure to push, the other cpus will eventually try to
pull the task when the conditions are proper. This also solves the
problem that we don't completely analyze all tasks due to encountering
an unpushable tasks. Now every task will have a push attempted (when
appropriate).
This reduces latency both by shorting the critical section of the
rq->lock for certain workloads, and by making sure the algorithm
considers all eligible tasks in the system.
[ rostedt: added a couple more BUG_ONs ]
Signed-off-by: Gregory Haskins <ghaskins@novell.com>
Acked-by: Steven Rostedt <srostedt@redhat.com>
2008-12-29 15:39:53 +01:00
|
|
|
/*
|
2008-12-29 15:39:53 +01:00
|
|
|
* Something has shifted, try again.
|
sched: create "pushable_tasks" list to limit pushing to one attempt
The RT scheduler employs a "push/pull" design to actively balance tasks
within the system (on a per disjoint cpuset basis). When a task is
awoken, it is immediately determined if there are any lower priority
cpus which should be preempted. This is opposed to the way normal
SCHED_OTHER tasks behave, which will wait for a periodic rebalancing
operation to occur before spreading out load.
When a particular RQ has more than 1 active RT task, it is said to
be in an "overloaded" state. Once this occurs, the system enters
the active balancing mode, where it will try to push the task away,
or persuade a different cpu to pull it over. The system will stay
in this state until the system falls back below the <= 1 queued RT
task per RQ.
However, the current implementation suffers from a limitation in the
push logic. Once overloaded, all tasks (other than current) on the
RQ are analyzed on every push operation, even if it was previously
unpushable (due to affinity, etc). Whats more, the operation stops
at the first task that is unpushable and will not look at items
lower in the queue. This causes two problems:
1) We can have the same tasks analyzed over and over again during each
push, which extends out the fast path in the scheduler for no
gain. Consider a RQ that has dozens of tasks that are bound to a
core. Each one of those tasks will be encountered and skipped
for each push operation while they are queued.
2) There may be lower-priority tasks under the unpushable task that
could have been successfully pushed, but will never be considered
until either the unpushable task is cleared, or a pull operation
succeeds. The net result is a potential latency source for mid
priority tasks.
This patch aims to rectify these two conditions by introducing a new
priority sorted list: "pushable_tasks". A task is added to the list
each time a task is activated or preempted. It is removed from the
list any time it is deactivated, made current, or fails to push.
This works because a task only needs to be attempted to push once.
After an initial failure to push, the other cpus will eventually try to
pull the task when the conditions are proper. This also solves the
problem that we don't completely analyze all tasks due to encountering
an unpushable tasks. Now every task will have a push attempted (when
appropriate).
This reduces latency both by shorting the critical section of the
rq->lock for certain workloads, and by making sure the algorithm
considers all eligible tasks in the system.
[ rostedt: added a couple more BUG_ONs ]
Signed-off-by: Gregory Haskins <ghaskins@novell.com>
Acked-by: Steven Rostedt <srostedt@redhat.com>
2008-12-29 15:39:53 +01:00
|
|
|
*/
|
2008-12-29 15:39:53 +01:00
|
|
|
put_task_struct(next_task);
|
|
|
|
next_task = task;
|
|
|
|
goto retry;
|
2008-01-25 21:08:05 +01:00
|
|
|
}
|
|
|
|
|
2008-01-25 21:08:09 +01:00
|
|
|
deactivate_task(rq, next_task, 0);
|
2008-01-25 21:08:05 +01:00
|
|
|
set_task_cpu(next_task, lowest_rq->cpu);
|
|
|
|
activate_task(lowest_rq, next_task, 0);
|
2011-06-17 03:55:20 +02:00
|
|
|
ret = 1;
|
2008-01-25 21:08:05 +01:00
|
|
|
|
|
|
|
resched_task(lowest_rq->curr);
|
|
|
|
|
2008-08-11 09:30:22 +02:00
|
|
|
double_unlock_balance(rq, lowest_rq);
|
2008-01-25 21:08:05 +01:00
|
|
|
|
|
|
|
out:
|
|
|
|
put_task_struct(next_task);
|
|
|
|
|
2011-06-17 03:55:20 +02:00
|
|
|
return ret;
|
2008-01-25 21:08:05 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
static void push_rt_tasks(struct rq *rq)
|
|
|
|
{
|
|
|
|
/* push_rt_task will return true if it moved an RT */
|
|
|
|
while (push_rt_task(rq))
|
|
|
|
;
|
|
|
|
}
|
|
|
|
|
2008-01-25 21:08:07 +01:00
|
|
|
static int pull_rt_task(struct rq *this_rq)
|
|
|
|
{
|
2008-01-25 21:08:17 +01:00
|
|
|
int this_cpu = this_rq->cpu, ret = 0, cpu;
|
2008-12-29 15:39:49 +01:00
|
|
|
struct task_struct *p;
|
2008-01-25 21:08:07 +01:00
|
|
|
struct rq *src_rq;
|
|
|
|
|
2008-01-25 21:08:18 +01:00
|
|
|
if (likely(!rt_overloaded(this_rq)))
|
2008-01-25 21:08:07 +01:00
|
|
|
return 0;
|
|
|
|
|
2013-10-15 12:35:07 +02:00
|
|
|
/*
|
|
|
|
* Match the barrier from rt_set_overloaded; this guarantees that if we
|
|
|
|
* see overloaded we must also see the rto_mask bit.
|
|
|
|
*/
|
|
|
|
smp_rmb();
|
|
|
|
|
2008-11-24 17:05:05 +01:00
|
|
|
for_each_cpu(cpu, this_rq->rd->rto_mask) {
|
2008-01-25 21:08:07 +01:00
|
|
|
if (this_cpu == cpu)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
src_rq = cpu_rq(cpu);
|
2008-12-29 15:39:50 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Don't bother taking the src_rq->lock if the next highest
|
|
|
|
* task is known to be lower-priority than our current task.
|
|
|
|
* This may look racy, but if this value is about to go
|
|
|
|
* logically higher, the src_rq will push this task away.
|
|
|
|
* And if its going logically lower, we do not care
|
|
|
|
*/
|
|
|
|
if (src_rq->rt.highest_prio.next >=
|
|
|
|
this_rq->rt.highest_prio.curr)
|
|
|
|
continue;
|
|
|
|
|
2008-01-25 21:08:07 +01:00
|
|
|
/*
|
|
|
|
* We can potentially drop this_rq's lock in
|
|
|
|
* double_lock_balance, and another CPU could
|
2008-12-29 15:39:49 +01:00
|
|
|
* alter this_rq
|
2008-01-25 21:08:07 +01:00
|
|
|
*/
|
2008-12-29 15:39:49 +01:00
|
|
|
double_lock_balance(this_rq, src_rq);
|
2008-01-25 21:08:07 +01:00
|
|
|
|
|
|
|
/*
|
2013-06-07 21:37:43 +02:00
|
|
|
* We can pull only a task, which is pushable
|
|
|
|
* on its rq, and no others.
|
2008-01-25 21:08:07 +01:00
|
|
|
*/
|
2013-06-07 21:37:43 +02:00
|
|
|
p = pick_highest_pushable_task(src_rq, this_cpu);
|
2008-01-25 21:08:07 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Do we have an RT task that preempts
|
|
|
|
* the to-be-scheduled task?
|
|
|
|
*/
|
2008-12-29 15:39:49 +01:00
|
|
|
if (p && (p->prio < this_rq->rt.highest_prio.curr)) {
|
2008-01-25 21:08:07 +01:00
|
|
|
WARN_ON(p == src_rq->curr);
|
2011-04-05 17:23:44 +02:00
|
|
|
WARN_ON(!p->on_rq);
|
2008-01-25 21:08:07 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* There's a chance that p is higher in priority
|
|
|
|
* than what's currently running on its cpu.
|
|
|
|
* This is just that p is wakeing up and hasn't
|
|
|
|
* had a chance to schedule. We only pull
|
|
|
|
* p if it is lower in priority than the
|
2008-12-29 15:39:49 +01:00
|
|
|
* current task on the run queue
|
2008-01-25 21:08:07 +01:00
|
|
|
*/
|
2008-12-29 15:39:49 +01:00
|
|
|
if (p->prio < src_rq->curr->prio)
|
2008-01-25 21:08:30 +01:00
|
|
|
goto skip;
|
2008-01-25 21:08:07 +01:00
|
|
|
|
|
|
|
ret = 1;
|
|
|
|
|
|
|
|
deactivate_task(src_rq, p, 0);
|
|
|
|
set_task_cpu(p, this_cpu);
|
|
|
|
activate_task(this_rq, p, 0);
|
|
|
|
/*
|
|
|
|
* We continue with the search, just in
|
|
|
|
* case there's an even higher prio task
|
2011-03-31 03:57:33 +02:00
|
|
|
* in another runqueue. (low likelihood
|
2008-01-25 21:08:07 +01:00
|
|
|
* but possible)
|
|
|
|
*/
|
|
|
|
}
|
2010-10-17 21:46:10 +02:00
|
|
|
skip:
|
2008-08-11 09:30:22 +02:00
|
|
|
double_unlock_balance(this_rq, src_rq);
|
2008-01-25 21:08:07 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2008-01-25 21:08:22 +01:00
|
|
|
static void pre_schedule_rt(struct rq *rq, struct task_struct *prev)
|
2008-01-25 21:08:07 +01:00
|
|
|
{
|
|
|
|
/* Try to pull RT tasks here if we lower this rq's prio */
|
2010-02-09 20:43:59 +01:00
|
|
|
if (rq->rt.highest_prio.curr > prev->prio)
|
2008-01-25 21:08:07 +01:00
|
|
|
pull_rt_task(rq);
|
|
|
|
}
|
|
|
|
|
2008-01-25 21:08:22 +01:00
|
|
|
static void post_schedule_rt(struct rq *rq)
|
2008-01-25 21:08:05 +01:00
|
|
|
{
|
2008-12-29 15:39:52 +01:00
|
|
|
push_rt_tasks(rq);
|
2008-01-25 21:08:05 +01:00
|
|
|
}
|
|
|
|
|
2008-04-23 13:13:29 +02:00
|
|
|
/*
|
|
|
|
* If we are not running and we are not going to reschedule soon, we should
|
|
|
|
* try to push tasks away now
|
|
|
|
*/
|
2009-12-16 18:04:40 +01:00
|
|
|
static void task_woken_rt(struct rq *rq, struct task_struct *p)
|
2008-01-25 21:08:07 +01:00
|
|
|
{
|
2008-01-25 21:08:22 +01:00
|
|
|
if (!task_running(rq, p) &&
|
2008-04-23 13:13:29 +02:00
|
|
|
!test_tsk_need_resched(rq->curr) &&
|
sched: create "pushable_tasks" list to limit pushing to one attempt
The RT scheduler employs a "push/pull" design to actively balance tasks
within the system (on a per disjoint cpuset basis). When a task is
awoken, it is immediately determined if there are any lower priority
cpus which should be preempted. This is opposed to the way normal
SCHED_OTHER tasks behave, which will wait for a periodic rebalancing
operation to occur before spreading out load.
When a particular RQ has more than 1 active RT task, it is said to
be in an "overloaded" state. Once this occurs, the system enters
the active balancing mode, where it will try to push the task away,
or persuade a different cpu to pull it over. The system will stay
in this state until the system falls back below the <= 1 queued RT
task per RQ.
However, the current implementation suffers from a limitation in the
push logic. Once overloaded, all tasks (other than current) on the
RQ are analyzed on every push operation, even if it was previously
unpushable (due to affinity, etc). Whats more, the operation stops
at the first task that is unpushable and will not look at items
lower in the queue. This causes two problems:
1) We can have the same tasks analyzed over and over again during each
push, which extends out the fast path in the scheduler for no
gain. Consider a RQ that has dozens of tasks that are bound to a
core. Each one of those tasks will be encountered and skipped
for each push operation while they are queued.
2) There may be lower-priority tasks under the unpushable task that
could have been successfully pushed, but will never be considered
until either the unpushable task is cleared, or a pull operation
succeeds. The net result is a potential latency source for mid
priority tasks.
This patch aims to rectify these two conditions by introducing a new
priority sorted list: "pushable_tasks". A task is added to the list
each time a task is activated or preempted. It is removed from the
list any time it is deactivated, made current, or fails to push.
This works because a task only needs to be attempted to push once.
After an initial failure to push, the other cpus will eventually try to
pull the task when the conditions are proper. This also solves the
problem that we don't completely analyze all tasks due to encountering
an unpushable tasks. Now every task will have a push attempted (when
appropriate).
This reduces latency both by shorting the critical section of the
rq->lock for certain workloads, and by making sure the algorithm
considers all eligible tasks in the system.
[ rostedt: added a couple more BUG_ONs ]
Signed-off-by: Gregory Haskins <ghaskins@novell.com>
Acked-by: Steven Rostedt <srostedt@redhat.com>
2008-12-29 15:39:53 +01:00
|
|
|
has_pushable_tasks(rq) &&
|
2012-04-23 12:11:21 +02:00
|
|
|
p->nr_cpus_allowed > 1 &&
|
sched/deadline: Add SCHED_DEADLINE SMP-related data structures & logic
Introduces data structures relevant for implementing dynamic
migration of -deadline tasks and the logic for checking if
runqueues are overloaded with -deadline tasks and for choosing
where a task should migrate, when it is the case.
Adds also dynamic migrations to SCHED_DEADLINE, so that tasks can
be moved among CPUs when necessary. It is also possible to bind a
task to a (set of) CPU(s), thus restricting its capability of
migrating, or forbidding migrations at all.
The very same approach used in sched_rt is utilised:
- -deadline tasks are kept into CPU-specific runqueues,
- -deadline tasks are migrated among runqueues to achieve the
following:
* on an M-CPU system the M earliest deadline ready tasks
are always running;
* affinity/cpusets settings of all the -deadline tasks is
always respected.
Therefore, this very special form of "load balancing" is done with
an active method, i.e., the scheduler pushes or pulls tasks between
runqueues when they are woken up and/or (de)scheduled.
IOW, every time a preemption occurs, the descheduled task might be sent
to some other CPU (depending on its deadline) to continue executing
(push). On the other hand, every time a CPU becomes idle, it might pull
the second earliest deadline ready task from some other CPU.
To enforce this, a pull operation is always attempted before taking any
scheduling decision (pre_schedule()), as well as a push one after each
scheduling decision (post_schedule()). In addition, when a task arrives
or wakes up, the best CPU where to resume it is selected taking into
account its affinity mask, the system topology, but also its deadline.
E.g., from the scheduling point of view, the best CPU where to wake
up (and also where to push) a task is the one which is running the task
with the latest deadline among the M executing ones.
In order to facilitate these decisions, per-runqueue "caching" of the
deadlines of the currently running and of the first ready task is used.
Queued but not running tasks are also parked in another rb-tree to
speed-up pushes.
Signed-off-by: Juri Lelli <juri.lelli@gmail.com>
Signed-off-by: Dario Faggioli <raistlin@linux.it>
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1383831828-15501-5-git-send-email-juri.lelli@gmail.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-11-07 14:43:38 +01:00
|
|
|
(dl_task(rq->curr) || rt_task(rq->curr)) &&
|
2012-04-23 12:11:21 +02:00
|
|
|
(rq->curr->nr_cpus_allowed < 2 ||
|
2011-09-12 16:28:04 +02:00
|
|
|
rq->curr->prio <= p->prio))
|
2008-01-25 21:08:07 +01:00
|
|
|
push_rt_tasks(rq);
|
|
|
|
}
|
|
|
|
|
2008-03-26 22:23:49 +01:00
|
|
|
static void set_cpus_allowed_rt(struct task_struct *p,
|
2008-11-24 17:05:14 +01:00
|
|
|
const struct cpumask *new_mask)
|
2008-01-25 21:08:07 +01:00
|
|
|
{
|
2012-04-11 07:06:04 +02:00
|
|
|
struct rq *rq;
|
|
|
|
int weight;
|
2008-01-25 21:08:07 +01:00
|
|
|
|
|
|
|
BUG_ON(!rt_task(p));
|
|
|
|
|
2012-04-11 07:06:04 +02:00
|
|
|
if (!p->on_rq)
|
|
|
|
return;
|
sched: create "pushable_tasks" list to limit pushing to one attempt
The RT scheduler employs a "push/pull" design to actively balance tasks
within the system (on a per disjoint cpuset basis). When a task is
awoken, it is immediately determined if there are any lower priority
cpus which should be preempted. This is opposed to the way normal
SCHED_OTHER tasks behave, which will wait for a periodic rebalancing
operation to occur before spreading out load.
When a particular RQ has more than 1 active RT task, it is said to
be in an "overloaded" state. Once this occurs, the system enters
the active balancing mode, where it will try to push the task away,
or persuade a different cpu to pull it over. The system will stay
in this state until the system falls back below the <= 1 queued RT
task per RQ.
However, the current implementation suffers from a limitation in the
push logic. Once overloaded, all tasks (other than current) on the
RQ are analyzed on every push operation, even if it was previously
unpushable (due to affinity, etc). Whats more, the operation stops
at the first task that is unpushable and will not look at items
lower in the queue. This causes two problems:
1) We can have the same tasks analyzed over and over again during each
push, which extends out the fast path in the scheduler for no
gain. Consider a RQ that has dozens of tasks that are bound to a
core. Each one of those tasks will be encountered and skipped
for each push operation while they are queued.
2) There may be lower-priority tasks under the unpushable task that
could have been successfully pushed, but will never be considered
until either the unpushable task is cleared, or a pull operation
succeeds. The net result is a potential latency source for mid
priority tasks.
This patch aims to rectify these two conditions by introducing a new
priority sorted list: "pushable_tasks". A task is added to the list
each time a task is activated or preempted. It is removed from the
list any time it is deactivated, made current, or fails to push.
This works because a task only needs to be attempted to push once.
After an initial failure to push, the other cpus will eventually try to
pull the task when the conditions are proper. This also solves the
problem that we don't completely analyze all tasks due to encountering
an unpushable tasks. Now every task will have a push attempted (when
appropriate).
This reduces latency both by shorting the critical section of the
rq->lock for certain workloads, and by making sure the algorithm
considers all eligible tasks in the system.
[ rostedt: added a couple more BUG_ONs ]
Signed-off-by: Gregory Haskins <ghaskins@novell.com>
Acked-by: Steven Rostedt <srostedt@redhat.com>
2008-12-29 15:39:53 +01:00
|
|
|
|
2012-04-11 07:06:04 +02:00
|
|
|
weight = cpumask_weight(new_mask);
|
sched: create "pushable_tasks" list to limit pushing to one attempt
The RT scheduler employs a "push/pull" design to actively balance tasks
within the system (on a per disjoint cpuset basis). When a task is
awoken, it is immediately determined if there are any lower priority
cpus which should be preempted. This is opposed to the way normal
SCHED_OTHER tasks behave, which will wait for a periodic rebalancing
operation to occur before spreading out load.
When a particular RQ has more than 1 active RT task, it is said to
be in an "overloaded" state. Once this occurs, the system enters
the active balancing mode, where it will try to push the task away,
or persuade a different cpu to pull it over. The system will stay
in this state until the system falls back below the <= 1 queued RT
task per RQ.
However, the current implementation suffers from a limitation in the
push logic. Once overloaded, all tasks (other than current) on the
RQ are analyzed on every push operation, even if it was previously
unpushable (due to affinity, etc). Whats more, the operation stops
at the first task that is unpushable and will not look at items
lower in the queue. This causes two problems:
1) We can have the same tasks analyzed over and over again during each
push, which extends out the fast path in the scheduler for no
gain. Consider a RQ that has dozens of tasks that are bound to a
core. Each one of those tasks will be encountered and skipped
for each push operation while they are queued.
2) There may be lower-priority tasks under the unpushable task that
could have been successfully pushed, but will never be considered
until either the unpushable task is cleared, or a pull operation
succeeds. The net result is a potential latency source for mid
priority tasks.
This patch aims to rectify these two conditions by introducing a new
priority sorted list: "pushable_tasks". A task is added to the list
each time a task is activated or preempted. It is removed from the
list any time it is deactivated, made current, or fails to push.
This works because a task only needs to be attempted to push once.
After an initial failure to push, the other cpus will eventually try to
pull the task when the conditions are proper. This also solves the
problem that we don't completely analyze all tasks due to encountering
an unpushable tasks. Now every task will have a push attempted (when
appropriate).
This reduces latency both by shorting the critical section of the
rq->lock for certain workloads, and by making sure the algorithm
considers all eligible tasks in the system.
[ rostedt: added a couple more BUG_ONs ]
Signed-off-by: Gregory Haskins <ghaskins@novell.com>
Acked-by: Steven Rostedt <srostedt@redhat.com>
2008-12-29 15:39:53 +01:00
|
|
|
|
2012-04-11 07:06:04 +02:00
|
|
|
/*
|
|
|
|
* Only update if the process changes its state from whether it
|
|
|
|
* can migrate or not.
|
|
|
|
*/
|
2012-04-23 12:11:21 +02:00
|
|
|
if ((p->nr_cpus_allowed > 1) == (weight > 1))
|
2012-04-11 07:06:04 +02:00
|
|
|
return;
|
sched: create "pushable_tasks" list to limit pushing to one attempt
The RT scheduler employs a "push/pull" design to actively balance tasks
within the system (on a per disjoint cpuset basis). When a task is
awoken, it is immediately determined if there are any lower priority
cpus which should be preempted. This is opposed to the way normal
SCHED_OTHER tasks behave, which will wait for a periodic rebalancing
operation to occur before spreading out load.
When a particular RQ has more than 1 active RT task, it is said to
be in an "overloaded" state. Once this occurs, the system enters
the active balancing mode, where it will try to push the task away,
or persuade a different cpu to pull it over. The system will stay
in this state until the system falls back below the <= 1 queued RT
task per RQ.
However, the current implementation suffers from a limitation in the
push logic. Once overloaded, all tasks (other than current) on the
RQ are analyzed on every push operation, even if it was previously
unpushable (due to affinity, etc). Whats more, the operation stops
at the first task that is unpushable and will not look at items
lower in the queue. This causes two problems:
1) We can have the same tasks analyzed over and over again during each
push, which extends out the fast path in the scheduler for no
gain. Consider a RQ that has dozens of tasks that are bound to a
core. Each one of those tasks will be encountered and skipped
for each push operation while they are queued.
2) There may be lower-priority tasks under the unpushable task that
could have been successfully pushed, but will never be considered
until either the unpushable task is cleared, or a pull operation
succeeds. The net result is a potential latency source for mid
priority tasks.
This patch aims to rectify these two conditions by introducing a new
priority sorted list: "pushable_tasks". A task is added to the list
each time a task is activated or preempted. It is removed from the
list any time it is deactivated, made current, or fails to push.
This works because a task only needs to be attempted to push once.
After an initial failure to push, the other cpus will eventually try to
pull the task when the conditions are proper. This also solves the
problem that we don't completely analyze all tasks due to encountering
an unpushable tasks. Now every task will have a push attempted (when
appropriate).
This reduces latency both by shorting the critical section of the
rq->lock for certain workloads, and by making sure the algorithm
considers all eligible tasks in the system.
[ rostedt: added a couple more BUG_ONs ]
Signed-off-by: Gregory Haskins <ghaskins@novell.com>
Acked-by: Steven Rostedt <srostedt@redhat.com>
2008-12-29 15:39:53 +01:00
|
|
|
|
2012-04-11 07:06:04 +02:00
|
|
|
rq = task_rq(p);
|
2008-01-25 21:08:07 +01:00
|
|
|
|
2012-04-11 07:06:04 +02:00
|
|
|
/*
|
|
|
|
* The process used to be able to migrate OR it can now migrate
|
|
|
|
*/
|
|
|
|
if (weight <= 1) {
|
|
|
|
if (!task_current(rq, p))
|
|
|
|
dequeue_pushable_task(rq, p);
|
|
|
|
BUG_ON(!rq->rt.rt_nr_migratory);
|
|
|
|
rq->rt.rt_nr_migratory--;
|
|
|
|
} else {
|
|
|
|
if (!task_current(rq, p))
|
|
|
|
enqueue_pushable_task(rq, p);
|
|
|
|
rq->rt.rt_nr_migratory++;
|
2008-01-25 21:08:07 +01:00
|
|
|
}
|
2012-04-11 07:06:04 +02:00
|
|
|
|
|
|
|
update_rt_migration(&rq->rt);
|
2008-01-25 21:08:07 +01:00
|
|
|
}
|
2008-01-25 21:08:15 +01:00
|
|
|
|
2008-01-25 21:08:18 +01:00
|
|
|
/* Assumes rq->lock is held */
|
2008-06-04 21:04:05 +02:00
|
|
|
static void rq_online_rt(struct rq *rq)
|
2008-01-25 21:08:18 +01:00
|
|
|
{
|
|
|
|
if (rq->rt.overloaded)
|
|
|
|
rt_set_overload(rq);
|
2008-05-12 21:21:01 +02:00
|
|
|
|
2008-06-05 14:49:58 +02:00
|
|
|
__enable_runtime(rq);
|
|
|
|
|
2008-12-29 15:39:49 +01:00
|
|
|
cpupri_set(&rq->rd->cpupri, rq->cpu, rq->rt.highest_prio.curr);
|
2008-01-25 21:08:18 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Assumes rq->lock is held */
|
2008-06-04 21:04:05 +02:00
|
|
|
static void rq_offline_rt(struct rq *rq)
|
2008-01-25 21:08:18 +01:00
|
|
|
{
|
|
|
|
if (rq->rt.overloaded)
|
|
|
|
rt_clear_overload(rq);
|
2008-05-12 21:21:01 +02:00
|
|
|
|
2008-06-05 14:49:58 +02:00
|
|
|
__disable_runtime(rq);
|
|
|
|
|
2008-05-12 21:21:01 +02:00
|
|
|
cpupri_set(&rq->rd->cpupri, rq->cpu, CPUPRI_INVALID);
|
2008-01-25 21:08:18 +01:00
|
|
|
}
|
2008-01-25 21:08:22 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* When switch from the rt queue, we bring ourselves to a position
|
|
|
|
* that we might want to pull RT tasks from other runqueues.
|
|
|
|
*/
|
2011-01-17 17:03:27 +01:00
|
|
|
static void switched_from_rt(struct rq *rq, struct task_struct *p)
|
2008-01-25 21:08:22 +01:00
|
|
|
{
|
|
|
|
/*
|
|
|
|
* If there are other RT tasks then we will reschedule
|
|
|
|
* and the scheduling of the other RT tasks will handle
|
|
|
|
* the balancing. But if we are the last RT task
|
|
|
|
* we may need to handle the pulling of RT tasks
|
|
|
|
* now.
|
|
|
|
*/
|
2012-11-22 21:02:15 +01:00
|
|
|
if (!p->on_rq || rq->rt.rt_nr_running)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (pull_rt_task(rq))
|
|
|
|
resched_task(rq->curr);
|
2008-01-25 21:08:22 +01:00
|
|
|
}
|
2008-11-25 00:28:41 +01:00
|
|
|
|
2011-10-25 10:00:11 +02:00
|
|
|
void init_sched_rt_class(void)
|
2008-11-25 00:28:41 +01:00
|
|
|
{
|
|
|
|
unsigned int i;
|
|
|
|
|
2011-10-25 10:00:11 +02:00
|
|
|
for_each_possible_cpu(i) {
|
2009-06-06 23:51:36 +02:00
|
|
|
zalloc_cpumask_var_node(&per_cpu(local_cpu_mask, i),
|
2009-01-01 03:08:45 +01:00
|
|
|
GFP_KERNEL, cpu_to_node(i));
|
2011-10-25 10:00:11 +02:00
|
|
|
}
|
2008-11-25 00:28:41 +01:00
|
|
|
}
|
2008-01-25 21:08:22 +01:00
|
|
|
#endif /* CONFIG_SMP */
|
|
|
|
|
|
|
|
/*
|
|
|
|
* When switching a task to RT, we may overload the runqueue
|
|
|
|
* with RT tasks. In this case we try to push them off to
|
|
|
|
* other runqueues.
|
|
|
|
*/
|
2011-01-17 17:03:27 +01:00
|
|
|
static void switched_to_rt(struct rq *rq, struct task_struct *p)
|
2008-01-25 21:08:22 +01:00
|
|
|
{
|
|
|
|
int check_resched = 1;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If we are already running, then there's nothing
|
|
|
|
* that needs to be done. But if we are not running
|
|
|
|
* we may need to preempt the current running task.
|
|
|
|
* If that current running task is also an RT task
|
|
|
|
* then see if we can move to another run queue.
|
|
|
|
*/
|
2011-04-05 17:23:44 +02:00
|
|
|
if (p->on_rq && rq->curr != p) {
|
2008-01-25 21:08:22 +01:00
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
if (rq->rt.overloaded && push_rt_task(rq) &&
|
|
|
|
/* Don't resched if we changed runqueues */
|
|
|
|
rq != task_rq(p))
|
|
|
|
check_resched = 0;
|
|
|
|
#endif /* CONFIG_SMP */
|
|
|
|
if (check_resched && p->prio < rq->curr->prio)
|
|
|
|
resched_task(rq->curr);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Priority of the task has changed. This may cause
|
|
|
|
* us to initiate a push or pull.
|
|
|
|
*/
|
2011-01-17 17:03:27 +01:00
|
|
|
static void
|
|
|
|
prio_changed_rt(struct rq *rq, struct task_struct *p, int oldprio)
|
2008-01-25 21:08:22 +01:00
|
|
|
{
|
2011-04-05 17:23:44 +02:00
|
|
|
if (!p->on_rq)
|
2011-01-17 17:03:27 +01:00
|
|
|
return;
|
|
|
|
|
|
|
|
if (rq->curr == p) {
|
2008-01-25 21:08:22 +01:00
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
/*
|
|
|
|
* If our priority decreases while running, we
|
|
|
|
* may need to pull tasks to this runqueue.
|
|
|
|
*/
|
|
|
|
if (oldprio < p->prio)
|
|
|
|
pull_rt_task(rq);
|
|
|
|
/*
|
|
|
|
* If there's a higher priority task waiting to run
|
2008-03-05 16:00:12 +01:00
|
|
|
* then reschedule. Note, the above pull_rt_task
|
|
|
|
* can release the rq lock and p could migrate.
|
|
|
|
* Only reschedule if p is still on the same runqueue.
|
2008-01-25 21:08:22 +01:00
|
|
|
*/
|
2008-12-29 15:39:49 +01:00
|
|
|
if (p->prio > rq->rt.highest_prio.curr && rq->curr == p)
|
2008-01-25 21:08:22 +01:00
|
|
|
resched_task(p);
|
|
|
|
#else
|
|
|
|
/* For UP simply resched on drop of prio */
|
|
|
|
if (oldprio < p->prio)
|
|
|
|
resched_task(p);
|
2008-01-25 21:08:05 +01:00
|
|
|
#endif /* CONFIG_SMP */
|
2008-01-25 21:08:22 +01:00
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* This task is not running, but if it is
|
|
|
|
* greater than the current running task
|
|
|
|
* then reschedule.
|
|
|
|
*/
|
|
|
|
if (p->prio < rq->curr->prio)
|
|
|
|
resched_task(rq->curr);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2008-01-25 21:08:27 +01:00
|
|
|
static void watchdog(struct rq *rq, struct task_struct *p)
|
|
|
|
{
|
|
|
|
unsigned long soft, hard;
|
|
|
|
|
2010-03-05 22:42:54 +01:00
|
|
|
/* max may change after cur was read, this will be fixed next tick */
|
|
|
|
soft = task_rlimit(p, RLIMIT_RTTIME);
|
|
|
|
hard = task_rlimit_max(p, RLIMIT_RTTIME);
|
2008-01-25 21:08:27 +01:00
|
|
|
|
|
|
|
if (soft != RLIM_INFINITY) {
|
|
|
|
unsigned long next;
|
|
|
|
|
sched/rt: Avoid updating RT entry timeout twice within one tick period
The issue below was found in 2.6.34-rt rather than mainline rt
kernel, but the issue still exists upstream as well.
So please let me describe how it was noticed on 2.6.34-rt:
On this version, each softirq has its own thread, it means there
is at least one RT FIFO task per cpu. The priority of these
tasks is set to 49 by default. If user launches an RT FIFO task
with priority lower than 49 of softirq RT tasks, it's possible
there are two RT FIFO tasks enqueued one cpu runqueue at one
moment. By current strategy of balancing RT tasks, when it comes
to RT tasks, we really need to put them off to a CPU that they
can run on as soon as possible. Even if it means a bit of cache
line flushing, we want RT tasks to be run with the least latency.
When the user RT FIFO task which just launched before is
running, the sched timer tick of the current cpu happens. In this
tick period, the timeout value of the user RT task will be
updated once. Subsequently, we try to wake up one softirq RT
task on its local cpu. As the priority of current user RT task
is lower than the softirq RT task, the current task will be
preempted by the higher priority softirq RT task. Before
preemption, we check to see if current can readily move to a
different cpu. If so, we will reschedule to allow the RT push logic
to try to move current somewhere else. Whenever the woken
softirq RT task runs, it first tries to migrate the user FIFO RT
task over to a cpu that is running a task of lesser priority. If
migration is done, it will send a reschedule request to the found
cpu by IPI interrupt. Once the target cpu responds the IPI
interrupt, it will pick the migrated user RT task to preempt its
current task. When the user RT task is running on the new cpu,
the sched timer tick of the cpu fires. So it will tick the user
RT task again. This also means the RT task timeout value will be
updated again. As the migration may be done in one tick period,
it means the user RT task timeout value will be updated twice
within one tick.
If we set a limit on the amount of cpu time for the user RT task
by setrlimit(RLIMIT_RTTIME), the SIGXCPU signal should be posted
upon reaching the soft limit.
But exactly when the SIGXCPU signal should be sent depends on the
RT task timeout value. In fact the timeout mechanism of sending
the SIGXCPU signal assumes the RT task timeout is increased once
every tick.
However, currently the timeout value may be added twice per
tick. So it results in the SIGXCPU signal being sent earlier
than expected.
To solve this issue, we prevent the timeout value from increasing
twice within one tick time by remembering the jiffies value of
last updating the timeout. As long as the RT task's jiffies is
different with the global jiffies value, we allow its timeout to
be updated.
Signed-off-by: Ying Xue <ying.xue@windriver.com>
Signed-off-by: Fan Du <fan.du@windriver.com>
Reviewed-by: Yong Zhang <yong.zhang0@gmail.com>
Acked-by: Steven Rostedt <rostedt@goodmis.org>
Cc: <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1342508623-2887-1-git-send-email-ying.xue@windriver.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2012-07-17 09:03:43 +02:00
|
|
|
if (p->rt.watchdog_stamp != jiffies) {
|
|
|
|
p->rt.timeout++;
|
|
|
|
p->rt.watchdog_stamp = jiffies;
|
|
|
|
}
|
|
|
|
|
2008-01-25 21:08:27 +01:00
|
|
|
next = DIV_ROUND_UP(min(soft, hard), USEC_PER_SEC/HZ);
|
2008-01-25 21:08:32 +01:00
|
|
|
if (p->rt.timeout > next)
|
timers: fix itimer/many thread hang
Overview
This patch reworks the handling of POSIX CPU timers, including the
ITIMER_PROF, ITIMER_VIRT timers and rlimit handling. It was put together
with the help of Roland McGrath, the owner and original writer of this code.
The problem we ran into, and the reason for this rework, has to do with using
a profiling timer in a process with a large number of threads. It appears
that the performance of the old implementation of run_posix_cpu_timers() was
at least O(n*3) (where "n" is the number of threads in a process) or worse.
Everything is fine with an increasing number of threads until the time taken
for that routine to run becomes the same as or greater than the tick time, at
which point things degrade rather quickly.
This patch fixes bug 9906, "Weird hang with NPTL and SIGPROF."
Code Changes
This rework corrects the implementation of run_posix_cpu_timers() to make it
run in constant time for a particular machine. (Performance may vary between
one machine and another depending upon whether the kernel is built as single-
or multiprocessor and, in the latter case, depending upon the number of
running processors.) To do this, at each tick we now update fields in
signal_struct as well as task_struct. The run_posix_cpu_timers() function
uses those fields to make its decisions.
We define a new structure, "task_cputime," to contain user, system and
scheduler times and use these in appropriate places:
struct task_cputime {
cputime_t utime;
cputime_t stime;
unsigned long long sum_exec_runtime;
};
This is included in the structure "thread_group_cputime," which is a new
substructure of signal_struct and which varies for uniprocessor versus
multiprocessor kernels. For uniprocessor kernels, it uses "task_cputime" as
a simple substructure, while for multiprocessor kernels it is a pointer:
struct thread_group_cputime {
struct task_cputime totals;
};
struct thread_group_cputime {
struct task_cputime *totals;
};
We also add a new task_cputime substructure directly to signal_struct, to
cache the earliest expiration of process-wide timers, and task_cputime also
replaces the it_*_expires fields of task_struct (used for earliest expiration
of thread timers). The "thread_group_cputime" structure contains process-wide
timers that are updated via account_user_time() and friends. In the non-SMP
case the structure is a simple aggregator; unfortunately in the SMP case that
simplicity was not achievable due to cache-line contention between CPUs (in
one measured case performance was actually _worse_ on a 16-cpu system than
the same test on a 4-cpu system, due to this contention). For SMP, the
thread_group_cputime counters are maintained as a per-cpu structure allocated
using alloc_percpu(). The timer functions update only the timer field in
the structure corresponding to the running CPU, obtained using per_cpu_ptr().
We define a set of inline functions in sched.h that we use to maintain the
thread_group_cputime structure and hide the differences between UP and SMP
implementations from the rest of the kernel. The thread_group_cputime_init()
function initializes the thread_group_cputime structure for the given task.
The thread_group_cputime_alloc() is a no-op for UP; for SMP it calls the
out-of-line function thread_group_cputime_alloc_smp() to allocate and fill
in the per-cpu structures and fields. The thread_group_cputime_free()
function, also a no-op for UP, in SMP frees the per-cpu structures. The
thread_group_cputime_clone_thread() function (also a UP no-op) for SMP calls
thread_group_cputime_alloc() if the per-cpu structures haven't yet been
allocated. The thread_group_cputime() function fills the task_cputime
structure it is passed with the contents of the thread_group_cputime fields;
in UP it's that simple but in SMP it must also safely check that tsk->signal
is non-NULL (if it is it just uses the appropriate fields of task_struct) and,
if so, sums the per-cpu values for each online CPU. Finally, the three
functions account_group_user_time(), account_group_system_time() and
account_group_exec_runtime() are used by timer functions to update the
respective fields of the thread_group_cputime structure.
Non-SMP operation is trivial and will not be mentioned further.
The per-cpu structure is always allocated when a task creates its first new
thread, via a call to thread_group_cputime_clone_thread() from copy_signal().
It is freed at process exit via a call to thread_group_cputime_free() from
cleanup_signal().
All functions that formerly summed utime/stime/sum_sched_runtime values from
from all threads in the thread group now use thread_group_cputime() to
snapshot the values in the thread_group_cputime structure or the values in
the task structure itself if the per-cpu structure hasn't been allocated.
Finally, the code in kernel/posix-cpu-timers.c has changed quite a bit.
The run_posix_cpu_timers() function has been split into a fast path and a
slow path; the former safely checks whether there are any expired thread
timers and, if not, just returns, while the slow path does the heavy lifting.
With the dedicated thread group fields, timers are no longer "rebalanced" and
the process_timer_rebalance() function and related code has gone away. All
summing loops are gone and all code that used them now uses the
thread_group_cputime() inline. When process-wide timers are set, the new
task_cputime structure in signal_struct is used to cache the earliest
expiration; this is checked in the fast path.
Performance
The fix appears not to add significant overhead to existing operations. It
generally performs the same as the current code except in two cases, one in
which it performs slightly worse (Case 5 below) and one in which it performs
very significantly better (Case 2 below). Overall it's a wash except in those
two cases.
I've since done somewhat more involved testing on a dual-core Opteron system.
Case 1: With no itimer running, for a test with 100,000 threads, the fixed
kernel took 1428.5 seconds, 513 seconds more than the unfixed system,
all of which was spent in the system. There were twice as many
voluntary context switches with the fix as without it.
Case 2: With an itimer running at .01 second ticks and 4000 threads (the most
an unmodified kernel can handle), the fixed kernel ran the test in
eight percent of the time (5.8 seconds as opposed to 70 seconds) and
had better tick accuracy (.012 seconds per tick as opposed to .023
seconds per tick).
Case 3: A 4000-thread test with an initial timer tick of .01 second and an
interval of 10,000 seconds (i.e. a timer that ticks only once) had
very nearly the same performance in both cases: 6.3 seconds elapsed
for the fixed kernel versus 5.5 seconds for the unfixed kernel.
With fewer threads (eight in these tests), the Case 1 test ran in essentially
the same time on both the modified and unmodified kernels (5.2 seconds versus
5.8 seconds). The Case 2 test ran in about the same time as well, 5.9 seconds
versus 5.4 seconds but again with much better tick accuracy, .013 seconds per
tick versus .025 seconds per tick for the unmodified kernel.
Since the fix affected the rlimit code, I also tested soft and hard CPU limits.
Case 4: With a hard CPU limit of 20 seconds and eight threads (and an itimer
running), the modified kernel was very slightly favored in that while
it killed the process in 19.997 seconds of CPU time (5.002 seconds of
wall time), only .003 seconds of that was system time, the rest was
user time. The unmodified kernel killed the process in 20.001 seconds
of CPU (5.014 seconds of wall time) of which .016 seconds was system
time. Really, though, the results were too close to call. The results
were essentially the same with no itimer running.
Case 5: With a soft limit of 20 seconds and a hard limit of 2000 seconds
(where the hard limit would never be reached) and an itimer running,
the modified kernel exhibited worse tick accuracy than the unmodified
kernel: .050 seconds/tick versus .028 seconds/tick. Otherwise,
performance was almost indistinguishable. With no itimer running this
test exhibited virtually identical behavior and times in both cases.
In times past I did some limited performance testing. those results are below.
On a four-cpu Opteron system without this fix, a sixteen-thread test executed
in 3569.991 seconds, of which user was 3568.435s and system was 1.556s. On
the same system with the fix, user and elapsed time were about the same, but
system time dropped to 0.007 seconds. Performance with eight, four and one
thread were comparable. Interestingly, the timer ticks with the fix seemed
more accurate: The sixteen-thread test with the fix received 149543 ticks
for 0.024 seconds per tick, while the same test without the fix received 58720
for 0.061 seconds per tick. Both cases were configured for an interval of
0.01 seconds. Again, the other tests were comparable. Each thread in this
test computed the primes up to 25,000,000.
I also did a test with a large number of threads, 100,000 threads, which is
impossible without the fix. In this case each thread computed the primes only
up to 10,000 (to make the runtime manageable). System time dominated, at
1546.968 seconds out of a total 2176.906 seconds (giving a user time of
629.938s). It received 147651 ticks for 0.015 seconds per tick, still quite
accurate. There is obviously no comparable test without the fix.
Signed-off-by: Frank Mayhar <fmayhar@google.com>
Cc: Roland McGrath <roland@redhat.com>
Cc: Alexey Dobriyan <adobriyan@gmail.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-09-12 18:54:39 +02:00
|
|
|
p->cputime_expires.sched_exp = p->se.sum_exec_runtime;
|
2008-01-25 21:08:27 +01:00
|
|
|
}
|
|
|
|
}
|
2007-07-09 18:51:58 +02:00
|
|
|
|
2008-01-25 21:08:29 +01:00
|
|
|
static void task_tick_rt(struct rq *rq, struct task_struct *p, int queued)
|
2007-07-09 18:51:58 +02:00
|
|
|
{
|
2012-05-17 06:34:23 +02:00
|
|
|
struct sched_rt_entity *rt_se = &p->rt;
|
|
|
|
|
2007-12-20 15:01:17 +01:00
|
|
|
update_curr_rt(rq);
|
|
|
|
|
2008-01-25 21:08:27 +01:00
|
|
|
watchdog(rq, p);
|
|
|
|
|
2007-07-09 18:51:58 +02:00
|
|
|
/*
|
|
|
|
* RR tasks need a special form of timeslice management.
|
|
|
|
* FIFO tasks have no timeslices.
|
|
|
|
*/
|
|
|
|
if (p->policy != SCHED_RR)
|
|
|
|
return;
|
|
|
|
|
2008-01-25 21:08:27 +01:00
|
|
|
if (--p->rt.time_slice)
|
2007-07-09 18:51:58 +02:00
|
|
|
return;
|
|
|
|
|
2013-02-07 16:47:04 +01:00
|
|
|
p->rt.time_slice = sched_rr_timeslice;
|
2007-07-09 18:51:58 +02:00
|
|
|
|
2007-08-24 20:39:10 +02:00
|
|
|
/*
|
2013-10-21 14:15:43 +02:00
|
|
|
* Requeue to the end of queue if we (and all of our ancestors) are not
|
|
|
|
* the only element on the queue
|
2007-08-24 20:39:10 +02:00
|
|
|
*/
|
2012-05-17 06:34:23 +02:00
|
|
|
for_each_sched_rt_entity(rt_se) {
|
|
|
|
if (rt_se->run_list.prev != rt_se->run_list.next) {
|
|
|
|
requeue_task_rt(rq, p, 0);
|
|
|
|
set_tsk_need_resched(p);
|
|
|
|
return;
|
|
|
|
}
|
2007-08-24 20:39:10 +02:00
|
|
|
}
|
2007-07-09 18:51:58 +02:00
|
|
|
}
|
|
|
|
|
2007-10-15 17:00:08 +02:00
|
|
|
static void set_curr_task_rt(struct rq *rq)
|
|
|
|
{
|
|
|
|
struct task_struct *p = rq->curr;
|
|
|
|
|
2013-04-12 01:51:02 +02:00
|
|
|
p->se.exec_start = rq_clock_task(rq);
|
sched: create "pushable_tasks" list to limit pushing to one attempt
The RT scheduler employs a "push/pull" design to actively balance tasks
within the system (on a per disjoint cpuset basis). When a task is
awoken, it is immediately determined if there are any lower priority
cpus which should be preempted. This is opposed to the way normal
SCHED_OTHER tasks behave, which will wait for a periodic rebalancing
operation to occur before spreading out load.
When a particular RQ has more than 1 active RT task, it is said to
be in an "overloaded" state. Once this occurs, the system enters
the active balancing mode, where it will try to push the task away,
or persuade a different cpu to pull it over. The system will stay
in this state until the system falls back below the <= 1 queued RT
task per RQ.
However, the current implementation suffers from a limitation in the
push logic. Once overloaded, all tasks (other than current) on the
RQ are analyzed on every push operation, even if it was previously
unpushable (due to affinity, etc). Whats more, the operation stops
at the first task that is unpushable and will not look at items
lower in the queue. This causes two problems:
1) We can have the same tasks analyzed over and over again during each
push, which extends out the fast path in the scheduler for no
gain. Consider a RQ that has dozens of tasks that are bound to a
core. Each one of those tasks will be encountered and skipped
for each push operation while they are queued.
2) There may be lower-priority tasks under the unpushable task that
could have been successfully pushed, but will never be considered
until either the unpushable task is cleared, or a pull operation
succeeds. The net result is a potential latency source for mid
priority tasks.
This patch aims to rectify these two conditions by introducing a new
priority sorted list: "pushable_tasks". A task is added to the list
each time a task is activated or preempted. It is removed from the
list any time it is deactivated, made current, or fails to push.
This works because a task only needs to be attempted to push once.
After an initial failure to push, the other cpus will eventually try to
pull the task when the conditions are proper. This also solves the
problem that we don't completely analyze all tasks due to encountering
an unpushable tasks. Now every task will have a push attempted (when
appropriate).
This reduces latency both by shorting the critical section of the
rq->lock for certain workloads, and by making sure the algorithm
considers all eligible tasks in the system.
[ rostedt: added a couple more BUG_ONs ]
Signed-off-by: Gregory Haskins <ghaskins@novell.com>
Acked-by: Steven Rostedt <srostedt@redhat.com>
2008-12-29 15:39:53 +01:00
|
|
|
|
|
|
|
/* The running task is never eligible for pushing */
|
|
|
|
dequeue_pushable_task(rq, p);
|
2007-10-15 17:00:08 +02:00
|
|
|
}
|
|
|
|
|
2010-01-14 04:21:52 +01:00
|
|
|
static unsigned int get_rr_interval_rt(struct rq *rq, struct task_struct *task)
|
2009-09-21 03:31:53 +02:00
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Time slice is 0 for SCHED_FIFO tasks
|
|
|
|
*/
|
|
|
|
if (task->policy == SCHED_RR)
|
2013-02-07 16:47:04 +01:00
|
|
|
return sched_rr_timeslice;
|
2009-09-21 03:31:53 +02:00
|
|
|
else
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2011-10-25 10:00:11 +02:00
|
|
|
const struct sched_class rt_sched_class = {
|
2007-10-15 17:00:12 +02:00
|
|
|
.next = &fair_sched_class,
|
2007-07-09 18:51:58 +02:00
|
|
|
.enqueue_task = enqueue_task_rt,
|
|
|
|
.dequeue_task = dequeue_task_rt,
|
|
|
|
.yield_task = yield_task_rt,
|
|
|
|
|
|
|
|
.check_preempt_curr = check_preempt_curr_rt,
|
|
|
|
|
|
|
|
.pick_next_task = pick_next_task_rt,
|
|
|
|
.put_prev_task = put_prev_task_rt,
|
|
|
|
|
2007-10-24 18:23:51 +02:00
|
|
|
#ifdef CONFIG_SMP
|
2008-10-22 09:25:26 +02:00
|
|
|
.select_task_rq = select_task_rq_rt,
|
|
|
|
|
2008-01-25 21:08:07 +01:00
|
|
|
.set_cpus_allowed = set_cpus_allowed_rt,
|
2008-06-04 21:04:05 +02:00
|
|
|
.rq_online = rq_online_rt,
|
|
|
|
.rq_offline = rq_offline_rt,
|
2008-01-25 21:08:22 +01:00
|
|
|
.pre_schedule = pre_schedule_rt,
|
|
|
|
.post_schedule = post_schedule_rt,
|
2009-12-16 18:04:40 +01:00
|
|
|
.task_woken = task_woken_rt,
|
2008-01-25 21:08:22 +01:00
|
|
|
.switched_from = switched_from_rt,
|
2007-10-24 18:23:51 +02:00
|
|
|
#endif
|
2007-07-09 18:51:58 +02:00
|
|
|
|
2007-10-15 17:00:08 +02:00
|
|
|
.set_curr_task = set_curr_task_rt,
|
2007-07-09 18:51:58 +02:00
|
|
|
.task_tick = task_tick_rt,
|
2008-01-25 21:08:22 +01:00
|
|
|
|
2009-09-21 03:31:53 +02:00
|
|
|
.get_rr_interval = get_rr_interval_rt,
|
|
|
|
|
2008-01-25 21:08:22 +01:00
|
|
|
.prio_changed = prio_changed_rt,
|
|
|
|
.switched_to = switched_to_rt,
|
2007-07-09 18:51:58 +02:00
|
|
|
};
|
2008-06-19 14:22:24 +02:00
|
|
|
|
|
|
|
#ifdef CONFIG_SCHED_DEBUG
|
|
|
|
extern void print_rt_rq(struct seq_file *m, int cpu, struct rt_rq *rt_rq);
|
|
|
|
|
2011-10-25 10:00:11 +02:00
|
|
|
void print_rt_stats(struct seq_file *m, int cpu)
|
2008-06-19 14:22:24 +02:00
|
|
|
{
|
2011-05-14 08:20:02 +02:00
|
|
|
rt_rq_iter_t iter;
|
2008-06-19 14:22:24 +02:00
|
|
|
struct rt_rq *rt_rq;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
2011-05-14 08:20:02 +02:00
|
|
|
for_each_rt_rq(rt_rq, iter, cpu_rq(cpu))
|
2008-06-19 14:22:24 +02:00
|
|
|
print_rt_rq(m, cpu, rt_rq);
|
|
|
|
rcu_read_unlock();
|
|
|
|
}
|
2008-06-24 20:09:43 +02:00
|
|
|
#endif /* CONFIG_SCHED_DEBUG */
|