OpenE2K
/
linux
6
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

KVM: use simple waitqueue for vcpu->wq 

The problem:

On -RT, an emulated LAPIC timer instances has the following path:

1) hard interrupt
2) ksoftirqd is scheduled
3) ksoftirqd wakes up vcpu thread
4) vcpu thread is scheduled

This extra context switch introduces unnecessary latency in the
LAPIC path for a KVM guest.

The solution:

Allow waking up vcpu thread from hardirq context,
thus avoiding the need for ksoftirqd to be scheduled.

Normal waitqueues make use of spinlocks, which on -RT
are sleepable locks. Therefore, waking up a waitqueue
waiter involves locking a sleeping lock, which
is not allowed from hard interrupt context.

cyclictest command line:

This patch reduces the average latency in my tests from 14us to 11us.

Cc: stable-rt@vger.kernel.org
Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
This commit is contained in:
Marcelo Tosatti 2015-04-08 20:33:24 -03:00 committed by Alibek Omarov
parent 74bfead8bc
commit ff8bba4038
9 changed files with 32 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
arch/arm/kvm/arm.c
View File

@ -458,9 +458,9 @@ static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu)
static void vcpu_pause(struct kvm_vcpu *vcpu)
{
wait_queue_head_t *wq = kvm_arch_vcpu_wq(vcpu);
struct swait_head *wq = kvm_arch_vcpu_wq(vcpu);
wait_event_interruptible(*wq, !vcpu->arch.pause);
swait_event_interruptible(*wq, !vcpu->arch.pause);
}
static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu)

4
arch/arm/kvm/psci.c
View File

@ -36,7 +36,7 @@ static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu)
{
struct kvm *kvm = source_vcpu->kvm;
struct kvm_vcpu *vcpu = NULL, *tmp;
wait_queue_head_t *wq;
struct swait_head *wq;
unsigned long cpu_id;
unsigned long mpidr;
phys_addr_t target_pc;
@ -80,7 +80,7 @@ static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu)
smp_mb(); /* Make sure the above is visible */
wq = kvm_arch_vcpu_wq(vcpu);
wake_up_interruptible(wq);
swait_wake_interruptible(wq);
return KVM_PSCI_RET_SUCCESS;
}

4
arch/powerpc/include/asm/kvm_host.h
View File

@ -295,7 +295,7 @@ struct kvmppc_vcore {
u8 in_guest;
struct list_head runnable_threads;
spinlock_t lock;
wait_queue_head_t wq;
struct swait_head wq;
u64 stolen_tb;
u64 preempt_tb;
struct kvm_vcpu *runner;
@ -612,7 +612,7 @@ struct kvm_vcpu_arch {
u8 prodded;
u32 last_inst;
wait_queue_head_t *wqp;
struct swait_head *wqp;
struct kvmppc_vcore *vcore;
int ret;
int trap;

20
arch/powerpc/kvm/book3s_hv.c
View File

@ -74,11 +74,11 @@ static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu)
{
int me;
int cpu = vcpu->cpu;
wait_queue_head_t *wqp;
struct swait_head *wqp;
wqp = kvm_arch_vcpu_wq(vcpu);
if (waitqueue_active(wqp)) {
wake_up_interruptible(wqp);
if (swaitqueue_active(wqp)) {
swait_wake_interruptible(wqp);
++vcpu->stat.halt_wakeup;
}
@ -583,8 +583,8 @@ int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu)
tvcpu->arch.prodded = 1;
smp_mb();
if (vcpu->arch.ceded) {
if (waitqueue_active(&vcpu->wq)) {
wake_up_interruptible(&vcpu->wq);
if (swaitqueue_active(&vcpu->wq)) {
swait_wake_interruptible(&vcpu->wq);
vcpu->stat.halt_wakeup++;
}
}
@ -1199,7 +1199,7 @@ static struct kvm_vcpu *kvmppc_core_vcpu_create_hv(struct kvm *kvm,
if (vcore) {
INIT_LIST_HEAD(&vcore->runnable_threads);
spin_lock_init(&vcore->lock);
init_waitqueue_head(&vcore->wq);
init_swait_head(&vcore->wq);
vcore->preempt_tb = TB_NIL;
vcore->lpcr = kvm->arch.lpcr;
vcore->first_vcpuid = core * threads_per_core;
@ -1566,13 +1566,13 @@ static void kvmppc_wait_for_exec(struct kvm_vcpu *vcpu, int wait_state)
*/
static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc)
{
DEFINE_WAIT(wait);
DEFINE_SWAITER(wait);
prepare_to_wait(&vc->wq, &wait, TASK_INTERRUPTIBLE);
swait_prepare(&vc->wq, &wait, TASK_INTERRUPTIBLE);
vc->vcore_state = VCORE_SLEEPING;
spin_unlock(&vc->lock);
schedule();
finish_wait(&vc->wq, &wait);
swait_finish(&vc->wq, &wait);
spin_lock(&vc->lock);
vc->vcore_state = VCORE_INACTIVE;
}
@ -1613,7 +1613,7 @@ static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
kvmppc_create_dtl_entry(vcpu, vc);
kvmppc_start_thread(vcpu);
} else if (vc->vcore_state == VCORE_SLEEPING) {
wake_up(&vc->wq);
swait_wake(&vc->wq);
}
}

2
arch/s390/include/asm/kvm_host.h
View File

@ -233,7 +233,7 @@ struct kvm_s390_local_interrupt {
atomic_t active;
struct kvm_s390_float_interrupt *float_int;
int timer_due; /* event indicator for waitqueue below */
wait_queue_head_t *wq;
struct swait_head *wq;
atomic_t *cpuflags;
unsigned int action_bits;
};

6
arch/x86/kvm/lapic.c
View File

@ -1567,7 +1567,7 @@ static enum hrtimer_restart apic_timer_fn(struct hrtimer *data)
struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer);
struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer);
struct kvm_vcpu *vcpu = apic->vcpu;
wait_queue_head_t *q = &vcpu->wq;
struct swait_head *q = &vcpu->wq;
/*
* There is a race window between reading and incrementing, but we do
@ -1581,8 +1581,8 @@ static enum hrtimer_restart apic_timer_fn(struct hrtimer *data)
kvm_make_request(KVM_REQ_PENDING_TIMER, vcpu);
}
if (waitqueue_active(q))
wake_up_interruptible(q);
if (swaitqueue_active(q))
swait_wake_interruptible(q);
if (lapic_is_periodic(apic)) {
hrtimer_add_expires_ns(&ktimer->timer, ktimer->period);

4
include/linux/kvm_host.h
View File

@ -231,7 +231,7 @@ struct kvm_vcpu {
int fpu_active;
int guest_fpu_loaded, guest_xcr0_loaded;
wait_queue_head_t wq;
struct swait_head wq;
struct pid *pid;
int sigset_active;
sigset_t sigset;
@ -677,7 +677,7 @@ static inline bool kvm_arch_has_noncoherent_dma(struct kvm *kvm)
}
#endif
static inline wait_queue_head_t *kvm_arch_vcpu_wq(struct kvm_vcpu *vcpu)
static inline struct swait_head *kvm_arch_vcpu_wq(struct kvm_vcpu *vcpu)
{
#ifdef __KVM_HAVE_ARCH_WQP
return vcpu->arch.wqp;

4
virt/kvm/async_pf.c
View File

@ -82,8 +82,8 @@ static void async_pf_execute(struct work_struct *work)
trace_kvm_async_pf_completed(addr, gva);
if (waitqueue_active(&vcpu->wq))
wake_up_interruptible(&vcpu->wq);
if (swaitqueue_active(&vcpu->wq))
swait_wake_interruptible(&vcpu->wq);
mmput(mm);
kvm_put_kvm(vcpu->kvm);

16
virt/kvm/kvm_main.c
View File

@ -219,7 +219,7 @@ int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
vcpu->kvm = kvm;
vcpu->vcpu_id = id;
vcpu->pid = NULL;
init_waitqueue_head(&vcpu->wq);
init_swait_head(&vcpu->wq);
kvm_async_pf_vcpu_init(vcpu);
page = alloc_page(GFP_KERNEL | __GFP_ZERO);
@ -1675,10 +1675,10 @@ EXPORT_SYMBOL_GPL(mark_page_dirty);
*/
void kvm_vcpu_block(struct kvm_vcpu *vcpu)
{
DEFINE_WAIT(wait);
DEFINE_SWAITER(wait);
for (;;) {
prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
swait_prepare(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
if (kvm_arch_vcpu_runnable(vcpu)) {
kvm_make_request(KVM_REQ_UNHALT, vcpu);
@ -1692,7 +1692,7 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu)
schedule();
}
finish_wait(&vcpu->wq, &wait);
swait_finish(&vcpu->wq, &wait);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_block);
@ -1704,11 +1704,11 @@ void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
{
int me;
int cpu = vcpu->cpu;
wait_queue_head_t *wqp;
struct swait_head *wqp;
wqp = kvm_arch_vcpu_wq(vcpu);
if (waitqueue_active(wqp)) {
wake_up_interruptible(wqp);
if (swaitqueue_active(wqp)) {
swait_wake_interruptible(wqp);
++vcpu->stat.halt_wakeup;
}
@ -1814,7 +1814,7 @@ void kvm_vcpu_on_spin(struct kvm_vcpu *me)
continue;
if (vcpu == me)
continue;
if (waitqueue_active(&vcpu->wq))
if (swaitqueue_active(&vcpu->wq))
continue;
if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
continue;