KVM: use yield_to instead of sleep in kvm_vcpu_on_spin

Instead of sleeping in kvm_vcpu_on_spin, which can cause gigantic
slowdowns of certain workloads, we instead use yield_to to get
another VCPU in the same KVM guest to run sooner.

This seems to give a 10-15% speedup in certain workloads.

Signed-off-by: Rik van Riel <riel@redhat.com>
Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>
Signed-off-by: Avi Kivity <avi@redhat.com>

1 files changed, 47 insertions, 10 deletions

diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index bc8bfd15ab71..2dc53a6dc285 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -1484,18 +1484,55 @@ void kvm_resched(struct kvm_vcpu *vcpu)
 }
 EXPORT_SYMBOL_GPL(kvm_resched);
 
-void kvm_vcpu_on_spin(struct kvm_vcpu *vcpu)
+void kvm_vcpu_on_spin(struct kvm_vcpu *me)
 {
-        ktime_t expires;
-        DEFINE_WAIT(wait);
-
-        prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
-
-        /* Sleep for 100 us, and hope lock-holder got scheduled */
-        expires = ktime_add_ns(ktime_get(), 100000UL);
-        schedule_hrtimeout(&expires, HRTIMER_MODE_ABS);
+        struct kvm *kvm = me->kvm;
+        struct kvm_vcpu *vcpu;
+        int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
+        int yielded = 0;
+        int pass;
+        int i;
 
-        finish_wait(&vcpu->wq, &wait);
+        /*
+         * We boost the priority of a VCPU that is runnable but not
+         * currently running, because it got preempted by something
+         * else and called schedule in __vcpu_run.  Hopefully that
+         * VCPU is holding the lock that we need and will release it.
+         * We approximate round-robin by starting at the last boosted VCPU.
+         */
+        for (pass = 0; pass < 2 && !yielded; pass++) {
+                kvm_for_each_vcpu(i, vcpu, kvm) {
+                        struct task_struct *task = NULL;
+                        struct pid *pid;
+                        if (!pass && i < last_boosted_vcpu) {
+                                i = last_boosted_vcpu;
+                                continue;
+                        } else if (pass && i > last_boosted_vcpu)
+                                break;
+                        if (vcpu == me)
+                                continue;
+                        if (waitqueue_active(&vcpu->wq))
+                                continue;
+                        rcu_read_lock();
+                        pid = rcu_dereference(vcpu->pid);
+                        if (pid)
+                                task = get_pid_task(vcpu->pid, PIDTYPE_PID);
+                        rcu_read_unlock();
+                        if (!task)
+                                continue;
+                        if (task->flags & PF_VCPU) {
+                                put_task_struct(task);
+                                continue;
+                        }
+                        if (yield_to(task, 1)) {
+                                put_task_struct(task);
+                                kvm->last_boosted_vcpu = i;
+                                yielded = 1;
+                                break;
+                        }
+                        put_task_struct(task);
+                }
+        }
 }
 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);