KVM: MMU: Concurrent guest walkers

Do not hold kvm->lock mutex across the entire pagefault code,
only acquire it in places where it is necessary, such as mmu
hash list, active list, rmap and parent pte handling.

Allow concurrent guest walkers by switching walk_addr() to use
mmap_sem in read-mode.

And get rid of the lockless __gfn_to_page.

[avi: move kvm_mmu_pte_write() locking inside the function]
[avi: add locking for real mode]
[avi: fix cmpxchg locking]

Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>
Signed-off-by: Avi Kivity <avi@qumranet.com>

1 files changed, 7 insertions, 1 deletions

diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h
index 56b88f7e83ef..7f83f5557d5e 100644
--- a/arch/x86/kvm/paging_tmpl.h
+++ b/arch/x86/kvm/paging_tmpl.h
@@ -368,11 +368,13 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
         if (r)
                 return r;
 
+        down_read(&current->mm->mmap_sem);
         /*
          * Look up the shadow pte for the faulting address.
          */
         r = FNAME(walk_addr)(&walker, vcpu, addr, write_fault, user_fault,
                              fetch_fault);
+        up_read(&current->mm->mmap_sem);
 
         /*
          * The page is not mapped by the guest.  Let the guest handle it.
@@ -384,6 +386,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
                 return 0;
         }
 
+        mutex_lock(&vcpu->kvm->lock);
         shadow_pte = FNAME(fetch)(vcpu, addr, &walker, user_fault, write_fault,
                                   &write_pt);
         pgprintk("%s: shadow pte %p %llx ptwrite %d\n", __FUNCTION__,
@@ -395,11 +398,14 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
         /*
          * mmio: emulate if accessible, otherwise its a guest fault.
          */
-        if (shadow_pte && is_io_pte(*shadow_pte))
+        if (shadow_pte && is_io_pte(*shadow_pte)) {
+                mutex_unlock(&vcpu->kvm->lock);
                 return 1;
+        }
 
         ++vcpu->stat.pf_fixed;
         kvm_mmu_audit(vcpu, "post page fault (fixed)");
+        mutex_unlock(&vcpu->kvm->lock);
 
         return write_pt;
 }