KVM: ppc: directly insert shadow mappings into the hardware TLB

Formerly, we used to maintain a per-vcpu shadow TLB and on every entry to the
guest would load this array into the hardware TLB. This consumed 1280 bytes of
memory (64 entries of 16 bytes plus a struct page pointer each), and also
required some assembly to loop over the array on every entry.

Instead of saving a copy in memory, we can just store shadow mappings directly
into the hardware TLB, accepting that the host kernel will clobber these as
part of the normal 440 TLB round robin. When we do that we need less than half
the memory, and we have decreased the exit handling time for all guest exits,
at the cost of increased number of TLB misses because the host overwrites some
guest entries.

These savings will be increased on processors with larger TLBs or which
implement intelligent flush instructions like tlbivax (which will avoid the
need to walk arrays in software).

In addition to that and to the code simplification, we have a greater chance of
leaving other host userspace mappings in the TLB, instead of forcing all
subsequent tasks to re-fault all their mappings.

Signed-off-by: Hollis Blanchard <hollisb@us.ibm.com>
Signed-off-by: Avi Kivity <avi@redhat.com>

1 files changed, 125 insertions, 131 deletions

diff --git a/arch/powerpc/kvm/44x_tlb.c b/arch/powerpc/kvm/44x_tlb.c
index d49dc66ab3c3..2981ebea3d1f 100644
--- a/arch/powerpc/kvm/44x_tlb.c
+++ b/arch/powerpc/kvm/44x_tlb.c
@@ -22,6 +22,8 @@
 #include <linux/kvm.h>
 #include <linux/kvm_host.h>
 #include <linux/highmem.h>
+
+#include <asm/tlbflush.h>
 #include <asm/mmu-44x.h>
 #include <asm/kvm_ppc.h>
 #include <asm/kvm_44x.h>
@@ -40,8 +42,6 @@
 #define PPC44x_TLB_USER_PERM_MASK (PPC44x_TLB_UX|PPC44x_TLB_UR|PPC44x_TLB_UW)
 #define PPC44x_TLB_SUPER_PERM_MASK (PPC44x_TLB_SX|PPC44x_TLB_SR|PPC44x_TLB_SW)
 
-static unsigned int kvmppc_tlb_44x_pos;
-
 #ifdef DEBUG
 void kvmppc_dump_tlbs(struct kvm_vcpu *vcpu)
 {
@@ -52,24 +52,49 @@ void kvmppc_dump_tlbs(struct kvm_vcpu *vcpu)
         printk("| %2s | %3s | %8s | %8s | %8s |\n",
                         "nr", "tid", "word0", "word1", "word2");
 
-        for (i = 0; i < PPC44x_TLB_SIZE; i++) {
+        for (i = 0; i < ARRAY_SIZE(vcpu_44x->guest_tlb); i++) {
                 tlbe = &vcpu_44x->guest_tlb[i];
                 if (tlbe->word0 & PPC44x_TLB_VALID)
                         printk(" G%2d |  %02X | %08X | %08X | %08X |\n",
                                i, tlbe->tid, tlbe->word0, tlbe->word1,
                                tlbe->word2);
         }
-
-        for (i = 0; i < PPC44x_TLB_SIZE; i++) {
-                tlbe = &vcpu_44x->shadow_tlb[i];
-                if (tlbe->word0 & PPC44x_TLB_VALID)
-                        printk(" S%2d | %02X | %08X | %08X | %08X |\n",
-                               i, tlbe->tid, tlbe->word0, tlbe->word1,
-                               tlbe->word2);
-        }
 }
 #endif
 
+static inline void kvmppc_44x_tlbie(unsigned int index)
+{
+        /* 0 <= index < 64, so the V bit is clear and we can use the index as
+         * word0. */
+        asm volatile(
+                "tlbwe %[index], %[index], 0\n"
+        :
+        : [index] "r"(index)
+        );
+}
+
+static inline void kvmppc_44x_tlbwe(unsigned int index,
+                                    struct kvmppc_44x_tlbe *stlbe)
+{
+        unsigned long tmp;
+
+        asm volatile(
+                "mfspr %[tmp], %[sprn_mmucr]\n"
+                "rlwimi %[tmp], %[tid], 0, 0xff\n"
+                "mtspr %[sprn_mmucr], %[tmp]\n"
+                "tlbwe %[word0], %[index], 0\n"
+                "tlbwe %[word1], %[index], 1\n"
+                "tlbwe %[word2], %[index], 2\n"
+                : [tmp]   "=&r"(tmp)
+                : [word0] "r"(stlbe->word0),
+                  [word1] "r"(stlbe->word1),
+                  [word2] "r"(stlbe->word2),
+                  [tid]   "r"(stlbe->tid),
+                  [index] "r"(index),
+                  [sprn_mmucr] "i"(SPRN_MMUCR)
+        );
+}
+
 static u32 kvmppc_44x_tlb_shadow_attrib(u32 attrib, int usermode)
 {
         /* We only care about the guest's permission and user bits. */
@@ -99,7 +124,7 @@ int kvmppc_44x_tlb_index(struct kvm_vcpu *vcpu, gva_t eaddr, unsigned int pid,
         int i;
 
         /* XXX Replace loop with fancy data structures. */
-        for (i = 0; i < PPC44x_TLB_SIZE; i++) {
+        for (i = 0; i < ARRAY_SIZE(vcpu_44x->guest_tlb); i++) {
                 struct kvmppc_44x_tlbe *tlbe = &vcpu_44x->guest_tlb[i];
                 unsigned int tid;
 
@@ -125,65 +150,53 @@ int kvmppc_44x_tlb_index(struct kvm_vcpu *vcpu, gva_t eaddr, unsigned int pid,
         return -1;
 }
 
-struct kvmppc_44x_tlbe *kvmppc_44x_itlb_search(struct kvm_vcpu *vcpu,
-                                               gva_t eaddr)
+int kvmppc_44x_itlb_index(struct kvm_vcpu *vcpu, gva_t eaddr)
 {
-        struct kvmppc_vcpu_44x *vcpu_44x = to_44x(vcpu);
         unsigned int as = !!(vcpu->arch.msr & MSR_IS);
-        unsigned int index;
 
-        index = kvmppc_44x_tlb_index(vcpu, eaddr, vcpu->arch.pid, as);
-        if (index == -1)
-                return NULL;
-        return &vcpu_44x->guest_tlb[index];
+        return kvmppc_44x_tlb_index(vcpu, eaddr, vcpu->arch.pid, as);
 }
 
-struct kvmppc_44x_tlbe *kvmppc_44x_dtlb_search(struct kvm_vcpu *vcpu,
-                                               gva_t eaddr)
+int kvmppc_44x_dtlb_index(struct kvm_vcpu *vcpu, gva_t eaddr)
 {
-        struct kvmppc_vcpu_44x *vcpu_44x = to_44x(vcpu);
         unsigned int as = !!(vcpu->arch.msr & MSR_DS);
-        unsigned int index;
 
-        index = kvmppc_44x_tlb_index(vcpu, eaddr, vcpu->arch.pid, as);
-        if (index == -1)
-                return NULL;
-        return &vcpu_44x->guest_tlb[index];
+        return kvmppc_44x_tlb_index(vcpu, eaddr, vcpu->arch.pid, as);
 }
 
-static int kvmppc_44x_tlbe_is_writable(struct kvmppc_44x_tlbe *tlbe)
+static void kvmppc_44x_shadow_release(struct kvmppc_vcpu_44x *vcpu_44x,
+                                      unsigned int stlb_index)
 {
-        return tlbe->word2 & (PPC44x_TLB_SW|PPC44x_TLB_UW);
-}
+        struct kvmppc_44x_shadow_ref *ref = &vcpu_44x->shadow_refs[stlb_index];
 
-static void kvmppc_44x_shadow_release(struct kvm_vcpu *vcpu,
-                                      unsigned int index)
-{
-        struct kvmppc_vcpu_44x *vcpu_44x = to_44x(vcpu);
-        struct kvmppc_44x_tlbe *stlbe = &vcpu_44x->shadow_tlb[index];
-        struct page *page = vcpu_44x->shadow_pages[index];
+        if (!ref->page)
+                return;
 
-        if (get_tlb_v(stlbe)) {
-                if (kvmppc_44x_tlbe_is_writable(stlbe))
-                        kvm_release_page_dirty(page);
-                else
-                        kvm_release_page_clean(page);
-        }
-}
+        /* Discard from the TLB. */
+        /* Note: we could actually invalidate a host mapping, if the host overwrote
+         * this TLB entry since we inserted a guest mapping. */
+        kvmppc_44x_tlbie(stlb_index);
 
-void kvmppc_core_destroy_mmu(struct kvm_vcpu *vcpu)
-{
-        int i;
+        /* Now release the page. */
+        if (ref->writeable)
+                kvm_release_page_dirty(ref->page);
+        else
+                kvm_release_page_clean(ref->page);
 
-        for (i = 0; i <= tlb_44x_hwater; i++)
-                kvmppc_44x_shadow_release(vcpu, i);
+        ref->page = NULL;
+
+        /* XXX set tlb_44x_index to stlb_index? */
+
+        KVMTRACE_1D(STLB_INVAL, &vcpu_44x->vcpu, stlb_index, handler);
 }
 
-void kvmppc_tlbe_set_modified(struct kvm_vcpu *vcpu, unsigned int i)
+void kvmppc_core_destroy_mmu(struct kvm_vcpu *vcpu)
 {
         struct kvmppc_vcpu_44x *vcpu_44x = to_44x(vcpu);
+        int i;
 
-        vcpu_44x->shadow_tlb_mod[i] = 1;
+        for (i = 0; i <= tlb_44x_hwater; i++)
+                kvmppc_44x_shadow_release(vcpu_44x, i);
 }
 
 /**
@@ -199,21 +212,24 @@ void kvmppc_tlbe_set_modified(struct kvm_vcpu *vcpu, unsigned int i)
  * the shadow TLB.
  */
 void kvmppc_mmu_map(struct kvm_vcpu *vcpu, u64 gvaddr, gpa_t gpaddr, u64 asid,
-                    u32 flags, u32 max_bytes)
+                    u32 flags, u32 max_bytes, unsigned int gtlb_index)
 {
+        struct kvmppc_44x_tlbe stlbe;
         struct kvmppc_vcpu_44x *vcpu_44x = to_44x(vcpu);
+        struct kvmppc_44x_shadow_ref *ref;
         struct page *new_page;
-        struct kvmppc_44x_tlbe *stlbe;
         hpa_t hpaddr;
         gfn_t gfn;
         unsigned int victim;
 
-        /* Future optimization: don't overwrite the TLB entry containing the
-         * current PC (or stack?). */
-        victim = kvmppc_tlb_44x_pos++;
-        if (kvmppc_tlb_44x_pos > tlb_44x_hwater)
-                kvmppc_tlb_44x_pos = 0;
-        stlbe = &vcpu_44x->shadow_tlb[victim];
+        /* Select TLB entry to clobber. Indirectly guard against races with the TLB
+         * miss handler by disabling interrupts. */
+        local_irq_disable();
+        victim = ++tlb_44x_index;
+        if (victim > tlb_44x_hwater)
+                victim = 0;
+        tlb_44x_index = victim;
+        local_irq_enable();
 
         /* Get reference to new page. */
         gfn = gpaddr >> PAGE_SHIFT;
@@ -225,10 +241,8 @@ void kvmppc_mmu_map(struct kvm_vcpu *vcpu, u64 gvaddr, gpa_t gpaddr, u64 asid,
         }
         hpaddr = page_to_phys(new_page);
 
-        /* Drop reference to old page. */
-        kvmppc_44x_shadow_release(vcpu, victim);
-
-        vcpu_44x->shadow_pages[victim] = new_page;
+        /* Invalidate any previous shadow mappings. */
+        kvmppc_44x_shadow_release(vcpu_44x, victim);
 
         /* XXX Make sure (va, size) doesn't overlap any other
          * entries. 440x6 user manual says the result would be
@@ -236,21 +250,19 @@ void kvmppc_mmu_map(struct kvm_vcpu *vcpu, u64 gvaddr, gpa_t gpaddr, u64 asid,
 
         /* XXX what about AS? */
 
-        stlbe->tid = !(asid & 0xff);
-
         /* Force TS=1 for all guest mappings. */
-        stlbe->word0 = PPC44x_TLB_VALID | PPC44x_TLB_TS;
+        stlbe.word0 = PPC44x_TLB_VALID | PPC44x_TLB_TS;
 
         if (max_bytes >= PAGE_SIZE) {
                 /* Guest mapping is larger than or equal to host page size. We can use
                  * a "native" host mapping. */
-                stlbe->word0 |= (gvaddr & PAGE_MASK) | PPC44x_TLBE_SIZE;
+                stlbe.word0 |= (gvaddr & PAGE_MASK) | PPC44x_TLBE_SIZE;
         } else {
                 /* Guest mapping is smaller than host page size. We must restrict the
                  * size of the mapping to be at most the smaller of the two, but for
                  * simplicity we fall back to a 4K mapping (this is probably what the
                  * guest is using anyways). */
-                stlbe->word0 |= (gvaddr & PAGE_MASK_4K) | PPC44x_TLB_4K;
+                stlbe.word0 |= (gvaddr & PAGE_MASK_4K) | PPC44x_TLB_4K;
 
                 /* 'hpaddr' is a host page, which is larger than the mapping we're
                  * inserting here. To compensate, we must add the in-page offset to the
@@ -258,47 +270,36 @@ void kvmppc_mmu_map(struct kvm_vcpu *vcpu, u64 gvaddr, gpa_t gpaddr, u64 asid,
                 hpaddr |= gpaddr & (PAGE_MASK ^ PAGE_MASK_4K);
         }
 
-        stlbe->word1 = (hpaddr & 0xfffffc00) | ((hpaddr >> 32) & 0xf);
-        stlbe->word2 = kvmppc_44x_tlb_shadow_attrib(flags,
+        stlbe.word1 = (hpaddr & 0xfffffc00) | ((hpaddr >> 32) & 0xf);
+        stlbe.word2 = kvmppc_44x_tlb_shadow_attrib(flags,
                                                     vcpu->arch.msr & MSR_PR);
-        kvmppc_tlbe_set_modified(vcpu, victim);
-
-        KVMTRACE_5D(STLB_WRITE, vcpu, victim,
-                        stlbe->tid, stlbe->word0, stlbe->word1, stlbe->word2,
-                        handler);
+        stlbe.tid = !(asid & 0xff);
+
+        /* Keep track of the reference so we can properly release it later. */
+        ref = &vcpu_44x->shadow_refs[victim];
+        ref->page = new_page;
+        ref->gtlb_index = gtlb_index;
+        ref->writeable = !!(stlbe.word2 & PPC44x_TLB_UW);
+        ref->tid = stlbe.tid;
+
+        /* Insert shadow mapping into hardware TLB. */
+        kvmppc_44x_tlbwe(victim, &stlbe);
+        KVMTRACE_5D(STLB_WRITE, vcpu, victim, stlbe.tid, stlbe.word0, stlbe.word1,
+                    stlbe.word2, handler);
 }
 
-static void kvmppc_mmu_invalidate(struct kvm_vcpu *vcpu, gva_t eaddr,
-                                  gva_t eend, u32 asid)
+/* For a particular guest TLB entry, invalidate the corresponding host TLB
+ * mappings and release the host pages. */
+static void kvmppc_44x_invalidate(struct kvm_vcpu *vcpu,
+                                  unsigned int gtlb_index)
 {
         struct kvmppc_vcpu_44x *vcpu_44x = to_44x(vcpu);
-        unsigned int pid = !(asid & 0xff);
         int i;
 
-        /* XXX Replace loop with fancy data structures. */
-        for (i = 0; i <= tlb_44x_hwater; i++) {
-                struct kvmppc_44x_tlbe *stlbe = &vcpu_44x->shadow_tlb[i];
-                unsigned int tid;
-
-                if (!get_tlb_v(stlbe))
-                        continue;
-
-                if (eend < get_tlb_eaddr(stlbe))
-                        continue;
-
-                if (eaddr > get_tlb_end(stlbe))
-                        continue;
-
-                tid = get_tlb_tid(stlbe);
-                if (tid && (tid != pid))
-                        continue;
-
-                kvmppc_44x_shadow_release(vcpu, i);
-                stlbe->word0 = 0;
-                kvmppc_tlbe_set_modified(vcpu, i);
-                KVMTRACE_5D(STLB_INVAL, vcpu, i,
-                                stlbe->tid, stlbe->word0, stlbe->word1,
-                                stlbe->word2, handler);
+        for (i = 0; i < ARRAY_SIZE(vcpu_44x->shadow_refs); i++) {
+                struct kvmppc_44x_shadow_ref *ref = &vcpu_44x->shadow_refs[i];
+                if (ref->gtlb_index == gtlb_index)
+                        kvmppc_44x_shadow_release(vcpu_44x, i);
         }
 }
 
@@ -321,14 +322,11 @@ void kvmppc_set_pid(struct kvm_vcpu *vcpu, u32 new_pid)
          * can't access guest kernel mappings (TID=1). When we switch to a new
          * guest PID, which will also use host PID=0, we must discard the old guest
          * userspace mappings. */
-        for (i = 0; i < ARRAY_SIZE(vcpu_44x->shadow_tlb); i++) {
-                struct kvmppc_44x_tlbe *stlbe = &vcpu_44x->shadow_tlb[i];
-
-                if (get_tlb_tid(stlbe) == 0) {
-                        kvmppc_44x_shadow_release(vcpu, i);
-                        stlbe->word0 = 0;
-                        kvmppc_tlbe_set_modified(vcpu, i);
-                }
+        for (i = 0; i < ARRAY_SIZE(vcpu_44x->shadow_refs); i++) {
+                struct kvmppc_44x_shadow_ref *ref = &vcpu_44x->shadow_refs[i];
+
+                if (ref->tid == 0)
+                        kvmppc_44x_shadow_release(vcpu_44x, i);
         }
 }
 
@@ -356,26 +354,21 @@ static int tlbe_is_host_safe(const struct kvm_vcpu *vcpu,
 int kvmppc_44x_emul_tlbwe(struct kvm_vcpu *vcpu, u8 ra, u8 rs, u8 ws)
 {
         struct kvmppc_vcpu_44x *vcpu_44x = to_44x(vcpu);
-        gva_t eaddr;
-        u64 asid;
         struct kvmppc_44x_tlbe *tlbe;
-        unsigned int index;
+        unsigned int gtlb_index;
 
-        index = vcpu->arch.gpr[ra];
-        if (index > PPC44x_TLB_SIZE) {
-                printk("%s: index %d\n", __func__, index);
+        gtlb_index = vcpu->arch.gpr[ra];
+        if (gtlb_index > KVM44x_GUEST_TLB_SIZE) {
+                printk("%s: index %d\n", __func__, gtlb_index);
                 kvmppc_dump_vcpu(vcpu);
                 return EMULATE_FAIL;
         }
 
-        tlbe = &vcpu_44x->guest_tlb[index];
+        tlbe = &vcpu_44x->guest_tlb[gtlb_index];
 
-        /* Invalidate shadow mappings for the about-to-be-clobbered TLBE. */
-        if (tlbe->word0 & PPC44x_TLB_VALID) {
-                eaddr = get_tlb_eaddr(tlbe);
-                asid = (tlbe->word0 & PPC44x_TLB_TS) | tlbe->tid;
-                kvmppc_mmu_invalidate(vcpu, eaddr, get_tlb_end(tlbe), asid);
-        }
+        /* Invalidate shadow mappings for the about-to-be-clobbered TLB entry. */
+        if (tlbe->word0 & PPC44x_TLB_VALID)
+                kvmppc_44x_invalidate(vcpu, gtlb_index);
 
         switch (ws) {
         case PPC44x_TLB_PAGEID:
@@ -396,6 +389,8 @@ int kvmppc_44x_emul_tlbwe(struct kvm_vcpu *vcpu, u8 ra, u8 rs, u8 ws)
         }
 
         if (tlbe_is_host_safe(vcpu, tlbe)) {
+                u64 asid;
+                gva_t eaddr;
                 gpa_t gpaddr;
                 u32 flags;
                 u32 bytes;
@@ -411,12 +406,11 @@ int kvmppc_44x_emul_tlbwe(struct kvm_vcpu *vcpu, u8 ra, u8 rs, u8 ws)
                 asid = (tlbe->word0 & PPC44x_TLB_TS) | tlbe->tid;
                 flags = tlbe->word2 & 0xffff;
 
-                kvmppc_mmu_map(vcpu, eaddr, gpaddr, asid, flags, bytes);
+                kvmppc_mmu_map(vcpu, eaddr, gpaddr, asid, flags, bytes, gtlb_index);
         }
 
-        KVMTRACE_5D(GTLB_WRITE, vcpu, index,
-                    tlbe->tid, tlbe->word0, tlbe->word1, tlbe->word2,
-                    handler);
+        KVMTRACE_5D(GTLB_WRITE, vcpu, gtlb_index, tlbe->tid, tlbe->word0,
+                    tlbe->word1, tlbe->word2, handler);
 
         return EMULATE_DONE;
 }
@@ -424,7 +418,7 @@ int kvmppc_44x_emul_tlbwe(struct kvm_vcpu *vcpu, u8 ra, u8 rs, u8 ws)
 int kvmppc_44x_emul_tlbsx(struct kvm_vcpu *vcpu, u8 rt, u8 ra, u8 rb, u8 rc)
 {
         u32 ea;
-        int index;
+        int gtlb_index;
         unsigned int as = get_mmucr_sts(vcpu);
         unsigned int pid = get_mmucr_stid(vcpu);
 
@@ -432,14 +426,14 @@ int kvmppc_44x_emul_tlbsx(struct kvm_vcpu *vcpu, u8 rt, u8 ra, u8 rb, u8 rc)
         if (ra)
                 ea += vcpu->arch.gpr[ra];
 
-        index = kvmppc_44x_tlb_index(vcpu, ea, pid, as);
+        gtlb_index = kvmppc_44x_tlb_index(vcpu, ea, pid, as);
         if (rc) {
-                if (index < 0)
+                if (gtlb_index < 0)
                         vcpu->arch.cr &= ~0x20000000;
                 else
                         vcpu->arch.cr |= 0x20000000;
         }
-        vcpu->arch.gpr[rt] = index;
+        vcpu->arch.gpr[rt] = gtlb_index;
 
         return EMULATE_DONE;
 }