aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/powerpc/kvm/Makefile9
-rw-r--r--arch/powerpc/kvm/e500.h1
-rw-r--r--arch/powerpc/kvm/e500_mmu.c (renamed from arch/powerpc/kvm/e500_tlb.c)659
-rw-r--r--arch/powerpc/kvm/e500_mmu_host.c699
-rw-r--r--arch/powerpc/kvm/e500_mmu_host.h18
5 files changed, 743 insertions, 643 deletions
diff --git a/arch/powerpc/kvm/Makefile b/arch/powerpc/kvm/Makefile
index 1e473d46322c..b772eded8c26 100644
--- a/arch/powerpc/kvm/Makefile
+++ b/arch/powerpc/kvm/Makefile
@@ -10,7 +10,8 @@ common-objs-y = $(addprefix ../../../virt/kvm/, kvm_main.o coalesced_mmio.o \
10 eventfd.o) 10 eventfd.o)
11 11
12CFLAGS_44x_tlb.o := -I. 12CFLAGS_44x_tlb.o := -I.
13CFLAGS_e500_tlb.o := -I. 13CFLAGS_e500_mmu.o := -I.
14CFLAGS_e500_mmu_host.o := -I.
14CFLAGS_emulate.o := -I. 15CFLAGS_emulate.o := -I.
15 16
16common-objs-y += powerpc.o emulate.o 17common-objs-y += powerpc.o emulate.o
@@ -35,7 +36,8 @@ kvm-e500-objs := \
35 booke_emulate.o \ 36 booke_emulate.o \
36 booke_interrupts.o \ 37 booke_interrupts.o \
37 e500.o \ 38 e500.o \
38 e500_tlb.o \ 39 e500_mmu.o \
40 e500_mmu_host.o \
39 e500_emulate.o 41 e500_emulate.o
40kvm-objs-$(CONFIG_KVM_E500V2) := $(kvm-e500-objs) 42kvm-objs-$(CONFIG_KVM_E500V2) := $(kvm-e500-objs)
41 43
@@ -45,7 +47,8 @@ kvm-e500mc-objs := \
45 booke_emulate.o \ 47 booke_emulate.o \
46 bookehv_interrupts.o \ 48 bookehv_interrupts.o \
47 e500mc.o \ 49 e500mc.o \
48 e500_tlb.o \ 50 e500_mmu.o \
51 e500_mmu_host.o \
49 e500_emulate.o 52 e500_emulate.o
50kvm-objs-$(CONFIG_KVM_E500MC) := $(kvm-e500mc-objs) 53kvm-objs-$(CONFIG_KVM_E500MC) := $(kvm-e500mc-objs)
51 54
diff --git a/arch/powerpc/kvm/e500.h b/arch/powerpc/kvm/e500.h
index c70d37ed770a..41cefd43655f 100644
--- a/arch/powerpc/kvm/e500.h
+++ b/arch/powerpc/kvm/e500.h
@@ -28,6 +28,7 @@
28 28
29#define E500_TLB_VALID 1 29#define E500_TLB_VALID 1
30#define E500_TLB_BITMAP 2 30#define E500_TLB_BITMAP 2
31#define E500_TLB_TLB0 (1 << 2)
31 32
32struct tlbe_ref { 33struct tlbe_ref {
33 pfn_t pfn; 34 pfn_t pfn;
diff --git a/arch/powerpc/kvm/e500_tlb.c b/arch/powerpc/kvm/e500_mmu.c
index cf3f18012371..5c4475983f78 100644
--- a/arch/powerpc/kvm/e500_tlb.c
+++ b/arch/powerpc/kvm/e500_mmu.c
@@ -1,10 +1,11 @@
1/* 1/*
2 * Copyright (C) 2008-2011 Freescale Semiconductor, Inc. All rights reserved. 2 * Copyright (C) 2008-2013 Freescale Semiconductor, Inc. All rights reserved.
3 * 3 *
4 * Author: Yu Liu, yu.liu@freescale.com 4 * Author: Yu Liu, yu.liu@freescale.com
5 * Scott Wood, scottwood@freescale.com 5 * Scott Wood, scottwood@freescale.com
6 * Ashish Kalra, ashish.kalra@freescale.com 6 * Ashish Kalra, ashish.kalra@freescale.com
7 * Varun Sethi, varun.sethi@freescale.com 7 * Varun Sethi, varun.sethi@freescale.com
8 * Alexander Graf, agraf@suse.de
8 * 9 *
9 * Description: 10 * Description:
10 * This file is based on arch/powerpc/kvm/44x_tlb.c, 11 * This file is based on arch/powerpc/kvm/44x_tlb.c,
@@ -33,10 +34,7 @@
33#include "e500.h" 34#include "e500.h"
34#include "trace.h" 35#include "trace.h"
35#include "timing.h" 36#include "timing.h"
36 37#include "e500_mmu_host.h"
37#define to_htlb1_esel(esel) (host_tlb_params[1].entries - (esel) - 1)
38
39static struct kvmppc_e500_tlb_params host_tlb_params[E500_TLB_NUM];
40 38
41static inline unsigned int gtlb0_get_next_victim( 39static inline unsigned int gtlb0_get_next_victim(
42 struct kvmppc_vcpu_e500 *vcpu_e500) 40 struct kvmppc_vcpu_e500 *vcpu_e500)
@@ -50,174 +48,6 @@ static inline unsigned int gtlb0_get_next_victim(
50 return victim; 48 return victim;
51} 49}
52 50
53static inline unsigned int tlb1_max_shadow_size(void)
54{
55 /* reserve one entry for magic page */
56 return host_tlb_params[1].entries - tlbcam_index - 1;
57}
58
59static inline int tlbe_is_writable(struct kvm_book3e_206_tlb_entry *tlbe)
60{
61 return tlbe->mas7_3 & (MAS3_SW|MAS3_UW);
62}
63
64static inline u32 e500_shadow_mas3_attrib(u32 mas3, int usermode)
65{
66 /* Mask off reserved bits. */
67 mas3 &= MAS3_ATTRIB_MASK;
68
69#ifndef CONFIG_KVM_BOOKE_HV
70 if (!usermode) {
71 /* Guest is in supervisor mode,
72 * so we need to translate guest
73 * supervisor permissions into user permissions. */
74 mas3 &= ~E500_TLB_USER_PERM_MASK;
75 mas3 |= (mas3 & E500_TLB_SUPER_PERM_MASK) << 1;
76 }
77 mas3 |= E500_TLB_SUPER_PERM_MASK;
78#endif
79 return mas3;
80}
81
82static inline u32 e500_shadow_mas2_attrib(u32 mas2, int usermode)
83{
84#ifdef CONFIG_SMP
85 return (mas2 & MAS2_ATTRIB_MASK) | MAS2_M;
86#else
87 return mas2 & MAS2_ATTRIB_MASK;
88#endif
89}
90
91/*
92 * writing shadow tlb entry to host TLB
93 */
94static inline void __write_host_tlbe(struct kvm_book3e_206_tlb_entry *stlbe,
95 uint32_t mas0)
96{
97 unsigned long flags;
98
99 local_irq_save(flags);
100 mtspr(SPRN_MAS0, mas0);
101 mtspr(SPRN_MAS1, stlbe->mas1);
102 mtspr(SPRN_MAS2, (unsigned long)stlbe->mas2);
103 mtspr(SPRN_MAS3, (u32)stlbe->mas7_3);
104 mtspr(SPRN_MAS7, (u32)(stlbe->mas7_3 >> 32));
105#ifdef CONFIG_KVM_BOOKE_HV
106 mtspr(SPRN_MAS8, stlbe->mas8);
107#endif
108 asm volatile("isync; tlbwe" : : : "memory");
109
110#ifdef CONFIG_KVM_BOOKE_HV
111 /* Must clear mas8 for other host tlbwe's */
112 mtspr(SPRN_MAS8, 0);
113 isync();
114#endif
115 local_irq_restore(flags);
116
117 trace_kvm_booke206_stlb_write(mas0, stlbe->mas8, stlbe->mas1,
118 stlbe->mas2, stlbe->mas7_3);
119}
120
121/*
122 * Acquire a mas0 with victim hint, as if we just took a TLB miss.
123 *
124 * We don't care about the address we're searching for, other than that it's
125 * in the right set and is not present in the TLB. Using a zero PID and a
126 * userspace address means we don't have to set and then restore MAS5, or
127 * calculate a proper MAS6 value.
128 */
129static u32 get_host_mas0(unsigned long eaddr)
130{
131 unsigned long flags;
132 u32 mas0;
133
134 local_irq_save(flags);
135 mtspr(SPRN_MAS6, 0);
136 asm volatile("tlbsx 0, %0" : : "b" (eaddr & ~CONFIG_PAGE_OFFSET));
137 mas0 = mfspr(SPRN_MAS0);
138 local_irq_restore(flags);
139
140 return mas0;
141}
142
143/* sesel is for tlb1 only */
144static inline void write_host_tlbe(struct kvmppc_vcpu_e500 *vcpu_e500,
145 int tlbsel, int sesel, struct kvm_book3e_206_tlb_entry *stlbe)
146{
147 u32 mas0;
148
149 if (tlbsel == 0) {
150 mas0 = get_host_mas0(stlbe->mas2);
151 __write_host_tlbe(stlbe, mas0);
152 } else {
153 __write_host_tlbe(stlbe,
154 MAS0_TLBSEL(1) |
155 MAS0_ESEL(to_htlb1_esel(sesel)));
156 }
157}
158
159#ifdef CONFIG_KVM_E500V2
160void kvmppc_map_magic(struct kvm_vcpu *vcpu)
161{
162 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
163 struct kvm_book3e_206_tlb_entry magic;
164 ulong shared_page = ((ulong)vcpu->arch.shared) & PAGE_MASK;
165 unsigned int stid;
166 pfn_t pfn;
167
168 pfn = (pfn_t)virt_to_phys((void *)shared_page) >> PAGE_SHIFT;
169 get_page(pfn_to_page(pfn));
170
171 preempt_disable();
172 stid = kvmppc_e500_get_sid(vcpu_e500, 0, 0, 0, 0);
173
174 magic.mas1 = MAS1_VALID | MAS1_TS | MAS1_TID(stid) |
175 MAS1_TSIZE(BOOK3E_PAGESZ_4K);
176 magic.mas2 = vcpu->arch.magic_page_ea | MAS2_M;
177 magic.mas7_3 = ((u64)pfn << PAGE_SHIFT) |
178 MAS3_SW | MAS3_SR | MAS3_UW | MAS3_UR;
179 magic.mas8 = 0;
180
181 __write_host_tlbe(&magic, MAS0_TLBSEL(1) | MAS0_ESEL(tlbcam_index));
182 preempt_enable();
183}
184#endif
185
186static void inval_gtlbe_on_host(struct kvmppc_vcpu_e500 *vcpu_e500,
187 int tlbsel, int esel)
188{
189 struct kvm_book3e_206_tlb_entry *gtlbe =
190 get_entry(vcpu_e500, tlbsel, esel);
191
192 if (tlbsel == 1 &&
193 vcpu_e500->gtlb_priv[1][esel].ref.flags & E500_TLB_BITMAP) {
194 u64 tmp = vcpu_e500->g2h_tlb1_map[esel];
195 int hw_tlb_indx;
196 unsigned long flags;
197
198 local_irq_save(flags);
199 while (tmp) {
200 hw_tlb_indx = __ilog2_u64(tmp & -tmp);
201 mtspr(SPRN_MAS0,
202 MAS0_TLBSEL(1) |
203 MAS0_ESEL(to_htlb1_esel(hw_tlb_indx)));
204 mtspr(SPRN_MAS1, 0);
205 asm volatile("tlbwe");
206 vcpu_e500->h2g_tlb1_rmap[hw_tlb_indx] = 0;
207 tmp &= tmp - 1;
208 }
209 mb();
210 vcpu_e500->g2h_tlb1_map[esel] = 0;
211 vcpu_e500->gtlb_priv[1][esel].ref.flags &= ~E500_TLB_BITMAP;
212 local_irq_restore(flags);
213
214 return;
215 }
216
217 /* Guest tlbe is backed by at most one host tlbe per shadow pid. */
218 kvmppc_e500_tlbil_one(vcpu_e500, gtlbe);
219}
220
221static int tlb0_set_base(gva_t addr, int sets, int ways) 51static int tlb0_set_base(gva_t addr, int sets, int ways)
222{ 52{
223 int set_base; 53 int set_base;
@@ -296,70 +126,6 @@ static int kvmppc_e500_tlb_index(struct kvmppc_vcpu_e500 *vcpu_e500,
296 return -1; 126 return -1;
297} 127}
298 128
299static inline void kvmppc_e500_ref_setup(struct tlbe_ref *ref,
300 struct kvm_book3e_206_tlb_entry *gtlbe,
301 pfn_t pfn)
302{
303 ref->pfn = pfn;
304 ref->flags = E500_TLB_VALID;
305
306 if (tlbe_is_writable(gtlbe))
307 kvm_set_pfn_dirty(pfn);
308}
309
310static inline void kvmppc_e500_ref_release(struct tlbe_ref *ref)
311{
312 if (ref->flags & E500_TLB_VALID) {
313 trace_kvm_booke206_ref_release(ref->pfn, ref->flags);
314 ref->flags = 0;
315 }
316}
317
318static void clear_tlb1_bitmap(struct kvmppc_vcpu_e500 *vcpu_e500)
319{
320 if (vcpu_e500->g2h_tlb1_map)
321 memset(vcpu_e500->g2h_tlb1_map, 0,
322 sizeof(u64) * vcpu_e500->gtlb_params[1].entries);
323 if (vcpu_e500->h2g_tlb1_rmap)
324 memset(vcpu_e500->h2g_tlb1_rmap, 0,
325 sizeof(unsigned int) * host_tlb_params[1].entries);
326}
327
328static void clear_tlb_privs(struct kvmppc_vcpu_e500 *vcpu_e500)
329{
330 int tlbsel = 0;
331 int i;
332
333 for (i = 0; i < vcpu_e500->gtlb_params[tlbsel].entries; i++) {
334 struct tlbe_ref *ref =
335 &vcpu_e500->gtlb_priv[tlbsel][i].ref;
336 kvmppc_e500_ref_release(ref);
337 }
338}
339
340static void clear_tlb_refs(struct kvmppc_vcpu_e500 *vcpu_e500)
341{
342 int stlbsel = 1;
343 int i;
344
345 kvmppc_e500_tlbil_all(vcpu_e500);
346
347 for (i = 0; i < host_tlb_params[stlbsel].entries; i++) {
348 struct tlbe_ref *ref =
349 &vcpu_e500->tlb_refs[stlbsel][i];
350 kvmppc_e500_ref_release(ref);
351 }
352
353 clear_tlb_privs(vcpu_e500);
354}
355
356void kvmppc_core_flush_tlb(struct kvm_vcpu *vcpu)
357{
358 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
359 clear_tlb_refs(vcpu_e500);
360 clear_tlb1_bitmap(vcpu_e500);
361}
362
363static inline void kvmppc_e500_deliver_tlb_miss(struct kvm_vcpu *vcpu, 129static inline void kvmppc_e500_deliver_tlb_miss(struct kvm_vcpu *vcpu,
364 unsigned int eaddr, int as) 130 unsigned int eaddr, int as)
365{ 131{
@@ -385,216 +151,6 @@ static inline void kvmppc_e500_deliver_tlb_miss(struct kvm_vcpu *vcpu,
385 | (as ? MAS6_SAS : 0); 151 | (as ? MAS6_SAS : 0);
386} 152}
387 153
388/* TID must be supplied by the caller */
389static inline void kvmppc_e500_setup_stlbe(
390 struct kvm_vcpu *vcpu,
391 struct kvm_book3e_206_tlb_entry *gtlbe,
392 int tsize, struct tlbe_ref *ref, u64 gvaddr,
393 struct kvm_book3e_206_tlb_entry *stlbe)
394{
395 pfn_t pfn = ref->pfn;
396 u32 pr = vcpu->arch.shared->msr & MSR_PR;
397
398 BUG_ON(!(ref->flags & E500_TLB_VALID));
399
400 /* Force IPROT=0 for all guest mappings. */
401 stlbe->mas1 = MAS1_TSIZE(tsize) | get_tlb_sts(gtlbe) | MAS1_VALID;
402 stlbe->mas2 = (gvaddr & MAS2_EPN) |
403 e500_shadow_mas2_attrib(gtlbe->mas2, pr);
404 stlbe->mas7_3 = ((u64)pfn << PAGE_SHIFT) |
405 e500_shadow_mas3_attrib(gtlbe->mas7_3, pr);
406
407#ifdef CONFIG_KVM_BOOKE_HV
408 stlbe->mas8 = MAS8_TGS | vcpu->kvm->arch.lpid;
409#endif
410}
411
412static inline void kvmppc_e500_shadow_map(struct kvmppc_vcpu_e500 *vcpu_e500,
413 u64 gvaddr, gfn_t gfn, struct kvm_book3e_206_tlb_entry *gtlbe,
414 int tlbsel, struct kvm_book3e_206_tlb_entry *stlbe,
415 struct tlbe_ref *ref)
416{
417 struct kvm_memory_slot *slot;
418 unsigned long pfn = 0; /* silence GCC warning */
419 unsigned long hva;
420 int pfnmap = 0;
421 int tsize = BOOK3E_PAGESZ_4K;
422
423 /*
424 * Translate guest physical to true physical, acquiring
425 * a page reference if it is normal, non-reserved memory.
426 *
427 * gfn_to_memslot() must succeed because otherwise we wouldn't
428 * have gotten this far. Eventually we should just pass the slot
429 * pointer through from the first lookup.
430 */
431 slot = gfn_to_memslot(vcpu_e500->vcpu.kvm, gfn);
432 hva = gfn_to_hva_memslot(slot, gfn);
433
434 if (tlbsel == 1) {
435 struct vm_area_struct *vma;
436 down_read(&current->mm->mmap_sem);
437
438 vma = find_vma(current->mm, hva);
439 if (vma && hva >= vma->vm_start &&
440 (vma->vm_flags & VM_PFNMAP)) {
441 /*
442 * This VMA is a physically contiguous region (e.g.
443 * /dev/mem) that bypasses normal Linux page
444 * management. Find the overlap between the
445 * vma and the memslot.
446 */
447
448 unsigned long start, end;
449 unsigned long slot_start, slot_end;
450
451 pfnmap = 1;
452
453 start = vma->vm_pgoff;
454 end = start +
455 ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT);
456
457 pfn = start + ((hva - vma->vm_start) >> PAGE_SHIFT);
458
459 slot_start = pfn - (gfn - slot->base_gfn);
460 slot_end = slot_start + slot->npages;
461
462 if (start < slot_start)
463 start = slot_start;
464 if (end > slot_end)
465 end = slot_end;
466
467 tsize = (gtlbe->mas1 & MAS1_TSIZE_MASK) >>
468 MAS1_TSIZE_SHIFT;
469
470 /*
471 * e500 doesn't implement the lowest tsize bit,
472 * or 1K pages.
473 */
474 tsize = max(BOOK3E_PAGESZ_4K, tsize & ~1);
475
476 /*
477 * Now find the largest tsize (up to what the guest
478 * requested) that will cover gfn, stay within the
479 * range, and for which gfn and pfn are mutually
480 * aligned.
481 */
482
483 for (; tsize > BOOK3E_PAGESZ_4K; tsize -= 2) {
484 unsigned long gfn_start, gfn_end, tsize_pages;
485 tsize_pages = 1 << (tsize - 2);
486
487 gfn_start = gfn & ~(tsize_pages - 1);
488 gfn_end = gfn_start + tsize_pages;
489
490 if (gfn_start + pfn - gfn < start)
491 continue;
492 if (gfn_end + pfn - gfn > end)
493 continue;
494 if ((gfn & (tsize_pages - 1)) !=
495 (pfn & (tsize_pages - 1)))
496 continue;
497
498 gvaddr &= ~((tsize_pages << PAGE_SHIFT) - 1);
499 pfn &= ~(tsize_pages - 1);
500 break;
501 }
502 } else if (vma && hva >= vma->vm_start &&
503 (vma->vm_flags & VM_HUGETLB)) {
504 unsigned long psize = vma_kernel_pagesize(vma);
505
506 tsize = (gtlbe->mas1 & MAS1_TSIZE_MASK) >>
507 MAS1_TSIZE_SHIFT;
508
509 /*
510 * Take the largest page size that satisfies both host
511 * and guest mapping
512 */
513 tsize = min(__ilog2(psize) - 10, tsize);
514
515 /*
516 * e500 doesn't implement the lowest tsize bit,
517 * or 1K pages.
518 */
519 tsize = max(BOOK3E_PAGESZ_4K, tsize & ~1);
520 }
521
522 up_read(&current->mm->mmap_sem);
523 }
524
525 if (likely(!pfnmap)) {
526 unsigned long tsize_pages = 1 << (tsize + 10 - PAGE_SHIFT);
527 pfn = gfn_to_pfn_memslot(slot, gfn);
528 if (is_error_noslot_pfn(pfn)) {
529 printk(KERN_ERR "Couldn't get real page for gfn %lx!\n",
530 (long)gfn);
531 return;
532 }
533
534 /* Align guest and physical address to page map boundaries */
535 pfn &= ~(tsize_pages - 1);
536 gvaddr &= ~((tsize_pages << PAGE_SHIFT) - 1);
537 }
538
539 /* Drop old ref and setup new one. */
540 kvmppc_e500_ref_release(ref);
541 kvmppc_e500_ref_setup(ref, gtlbe, pfn);
542
543 kvmppc_e500_setup_stlbe(&vcpu_e500->vcpu, gtlbe, tsize,
544 ref, gvaddr, stlbe);
545
546 /* Clear i-cache for new pages */
547 kvmppc_mmu_flush_icache(pfn);
548
549 /* Drop refcount on page, so that mmu notifiers can clear it */
550 kvm_release_pfn_clean(pfn);
551}
552
553/* XXX only map the one-one case, for now use TLB0 */
554static void kvmppc_e500_tlb0_map(struct kvmppc_vcpu_e500 *vcpu_e500,
555 int esel,
556 struct kvm_book3e_206_tlb_entry *stlbe)
557{
558 struct kvm_book3e_206_tlb_entry *gtlbe;
559 struct tlbe_ref *ref;
560
561 gtlbe = get_entry(vcpu_e500, 0, esel);
562 ref = &vcpu_e500->gtlb_priv[0][esel].ref;
563
564 kvmppc_e500_shadow_map(vcpu_e500, get_tlb_eaddr(gtlbe),
565 get_tlb_raddr(gtlbe) >> PAGE_SHIFT,
566 gtlbe, 0, stlbe, ref);
567}
568
569/* Caller must ensure that the specified guest TLB entry is safe to insert into
570 * the shadow TLB. */
571/* XXX for both one-one and one-to-many , for now use TLB1 */
572static int kvmppc_e500_tlb1_map(struct kvmppc_vcpu_e500 *vcpu_e500,
573 u64 gvaddr, gfn_t gfn, struct kvm_book3e_206_tlb_entry *gtlbe,
574 struct kvm_book3e_206_tlb_entry *stlbe, int esel)
575{
576 struct tlbe_ref *ref;
577 unsigned int victim;
578
579 victim = vcpu_e500->host_tlb1_nv++;
580
581 if (unlikely(vcpu_e500->host_tlb1_nv >= tlb1_max_shadow_size()))
582 vcpu_e500->host_tlb1_nv = 0;
583
584 ref = &vcpu_e500->tlb_refs[1][victim];
585 kvmppc_e500_shadow_map(vcpu_e500, gvaddr, gfn, gtlbe, 1, stlbe, ref);
586
587 vcpu_e500->g2h_tlb1_map[esel] |= (u64)1 << victim;
588 vcpu_e500->gtlb_priv[1][esel].ref.flags |= E500_TLB_BITMAP;
589 if (vcpu_e500->h2g_tlb1_rmap[victim]) {
590 unsigned int idx = vcpu_e500->h2g_tlb1_rmap[victim];
591 vcpu_e500->g2h_tlb1_map[idx] &= ~(1ULL << victim);
592 }
593 vcpu_e500->h2g_tlb1_rmap[victim] = esel;
594
595 return victim;
596}
597
598static void kvmppc_recalc_tlb1map_range(struct kvmppc_vcpu_e500 *vcpu_e500) 154static void kvmppc_recalc_tlb1map_range(struct kvmppc_vcpu_e500 *vcpu_e500)
599{ 155{
600 int size = vcpu_e500->gtlb_params[1].entries; 156 int size = vcpu_e500->gtlb_params[1].entries;
@@ -683,8 +239,8 @@ int kvmppc_e500_emul_mt_mmucsr0(struct kvmppc_vcpu_e500 *vcpu_e500, ulong value)
683 for (esel = 0; esel < vcpu_e500->gtlb_params[1].entries; esel++) 239 for (esel = 0; esel < vcpu_e500->gtlb_params[1].entries; esel++)
684 kvmppc_e500_gtlbe_invalidate(vcpu_e500, 1, esel); 240 kvmppc_e500_gtlbe_invalidate(vcpu_e500, 1, esel);
685 241
686 /* Invalidate all vcpu id mappings */ 242 /* Invalidate all host shadow mappings */
687 kvmppc_e500_tlbil_all(vcpu_e500); 243 kvmppc_core_flush_tlb(&vcpu_e500->vcpu);
688 244
689 return EMULATE_DONE; 245 return EMULATE_DONE;
690} 246}
@@ -713,8 +269,8 @@ int kvmppc_e500_emul_tlbivax(struct kvm_vcpu *vcpu, gva_t ea)
713 kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel); 269 kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel);
714 } 270 }
715 271
716 /* Invalidate all vcpu id mappings */ 272 /* Invalidate all host shadow mappings */
717 kvmppc_e500_tlbil_all(vcpu_e500); 273 kvmppc_core_flush_tlb(&vcpu_e500->vcpu);
718 274
719 return EMULATE_DONE; 275 return EMULATE_DONE;
720} 276}
@@ -834,27 +390,11 @@ int kvmppc_e500_emul_tlbsx(struct kvm_vcpu *vcpu, gva_t ea)
834 return EMULATE_DONE; 390 return EMULATE_DONE;
835} 391}
836 392
837/* sesel is for tlb1 only */
838static void write_stlbe(struct kvmppc_vcpu_e500 *vcpu_e500,
839 struct kvm_book3e_206_tlb_entry *gtlbe,
840 struct kvm_book3e_206_tlb_entry *stlbe,
841 int stlbsel, int sesel)
842{
843 int stid;
844
845 preempt_disable();
846 stid = kvmppc_e500_get_tlb_stid(&vcpu_e500->vcpu, gtlbe);
847
848 stlbe->mas1 |= MAS1_TID(stid);
849 write_host_tlbe(vcpu_e500, stlbsel, sesel, stlbe);
850 preempt_enable();
851}
852
853int kvmppc_e500_emul_tlbwe(struct kvm_vcpu *vcpu) 393int kvmppc_e500_emul_tlbwe(struct kvm_vcpu *vcpu)
854{ 394{
855 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 395 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
856 struct kvm_book3e_206_tlb_entry *gtlbe, stlbe; 396 struct kvm_book3e_206_tlb_entry *gtlbe;
857 int tlbsel, esel, stlbsel, sesel; 397 int tlbsel, esel;
858 int recal = 0; 398 int recal = 0;
859 399
860 tlbsel = get_tlb_tlbsel(vcpu); 400 tlbsel = get_tlb_tlbsel(vcpu);
@@ -892,40 +432,16 @@ int kvmppc_e500_emul_tlbwe(struct kvm_vcpu *vcpu)
892 432
893 /* Invalidate shadow mappings for the about-to-be-clobbered TLBE. */ 433 /* Invalidate shadow mappings for the about-to-be-clobbered TLBE. */
894 if (tlbe_is_host_safe(vcpu, gtlbe)) { 434 if (tlbe_is_host_safe(vcpu, gtlbe)) {
895 u64 eaddr; 435 u64 eaddr = get_tlb_eaddr(gtlbe);
896 u64 raddr; 436 u64 raddr = get_tlb_raddr(gtlbe);
897 437
898 switch (tlbsel) { 438 if (tlbsel == 0) {
899 case 0:
900 /* TLB0 */
901 gtlbe->mas1 &= ~MAS1_TSIZE(~0); 439 gtlbe->mas1 &= ~MAS1_TSIZE(~0);
902 gtlbe->mas1 |= MAS1_TSIZE(BOOK3E_PAGESZ_4K); 440 gtlbe->mas1 |= MAS1_TSIZE(BOOK3E_PAGESZ_4K);
903
904 stlbsel = 0;
905 kvmppc_e500_tlb0_map(vcpu_e500, esel, &stlbe);
906 sesel = 0; /* unused */
907
908 break;
909
910 case 1:
911 /* TLB1 */
912 eaddr = get_tlb_eaddr(gtlbe);
913 raddr = get_tlb_raddr(gtlbe);
914
915 /* Create a 4KB mapping on the host.
916 * If the guest wanted a large page,
917 * only the first 4KB is mapped here and the rest
918 * are mapped on the fly. */
919 stlbsel = 1;
920 sesel = kvmppc_e500_tlb1_map(vcpu_e500, eaddr,
921 raddr >> PAGE_SHIFT, gtlbe, &stlbe, esel);
922 break;
923
924 default:
925 BUG();
926 } 441 }
927 442
928 write_stlbe(vcpu_e500, gtlbe, &stlbe, stlbsel, sesel); 443 /* Premap the faulting page */
444 kvmppc_mmu_map(vcpu, eaddr, raddr, index_of(tlbsel, esel));
929 } 445 }
930 446
931 kvmppc_set_exit_type(vcpu, EMULATED_TLBWE_EXITS); 447 kvmppc_set_exit_type(vcpu, EMULATED_TLBWE_EXITS);
@@ -1019,100 +535,14 @@ void kvmppc_mmu_destroy(struct kvm_vcpu *vcpu)
1019{ 535{
1020} 536}
1021 537
1022void kvmppc_mmu_map(struct kvm_vcpu *vcpu, u64 eaddr, gpa_t gpaddr,
1023 unsigned int index)
1024{
1025 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
1026 struct tlbe_priv *priv;
1027 struct kvm_book3e_206_tlb_entry *gtlbe, stlbe;
1028 int tlbsel = tlbsel_of(index);
1029 int esel = esel_of(index);
1030 int stlbsel, sesel;
1031
1032 gtlbe = get_entry(vcpu_e500, tlbsel, esel);
1033
1034 switch (tlbsel) {
1035 case 0:
1036 stlbsel = 0;
1037 sesel = 0; /* unused */
1038 priv = &vcpu_e500->gtlb_priv[tlbsel][esel];
1039
1040 /* Only triggers after clear_tlb_refs */
1041 if (unlikely(!(priv->ref.flags & E500_TLB_VALID)))
1042 kvmppc_e500_tlb0_map(vcpu_e500, esel, &stlbe);
1043 else
1044 kvmppc_e500_setup_stlbe(vcpu, gtlbe, BOOK3E_PAGESZ_4K,
1045 &priv->ref, eaddr, &stlbe);
1046 break;
1047
1048 case 1: {
1049 gfn_t gfn = gpaddr >> PAGE_SHIFT;
1050
1051 stlbsel = 1;
1052 sesel = kvmppc_e500_tlb1_map(vcpu_e500, eaddr, gfn,
1053 gtlbe, &stlbe, esel);
1054 break;
1055 }
1056
1057 default:
1058 BUG();
1059 break;
1060 }
1061
1062 write_stlbe(vcpu_e500, gtlbe, &stlbe, stlbsel, sesel);
1063}
1064
1065/************* MMU Notifiers *************/
1066
1067int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
1068{
1069 trace_kvm_unmap_hva(hva);
1070
1071 /*
1072 * Flush all shadow tlb entries everywhere. This is slow, but
1073 * we are 100% sure that we catch the to be unmapped page
1074 */
1075 kvm_flush_remote_tlbs(kvm);
1076
1077 return 0;
1078}
1079
1080int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end)
1081{
1082 /* kvm_unmap_hva flushes everything anyways */
1083 kvm_unmap_hva(kvm, start);
1084
1085 return 0;
1086}
1087
1088int kvm_age_hva(struct kvm *kvm, unsigned long hva)
1089{
1090 /* XXX could be more clever ;) */
1091 return 0;
1092}
1093
1094int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
1095{
1096 /* XXX could be more clever ;) */
1097 return 0;
1098}
1099
1100void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
1101{
1102 /* The page will get remapped properly on its next fault */
1103 kvm_unmap_hva(kvm, hva);
1104}
1105
1106/*****************************************/ 538/*****************************************/
1107 539
1108static void free_gtlb(struct kvmppc_vcpu_e500 *vcpu_e500) 540static void free_gtlb(struct kvmppc_vcpu_e500 *vcpu_e500)
1109{ 541{
1110 int i; 542 int i;
1111 543
1112 clear_tlb1_bitmap(vcpu_e500); 544 kvmppc_core_flush_tlb(&vcpu_e500->vcpu);
1113 kfree(vcpu_e500->g2h_tlb1_map); 545 kfree(vcpu_e500->g2h_tlb1_map);
1114
1115 clear_tlb_refs(vcpu_e500);
1116 kfree(vcpu_e500->gtlb_priv[0]); 546 kfree(vcpu_e500->gtlb_priv[0]);
1117 kfree(vcpu_e500->gtlb_priv[1]); 547 kfree(vcpu_e500->gtlb_priv[1]);
1118 548
@@ -1303,7 +733,7 @@ int kvm_vcpu_ioctl_dirty_tlb(struct kvm_vcpu *vcpu,
1303{ 733{
1304 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 734 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
1305 kvmppc_recalc_tlb1map_range(vcpu_e500); 735 kvmppc_recalc_tlb1map_range(vcpu_e500);
1306 clear_tlb_refs(vcpu_e500); 736 kvmppc_core_flush_tlb(vcpu);
1307 return 0; 737 return 0;
1308} 738}
1309 739
@@ -1313,37 +743,8 @@ int kvmppc_e500_tlb_init(struct kvmppc_vcpu_e500 *vcpu_e500)
1313 int entry_size = sizeof(struct kvm_book3e_206_tlb_entry); 743 int entry_size = sizeof(struct kvm_book3e_206_tlb_entry);
1314 int entries = KVM_E500_TLB0_SIZE + KVM_E500_TLB1_SIZE; 744 int entries = KVM_E500_TLB0_SIZE + KVM_E500_TLB1_SIZE;
1315 745
1316 host_tlb_params[0].entries = mfspr(SPRN_TLB0CFG) & TLBnCFG_N_ENTRY; 746 if (e500_mmu_host_init(vcpu_e500))
1317 host_tlb_params[1].entries = mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY; 747 goto err;
1318
1319 /*
1320 * This should never happen on real e500 hardware, but is
1321 * architecturally possible -- e.g. in some weird nested
1322 * virtualization case.
1323 */
1324 if (host_tlb_params[0].entries == 0 ||
1325 host_tlb_params[1].entries == 0) {
1326 pr_err("%s: need to know host tlb size\n", __func__);
1327 return -ENODEV;
1328 }
1329
1330 host_tlb_params[0].ways = (mfspr(SPRN_TLB0CFG) & TLBnCFG_ASSOC) >>
1331 TLBnCFG_ASSOC_SHIFT;
1332 host_tlb_params[1].ways = host_tlb_params[1].entries;
1333
1334 if (!is_power_of_2(host_tlb_params[0].entries) ||
1335 !is_power_of_2(host_tlb_params[0].ways) ||
1336 host_tlb_params[0].entries < host_tlb_params[0].ways ||
1337 host_tlb_params[0].ways == 0) {
1338 pr_err("%s: bad tlb0 host config: %u entries %u ways\n",
1339 __func__, host_tlb_params[0].entries,
1340 host_tlb_params[0].ways);
1341 return -ENODEV;
1342 }
1343
1344 host_tlb_params[0].sets =
1345 host_tlb_params[0].entries / host_tlb_params[0].ways;
1346 host_tlb_params[1].sets = 1;
1347 748
1348 vcpu_e500->gtlb_params[0].entries = KVM_E500_TLB0_SIZE; 749 vcpu_e500->gtlb_params[0].entries = KVM_E500_TLB0_SIZE;
1349 vcpu_e500->gtlb_params[1].entries = KVM_E500_TLB1_SIZE; 750 vcpu_e500->gtlb_params[1].entries = KVM_E500_TLB1_SIZE;
@@ -1362,18 +763,6 @@ int kvmppc_e500_tlb_init(struct kvmppc_vcpu_e500 *vcpu_e500)
1362 vcpu_e500->gtlb_offset[0] = 0; 763 vcpu_e500->gtlb_offset[0] = 0;
1363 vcpu_e500->gtlb_offset[1] = KVM_E500_TLB0_SIZE; 764 vcpu_e500->gtlb_offset[1] = KVM_E500_TLB0_SIZE;
1364 765
1365 vcpu_e500->tlb_refs[0] =
1366 kzalloc(sizeof(struct tlbe_ref) * host_tlb_params[0].entries,
1367 GFP_KERNEL);
1368 if (!vcpu_e500->tlb_refs[0])
1369 goto err;
1370
1371 vcpu_e500->tlb_refs[1] =
1372 kzalloc(sizeof(struct tlbe_ref) * host_tlb_params[1].entries,
1373 GFP_KERNEL);
1374 if (!vcpu_e500->tlb_refs[1])
1375 goto err;
1376
1377 vcpu_e500->gtlb_priv[0] = kzalloc(sizeof(struct tlbe_ref) * 766 vcpu_e500->gtlb_priv[0] = kzalloc(sizeof(struct tlbe_ref) *
1378 vcpu_e500->gtlb_params[0].entries, 767 vcpu_e500->gtlb_params[0].entries,
1379 GFP_KERNEL); 768 GFP_KERNEL);
@@ -1392,12 +781,6 @@ int kvmppc_e500_tlb_init(struct kvmppc_vcpu_e500 *vcpu_e500)
1392 if (!vcpu_e500->g2h_tlb1_map) 781 if (!vcpu_e500->g2h_tlb1_map)
1393 goto err; 782 goto err;
1394 783
1395 vcpu_e500->h2g_tlb1_rmap = kzalloc(sizeof(unsigned int) *
1396 host_tlb_params[1].entries,
1397 GFP_KERNEL);
1398 if (!vcpu_e500->h2g_tlb1_rmap)
1399 goto err;
1400
1401 /* Init TLB configuration register */ 784 /* Init TLB configuration register */
1402 vcpu->arch.tlbcfg[0] = mfspr(SPRN_TLB0CFG) & 785 vcpu->arch.tlbcfg[0] = mfspr(SPRN_TLB0CFG) &
1403 ~(TLBnCFG_N_ENTRY | TLBnCFG_ASSOC); 786 ~(TLBnCFG_N_ENTRY | TLBnCFG_ASSOC);
@@ -1416,15 +799,11 @@ int kvmppc_e500_tlb_init(struct kvmppc_vcpu_e500 *vcpu_e500)
1416 799
1417err: 800err:
1418 free_gtlb(vcpu_e500); 801 free_gtlb(vcpu_e500);
1419 kfree(vcpu_e500->tlb_refs[0]);
1420 kfree(vcpu_e500->tlb_refs[1]);
1421 return -1; 802 return -1;
1422} 803}
1423 804
1424void kvmppc_e500_tlb_uninit(struct kvmppc_vcpu_e500 *vcpu_e500) 805void kvmppc_e500_tlb_uninit(struct kvmppc_vcpu_e500 *vcpu_e500)
1425{ 806{
1426 free_gtlb(vcpu_e500); 807 free_gtlb(vcpu_e500);
1427 kfree(vcpu_e500->h2g_tlb1_rmap); 808 e500_mmu_host_uninit(vcpu_e500);
1428 kfree(vcpu_e500->tlb_refs[0]);
1429 kfree(vcpu_e500->tlb_refs[1]);
1430} 809}
diff --git a/arch/powerpc/kvm/e500_mmu_host.c b/arch/powerpc/kvm/e500_mmu_host.c
new file mode 100644
index 000000000000..a222edfb9a9b
--- /dev/null
+++ b/arch/powerpc/kvm/e500_mmu_host.c
@@ -0,0 +1,699 @@
1/*
2 * Copyright (C) 2008-2013 Freescale Semiconductor, Inc. All rights reserved.
3 *
4 * Author: Yu Liu, yu.liu@freescale.com
5 * Scott Wood, scottwood@freescale.com
6 * Ashish Kalra, ashish.kalra@freescale.com
7 * Varun Sethi, varun.sethi@freescale.com
8 * Alexander Graf, agraf@suse.de
9 *
10 * Description:
11 * This file is based on arch/powerpc/kvm/44x_tlb.c,
12 * by Hollis Blanchard <hollisb@us.ibm.com>.
13 *
14 * This program is free software; you can redistribute it and/or modify
15 * it under the terms of the GNU General Public License, version 2, as
16 * published by the Free Software Foundation.
17 */
18
19#include <linux/kernel.h>
20#include <linux/types.h>
21#include <linux/slab.h>
22#include <linux/string.h>
23#include <linux/kvm.h>
24#include <linux/kvm_host.h>
25#include <linux/highmem.h>
26#include <linux/log2.h>
27#include <linux/uaccess.h>
28#include <linux/sched.h>
29#include <linux/rwsem.h>
30#include <linux/vmalloc.h>
31#include <linux/hugetlb.h>
32#include <asm/kvm_ppc.h>
33
34#include "e500.h"
35#include "trace.h"
36#include "timing.h"
37#include "e500_mmu_host.h"
38
39#define to_htlb1_esel(esel) (host_tlb_params[1].entries - (esel) - 1)
40
41static struct kvmppc_e500_tlb_params host_tlb_params[E500_TLB_NUM];
42
43static inline unsigned int tlb1_max_shadow_size(void)
44{
45 /* reserve one entry for magic page */
46 return host_tlb_params[1].entries - tlbcam_index - 1;
47}
48
49static inline u32 e500_shadow_mas3_attrib(u32 mas3, int usermode)
50{
51 /* Mask off reserved bits. */
52 mas3 &= MAS3_ATTRIB_MASK;
53
54#ifndef CONFIG_KVM_BOOKE_HV
55 if (!usermode) {
56 /* Guest is in supervisor mode,
57 * so we need to translate guest
58 * supervisor permissions into user permissions. */
59 mas3 &= ~E500_TLB_USER_PERM_MASK;
60 mas3 |= (mas3 & E500_TLB_SUPER_PERM_MASK) << 1;
61 }
62 mas3 |= E500_TLB_SUPER_PERM_MASK;
63#endif
64 return mas3;
65}
66
67static inline u32 e500_shadow_mas2_attrib(u32 mas2, int usermode)
68{
69#ifdef CONFIG_SMP
70 return (mas2 & MAS2_ATTRIB_MASK) | MAS2_M;
71#else
72 return mas2 & MAS2_ATTRIB_MASK;
73#endif
74}
75
76/*
77 * writing shadow tlb entry to host TLB
78 */
79static inline void __write_host_tlbe(struct kvm_book3e_206_tlb_entry *stlbe,
80 uint32_t mas0)
81{
82 unsigned long flags;
83
84 local_irq_save(flags);
85 mtspr(SPRN_MAS0, mas0);
86 mtspr(SPRN_MAS1, stlbe->mas1);
87 mtspr(SPRN_MAS2, (unsigned long)stlbe->mas2);
88 mtspr(SPRN_MAS3, (u32)stlbe->mas7_3);
89 mtspr(SPRN_MAS7, (u32)(stlbe->mas7_3 >> 32));
90#ifdef CONFIG_KVM_BOOKE_HV
91 mtspr(SPRN_MAS8, stlbe->mas8);
92#endif
93 asm volatile("isync; tlbwe" : : : "memory");
94
95#ifdef CONFIG_KVM_BOOKE_HV
96 /* Must clear mas8 for other host tlbwe's */
97 mtspr(SPRN_MAS8, 0);
98 isync();
99#endif
100 local_irq_restore(flags);
101
102 trace_kvm_booke206_stlb_write(mas0, stlbe->mas8, stlbe->mas1,
103 stlbe->mas2, stlbe->mas7_3);
104}
105
106/*
107 * Acquire a mas0 with victim hint, as if we just took a TLB miss.
108 *
109 * We don't care about the address we're searching for, other than that it's
110 * in the right set and is not present in the TLB. Using a zero PID and a
111 * userspace address means we don't have to set and then restore MAS5, or
112 * calculate a proper MAS6 value.
113 */
114static u32 get_host_mas0(unsigned long eaddr)
115{
116 unsigned long flags;
117 u32 mas0;
118
119 local_irq_save(flags);
120 mtspr(SPRN_MAS6, 0);
121 asm volatile("tlbsx 0, %0" : : "b" (eaddr & ~CONFIG_PAGE_OFFSET));
122 mas0 = mfspr(SPRN_MAS0);
123 local_irq_restore(flags);
124
125 return mas0;
126}
127
128/* sesel is for tlb1 only */
129static inline void write_host_tlbe(struct kvmppc_vcpu_e500 *vcpu_e500,
130 int tlbsel, int sesel, struct kvm_book3e_206_tlb_entry *stlbe)
131{
132 u32 mas0;
133
134 if (tlbsel == 0) {
135 mas0 = get_host_mas0(stlbe->mas2);
136 __write_host_tlbe(stlbe, mas0);
137 } else {
138 __write_host_tlbe(stlbe,
139 MAS0_TLBSEL(1) |
140 MAS0_ESEL(to_htlb1_esel(sesel)));
141 }
142}
143
144/* sesel is for tlb1 only */
145static void write_stlbe(struct kvmppc_vcpu_e500 *vcpu_e500,
146 struct kvm_book3e_206_tlb_entry *gtlbe,
147 struct kvm_book3e_206_tlb_entry *stlbe,
148 int stlbsel, int sesel)
149{
150 int stid;
151
152 preempt_disable();
153 stid = kvmppc_e500_get_tlb_stid(&vcpu_e500->vcpu, gtlbe);
154
155 stlbe->mas1 |= MAS1_TID(stid);
156 write_host_tlbe(vcpu_e500, stlbsel, sesel, stlbe);
157 preempt_enable();
158}
159
160#ifdef CONFIG_KVM_E500V2
161/* XXX should be a hook in the gva2hpa translation */
162void kvmppc_map_magic(struct kvm_vcpu *vcpu)
163{
164 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
165 struct kvm_book3e_206_tlb_entry magic;
166 ulong shared_page = ((ulong)vcpu->arch.shared) & PAGE_MASK;
167 unsigned int stid;
168 pfn_t pfn;
169
170 pfn = (pfn_t)virt_to_phys((void *)shared_page) >> PAGE_SHIFT;
171 get_page(pfn_to_page(pfn));
172
173 preempt_disable();
174 stid = kvmppc_e500_get_sid(vcpu_e500, 0, 0, 0, 0);
175
176 magic.mas1 = MAS1_VALID | MAS1_TS | MAS1_TID(stid) |
177 MAS1_TSIZE(BOOK3E_PAGESZ_4K);
178 magic.mas2 = vcpu->arch.magic_page_ea | MAS2_M;
179 magic.mas7_3 = ((u64)pfn << PAGE_SHIFT) |
180 MAS3_SW | MAS3_SR | MAS3_UW | MAS3_UR;
181 magic.mas8 = 0;
182
183 __write_host_tlbe(&magic, MAS0_TLBSEL(1) | MAS0_ESEL(tlbcam_index));
184 preempt_enable();
185}
186#endif
187
188void inval_gtlbe_on_host(struct kvmppc_vcpu_e500 *vcpu_e500, int tlbsel,
189 int esel)
190{
191 struct kvm_book3e_206_tlb_entry *gtlbe =
192 get_entry(vcpu_e500, tlbsel, esel);
193 struct tlbe_ref *ref = &vcpu_e500->gtlb_priv[tlbsel][esel].ref;
194
195 /* Don't bother with unmapped entries */
196 if (!(ref->flags & E500_TLB_VALID))
197 return;
198
199 if (tlbsel == 1 && ref->flags & E500_TLB_BITMAP) {
200 u64 tmp = vcpu_e500->g2h_tlb1_map[esel];
201 int hw_tlb_indx;
202 unsigned long flags;
203
204 local_irq_save(flags);
205 while (tmp) {
206 hw_tlb_indx = __ilog2_u64(tmp & -tmp);
207 mtspr(SPRN_MAS0,
208 MAS0_TLBSEL(1) |
209 MAS0_ESEL(to_htlb1_esel(hw_tlb_indx)));
210 mtspr(SPRN_MAS1, 0);
211 asm volatile("tlbwe");
212 vcpu_e500->h2g_tlb1_rmap[hw_tlb_indx] = 0;
213 tmp &= tmp - 1;
214 }
215 mb();
216 vcpu_e500->g2h_tlb1_map[esel] = 0;
217 ref->flags &= ~(E500_TLB_BITMAP | E500_TLB_VALID);
218 local_irq_restore(flags);
219 }
220
221 if (tlbsel == 1 && ref->flags & E500_TLB_TLB0) {
222 /*
223 * TLB1 entry is backed by 4k pages. This should happen
224 * rarely and is not worth optimizing. Invalidate everything.
225 */
226 kvmppc_e500_tlbil_all(vcpu_e500);
227 ref->flags &= ~(E500_TLB_TLB0 | E500_TLB_VALID);
228 }
229
230 /* Already invalidated in between */
231 if (!(ref->flags & E500_TLB_VALID))
232 return;
233
234 /* Guest tlbe is backed by at most one host tlbe per shadow pid. */
235 kvmppc_e500_tlbil_one(vcpu_e500, gtlbe);
236
237 /* Mark the TLB as not backed by the host anymore */
238 ref->flags &= ~E500_TLB_VALID;
239}
240
241static inline int tlbe_is_writable(struct kvm_book3e_206_tlb_entry *tlbe)
242{
243 return tlbe->mas7_3 & (MAS3_SW|MAS3_UW);
244}
245
246static inline void kvmppc_e500_ref_setup(struct tlbe_ref *ref,
247 struct kvm_book3e_206_tlb_entry *gtlbe,
248 pfn_t pfn)
249{
250 ref->pfn = pfn;
251 ref->flags = E500_TLB_VALID;
252
253 if (tlbe_is_writable(gtlbe))
254 kvm_set_pfn_dirty(pfn);
255}
256
257static inline void kvmppc_e500_ref_release(struct tlbe_ref *ref)
258{
259 if (ref->flags & E500_TLB_VALID) {
260 trace_kvm_booke206_ref_release(ref->pfn, ref->flags);
261 ref->flags = 0;
262 }
263}
264
265static void clear_tlb1_bitmap(struct kvmppc_vcpu_e500 *vcpu_e500)
266{
267 if (vcpu_e500->g2h_tlb1_map)
268 memset(vcpu_e500->g2h_tlb1_map, 0,
269 sizeof(u64) * vcpu_e500->gtlb_params[1].entries);
270 if (vcpu_e500->h2g_tlb1_rmap)
271 memset(vcpu_e500->h2g_tlb1_rmap, 0,
272 sizeof(unsigned int) * host_tlb_params[1].entries);
273}
274
275static void clear_tlb_privs(struct kvmppc_vcpu_e500 *vcpu_e500)
276{
277 int tlbsel = 0;
278 int i;
279
280 for (i = 0; i < vcpu_e500->gtlb_params[tlbsel].entries; i++) {
281 struct tlbe_ref *ref =
282 &vcpu_e500->gtlb_priv[tlbsel][i].ref;
283 kvmppc_e500_ref_release(ref);
284 }
285}
286
287static void clear_tlb_refs(struct kvmppc_vcpu_e500 *vcpu_e500)
288{
289 int stlbsel = 1;
290 int i;
291
292 kvmppc_e500_tlbil_all(vcpu_e500);
293
294 for (i = 0; i < host_tlb_params[stlbsel].entries; i++) {
295 struct tlbe_ref *ref =
296 &vcpu_e500->tlb_refs[stlbsel][i];
297 kvmppc_e500_ref_release(ref);
298 }
299
300 clear_tlb_privs(vcpu_e500);
301}
302
303void kvmppc_core_flush_tlb(struct kvm_vcpu *vcpu)
304{
305 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
306 clear_tlb_refs(vcpu_e500);
307 clear_tlb1_bitmap(vcpu_e500);
308}
309
310/* TID must be supplied by the caller */
311static void kvmppc_e500_setup_stlbe(
312 struct kvm_vcpu *vcpu,
313 struct kvm_book3e_206_tlb_entry *gtlbe,
314 int tsize, struct tlbe_ref *ref, u64 gvaddr,
315 struct kvm_book3e_206_tlb_entry *stlbe)
316{
317 pfn_t pfn = ref->pfn;
318 u32 pr = vcpu->arch.shared->msr & MSR_PR;
319
320 BUG_ON(!(ref->flags & E500_TLB_VALID));
321
322 /* Force IPROT=0 for all guest mappings. */
323 stlbe->mas1 = MAS1_TSIZE(tsize) | get_tlb_sts(gtlbe) | MAS1_VALID;
324 stlbe->mas2 = (gvaddr & MAS2_EPN) |
325 e500_shadow_mas2_attrib(gtlbe->mas2, pr);
326 stlbe->mas7_3 = ((u64)pfn << PAGE_SHIFT) |
327 e500_shadow_mas3_attrib(gtlbe->mas7_3, pr);
328
329#ifdef CONFIG_KVM_BOOKE_HV
330 stlbe->mas8 = MAS8_TGS | vcpu->kvm->arch.lpid;
331#endif
332}
333
334static inline int kvmppc_e500_shadow_map(struct kvmppc_vcpu_e500 *vcpu_e500,
335 u64 gvaddr, gfn_t gfn, struct kvm_book3e_206_tlb_entry *gtlbe,
336 int tlbsel, struct kvm_book3e_206_tlb_entry *stlbe,
337 struct tlbe_ref *ref)
338{
339 struct kvm_memory_slot *slot;
340 unsigned long pfn = 0; /* silence GCC warning */
341 unsigned long hva;
342 int pfnmap = 0;
343 int tsize = BOOK3E_PAGESZ_4K;
344
345 /*
346 * Translate guest physical to true physical, acquiring
347 * a page reference if it is normal, non-reserved memory.
348 *
349 * gfn_to_memslot() must succeed because otherwise we wouldn't
350 * have gotten this far. Eventually we should just pass the slot
351 * pointer through from the first lookup.
352 */
353 slot = gfn_to_memslot(vcpu_e500->vcpu.kvm, gfn);
354 hva = gfn_to_hva_memslot(slot, gfn);
355
356 if (tlbsel == 1) {
357 struct vm_area_struct *vma;
358 down_read(&current->mm->mmap_sem);
359
360 vma = find_vma(current->mm, hva);
361 if (vma && hva >= vma->vm_start &&
362 (vma->vm_flags & VM_PFNMAP)) {
363 /*
364 * This VMA is a physically contiguous region (e.g.
365 * /dev/mem) that bypasses normal Linux page
366 * management. Find the overlap between the
367 * vma and the memslot.
368 */
369
370 unsigned long start, end;
371 unsigned long slot_start, slot_end;
372
373 pfnmap = 1;
374
375 start = vma->vm_pgoff;
376 end = start +
377 ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT);
378
379 pfn = start + ((hva - vma->vm_start) >> PAGE_SHIFT);
380
381 slot_start = pfn - (gfn - slot->base_gfn);
382 slot_end = slot_start + slot->npages;
383
384 if (start < slot_start)
385 start = slot_start;
386 if (end > slot_end)
387 end = slot_end;
388
389 tsize = (gtlbe->mas1 & MAS1_TSIZE_MASK) >>
390 MAS1_TSIZE_SHIFT;
391
392 /*
393 * e500 doesn't implement the lowest tsize bit,
394 * or 1K pages.
395 */
396 tsize = max(BOOK3E_PAGESZ_4K, tsize & ~1);
397
398 /*
399 * Now find the largest tsize (up to what the guest
400 * requested) that will cover gfn, stay within the
401 * range, and for which gfn and pfn are mutually
402 * aligned.
403 */
404
405 for (; tsize > BOOK3E_PAGESZ_4K; tsize -= 2) {
406 unsigned long gfn_start, gfn_end, tsize_pages;
407 tsize_pages = 1 << (tsize - 2);
408
409 gfn_start = gfn & ~(tsize_pages - 1);
410 gfn_end = gfn_start + tsize_pages;
411
412 if (gfn_start + pfn - gfn < start)
413 continue;
414 if (gfn_end + pfn - gfn > end)
415 continue;
416 if ((gfn & (tsize_pages - 1)) !=
417 (pfn & (tsize_pages - 1)))
418 continue;
419
420 gvaddr &= ~((tsize_pages << PAGE_SHIFT) - 1);
421 pfn &= ~(tsize_pages - 1);
422 break;
423 }
424 } else if (vma && hva >= vma->vm_start &&
425 (vma->vm_flags & VM_HUGETLB)) {
426 unsigned long psize = vma_kernel_pagesize(vma);
427
428 tsize = (gtlbe->mas1 & MAS1_TSIZE_MASK) >>
429 MAS1_TSIZE_SHIFT;
430
431 /*
432 * Take the largest page size that satisfies both host
433 * and guest mapping
434 */
435 tsize = min(__ilog2(psize) - 10, tsize);
436
437 /*
438 * e500 doesn't implement the lowest tsize bit,
439 * or 1K pages.
440 */
441 tsize = max(BOOK3E_PAGESZ_4K, tsize & ~1);
442 }
443
444 up_read(&current->mm->mmap_sem);
445 }
446
447 if (likely(!pfnmap)) {
448 unsigned long tsize_pages = 1 << (tsize + 10 - PAGE_SHIFT);
449 pfn = gfn_to_pfn_memslot(slot, gfn);
450 if (is_error_noslot_pfn(pfn)) {
451 printk(KERN_ERR "Couldn't get real page for gfn %lx!\n",
452 (long)gfn);
453 return -EINVAL;
454 }
455
456 /* Align guest and physical address to page map boundaries */
457 pfn &= ~(tsize_pages - 1);
458 gvaddr &= ~((tsize_pages << PAGE_SHIFT) - 1);
459 }
460
461 /* Drop old ref and setup new one. */
462 kvmppc_e500_ref_release(ref);
463 kvmppc_e500_ref_setup(ref, gtlbe, pfn);
464
465 kvmppc_e500_setup_stlbe(&vcpu_e500->vcpu, gtlbe, tsize,
466 ref, gvaddr, stlbe);
467
468 /* Clear i-cache for new pages */
469 kvmppc_mmu_flush_icache(pfn);
470
471 /* Drop refcount on page, so that mmu notifiers can clear it */
472 kvm_release_pfn_clean(pfn);
473
474 return 0;
475}
476
477/* XXX only map the one-one case, for now use TLB0 */
478static int kvmppc_e500_tlb0_map(struct kvmppc_vcpu_e500 *vcpu_e500, int esel,
479 struct kvm_book3e_206_tlb_entry *stlbe)
480{
481 struct kvm_book3e_206_tlb_entry *gtlbe;
482 struct tlbe_ref *ref;
483 int stlbsel = 0;
484 int sesel = 0;
485 int r;
486
487 gtlbe = get_entry(vcpu_e500, 0, esel);
488 ref = &vcpu_e500->gtlb_priv[0][esel].ref;
489
490 r = kvmppc_e500_shadow_map(vcpu_e500, get_tlb_eaddr(gtlbe),
491 get_tlb_raddr(gtlbe) >> PAGE_SHIFT,
492 gtlbe, 0, stlbe, ref);
493 if (r)
494 return r;
495
496 write_stlbe(vcpu_e500, gtlbe, stlbe, stlbsel, sesel);
497
498 return 0;
499}
500
501static int kvmppc_e500_tlb1_map_tlb1(struct kvmppc_vcpu_e500 *vcpu_e500,
502 struct tlbe_ref *ref,
503 int esel)
504{
505 unsigned int sesel = vcpu_e500->host_tlb1_nv++;
506
507 if (unlikely(vcpu_e500->host_tlb1_nv >= tlb1_max_shadow_size()))
508 vcpu_e500->host_tlb1_nv = 0;
509
510 vcpu_e500->tlb_refs[1][sesel] = *ref;
511 vcpu_e500->g2h_tlb1_map[esel] |= (u64)1 << sesel;
512 vcpu_e500->gtlb_priv[1][esel].ref.flags |= E500_TLB_BITMAP;
513 if (vcpu_e500->h2g_tlb1_rmap[sesel]) {
514 unsigned int idx = vcpu_e500->h2g_tlb1_rmap[sesel];
515 vcpu_e500->g2h_tlb1_map[idx] &= ~(1ULL << sesel);
516 }
517 vcpu_e500->h2g_tlb1_rmap[sesel] = esel;
518
519 return sesel;
520}
521
522/* Caller must ensure that the specified guest TLB entry is safe to insert into
523 * the shadow TLB. */
524/* For both one-one and one-to-many */
525static int kvmppc_e500_tlb1_map(struct kvmppc_vcpu_e500 *vcpu_e500,
526 u64 gvaddr, gfn_t gfn, struct kvm_book3e_206_tlb_entry *gtlbe,
527 struct kvm_book3e_206_tlb_entry *stlbe, int esel)
528{
529 struct tlbe_ref ref;
530 int sesel;
531 int r;
532
533 ref.flags = 0;
534 r = kvmppc_e500_shadow_map(vcpu_e500, gvaddr, gfn, gtlbe, 1, stlbe,
535 &ref);
536 if (r)
537 return r;
538
539 /* Use TLB0 when we can only map a page with 4k */
540 if (get_tlb_tsize(stlbe) == BOOK3E_PAGESZ_4K) {
541 vcpu_e500->gtlb_priv[1][esel].ref.flags |= E500_TLB_TLB0;
542 write_stlbe(vcpu_e500, gtlbe, stlbe, 0, 0);
543 return 0;
544 }
545
546 /* Otherwise map into TLB1 */
547 sesel = kvmppc_e500_tlb1_map_tlb1(vcpu_e500, &ref, esel);
548 write_stlbe(vcpu_e500, gtlbe, stlbe, 1, sesel);
549
550 return 0;
551}
552
553void kvmppc_mmu_map(struct kvm_vcpu *vcpu, u64 eaddr, gpa_t gpaddr,
554 unsigned int index)
555{
556 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
557 struct tlbe_priv *priv;
558 struct kvm_book3e_206_tlb_entry *gtlbe, stlbe;
559 int tlbsel = tlbsel_of(index);
560 int esel = esel_of(index);
561
562 gtlbe = get_entry(vcpu_e500, tlbsel, esel);
563
564 switch (tlbsel) {
565 case 0:
566 priv = &vcpu_e500->gtlb_priv[tlbsel][esel];
567
568 /* Triggers after clear_tlb_refs or on initial mapping */
569 if (!(priv->ref.flags & E500_TLB_VALID)) {
570 kvmppc_e500_tlb0_map(vcpu_e500, esel, &stlbe);
571 } else {
572 kvmppc_e500_setup_stlbe(vcpu, gtlbe, BOOK3E_PAGESZ_4K,
573 &priv->ref, eaddr, &stlbe);
574 write_stlbe(vcpu_e500, gtlbe, &stlbe, 0, 0);
575 }
576 break;
577
578 case 1: {
579 gfn_t gfn = gpaddr >> PAGE_SHIFT;
580 kvmppc_e500_tlb1_map(vcpu_e500, eaddr, gfn, gtlbe, &stlbe,
581 esel);
582 break;
583 }
584
585 default:
586 BUG();
587 break;
588 }
589}
590
591/************* MMU Notifiers *************/
592
593int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
594{
595 trace_kvm_unmap_hva(hva);
596
597 /*
598 * Flush all shadow tlb entries everywhere. This is slow, but
599 * we are 100% sure that we catch the to be unmapped page
600 */
601 kvm_flush_remote_tlbs(kvm);
602
603 return 0;
604}
605
606int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end)
607{
608 /* kvm_unmap_hva flushes everything anyways */
609 kvm_unmap_hva(kvm, start);
610
611 return 0;
612}
613
614int kvm_age_hva(struct kvm *kvm, unsigned long hva)
615{
616 /* XXX could be more clever ;) */
617 return 0;
618}
619
620int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
621{
622 /* XXX could be more clever ;) */
623 return 0;
624}
625
626void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
627{
628 /* The page will get remapped properly on its next fault */
629 kvm_unmap_hva(kvm, hva);
630}
631
632/*****************************************/
633
634int e500_mmu_host_init(struct kvmppc_vcpu_e500 *vcpu_e500)
635{
636 host_tlb_params[0].entries = mfspr(SPRN_TLB0CFG) & TLBnCFG_N_ENTRY;
637 host_tlb_params[1].entries = mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY;
638
639 /*
640 * This should never happen on real e500 hardware, but is
641 * architecturally possible -- e.g. in some weird nested
642 * virtualization case.
643 */
644 if (host_tlb_params[0].entries == 0 ||
645 host_tlb_params[1].entries == 0) {
646 pr_err("%s: need to know host tlb size\n", __func__);
647 return -ENODEV;
648 }
649
650 host_tlb_params[0].ways = (mfspr(SPRN_TLB0CFG) & TLBnCFG_ASSOC) >>
651 TLBnCFG_ASSOC_SHIFT;
652 host_tlb_params[1].ways = host_tlb_params[1].entries;
653
654 if (!is_power_of_2(host_tlb_params[0].entries) ||
655 !is_power_of_2(host_tlb_params[0].ways) ||
656 host_tlb_params[0].entries < host_tlb_params[0].ways ||
657 host_tlb_params[0].ways == 0) {
658 pr_err("%s: bad tlb0 host config: %u entries %u ways\n",
659 __func__, host_tlb_params[0].entries,
660 host_tlb_params[0].ways);
661 return -ENODEV;
662 }
663
664 host_tlb_params[0].sets =
665 host_tlb_params[0].entries / host_tlb_params[0].ways;
666 host_tlb_params[1].sets = 1;
667
668 vcpu_e500->tlb_refs[0] =
669 kzalloc(sizeof(struct tlbe_ref) * host_tlb_params[0].entries,
670 GFP_KERNEL);
671 if (!vcpu_e500->tlb_refs[0])
672 goto err;
673
674 vcpu_e500->tlb_refs[1] =
675 kzalloc(sizeof(struct tlbe_ref) * host_tlb_params[1].entries,
676 GFP_KERNEL);
677 if (!vcpu_e500->tlb_refs[1])
678 goto err;
679
680 vcpu_e500->h2g_tlb1_rmap = kzalloc(sizeof(unsigned int) *
681 host_tlb_params[1].entries,
682 GFP_KERNEL);
683 if (!vcpu_e500->h2g_tlb1_rmap)
684 goto err;
685
686 return 0;
687
688err:
689 kfree(vcpu_e500->tlb_refs[0]);
690 kfree(vcpu_e500->tlb_refs[1]);
691 return -EINVAL;
692}
693
694void e500_mmu_host_uninit(struct kvmppc_vcpu_e500 *vcpu_e500)
695{
696 kfree(vcpu_e500->h2g_tlb1_rmap);
697 kfree(vcpu_e500->tlb_refs[0]);
698 kfree(vcpu_e500->tlb_refs[1]);
699}
diff --git a/arch/powerpc/kvm/e500_mmu_host.h b/arch/powerpc/kvm/e500_mmu_host.h
new file mode 100644
index 000000000000..7624835b76c7
--- /dev/null
+++ b/arch/powerpc/kvm/e500_mmu_host.h
@@ -0,0 +1,18 @@
1/*
2 * Copyright (C) 2008-2013 Freescale Semiconductor, Inc. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License, version 2, as
6 * published by the Free Software Foundation.
7 */
8
9#ifndef KVM_E500_MMU_HOST_H
10#define KVM_E500_MMU_HOST_H
11
12void inval_gtlbe_on_host(struct kvmppc_vcpu_e500 *vcpu_e500, int tlbsel,
13 int esel);
14
15int e500_mmu_host_init(struct kvmppc_vcpu_e500 *vcpu_e500);
16void e500_mmu_host_uninit(struct kvmppc_vcpu_e500 *vcpu_e500);
17
18#endif /* KVM_E500_MMU_HOST_H */
generated by cgit v1.2.2 (git 2.25.0) at 2025-10-01 09:56:18 -0400